From d8bbc7858622b6d9c278469aab701ca0b609cddf Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Wed, 15 May 2024 05:35:49 +0200 Subject: Merging upstream version 126.0. Signed-off-by: Daniel Baumann --- .../audio_thread_priority/.cargo-checksum.json | 2 +- third_party/rust/audio_thread_priority/Cargo.toml | 4 +- third_party/rust/audio_thread_priority/src/lib.rs | 9 + .../rust/audio_thread_priority/src/rt_win.rs | 233 +++- .../rust/audioipc2-client/.cargo-checksum.json | 2 +- third_party/rust/audioipc2-client/Cargo.toml | 2 +- .../rust/audioipc2-server/.cargo-checksum.json | 2 +- third_party/rust/audioipc2-server/Cargo.toml | 2 +- third_party/rust/audioipc2-server/src/lib.rs | 4 +- third_party/rust/audioipc2-server/src/server.rs | 2 +- third_party/rust/audioipc2/.cargo-checksum.json | 2 +- third_party/rust/audioipc2/Cargo.toml | 2 +- third_party/rust/audioipc2/src/codec.rs | 2 +- third_party/rust/audioipc2/src/messages.rs | 2 +- third_party/rust/audioipc2/src/sys/unix/cmsg.rs | 2 +- third_party/rust/bumpalo/.cargo-checksum.json | 2 +- third_party/rust/bumpalo/CHANGELOG.md | 124 ++- third_party/rust/bumpalo/Cargo.toml | 11 +- third_party/rust/bumpalo/README.md | 23 +- third_party/rust/bumpalo/src/alloc.rs | 2 +- .../rust/bumpalo/src/collections/raw_vec.rs | 86 +- third_party/rust/bumpalo/src/collections/string.rs | 17 +- third_party/rust/bumpalo/src/collections/vec.rs | 148 +++ third_party/rust/bumpalo/src/lib.rs | 192 ++-- .../rust/coreaudio-sys-utils/.cargo-checksum.json | 2 +- .../src/audio_device_extensions.rs | 2 + .../rust/coreaudio-sys-utils/src/audio_object.rs | 4 + .../rust/coreaudio-sys-utils/src/audio_unit.rs | 13 + .../rust/coreaudio-sys-utils/src/dispatch.rs | 268 ++++- .../rust/cubeb-coreaudio/.cargo-checksum.json | 2 +- .../cubeb-coreaudio/.github/workflows/test.yml | 30 +- third_party/rust/cubeb-coreaudio/Cargo.toml | 1 + .../rust/cubeb-coreaudio/run_device_tests.sh | 2 - third_party/rust/cubeb-coreaudio/run_tests.sh | 5 +- .../src/backend/aggregate_device.rs | 45 +- .../rust/cubeb-coreaudio/src/backend/mod.rs | 834 ++++++++++++--- .../src/backend/tests/aggregate_device.rs | 347 +++--- .../rust/cubeb-coreaudio/src/backend/tests/api.rs | 444 ++++++-- .../src/backend/tests/device_change.rs | 36 +- .../src/backend/tests/device_property.rs | 176 ++-- .../src/backend/tests/interfaces.rs | 697 +++++++++++- .../cubeb-coreaudio/src/backend/tests/manual.rs | 288 +++-- .../cubeb-coreaudio/src/backend/tests/parallel.rs | 18 +- .../rust/cubeb-coreaudio/src/backend/tests/tone.rs | 4 +- .../cubeb-coreaudio/src/backend/tests/utils.rs | 507 +++++---- third_party/rust/d3d12/.cargo-checksum.json | 2 +- third_party/rust/d3d12/Cargo.toml | 2 +- .../rust/embed-manifest/.cargo-checksum.json | 1 + third_party/rust/embed-manifest/CHANGELOG.md | 59 ++ third_party/rust/embed-manifest/Cargo.toml | 38 + third_party/rust/embed-manifest/LICENSE | 23 + third_party/rust/embed-manifest/README.md | 62 ++ third_party/rust/embed-manifest/rustfmt.toml | 2 + third_party/rust/embed-manifest/src/embed/coff.rs | 192 ++++ third_party/rust/embed-manifest/src/embed/error.rs | 57 + third_party/rust/embed-manifest/src/embed/mod.rs | 139 +++ third_party/rust/embed-manifest/src/embed/test.rs | 173 +++ third_party/rust/embed-manifest/src/lib.rs | 134 +++ .../rust/embed-manifest/src/manifest/mod.rs | 882 ++++++++++++++++ .../rust/embed-manifest/src/manifest/test.rs | 117 ++ .../rust/embed-manifest/src/manifest/xml.rs | 140 +++ .../embed-manifest/testdata/sample.exe.manifest | 11 + .../rust/error-support/.cargo-checksum.json | 2 +- third_party/rust/error-support/Cargo.toml | 4 +- third_party/rust/glean-core/.cargo-checksum.json | 2 +- third_party/rust/glean-core/Cargo.toml | 6 +- .../rust/glean-core/src/common_metric_data.rs | 1 - third_party/rust/glean-core/src/core/mod.rs | 9 +- third_party/rust/glean-core/src/database/mod.rs | 1 - third_party/rust/glean-core/src/debug.rs | 1 - third_party/rust/glean-core/src/dispatcher/mod.rs | 5 +- third_party/rust/glean-core/src/error_recording.rs | 1 - .../rust/glean-core/src/event_database/mod.rs | 3 +- third_party/rust/glean-core/src/glean.udl | 1 + third_party/rust/glean-core/src/histogram/mod.rs | 1 - third_party/rust/glean-core/src/internal_pings.rs | 11 +- third_party/rust/glean-core/src/lib.rs | 5 +- third_party/rust/glean-core/src/lib_unit_tests.rs | 82 +- third_party/rust/glean-core/src/metrics/event.rs | 16 +- .../rust/glean-core/src/metrics/memory_unit.rs | 2 - .../src/metrics/metrics_enabled_config.rs | 2 +- third_party/rust/glean-core/src/metrics/ping.rs | 31 + third_party/rust/glean-core/src/metrics/string.rs | 2 - third_party/rust/glean-core/src/metrics/text.rs | 2 - .../rust/glean-core/src/metrics/time_unit.rs | 1 - .../rust/glean-core/src/metrics/timespan.rs | 1 - .../glean-core/src/metrics/timing_distribution.rs | 2 +- third_party/rust/glean-core/src/metrics/url.rs | 2 - third_party/rust/glean-core/src/storage/mod.rs | 4 +- third_party/rust/glean-core/src/traits/event.rs | 1 - .../rust/glean-core/src/upload/directory.rs | 2 - third_party/rust/glean-core/src/upload/mod.rs | 4 - third_party/rust/glean-core/src/upload/request.rs | 2 +- third_party/rust/glean-core/tests/common/mod.rs | 1 + third_party/rust/glean-core/tests/event.rs | 1 + third_party/rust/glean-core/tests/ping_maker.rs | 2 + third_party/rust/glean/.cargo-checksum.json | 2 +- third_party/rust/glean/Cargo.toml | 4 +- third_party/rust/glean/src/common_test.rs | 1 - third_party/rust/glean/src/configuration.rs | 12 + third_party/rust/glean/src/lib.rs | 1 + third_party/rust/glean/tests/schema.rs | 3 +- third_party/rust/goblin/.cargo-checksum.json | 2 +- third_party/rust/goblin/CHANGELOG.md | 27 +- third_party/rust/goblin/Cargo.toml | 10 +- third_party/rust/goblin/README.md | 7 +- third_party/rust/goblin/src/elf/reloc.rs | 2 +- third_party/rust/goblin/src/error.rs | 7 + third_party/rust/goblin/src/lib.rs | 24 +- third_party/rust/goblin/src/mach/load_command.rs | 3 + third_party/rust/goblin/src/pe/authenticode.rs | 200 +++- .../rust/goblin/src/pe/certificate_table.rs | 28 +- third_party/rust/goblin/src/pe/data_directories.rs | 175 +-- third_party/rust/goblin/src/pe/header.rs | 150 ++- third_party/rust/goblin/src/pe/mod.rs | 249 ++++- third_party/rust/goblin/src/pe/optional_header.rs | 83 ++ third_party/rust/goblin/src/pe/options.rs | 10 +- third_party/rust/goblin/src/pe/section_table.rs | 41 +- third_party/rust/goblin/src/pe/symbol.rs | 9 + third_party/rust/goblin/src/pe/utils.rs | 16 + third_party/rust/goblin/src/strtab.rs | 5 + third_party/rust/naga/.cargo-checksum.json | 2 +- third_party/rust/naga/Cargo.toml | 4 +- third_party/rust/naga/README.md | 4 +- third_party/rust/naga/src/back/mod.rs | 58 +- third_party/rust/naga/src/back/spv/index.rs | 116 +- third_party/rust/naga/src/back/spv/mod.rs | 9 + third_party/rust/naga/src/back/spv/writer.rs | 70 +- third_party/rust/naga/src/front/glsl/variables.rs | 2 +- third_party/rust/naga/src/span.rs | 9 +- third_party/rust/naga/src/valid/analyzer.rs | 69 +- third_party/rust/naga/src/valid/type.rs | 2 +- third_party/rust/neqo-common/.cargo-checksum.json | 2 +- third_party/rust/neqo-common/Cargo.toml | 28 +- third_party/rust/neqo-common/benches/timer.rs | 39 + third_party/rust/neqo-common/src/datagram.rs | 15 +- third_party/rust/neqo-common/src/lib.rs | 2 - third_party/rust/neqo-common/src/log.rs | 21 +- third_party/rust/neqo-common/src/timer.rs | 46 +- third_party/rust/neqo-common/src/tos.rs | 48 +- third_party/rust/neqo-common/src/udp.rs | 222 ---- third_party/rust/neqo-crypto/.cargo-checksum.json | 2 +- third_party/rust/neqo-crypto/Cargo.toml | 8 +- .../rust/neqo-crypto/bindings/bindings.toml | 5 - third_party/rust/neqo-crypto/bindings/mozpkix.hpp | 1 - third_party/rust/neqo-crypto/build.rs | 105 +- third_party/rust/neqo-crypto/min_version.txt | 1 + third_party/rust/neqo-crypto/src/aead.rs | 8 +- third_party/rust/neqo-crypto/src/aead_fuzzing.rs | 103 -- third_party/rust/neqo-crypto/src/aead_null.rs | 78 ++ third_party/rust/neqo-crypto/src/agent.rs | 11 +- third_party/rust/neqo-crypto/src/err.rs | 30 +- third_party/rust/neqo-crypto/src/lib.rs | 68 +- third_party/rust/neqo-crypto/src/min_version.rs | 9 + third_party/rust/neqo-crypto/src/selfencrypt.rs | 2 +- third_party/rust/neqo-crypto/src/time.rs | 4 +- third_party/rust/neqo-crypto/tests/aead.rs | 3 +- third_party/rust/neqo-crypto/tests/init.rs | 51 +- third_party/rust/neqo-crypto/tests/selfencrypt.rs | 6 +- third_party/rust/neqo-http3/.cargo-checksum.json | 2 +- third_party/rust/neqo-http3/Cargo.toml | 8 +- third_party/rust/neqo-http3/src/connection.rs | 20 +- .../rust/neqo-http3/src/connection_client.rs | 17 +- .../rust/neqo-http3/src/connection_server.rs | 17 +- third_party/rust/neqo-http3/src/recv_message.rs | 2 +- third_party/rust/neqo-http3/src/send_message.rs | 6 +- third_party/rust/neqo-http3/src/server_events.rs | 8 +- third_party/rust/neqo-qpack/.cargo-checksum.json | 2 +- third_party/rust/neqo-qpack/Cargo.toml | 2 +- third_party/rust/neqo-qpack/src/table.rs | 2 +- .../rust/neqo-transport/.cargo-checksum.json | 2 +- third_party/rust/neqo-transport/Cargo.toml | 4 +- .../rust/neqo-transport/benches/range_tracker.rs | 16 +- .../neqo-transport/benches/rx_stream_orderer.rs | 4 +- .../rust/neqo-transport/benches/transfer.rs | 16 +- .../rust/neqo-transport/src/cc/classic_cc.rs | 27 +- .../rust/neqo-transport/src/connection/dump.rs | 16 +- .../rust/neqo-transport/src/connection/mod.rs | 160 ++- .../rust/neqo-transport/src/connection/params.rs | 13 - .../rust/neqo-transport/src/connection/state.rs | 5 +- .../neqo-transport/src/connection/tests/fuzzing.rs | 42 - .../src/connection/tests/handshake.rs | 11 +- .../neqo-transport/src/connection/tests/mod.rs | 85 +- .../neqo-transport/src/connection/tests/null.rs | 42 + .../neqo-transport/src/connection/tests/stream.rs | 6 - third_party/rust/neqo-transport/src/crypto.rs | 98 +- third_party/rust/neqo-transport/src/frame.rs | 86 +- third_party/rust/neqo-transport/src/lib.rs | 8 +- third_party/rust/neqo-transport/src/packet/mod.rs | 99 +- .../rust/neqo-transport/src/packet/retry.rs | 1 - third_party/rust/neqo-transport/src/path.rs | 2 +- third_party/rust/neqo-transport/src/qlog.rs | 347 +++--- third_party/rust/neqo-transport/src/stats.rs | 6 +- .../rust/neqo-transport/tests/common/mod.rs | 9 +- .../rust/neqo-transport/tests/conn_vectors.rs | 2 +- .../rust/neqo-transport/tests/connection.rs | 70 ++ third_party/rust/neqo-transport/tests/retry.rs | 2 +- .../rust/oneshot-uniffi/.cargo-checksum.json | 2 +- third_party/rust/oneshot-uniffi/CHANGELOG.md | 7 + third_party/rust/oneshot-uniffi/Cargo.lock | 2 +- third_party/rust/oneshot-uniffi/Cargo.toml | 2 +- third_party/rust/oneshot-uniffi/src/errors.rs | 15 +- third_party/rust/oneshot-uniffi/src/lib.rs | 51 +- third_party/rust/oneshot-uniffi/tests/raw.rs | 46 + third_party/rust/relevancy/.cargo-checksum.json | 1 + third_party/rust/relevancy/Cargo.toml | 47 + third_party/rust/relevancy/build.rs | 8 + .../rust/relevancy/src/bin/generate-test-data.rs | 43 + third_party/rust/relevancy/src/db.rs | 118 +++ third_party/rust/relevancy/src/error.rs | 44 + third_party/rust/relevancy/src/interest.rs | 167 +++ third_party/rust/relevancy/src/lib.rs | 81 ++ .../rust/relevancy/src/populate_interests.rs | 157 +++ third_party/rust/relevancy/src/relevancy.udl | 106 ++ third_party/rust/relevancy/src/schema.rs | 53 + third_party/rust/relevancy/src/url_hash.rs | 63 ++ third_party/rust/relevancy/test-data | Bin 0 -> 192 bytes .../rust/remote_settings/.cargo-checksum.json | 2 +- third_party/rust/remote_settings/Cargo.toml | 4 +- third_party/rust/remote_settings/src/client.rs | 54 +- third_party/rust/rure/src/lib.rs | 4 + third_party/rust/scroll/.cargo-checksum.json | 2 +- third_party/rust/scroll/CHANGELOG.md | 17 - third_party/rust/scroll/Cargo.lock | 205 ---- third_party/rust/scroll/Cargo.toml | 33 +- third_party/rust/scroll/README.md | 13 +- third_party/rust/scroll/benches/bench.rs | 157 --- third_party/rust/scroll/examples/data_ctx.rs | 24 - third_party/rust/scroll/src/ctx.rs | 107 +- third_party/rust/scroll/src/endian.rs | 5 +- third_party/rust/scroll/src/error.rs | 26 +- third_party/rust/scroll/src/leb128.rs | 18 +- third_party/rust/scroll/src/lesser.rs | 7 +- third_party/rust/scroll/src/lib.rs | 67 +- third_party/rust/scroll/src/pread.rs | 7 +- third_party/rust/scroll/src/pwrite.rs | 9 +- third_party/rust/scroll/tests/api.rs | 292 ----- .../rust/scroll_derive/.cargo-checksum.json | 2 +- third_party/rust/scroll_derive/Cargo.toml | 2 +- third_party/rust/scroll_derive/README.md | 2 +- third_party/rust/smawk/.cargo-checksum.json | 1 + third_party/rust/smawk/Cargo.toml | 53 + third_party/rust/smawk/LICENSE | 21 + third_party/rust/smawk/README.md | 151 +++ third_party/rust/smawk/dprint.json | 19 + third_party/rust/smawk/rustfmt.toml | 2 + third_party/rust/smawk/src/brute_force.rs | 150 +++ third_party/rust/smawk/src/lib.rs | 570 ++++++++++ third_party/rust/smawk/src/monge.rs | 121 +++ third_party/rust/smawk/src/recursive.rs | 191 ++++ third_party/rust/smawk/tests/agreement.rs | 104 ++ third_party/rust/smawk/tests/complexity.rs | 83 ++ third_party/rust/smawk/tests/monge.rs | 83 ++ third_party/rust/smawk/tests/random_monge/mod.rs | 83 ++ third_party/rust/smawk/tests/version-numbers.rs | 9 + third_party/rust/sql-support/.cargo-checksum.json | 2 +- third_party/rust/sql-support/src/open_database.rs | 6 + third_party/rust/suggest/.cargo-checksum.json | 2 +- third_party/rust/suggest/Cargo.toml | 28 +- third_party/rust/suggest/README.md | 110 +- third_party/rust/suggest/benches/benchmark_all.rs | 25 + third_party/rust/suggest/src/benchmarks/README.md | 28 + third_party/rust/suggest/src/benchmarks/client.rs | 97 ++ third_party/rust/suggest/src/benchmarks/ingest.rs | 116 ++ third_party/rust/suggest/src/benchmarks/mod.rs | 40 + .../rust/suggest/src/bin/debug_ingestion_sizes.rs | 9 + third_party/rust/suggest/src/config.rs | 5 + third_party/rust/suggest/src/db.rs | 720 +++++++------ third_party/rust/suggest/src/lib.rs | 4 +- third_party/rust/suggest/src/pocket.rs | 2 +- third_party/rust/suggest/src/provider.rs | 48 + third_party/rust/suggest/src/rs.rs | 223 +++- third_party/rust/suggest/src/schema.rs | 187 +++- third_party/rust/suggest/src/store.rs | 1113 ++++++++++++++++++-- third_party/rust/suggest/src/suggest.udl | 11 + third_party/rust/suggest/src/suggestion.rs | 34 + third_party/rust/suggest/src/yelp.rs | 28 +- third_party/rust/sync15/.cargo-checksum.json | 2 +- third_party/rust/sync15/Cargo.toml | 4 +- third_party/rust/tabs/.cargo-checksum.json | 2 +- third_party/rust/tabs/Cargo.toml | 4 +- third_party/rust/textwrap/.cargo-checksum.json | 1 + third_party/rust/textwrap/CHANGELOG.md | 616 +++++++++++ third_party/rust/textwrap/Cargo.lock | 657 ++++++++++++ third_party/rust/textwrap/Cargo.toml | 91 ++ third_party/rust/textwrap/LICENSE | 21 + third_party/rust/textwrap/README.md | 176 ++++ third_party/rust/textwrap/rustfmt.toml | 1 + third_party/rust/textwrap/src/columns.rs | 193 ++++ third_party/rust/textwrap/src/core.rs | 461 ++++++++ third_party/rust/textwrap/src/fill.rs | 298 ++++++ third_party/rust/textwrap/src/fuzzing.rs | 23 + third_party/rust/textwrap/src/indentation.rs | 347 ++++++ third_party/rust/textwrap/src/lib.rs | 235 +++++ third_party/rust/textwrap/src/line_ending.rs | 88 ++ third_party/rust/textwrap/src/options.rs | 300 ++++++ third_party/rust/textwrap/src/refill.rs | 352 +++++++ third_party/rust/textwrap/src/termwidth.rs | 52 + third_party/rust/textwrap/src/word_separators.rs | 481 +++++++++ third_party/rust/textwrap/src/word_splitters.rs | 314 ++++++ third_party/rust/textwrap/src/wrap.rs | 686 ++++++++++++ third_party/rust/textwrap/src/wrap_algorithms.rs | 413 ++++++++ .../textwrap/src/wrap_algorithms/optimal_fit.rs | 433 ++++++++ third_party/rust/textwrap/tests/indent.rs | 88 ++ third_party/rust/textwrap/tests/version-numbers.rs | 22 + .../rust/unicode-linebreak/.cargo-checksum.json | 1 + third_party/rust/unicode-linebreak/Cargo.toml | 32 + third_party/rust/unicode-linebreak/LICENSE | 201 ++++ third_party/rust/unicode-linebreak/src/lib.rs | 160 +++ third_party/rust/unicode-linebreak/src/shared.rs | 134 +++ third_party/rust/unicode-linebreak/src/tables.rs | 10 + .../uniffi-example-arithmetic/.cargo-checksum.json | 2 +- .../rust/uniffi-example-arithmetic/Cargo.toml | 6 +- .../uniffi-example-geometry/.cargo-checksum.json | 2 +- .../rust/uniffi-example-geometry/Cargo.toml | 6 +- .../tests/bindings/test_geometry.py | 6 +- .../tests/bindings/test_geometry.rb | 6 +- .../uniffi-example-rondpoint/.cargo-checksum.json | 2 +- .../rust/uniffi-example-rondpoint/Cargo.toml | 6 +- .../tests/bindings/test_rondpoint.py | 2 +- .../tests/bindings/test_rondpoint.rb | 7 +- .../uniffi-example-sprites/.cargo-checksum.json | 2 +- third_party/rust/uniffi-example-sprites/Cargo.toml | 6 +- .../tests/bindings/test_sprites.py | 18 +- .../tests/bindings/test_sprites.rb | 18 +- .../uniffi-example-todolist/.cargo-checksum.json | 2 +- .../rust/uniffi-example-todolist/Cargo.toml | 6 +- .../tests/bindings/test_todolist.py | 4 +- .../tests/bindings/test_todolist.rb | 6 +- third_party/rust/uniffi/.cargo-checksum.json | 2 +- third_party/rust/uniffi/Cargo.toml | 11 +- third_party/rust/uniffi/README.md | 81 ++ third_party/rust/uniffi/src/cli.rs | 21 +- third_party/rust/uniffi/src/lib.rs | 7 +- .../rust/uniffi_bindgen/.cargo-checksum.json | 2 +- third_party/rust/uniffi_bindgen/Cargo.toml | 15 +- third_party/rust/uniffi_bindgen/README.md | 81 ++ .../rust/uniffi_bindgen/src/backend/filters.rs | 4 +- .../kotlin/gen_kotlin/callback_interface.rs | 2 +- .../src/bindings/kotlin/gen_kotlin/compounds.rs | 115 +- .../src/bindings/kotlin/gen_kotlin/executor.rs | 24 - .../src/bindings/kotlin/gen_kotlin/external.rs | 4 +- .../src/bindings/kotlin/gen_kotlin/mod.rs | 251 ++++- .../src/bindings/kotlin/gen_kotlin/object.rs | 19 +- .../src/bindings/kotlin/gen_kotlin/primitives.rs | 10 +- .../src/bindings/kotlin/templates/Async.kt | 107 +- .../src/bindings/kotlin/templates/BooleanHelper.kt | 2 +- .../bindings/kotlin/templates/ByteArrayHelper.kt | 4 +- .../kotlin/templates/CallbackInterfaceImpl.kt | 117 ++ .../kotlin/templates/CallbackInterfaceRuntime.kt | 65 +- .../kotlin/templates/CallbackInterfaceTemplate.kt | 140 +-- .../kotlin/templates/CustomTypeTemplate.kt | 2 +- .../bindings/kotlin/templates/DurationHelper.kt | 2 +- .../src/bindings/kotlin/templates/EnumTemplate.kt | 30 +- .../src/bindings/kotlin/templates/ErrorTemplate.kt | 18 +- .../kotlin/templates/ExternalTypeTemplate.kt | 2 +- .../kotlin/templates/FfiConverterTemplate.kt | 6 +- .../src/bindings/kotlin/templates/Float32Helper.kt | 2 +- .../src/bindings/kotlin/templates/Float64Helper.kt | 2 +- .../kotlin/templates/ForeignExecutorTemplate.kt | 83 -- .../src/bindings/kotlin/templates/HandleMap.kt | 27 + .../src/bindings/kotlin/templates/Helpers.kt | 152 +-- .../src/bindings/kotlin/templates/Int16Helper.kt | 2 +- .../src/bindings/kotlin/templates/Int32Helper.kt | 2 +- .../src/bindings/kotlin/templates/Int64Helper.kt | 2 +- .../src/bindings/kotlin/templates/Int8Helper.kt | 2 +- .../src/bindings/kotlin/templates/Interface.kt | 14 + .../src/bindings/kotlin/templates/MapTemplate.kt | 4 +- .../kotlin/templates/NamespaceLibraryTemplate.kt | 61 +- .../kotlin/templates/ObjectCleanerHelper.kt | 40 + .../kotlin/templates/ObjectCleanerHelperAndroid.kt | 26 + .../kotlin/templates/ObjectCleanerHelperJvm.kt | 25 + .../src/bindings/kotlin/templates/ObjectRuntime.kt | 161 --- .../bindings/kotlin/templates/ObjectTemplate.kt | 343 ++++-- .../bindings/kotlin/templates/OptionalTemplate.kt | 6 +- .../src/bindings/kotlin/templates/README.md | 13 + .../bindings/kotlin/templates/RecordTemplate.kt | 29 +- .../kotlin/templates/RustBufferTemplate.kt | 39 +- .../bindings/kotlin/templates/SequenceTemplate.kt | 6 +- .../src/bindings/kotlin/templates/StringHelper.kt | 10 +- .../bindings/kotlin/templates/TimestampHelper.kt | 2 +- .../kotlin/templates/TopLevelFunctionTemplate.kt | 52 +- .../src/bindings/kotlin/templates/Types.kt | 40 +- .../src/bindings/kotlin/templates/UInt16Helper.kt | 2 +- .../src/bindings/kotlin/templates/UInt32Helper.kt | 2 +- .../src/bindings/kotlin/templates/UInt64Helper.kt | 2 +- .../src/bindings/kotlin/templates/UInt8Helper.kt | 2 +- .../src/bindings/kotlin/templates/macros.kt | 140 ++- .../src/bindings/kotlin/templates/wrapper.kt | 4 + .../uniffi_bindgen/src/bindings/kotlin/test.rs | 16 +- .../src/bindings/python/gen_python/compounds.rs | 16 +- .../src/bindings/python/gen_python/executor.rs | 18 - .../src/bindings/python/gen_python/external.rs | 2 +- .../src/bindings/python/gen_python/mod.rs | 177 +++- .../src/bindings/python/templates/Async.py | 86 +- .../src/bindings/python/templates/BooleanHelper.py | 18 +- .../src/bindings/python/templates/BytesHelper.py | 5 +- .../python/templates/CallbackInterfaceImpl.py | 98 ++ .../python/templates/CallbackInterfaceRuntime.py | 59 +- .../python/templates/CallbackInterfaceTemplate.py | 112 +- .../src/bindings/python/templates/CustomType.py | 9 + .../bindings/python/templates/DurationHelper.py | 8 +- .../src/bindings/python/templates/EnumTemplate.py | 71 +- .../src/bindings/python/templates/ErrorTemplate.py | 19 + .../bindings/python/templates/ExternalTemplate.py | 8 +- .../src/bindings/python/templates/Float32Helper.py | 2 +- .../src/bindings/python/templates/Float64Helper.py | 2 +- .../python/templates/ForeignExecutorTemplate.py | 63 -- .../src/bindings/python/templates/HandleMap.py | 33 + .../src/bindings/python/templates/Helpers.py | 36 +- .../src/bindings/python/templates/Int16Helper.py | 2 +- .../src/bindings/python/templates/Int32Helper.py | 2 +- .../src/bindings/python/templates/Int64Helper.py | 2 +- .../src/bindings/python/templates/Int8Helper.py | 2 +- .../src/bindings/python/templates/MapTemplate.py | 6 + .../python/templates/NamespaceLibraryTemplate.py | 45 +- .../bindings/python/templates/ObjectTemplate.py | 124 ++- .../bindings/python/templates/OptionalTemplate.py | 5 + .../bindings/python/templates/PointerManager.py | 68 -- .../src/bindings/python/templates/Protocol.py | 9 + .../bindings/python/templates/RecordTemplate.py | 24 +- .../bindings/python/templates/RustBufferHelper.py | 18 +- .../python/templates/RustBufferTemplate.py | 11 +- .../bindings/python/templates/SequenceTemplate.py | 5 + .../src/bindings/python/templates/StringHelper.py | 4 +- .../bindings/python/templates/TimestampHelper.py | 4 + .../python/templates/TopLevelFunctionTemplate.py | 24 +- .../src/bindings/python/templates/Types.py | 5 +- .../src/bindings/python/templates/UInt16Helper.py | 2 +- .../src/bindings/python/templates/UInt32Helper.py | 2 +- .../src/bindings/python/templates/UInt64Helper.py | 2 +- .../src/bindings/python/templates/UInt8Helper.py | 2 +- .../src/bindings/python/templates/macros.py | 95 +- .../src/bindings/python/templates/wrapper.py | 17 +- .../uniffi_bindgen/src/bindings/python/test.rs | 14 +- .../src/bindings/ruby/gen_ruby/mod.rs | 79 +- .../src/bindings/ruby/templates/ObjectTemplate.rb | 36 +- .../src/bindings/ruby/templates/RecordTemplate.rb | 7 +- .../bindings/ruby/templates/RustBufferBuilder.rb | 2 +- .../bindings/ruby/templates/RustBufferStream.rb | 4 +- .../bindings/ruby/templates/RustBufferTemplate.rb | 42 +- .../ruby/templates/TopLevelFunctionTemplate.rb | 4 +- .../src/bindings/ruby/templates/macros.rb | 10 +- .../rust/uniffi_bindgen/src/bindings/ruby/test.rs | 15 +- .../bindings/swift/gen_swift/callback_interface.rs | 12 +- .../src/bindings/swift/gen_swift/compounds.rs | 5 +- .../src/bindings/swift/gen_swift/executor.rs | 23 - .../src/bindings/swift/gen_swift/external.rs | 2 +- .../src/bindings/swift/gen_swift/mod.rs | 209 +++- .../src/bindings/swift/gen_swift/object.rs | 18 +- .../src/bindings/swift/gen_swift/primitives.rs | 8 +- .../src/bindings/swift/templates/Async.swift | 100 +- .../swift/templates/BridgingHeaderTemplate.h | 48 +- .../swift/templates/CallbackInterfaceImpl.swift | 113 ++ .../swift/templates/CallbackInterfaceRuntime.swift | 57 - .../templates/CallbackInterfaceTemplate.swift | 145 +-- .../bindings/swift/templates/EnumTemplate.swift | 27 +- .../bindings/swift/templates/ErrorTemplate.swift | 12 +- .../swift/templates/ForeignExecutorTemplate.swift | 69 -- .../src/bindings/swift/templates/HandleMap.swift | 40 + .../src/bindings/swift/templates/Helpers.swift | 44 +- .../bindings/swift/templates/ObjectTemplate.swift | 201 ++-- .../src/bindings/swift/templates/Protocol.swift | 12 + .../bindings/swift/templates/RecordTemplate.swift | 17 +- .../swift/templates/RustBufferTemplate.swift | 4 + .../swift/templates/TopLevelFunctionTemplate.swift | 49 +- .../src/bindings/swift/templates/Types.swift | 3 - .../src/bindings/swift/templates/macros.swift | 125 ++- .../src/bindings/swift/templates/wrapper.swift | 8 + .../rust/uniffi_bindgen/src/bindings/swift/test.rs | 24 +- .../rust/uniffi_bindgen/src/interface/callbacks.rs | 207 +++- .../rust/uniffi_bindgen/src/interface/enum_.rs | 224 +++- .../rust/uniffi_bindgen/src/interface/ffi.rs | 195 +++- .../rust/uniffi_bindgen/src/interface/function.rs | 32 + .../rust/uniffi_bindgen/src/interface/mod.rs | 342 +++--- .../rust/uniffi_bindgen/src/interface/object.rs | 199 +++- .../rust/uniffi_bindgen/src/interface/record.rs | 53 + .../rust/uniffi_bindgen/src/interface/universe.rs | 5 +- third_party/rust/uniffi_bindgen/src/lib.rs | 138 ++- .../rust/uniffi_bindgen/src/library_mode.rs | 40 +- .../rust/uniffi_bindgen/src/macro_metadata/ci.rs | 21 +- .../uniffi_bindgen/src/macro_metadata/extract.rs | 2 +- .../rust/uniffi_bindgen/src/scaffolding/mod.rs | 36 +- .../templates/CallbackInterfaceTemplate.rs | 93 +- .../src/scaffolding/templates/EnumTemplate.rs | 13 +- .../src/scaffolding/templates/ErrorTemplate.rs | 10 +- .../scaffolding/templates/ExternalTypesTemplate.rs | 2 + .../src/scaffolding/templates/ObjectTemplate.rs | 34 +- .../src/scaffolding/templates/RecordTemplate.rs | 8 +- .../templates/TopLevelFunctionTemplate.rs | 5 +- third_party/rust/uniffi_build/.cargo-checksum.json | 2 +- third_party/rust/uniffi_build/Cargo.toml | 5 +- third_party/rust/uniffi_build/README.md | 81 ++ .../uniffi_checksum_derive/.cargo-checksum.json | 2 +- third_party/rust/uniffi_checksum_derive/Cargo.toml | 3 +- third_party/rust/uniffi_checksum_derive/README.md | 81 ++ third_party/rust/uniffi_core/.cargo-checksum.json | 2 +- third_party/rust/uniffi_core/Cargo.toml | 6 +- third_party/rust/uniffi_core/README.md | 81 ++ .../rust/uniffi_core/src/ffi/callbackinterface.rs | 125 +-- third_party/rust/uniffi_core/src/ffi/ffidefault.rs | 11 +- .../rust/uniffi_core/src/ffi/foreigncallbacks.rs | 116 +- .../rust/uniffi_core/src/ffi/foreignexecutor.rs | 487 --------- .../rust/uniffi_core/src/ffi/foreignfuture.rs | 241 +++++ third_party/rust/uniffi_core/src/ffi/handle.rs | 46 + third_party/rust/uniffi_core/src/ffi/mod.rs | 6 +- third_party/rust/uniffi_core/src/ffi/rustbuffer.rs | 122 +-- third_party/rust/uniffi_core/src/ffi/rustcalls.rs | 9 +- third_party/rust/uniffi_core/src/ffi/rustfuture.rs | 735 ------------- .../rust/uniffi_core/src/ffi/rustfuture/future.rs | 320 ++++++ .../rust/uniffi_core/src/ffi/rustfuture/mod.rs | 141 +++ .../uniffi_core/src/ffi/rustfuture/scheduler.rs | 96 ++ .../rust/uniffi_core/src/ffi/rustfuture/tests.rs | 223 ++++ .../rust/uniffi_core/src/ffi_converter_impls.rs | 75 +- .../rust/uniffi_core/src/ffi_converter_traits.rs | 151 ++- third_party/rust/uniffi_core/src/lib.rs | 6 +- third_party/rust/uniffi_core/src/metadata.rs | 53 +- .../rust/uniffi_macros/.cargo-checksum.json | 2 +- third_party/rust/uniffi_macros/Cargo.toml | 10 +- third_party/rust/uniffi_macros/README.md | 81 ++ third_party/rust/uniffi_macros/src/default.rs | 133 +++ third_party/rust/uniffi_macros/src/enum_.rs | 308 ++++-- third_party/rust/uniffi_macros/src/error.rs | 101 +- third_party/rust/uniffi_macros/src/export.rs | 211 +--- .../rust/uniffi_macros/src/export/attributes.rs | 306 +++++- .../uniffi_macros/src/export/callback_interface.rs | 204 +++- third_party/rust/uniffi_macros/src/export/item.rs | 88 +- .../rust/uniffi_macros/src/export/scaffolding.rs | 123 ++- .../uniffi_macros/src/export/trait_interface.rs | 183 ++++ .../rust/uniffi_macros/src/export/utrait.rs | 23 +- third_party/rust/uniffi_macros/src/fnsig.rs | 248 +++-- third_party/rust/uniffi_macros/src/lib.rs | 73 +- third_party/rust/uniffi_macros/src/object.rs | 76 +- third_party/rust/uniffi_macros/src/record.rs | 113 +- .../rust/uniffi_macros/src/setup_scaffolding.rs | 57 +- third_party/rust/uniffi_macros/src/util.rs | 47 +- third_party/rust/uniffi_meta/.cargo-checksum.json | 2 +- third_party/rust/uniffi_meta/Cargo.toml | 5 +- third_party/rust/uniffi_meta/README.md | 81 ++ third_party/rust/uniffi_meta/src/ffi_names.rs | 13 +- third_party/rust/uniffi_meta/src/group.rs | 9 +- third_party/rust/uniffi_meta/src/lib.rs | 65 +- third_party/rust/uniffi_meta/src/metadata.rs | 13 +- third_party/rust/uniffi_meta/src/reader.rs | 148 ++- third_party/rust/uniffi_meta/src/types.rs | 22 +- .../rust/uniffi_testing/.cargo-checksum.json | 2 +- third_party/rust/uniffi_testing/Cargo.toml | 2 +- third_party/rust/uniffi_udl/.cargo-checksum.json | 2 +- third_party/rust/uniffi_udl/Cargo.toml | 12 +- third_party/rust/uniffi_udl/README.md | 81 ++ third_party/rust/uniffi_udl/src/attributes.rs | 199 +++- third_party/rust/uniffi_udl/src/collectors.rs | 18 +- .../rust/uniffi_udl/src/converters/callables.rs | 16 +- .../rust/uniffi_udl/src/converters/enum_.rs | 79 +- .../rust/uniffi_udl/src/converters/interface.rs | 6 +- third_party/rust/uniffi_udl/src/converters/mod.rs | 10 + third_party/rust/uniffi_udl/src/finder.rs | 60 +- third_party/rust/uniffi_udl/src/lib.rs | 2 +- third_party/rust/uniffi_udl/src/literal.rs | 8 +- third_party/rust/uniffi_udl/src/resolver.rs | 1 - third_party/rust/wasm-encoder/.cargo-checksum.json | 2 +- third_party/rust/wasm-encoder/Cargo.toml | 10 +- .../rust/wasm-encoder/src/component/names.rs | 2 +- third_party/rust/wasm-encoder/src/core/code.rs | 89 ++ third_party/rust/wasm-smith/.cargo-checksum.json | 2 +- third_party/rust/wasm-smith/Cargo.toml | 18 +- third_party/rust/wasm-smith/src/component.rs | 53 +- third_party/rust/wasm-smith/src/config.rs | 108 +- third_party/rust/wasm-smith/src/core.rs | 806 +++++++++----- .../rust/wasm-smith/src/core/code_builder.rs | 141 ++- .../wasm-smith/src/core/code_builder/no_traps.rs | 2 +- third_party/rust/wasm-smith/src/core/encode.rs | 41 +- third_party/rust/wasm-smith/src/core/terminate.rs | 30 +- third_party/rust/wasm-smith/src/lib.rs | 7 +- .../rust/wasm-smith/tests/available_imports.rs | 46 +- third_party/rust/wasm-smith/tests/common/mod.rs | 43 + third_party/rust/wasm-smith/tests/core.rs | 3 +- third_party/rust/wasm-smith/tests/exports.rs | 147 +++ third_party/rust/wast/.cargo-checksum.json | 2 +- third_party/rust/wast/Cargo.toml | 13 +- third_party/rust/wast/src/component/component.rs | 1 + third_party/rust/wast/src/core/binary.rs | 124 ++- third_party/rust/wast/src/core/expr.rs | 53 +- third_party/rust/wast/src/core/memory.rs | 2 + third_party/rust/wast/src/core/module.rs | 1 + .../src/core/resolve/deinline_import_export.rs | 2 + third_party/rust/wast/src/core/table.rs | 1 + third_party/rust/wast/src/lib.rs | 1 + third_party/rust/wast/src/parser.rs | 12 +- third_party/rust/wast/src/wat.rs | 1 + .../rust/webext-storage/.cargo-checksum.json | 2 +- third_party/rust/webext-storage/Cargo.toml | 4 +- third_party/rust/weedle2/.cargo-checksum.json | 2 +- third_party/rust/weedle2/Cargo.toml | 5 +- third_party/rust/weedle2/README.md | 2 +- third_party/rust/weedle2/src/common.rs | 45 + third_party/rust/weedle2/src/dictionary.rs | 3 +- third_party/rust/weedle2/src/interface.rs | 4 +- third_party/rust/weedle2/src/lib.rs | 16 +- third_party/rust/weedle2/src/namespace.rs | 4 +- third_party/rust/weedle2/src/whitespace.rs | 1 + third_party/rust/wgpu-core/.cargo-checksum.json | 2 +- third_party/rust/wgpu-core/Cargo.toml | 4 +- third_party/rust/wgpu-core/src/command/bundle.rs | 21 +- third_party/rust/wgpu-core/src/command/clear.rs | 7 +- third_party/rust/wgpu-core/src/command/compute.rs | 9 +- .../rust/wgpu-core/src/command/memory_init.rs | 11 +- third_party/rust/wgpu-core/src/command/render.rs | 15 +- third_party/rust/wgpu-core/src/command/transfer.rs | 19 +- third_party/rust/wgpu-core/src/device/global.rs | 33 +- third_party/rust/wgpu-core/src/device/life.rs | 15 +- third_party/rust/wgpu-core/src/device/mod.rs | 11 +- third_party/rust/wgpu-core/src/device/queue.rs | 17 +- third_party/rust/wgpu-core/src/device/resource.rs | 20 +- third_party/rust/wgpu-core/src/snatch.rs | 35 +- third_party/rust/wgpu-core/src/track/buffer.rs | 12 +- third_party/rust/wgpu-core/src/track/metadata.rs | 5 + third_party/rust/wgpu-core/src/track/mod.rs | 50 +- third_party/rust/wgpu-core/src/track/texture.rs | 13 +- third_party/rust/wgpu-hal/.cargo-checksum.json | 2 +- third_party/rust/wgpu-hal/Cargo.toml | 12 +- third_party/rust/wgpu-hal/src/dx12/adapter.rs | 4 +- third_party/rust/wgpu-hal/src/dx12/command.rs | 4 +- third_party/rust/wgpu-hal/src/dx12/device.rs | 15 +- third_party/rust/wgpu-hal/src/dx12/instance.rs | 4 +- third_party/rust/wgpu-hal/src/dx12/mod.rs | 8 +- third_party/rust/wgpu-hal/src/empty.rs | 24 +- third_party/rust/wgpu-hal/src/gles/adapter.rs | 4 +- third_party/rust/wgpu-hal/src/gles/command.rs | 4 +- third_party/rust/wgpu-hal/src/gles/device.rs | 21 +- third_party/rust/wgpu-hal/src/gles/egl.rs | 8 +- third_party/rust/wgpu-hal/src/gles/queue.rs | 4 +- third_party/rust/wgpu-hal/src/gles/web.rs | 8 +- third_party/rust/wgpu-hal/src/gles/wgl.rs | 8 +- third_party/rust/wgpu-hal/src/lib.rs | 253 +++-- third_party/rust/wgpu-hal/src/metal/adapter.rs | 4 +- third_party/rust/wgpu-hal/src/metal/command.rs | 4 +- third_party/rust/wgpu-hal/src/metal/device.rs | 15 +- third_party/rust/wgpu-hal/src/metal/mod.rs | 8 +- third_party/rust/wgpu-hal/src/metal/surface.rs | 4 +- third_party/rust/wgpu-hal/src/vulkan/adapter.rs | 181 +++- third_party/rust/wgpu-hal/src/vulkan/command.rs | 4 +- third_party/rust/wgpu-hal/src/vulkan/device.rs | 4 +- third_party/rust/wgpu-hal/src/vulkan/instance.rs | 8 +- third_party/rust/wgpu-hal/src/vulkan/mod.rs | 6 +- third_party/rust/wgpu-types/.cargo-checksum.json | 2 +- third_party/rust/wgpu-types/Cargo.toml | 6 +- third_party/rust/wgpu-types/src/lib.rs | 29 +- third_party/rust/zip/src/read.rs | 9 +- third_party/rust/zip/src/types.rs | 2 - third_party/rust/zip/src/write.rs | 1 + 651 files changed, 30943 insertions(+), 9377 deletions(-) mode change 100644 => 100755 third_party/rust/bumpalo/src/lib.rs create mode 100644 third_party/rust/embed-manifest/.cargo-checksum.json create mode 100644 third_party/rust/embed-manifest/CHANGELOG.md create mode 100644 third_party/rust/embed-manifest/Cargo.toml create mode 100644 third_party/rust/embed-manifest/LICENSE create mode 100644 third_party/rust/embed-manifest/README.md create mode 100644 third_party/rust/embed-manifest/rustfmt.toml create mode 100644 third_party/rust/embed-manifest/src/embed/coff.rs create mode 100644 third_party/rust/embed-manifest/src/embed/error.rs create mode 100644 third_party/rust/embed-manifest/src/embed/mod.rs create mode 100644 third_party/rust/embed-manifest/src/embed/test.rs create mode 100644 third_party/rust/embed-manifest/src/lib.rs create mode 100644 third_party/rust/embed-manifest/src/manifest/mod.rs create mode 100644 third_party/rust/embed-manifest/src/manifest/test.rs create mode 100644 third_party/rust/embed-manifest/src/manifest/xml.rs create mode 100644 third_party/rust/embed-manifest/testdata/sample.exe.manifest create mode 100644 third_party/rust/neqo-common/benches/timer.rs delete mode 100644 third_party/rust/neqo-common/src/udp.rs delete mode 100644 third_party/rust/neqo-crypto/bindings/mozpkix.hpp create mode 100644 third_party/rust/neqo-crypto/min_version.txt delete mode 100644 third_party/rust/neqo-crypto/src/aead_fuzzing.rs create mode 100644 third_party/rust/neqo-crypto/src/aead_null.rs create mode 100644 third_party/rust/neqo-crypto/src/min_version.rs delete mode 100644 third_party/rust/neqo-transport/src/connection/tests/fuzzing.rs create mode 100644 third_party/rust/neqo-transport/src/connection/tests/null.rs create mode 100644 third_party/rust/oneshot-uniffi/tests/raw.rs create mode 100644 third_party/rust/relevancy/.cargo-checksum.json create mode 100644 third_party/rust/relevancy/Cargo.toml create mode 100644 third_party/rust/relevancy/build.rs create mode 100644 third_party/rust/relevancy/src/bin/generate-test-data.rs create mode 100644 third_party/rust/relevancy/src/db.rs create mode 100644 third_party/rust/relevancy/src/error.rs create mode 100644 third_party/rust/relevancy/src/interest.rs create mode 100644 third_party/rust/relevancy/src/lib.rs create mode 100644 third_party/rust/relevancy/src/populate_interests.rs create mode 100644 third_party/rust/relevancy/src/relevancy.udl create mode 100644 third_party/rust/relevancy/src/schema.rs create mode 100644 third_party/rust/relevancy/src/url_hash.rs create mode 100644 third_party/rust/relevancy/test-data delete mode 100644 third_party/rust/scroll/CHANGELOG.md delete mode 100644 third_party/rust/scroll/Cargo.lock delete mode 100644 third_party/rust/scroll/benches/bench.rs delete mode 100644 third_party/rust/scroll/examples/data_ctx.rs delete mode 100644 third_party/rust/scroll/tests/api.rs create mode 100644 third_party/rust/smawk/.cargo-checksum.json create mode 100644 third_party/rust/smawk/Cargo.toml create mode 100644 third_party/rust/smawk/LICENSE create mode 100644 third_party/rust/smawk/README.md create mode 100644 third_party/rust/smawk/dprint.json create mode 100644 third_party/rust/smawk/rustfmt.toml create mode 100644 third_party/rust/smawk/src/brute_force.rs create mode 100644 third_party/rust/smawk/src/lib.rs create mode 100644 third_party/rust/smawk/src/monge.rs create mode 100644 third_party/rust/smawk/src/recursive.rs create mode 100644 third_party/rust/smawk/tests/agreement.rs create mode 100644 third_party/rust/smawk/tests/complexity.rs create mode 100644 third_party/rust/smawk/tests/monge.rs create mode 100644 third_party/rust/smawk/tests/random_monge/mod.rs create mode 100644 third_party/rust/smawk/tests/version-numbers.rs create mode 100644 third_party/rust/suggest/benches/benchmark_all.rs create mode 100644 third_party/rust/suggest/src/benchmarks/README.md create mode 100644 third_party/rust/suggest/src/benchmarks/client.rs create mode 100644 third_party/rust/suggest/src/benchmarks/ingest.rs create mode 100644 third_party/rust/suggest/src/benchmarks/mod.rs create mode 100644 third_party/rust/suggest/src/bin/debug_ingestion_sizes.rs create mode 100644 third_party/rust/textwrap/.cargo-checksum.json create mode 100644 third_party/rust/textwrap/CHANGELOG.md create mode 100644 third_party/rust/textwrap/Cargo.lock create mode 100644 third_party/rust/textwrap/Cargo.toml create mode 100644 third_party/rust/textwrap/LICENSE create mode 100644 third_party/rust/textwrap/README.md create mode 100644 third_party/rust/textwrap/rustfmt.toml create mode 100644 third_party/rust/textwrap/src/columns.rs create mode 100644 third_party/rust/textwrap/src/core.rs create mode 100644 third_party/rust/textwrap/src/fill.rs create mode 100644 third_party/rust/textwrap/src/fuzzing.rs create mode 100644 third_party/rust/textwrap/src/indentation.rs create mode 100644 third_party/rust/textwrap/src/lib.rs create mode 100644 third_party/rust/textwrap/src/line_ending.rs create mode 100644 third_party/rust/textwrap/src/options.rs create mode 100644 third_party/rust/textwrap/src/refill.rs create mode 100644 third_party/rust/textwrap/src/termwidth.rs create mode 100644 third_party/rust/textwrap/src/word_separators.rs create mode 100644 third_party/rust/textwrap/src/word_splitters.rs create mode 100644 third_party/rust/textwrap/src/wrap.rs create mode 100644 third_party/rust/textwrap/src/wrap_algorithms.rs create mode 100644 third_party/rust/textwrap/src/wrap_algorithms/optimal_fit.rs create mode 100644 third_party/rust/textwrap/tests/indent.rs create mode 100644 third_party/rust/textwrap/tests/version-numbers.rs create mode 100644 third_party/rust/unicode-linebreak/.cargo-checksum.json create mode 100644 third_party/rust/unicode-linebreak/Cargo.toml create mode 100644 third_party/rust/unicode-linebreak/LICENSE create mode 100644 third_party/rust/unicode-linebreak/src/lib.rs create mode 100644 third_party/rust/unicode-linebreak/src/shared.rs create mode 100644 third_party/rust/unicode-linebreak/src/tables.rs create mode 100644 third_party/rust/uniffi/README.md create mode 100644 third_party/rust/uniffi_bindgen/README.md delete mode 100644 third_party/rust/uniffi_bindgen/src/bindings/kotlin/gen_kotlin/executor.rs create mode 100644 third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/CallbackInterfaceImpl.kt delete mode 100644 third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/ForeignExecutorTemplate.kt create mode 100644 third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/HandleMap.kt create mode 100644 third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/Interface.kt create mode 100644 third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/ObjectCleanerHelper.kt create mode 100644 third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/ObjectCleanerHelperAndroid.kt create mode 100644 third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/ObjectCleanerHelperJvm.kt delete mode 100644 third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/ObjectRuntime.kt create mode 100644 third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/README.md delete mode 100644 third_party/rust/uniffi_bindgen/src/bindings/python/gen_python/executor.rs create mode 100644 third_party/rust/uniffi_bindgen/src/bindings/python/templates/CallbackInterfaceImpl.py delete mode 100644 third_party/rust/uniffi_bindgen/src/bindings/python/templates/ForeignExecutorTemplate.py create mode 100644 third_party/rust/uniffi_bindgen/src/bindings/python/templates/HandleMap.py delete mode 100644 third_party/rust/uniffi_bindgen/src/bindings/python/templates/PointerManager.py create mode 100644 third_party/rust/uniffi_bindgen/src/bindings/python/templates/Protocol.py delete mode 100644 third_party/rust/uniffi_bindgen/src/bindings/swift/gen_swift/executor.rs create mode 100644 third_party/rust/uniffi_bindgen/src/bindings/swift/templates/CallbackInterfaceImpl.swift delete mode 100644 third_party/rust/uniffi_bindgen/src/bindings/swift/templates/ForeignExecutorTemplate.swift create mode 100644 third_party/rust/uniffi_bindgen/src/bindings/swift/templates/HandleMap.swift create mode 100644 third_party/rust/uniffi_bindgen/src/bindings/swift/templates/Protocol.swift create mode 100644 third_party/rust/uniffi_build/README.md create mode 100644 third_party/rust/uniffi_checksum_derive/README.md create mode 100644 third_party/rust/uniffi_core/README.md delete mode 100644 third_party/rust/uniffi_core/src/ffi/foreignexecutor.rs create mode 100644 third_party/rust/uniffi_core/src/ffi/foreignfuture.rs create mode 100644 third_party/rust/uniffi_core/src/ffi/handle.rs delete mode 100644 third_party/rust/uniffi_core/src/ffi/rustfuture.rs create mode 100644 third_party/rust/uniffi_core/src/ffi/rustfuture/future.rs create mode 100644 third_party/rust/uniffi_core/src/ffi/rustfuture/mod.rs create mode 100644 third_party/rust/uniffi_core/src/ffi/rustfuture/scheduler.rs create mode 100644 third_party/rust/uniffi_core/src/ffi/rustfuture/tests.rs create mode 100644 third_party/rust/uniffi_macros/README.md create mode 100644 third_party/rust/uniffi_macros/src/default.rs create mode 100644 third_party/rust/uniffi_macros/src/export/trait_interface.rs create mode 100644 third_party/rust/uniffi_meta/README.md create mode 100644 third_party/rust/uniffi_udl/README.md create mode 100644 third_party/rust/wasm-smith/tests/common/mod.rs create mode 100644 third_party/rust/wasm-smith/tests/exports.rs (limited to 'third_party/rust') diff --git a/third_party/rust/audio_thread_priority/.cargo-checksum.json b/third_party/rust/audio_thread_priority/.cargo-checksum.json index e1a42d3fe7..fc49e6d1a8 100644 --- a/third_party/rust/audio_thread_priority/.cargo-checksum.json +++ b/third_party/rust/audio_thread_priority/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"Cargo.toml":"6bceb8b41d1646bd255b39c13dcca5bbcc0984f0f2aca73decb7ee5627443387","LICENSE":"32ee9dbf6196874fc9d406c54a888a6c4cbb9aa4a7f35b46befeaff43a78fe85","Makefile":"0f9a771cfb30c7c4b9961d82fdca4e9e229a955bb2e636474a4101389e18e938","README.md":"c123692b3b50dd621b896a8269814d609cbf1e532b461bf4a77854ddd607eb7a","atp_test.cpp":"8075a040941a65fb9e3f7cbf0535853ca6661c3ac442ec35569b42b24bbec797","audio_thread_priority.h":"f0ecaf1b674f794cde0dc834028e074d4e4675d22ae96acf08b2ae1dceb3474e","generate_osx_bindings.sh":"06e4e03450f788ced18d31fff5660919e6f6ec1119ddace363ffeb82f0518a71","src/lib.rs":"1d5c629d1ee2dab80f7bf422a3add07eaca203f5c1557fdd379361d632e14224","src/mach_sys.rs":"352560fcb9b41d877cff92e5b3b04d6dc68b1f30508ce4b9aed78940120a883e","src/rt_linux.rs":"e77db6d66fb76772dde6c12850ef2c50730cfe00decc0c67211e63f3e8a59c9f","src/rt_mach.rs":"29f8c0397f14cecbac1f76394c2abfe0e05903b54486cf735f9a94a10c168643","src/rt_win.rs":"f2ba097cebf65252c27d9d6dcfbe1fcc041c4b312724bd98e080e114a4de3bb6"},"package":"17fd24082f432a11ad4fde564af75fc9a0beef2f4199e7691b090ff0e065c4d0"} \ No newline at end of file +{"files":{"Cargo.toml":"2dd36097338035819ea7fdb1de6d37009571d761fb659194423f7143a3ba25c7","LICENSE":"32ee9dbf6196874fc9d406c54a888a6c4cbb9aa4a7f35b46befeaff43a78fe85","Makefile":"0f9a771cfb30c7c4b9961d82fdca4e9e229a955bb2e636474a4101389e18e938","README.md":"c123692b3b50dd621b896a8269814d609cbf1e532b461bf4a77854ddd607eb7a","atp_test.cpp":"8075a040941a65fb9e3f7cbf0535853ca6661c3ac442ec35569b42b24bbec797","audio_thread_priority.h":"f0ecaf1b674f794cde0dc834028e074d4e4675d22ae96acf08b2ae1dceb3474e","generate_osx_bindings.sh":"06e4e03450f788ced18d31fff5660919e6f6ec1119ddace363ffeb82f0518a71","src/lib.rs":"142a4f40a0deba0544673c5c8cb4398525a7ead2cc5addb4e69dd9a81e771ded","src/mach_sys.rs":"352560fcb9b41d877cff92e5b3b04d6dc68b1f30508ce4b9aed78940120a883e","src/rt_linux.rs":"e77db6d66fb76772dde6c12850ef2c50730cfe00decc0c67211e63f3e8a59c9f","src/rt_mach.rs":"29f8c0397f14cecbac1f76394c2abfe0e05903b54486cf735f9a94a10c168643","src/rt_win.rs":"0c85d771c544e4bb2b8fb60bdffe5a3080c112896f42360eb92cd3b85c010ca0"},"package":"e3632611da7e79f8fc8fd75840f1ccfa7792dbf1e25d00791344a4450dd8834f"} \ No newline at end of file diff --git a/third_party/rust/audio_thread_priority/Cargo.toml b/third_party/rust/audio_thread_priority/Cargo.toml index 30861cdafd..e1fa1a342f 100644 --- a/third_party/rust/audio_thread_priority/Cargo.toml +++ b/third_party/rust/audio_thread_priority/Cargo.toml @@ -12,7 +12,7 @@ [package] edition = "2018" name = "audio_thread_priority" -version = "0.31.0" +version = "0.32.0" authors = ["Paul Adenot "] description = "Bump a thread to real-time priority, for audio work, on Linux, Windows and macOS" readme = "README.md" @@ -61,5 +61,5 @@ version = "0.3" version = "0.52" features = [ "Win32_Foundation", - "Win32_System_Threading", + "Win32_System_LibraryLoader", ] diff --git a/third_party/rust/audio_thread_priority/src/lib.rs b/third_party/rust/audio_thread_priority/src/lib.rs index eaae33d9d1..6006f7966e 100644 --- a/third_party/rust/audio_thread_priority/src/lib.rs +++ b/third_party/rust/audio_thread_priority/src/lib.rs @@ -647,6 +647,15 @@ mod tests { // automatically deallocated, but not demoted until the thread exits. } } + + #[test] + fn it_works_in_different_threads() { + let handles: Vec<_> = (0..32).map(|_| std::thread::spawn(it_works)).collect(); + for handle in handles { + handle.join().unwrap() + } + } + cfg_if! { if #[cfg(target_os = "linux")] { use nix::unistd::*; diff --git a/third_party/rust/audio_thread_priority/src/rt_win.rs b/third_party/rust/audio_thread_priority/src/rt_win.rs index 3a2bbf29d9..480c5bd75c 100644 --- a/third_party/rust/audio_thread_priority/src/rt_win.rs +++ b/third_party/rust/audio_thread_priority/src/rt_win.rs @@ -1,14 +1,11 @@ -use windows_sys::s; -use windows_sys::Win32::Foundation::GetLastError; -use windows_sys::Win32::Foundation::FALSE; -use windows_sys::Win32::Foundation::HANDLE; -use windows_sys::Win32::System::Threading::{ - AvRevertMmThreadCharacteristics, AvSetMmThreadCharacteristicsA, -}; - +use self::avrt_lib::AvRtLibrary; use crate::AudioThreadPriorityError; - use log::info; +use std::sync::OnceLock; +use windows_sys::{ + w, + Win32::Foundation::{HANDLE, WIN32_ERROR}, +}; #[derive(Debug)] pub struct RtPriorityHandleInternal { @@ -17,7 +14,7 @@ pub struct RtPriorityHandleInternal { } impl RtPriorityHandleInternal { - pub fn new(mmcss_task_index: u32, task_handle: HANDLE) -> RtPriorityHandleInternal { + fn new(mmcss_task_index: u32, task_handle: HANDLE) -> RtPriorityHandleInternal { RtPriorityHandleInternal { mmcss_task_index, task_handle, @@ -25,45 +22,205 @@ impl RtPriorityHandleInternal { } } +fn avrt() -> Result<&'static AvRtLibrary, AudioThreadPriorityError> { + static AV_RT_LIBRARY: OnceLock> = OnceLock::new(); + AV_RT_LIBRARY + .get_or_init(AvRtLibrary::try_new) + .as_ref() + .map_err(|win32_error| { + AudioThreadPriorityError::new(&format!("Unable to load avrt.dll ({win32_error})")) + }) +} + +pub fn promote_current_thread_to_real_time_internal( + _audio_buffer_frames: u32, + _audio_samplerate_hz: u32, +) -> Result { + avrt()? + .set_mm_thread_characteristics(w!("Audio")) + .map(|(mmcss_task_index, task_handle)| { + info!("task {mmcss_task_index} bumped to real time priority."); + RtPriorityHandleInternal::new(mmcss_task_index, task_handle) + }) + .map_err(|win32_error| { + AudioThreadPriorityError::new(&format!( + "Unable to bump the thread priority ({win32_error})" + )) + }) +} + pub fn demote_current_thread_from_real_time_internal( rt_priority_handle: RtPriorityHandleInternal, ) -> Result<(), AudioThreadPriorityError> { - let rv = unsafe { AvRevertMmThreadCharacteristics(rt_priority_handle.task_handle) }; - if rv == FALSE { - return Err(AudioThreadPriorityError::new(&format!( - "Unable to restore the thread priority ({:?})", - unsafe { GetLastError() } - ))); + let RtPriorityHandleInternal { + mmcss_task_index, + task_handle, + } = rt_priority_handle; + avrt()? + .revert_mm_thread_characteristics(task_handle) + .map(|_| { + info!("task {mmcss_task_index} priority restored."); + }) + .map_err(|win32_error| { + AudioThreadPriorityError::new(&format!( + "Unable to restore the thread priority for task {mmcss_task_index} ({win32_error})" + )) + }) +} + +mod avrt_lib { + use super::win32_utils::{win32_error_if, OwnedLibrary}; + use std::sync::Once; + use windows_sys::{ + core::PCWSTR, + s, w, + Win32::Foundation::{BOOL, FALSE, HANDLE, WIN32_ERROR}, + }; + + type AvSetMmThreadCharacteristicsWFn = unsafe extern "system" fn(PCWSTR, *mut u32) -> HANDLE; + type AvRevertMmThreadCharacteristicsFn = unsafe extern "system" fn(HANDLE) -> BOOL; + + #[derive(Debug)] + pub(super) struct AvRtLibrary { + // This field is never read because only used for its Drop behavior + #[allow(dead_code)] + module: OwnedLibrary, + + av_set_mm_thread_characteristics_w: AvSetMmThreadCharacteristicsWFn, + av_revert_mm_thread_characteristics: AvRevertMmThreadCharacteristicsFn, } - info!( - "task {} priority restored.", - rt_priority_handle.mmcss_task_index - ); + impl AvRtLibrary { + pub(super) fn try_new() -> Result { + let module = OwnedLibrary::try_new(w!("avrt.dll"))?; + let av_set_mm_thread_characteristics_w = unsafe { + std::mem::transmute::<_, AvSetMmThreadCharacteristicsWFn>( + module.get_proc(s!("AvSetMmThreadCharacteristicsW"))?, + ) + }; + let av_revert_mm_thread_characteristics = unsafe { + std::mem::transmute::<_, AvRevertMmThreadCharacteristicsFn>( + module.get_proc(s!("AvRevertMmThreadCharacteristics"))?, + ) + }; + Ok(Self { + module, + av_set_mm_thread_characteristics_w, + av_revert_mm_thread_characteristics, + }) + } + + pub(super) fn set_mm_thread_characteristics( + &self, + task_name: PCWSTR, + ) -> Result<(u32, HANDLE), WIN32_ERROR> { + // Ensure that the first call never runs in parallel with other calls. This + // seems necessary to guarantee the success of these other calls. We saw them + // fail with an error code of ERROR_PATH_NOT_FOUND in tests, presumably on a + // machine where the MMCSS service was initially inactive. + static FIRST_CALL: Once = Once::new(); + let mut first_call_result = None; + FIRST_CALL.call_once(|| { + first_call_result = Some(self.set_mm_thread_characteristics_internal(task_name)) + }); + first_call_result + .unwrap_or_else(|| self.set_mm_thread_characteristics_internal(task_name)) + } + + fn set_mm_thread_characteristics_internal( + &self, + task_name: PCWSTR, + ) -> Result<(u32, HANDLE), WIN32_ERROR> { + let mut mmcss_task_index = 0u32; + let task_handle = unsafe { + (self.av_set_mm_thread_characteristics_w)(task_name, &mut mmcss_task_index) + }; + win32_error_if(task_handle == 0)?; + Ok((mmcss_task_index, task_handle)) + } - Ok(()) + pub(super) fn revert_mm_thread_characteristics( + &self, + handle: HANDLE, + ) -> Result<(), WIN32_ERROR> { + let rv = unsafe { (self.av_revert_mm_thread_characteristics)(handle) }; + win32_error_if(rv == FALSE) + } + } } -pub fn promote_current_thread_to_real_time_internal( - _audio_buffer_frames: u32, - _audio_samplerate_hz: u32, -) -> Result { - let mut task_index = 0u32; +mod win32_utils { + use windows_sys::{ + core::{PCSTR, PCWSTR}, + Win32::{ + Foundation::{FreeLibrary, GetLastError, HMODULE, WIN32_ERROR}, + System::LibraryLoader::{GetProcAddress, LoadLibraryW}, + }, + }; + + pub(super) fn win32_error_if(condition: bool) -> Result<(), WIN32_ERROR> { + if condition { + Err(unsafe { GetLastError() }) + } else { + Ok(()) + } + } + + #[derive(Debug)] + pub(super) struct OwnedLibrary(HMODULE); + + impl OwnedLibrary { + pub(super) fn try_new(lib_file_name: PCWSTR) -> Result { + let module = unsafe { LoadLibraryW(lib_file_name) }; + win32_error_if(module == 0)?; + Ok(Self(module)) + } - let handle = unsafe { AvSetMmThreadCharacteristicsA(s!("Audio"), &mut task_index) }; - let handle = RtPriorityHandleInternal::new(task_index, handle); + fn raw(&self) -> HMODULE { + self.0 + } - if handle.task_handle == 0 { - return Err(AudioThreadPriorityError::new(&format!( - "Unable to restore the thread priority ({:?})", - unsafe { GetLastError() } - ))); + /// SAFETY: The caller must transmute the value wrapped in a Ok(_) to the correct + /// function type, with the correct extern specifier. + pub(super) unsafe fn get_proc( + &self, + proc_name: PCSTR, + ) -> Result isize, WIN32_ERROR> { + let proc = unsafe { GetProcAddress(self.raw(), proc_name) }; + win32_error_if(proc.is_none())?; + Ok(proc.unwrap()) + } } - info!( - "task {} bumped to real time priority.", - handle.mmcss_task_index - ); + impl Drop for OwnedLibrary { + fn drop(&mut self) { + unsafe { + FreeLibrary(self.raw()); + } + } + } +} - Ok(handle) +#[cfg(test)] +mod tests { + use super::{ + avrt, demote_current_thread_from_real_time_internal, + promote_current_thread_to_real_time_internal, + }; + + #[test] + fn test_successful_avrt_library_load() { + assert!(avrt().is_ok()) + } + + #[test] + fn test_successful_api_use() { + let handle = promote_current_thread_to_real_time_internal(512, 44100); + println!("handle: {handle:?}"); + assert!(handle.is_ok()); + + let result = demote_current_thread_from_real_time_internal(handle.unwrap()); + println!("result: {result:?}"); + assert!(result.is_ok()); + } } diff --git a/third_party/rust/audioipc2-client/.cargo-checksum.json b/third_party/rust/audioipc2-client/.cargo-checksum.json index 00f6ff0c0e..c524aec67f 100644 --- a/third_party/rust/audioipc2-client/.cargo-checksum.json +++ b/third_party/rust/audioipc2-client/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"Cargo.toml":"f4852af58bcfd24732469ffbdddfc9404bc0d8c1d2cef279f6a6b6ddd3a080a8","cbindgen.toml":"fb6abe1671497f432a06e40b1db7ed7cd2cceecbd9a2382193ad7534e8855e34","src/context.rs":"a0559e92b554ef3156ab2bf2f1424555c8ef4a7977b9f43ac8500a9f399f8d99","src/lib.rs":"c87d9d57a16a9286cde730978db692df0fbc70cc69dd4f4677198d6843031fd8","src/send_recv.rs":"859abe75b521eb4297c84b30423814b5b87f3c7741ad16fe72189212e123e1ac","src/stream.rs":"90dc6a85552f3569ab1847de4247a46bcff2f5aef0c4d43fa2376589df015b25"},"package":null} \ No newline at end of file +{"files":{"Cargo.toml":"e94c46bbd290f02adccc7ae932285416d7e021bfde80abb2fb31a2c05426e732","cbindgen.toml":"fb6abe1671497f432a06e40b1db7ed7cd2cceecbd9a2382193ad7534e8855e34","src/context.rs":"a0559e92b554ef3156ab2bf2f1424555c8ef4a7977b9f43ac8500a9f399f8d99","src/lib.rs":"c87d9d57a16a9286cde730978db692df0fbc70cc69dd4f4677198d6843031fd8","src/send_recv.rs":"859abe75b521eb4297c84b30423814b5b87f3c7741ad16fe72189212e123e1ac","src/stream.rs":"90dc6a85552f3569ab1847de4247a46bcff2f5aef0c4d43fa2376589df015b25"},"package":null} \ No newline at end of file diff --git a/third_party/rust/audioipc2-client/Cargo.toml b/third_party/rust/audioipc2-client/Cargo.toml index 2002062c27..b776fb47e0 100644 --- a/third_party/rust/audioipc2-client/Cargo.toml +++ b/third_party/rust/audioipc2-client/Cargo.toml @@ -25,7 +25,7 @@ cubeb-backend = "0.12" log = "0.4" [dependencies.audio_thread_priority] -version = "0.31" +version = "0.32" default-features = false [dependencies.audioipc] diff --git a/third_party/rust/audioipc2-server/.cargo-checksum.json b/third_party/rust/audioipc2-server/.cargo-checksum.json index 907a296910..092d47fc83 100644 --- a/third_party/rust/audioipc2-server/.cargo-checksum.json +++ b/third_party/rust/audioipc2-server/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"Cargo.toml":"03e2fd801658feeb4bc399194a001acd1f4880967bc2993ef3f1ef20d5545bb9","cbindgen.toml":"fb6abe1671497f432a06e40b1db7ed7cd2cceecbd9a2382193ad7534e8855e34","src/lib.rs":"06aff4fd1326aeabb16b01f81a6f3c59c1717ebe96285a063724830cdf30303a","src/server.rs":"31e0c763527a16dfc281399e8a6c9eb8c1bac71ab9d7866288ec87297f4316e0"},"package":null} \ No newline at end of file +{"files":{"Cargo.toml":"77997660e305851d9c0e656aac7159b999452a36f3436d8b2f402edd36fef853","cbindgen.toml":"fb6abe1671497f432a06e40b1db7ed7cd2cceecbd9a2382193ad7534e8855e34","src/lib.rs":"d70079c66de72c3469504f1f0c9cf5e510644cac17f2d8300b8d12218740e07b","src/server.rs":"187e2236aa9f2fb6cc4a533d40714a71504afa5ef9d849ac28b7f26032859c29"},"package":null} \ No newline at end of file diff --git a/third_party/rust/audioipc2-server/Cargo.toml b/third_party/rust/audioipc2-server/Cargo.toml index 26c5d8bdc8..e5436e4e6a 100644 --- a/third_party/rust/audioipc2-server/Cargo.toml +++ b/third_party/rust/audioipc2-server/Cargo.toml @@ -27,7 +27,7 @@ once_cell = "1.2.0" slab = "0.4" [dependencies.audio_thread_priority] -version = "0.31" +version = "0.32" default-features = false [dependencies.audioipc] diff --git a/third_party/rust/audioipc2-server/src/lib.rs b/third_party/rust/audioipc2-server/src/lib.rs index c90de38c66..05396e8e29 100644 --- a/third_party/rust/audioipc2-server/src/lib.rs +++ b/third_party/rust/audioipc2-server/src/lib.rs @@ -35,8 +35,8 @@ static G_CUBEB_CONTEXT_PARAMS: Lazy> = Lazy::new(|| { }); #[allow(deprecated)] +#[allow(clippy::upper_case_acronyms)] pub mod errors { - #![allow(clippy::upper_case_acronyms)] error_chain! { links { AudioIPC(::audioipc::errors::Error, ::audioipc::errors::ErrorKind); @@ -159,7 +159,7 @@ pub unsafe extern "C" fn audioipc2_server_start( assert!(!init_params.is_null()); let mut params = G_CUBEB_CONTEXT_PARAMS.lock().unwrap(); if !context_name.is_null() { - params.context_name = CStr::from_ptr(context_name).to_owned(); + CStr::from_ptr(context_name).clone_into(&mut params.context_name); } if !backend_name.is_null() { let backend_string = CStr::from_ptr(backend_name).to_owned(); diff --git a/third_party/rust/audioipc2-server/src/server.rs b/third_party/rust/audioipc2-server/src/server.rs index 256540a6fd..96a8fc3883 100644 --- a/third_party/rust/audioipc2-server/src/server.rs +++ b/third_party/rust/audioipc2-server/src/server.rs @@ -16,7 +16,7 @@ use audioipc::{ipccore, rpccore, sys, PlatformHandle}; use cubeb::InputProcessingParams; use cubeb_core as cubeb; use cubeb_core::ffi; -use std::convert::{From, TryInto}; +use std::convert::TryInto; use std::ffi::CStr; use std::mem::size_of; use std::os::raw::{c_long, c_void}; diff --git a/third_party/rust/audioipc2/.cargo-checksum.json b/third_party/rust/audioipc2/.cargo-checksum.json index f7e319830e..92cc622efe 100644 --- a/third_party/rust/audioipc2/.cargo-checksum.json +++ b/third_party/rust/audioipc2/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"Cargo.toml":"4cdefda1488b3ec6e65d11f02d228b1ccd5e190a802d792e524df99450de4832","benches/serialization.rs":"d56855d868dab6aa22c8b03a61084535351b76c94b68d8b1d20764e352fe473f","build.rs":"65df9a97c6cdaa3faf72581f04ac289197b0b1797d69d22c1796e957ff1089e2","src/codec.rs":"38408b512d935cd7889a03b25dd14b36083ec4e6d2fcabd636182cf45e3d50bc","src/errors.rs":"67a4a994d0724397657581cde153bdfc05ce86e7efc467f23fafc8f64df80fa4","src/ipccore.rs":"db73e916468c54d3497d75ffcab3bf23067771ed7b2e1a23c714429f56f59ec3","src/lib.rs":"a6fcac8b44318435db60313d3ef32ff3fada390bea8978c8414c40744998b98b","src/messages.rs":"f1d5568c3095ee9f557412c2f4c217abfa5696a5035e4f7cb2dbc8cce700f1c4","src/rpccore.rs":"025b6614f1c42b96b0a8e74fd7881032d338c66e0d67ec0af70f910a9e30ebe1","src/shm.rs":"c00d16f4af510d12e704ae865f7348ad64ddef180e42b18e7dd95c4be35a9c80","src/sys/mod.rs":"e6fa1d260abf093e1f7b50185195e2d3aee0eb8c9774c6f253953b5896d838f3","src/sys/unix/cmsg.rs":"6f0236bf6cd66ccd237b268348a826ae1d266073c2adadcfaae703c556230065","src/sys/unix/cmsghdr.c":"d7344b3dc15cdce410c68669b848bb81f7fe36362cd3699668cb613fa05180f8","src/sys/unix/mod.rs":"59835f0d5509940078b1820a54f49fc5514adeb3e45e7d21e3ab917431da2e74","src/sys/unix/msg.rs":"0e297d73bae9414184f85c2209cca0a3fde6d999a3f1d3f42faa3f56b6d57233","src/sys/windows/mod.rs":"7eaabb76e62c6962b636320e2bbf79a78fce61659c799a798f7dd6d56b0be8a1"},"package":null} \ No newline at end of file +{"files":{"Cargo.toml":"dff67ba050be15275d357b7f001df6caa6ae1f93b2acd61ac7d6ee269a1f5960","benches/serialization.rs":"d56855d868dab6aa22c8b03a61084535351b76c94b68d8b1d20764e352fe473f","build.rs":"65df9a97c6cdaa3faf72581f04ac289197b0b1797d69d22c1796e957ff1089e2","src/codec.rs":"86068272e220696d8d7e369072326349e7598e5a24223d98179c3251bb7b3ff1","src/errors.rs":"67a4a994d0724397657581cde153bdfc05ce86e7efc467f23fafc8f64df80fa4","src/ipccore.rs":"db73e916468c54d3497d75ffcab3bf23067771ed7b2e1a23c714429f56f59ec3","src/lib.rs":"a6fcac8b44318435db60313d3ef32ff3fada390bea8978c8414c40744998b98b","src/messages.rs":"d4f6d4f41b7fd3cc7deae726657e1100f315f4cd10c5fe6ce8a57c03c8e26ca9","src/rpccore.rs":"025b6614f1c42b96b0a8e74fd7881032d338c66e0d67ec0af70f910a9e30ebe1","src/shm.rs":"c00d16f4af510d12e704ae865f7348ad64ddef180e42b18e7dd95c4be35a9c80","src/sys/mod.rs":"e6fa1d260abf093e1f7b50185195e2d3aee0eb8c9774c6f253953b5896d838f3","src/sys/unix/cmsg.rs":"9529e8f8429db86f7c5df132953d3054e603852270f3c6938cdb5f630b2711f1","src/sys/unix/cmsghdr.c":"d7344b3dc15cdce410c68669b848bb81f7fe36362cd3699668cb613fa05180f8","src/sys/unix/mod.rs":"59835f0d5509940078b1820a54f49fc5514adeb3e45e7d21e3ab917431da2e74","src/sys/unix/msg.rs":"0e297d73bae9414184f85c2209cca0a3fde6d999a3f1d3f42faa3f56b6d57233","src/sys/windows/mod.rs":"7eaabb76e62c6962b636320e2bbf79a78fce61659c799a798f7dd6d56b0be8a1"},"package":null} \ No newline at end of file diff --git a/third_party/rust/audioipc2/Cargo.toml b/third_party/rust/audioipc2/Cargo.toml index d6b66fd4e7..62be642df5 100644 --- a/third_party/rust/audioipc2/Cargo.toml +++ b/third_party/rust/audioipc2/Cargo.toml @@ -63,7 +63,7 @@ cc = "1.0" ashmem = "0.1.2" [target."cfg(target_os = \"linux\")".dependencies.audio_thread_priority] -version = "0.31" +version = "0.32" default-features = false [target."cfg(unix)".dependencies] diff --git a/third_party/rust/audioipc2/src/codec.rs b/third_party/rust/audioipc2/src/codec.rs index 2c61faa6ea..eb172fbc51 100644 --- a/third_party/rust/audioipc2/src/codec.rs +++ b/third_party/rust/audioipc2/src/codec.rs @@ -10,7 +10,7 @@ // This should be fixed in Rust 1.68, after which the following `allow` can be deleted. #![allow(clippy::uninlined_format_args)] -use bincode::{self, Options}; +use bincode::Options; use byteorder::{ByteOrder, LittleEndian}; use bytes::{Buf, BufMut, BytesMut}; use serde::de::DeserializeOwned; diff --git a/third_party/rust/audioipc2/src/messages.rs b/third_party/rust/audioipc2/src/messages.rs index 9ba9d8df3c..ef99bd4149 100644 --- a/third_party/rust/audioipc2/src/messages.rs +++ b/third_party/rust/audioipc2/src/messages.rs @@ -8,7 +8,7 @@ use crate::PlatformHandleType; use crate::INVALID_HANDLE_VALUE; #[cfg(target_os = "linux")] use audio_thread_priority::RtPriorityThreadInfo; -use cubeb::{self, ffi}; +use cubeb::ffi; use serde_derive::Deserialize; use serde_derive::Serialize; use std::ffi::{CStr, CString}; diff --git a/third_party/rust/audioipc2/src/sys/unix/cmsg.rs b/third_party/rust/audioipc2/src/sys/unix/cmsg.rs index 88a133772b..48f202957c 100644 --- a/third_party/rust/audioipc2/src/sys/unix/cmsg.rs +++ b/third_party/rust/audioipc2/src/sys/unix/cmsg.rs @@ -5,7 +5,7 @@ use crate::sys::HANDLE_QUEUE_LIMIT; use bytes::{BufMut, BytesMut}; -use libc::{self, cmsghdr}; +use libc::cmsghdr; use std::convert::TryInto; use std::os::unix::io::RawFd; use std::{mem, slice}; diff --git a/third_party/rust/bumpalo/.cargo-checksum.json b/third_party/rust/bumpalo/.cargo-checksum.json index d747ed0ac6..c9c971e9d8 100644 --- a/third_party/rust/bumpalo/.cargo-checksum.json +++ b/third_party/rust/bumpalo/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"CHANGELOG.md":"8b5a7a49c720ba2678c07184f50b3608e2165fbf6704da494fba23c864e691e0","Cargo.toml":"8d5fd21d2b3ed1d7149e864d43f843fd469ccdcd9893ac3c2bef8518294a61dd","LICENSE-APACHE":"a60eea817514531668d7e00765731449fe14d059d3249e0bc93b36de45f759f2","LICENSE-MIT":"65f94e99ddaf4f5d1782a6dae23f35d4293a9a01444a13135a6887017d353cee","README.md":"00c9224790248ec71d1505615429699fd685b0290a0c2b6d7c0df0214e7f80eb","src/alloc.rs":"ab0f23fa11c26efdd8f0596ebdf0e3faa75d097881fb59639b0fb23340c106bc","src/boxed.rs":"5fc935f8e1a7bc1b8f6a39b2bcc4355a2be4743f2308fe3ffd557455a3a27cb2","src/collections/collect_in.rs":"0588a4ff3967a4323abb4218bbd615af4b123639ab4fae9130c6590c258b3d15","src/collections/mod.rs":"d58dc46eb4f9fcdde574f09bc5b8646f53e42d49c169561d98e0c23e5b36848a","src/collections/raw_vec.rs":"8829cc9a693fde38aa93e47a7bbbc2dac247620d07f60519f2e6cb44f5494bc5","src/collections/str/lossy.rs":"c5d62b16e01071e2a574ae41ef6693ad12f1e6c786c5d38f7a13ebd6cb23c088","src/collections/str/mod.rs":"d82a8bd417fbf52a589d89a16ea2a0ac4f6ac920c3976ab1f5b6ac0c8493c4f2","src/collections/string.rs":"388d39b999788baf5c14ccc3f5cb57da728060ea3295ddfc28f0f2e1ca5858ec","src/collections/vec.rs":"2eaf52e085e6d04767e97b224e82688dd0debd231c6536d6034f431376aa8bf0","src/lib.rs":"9eb2bdb8359b368a6f3091a66b3a5eb1216672ec1605cb18d5da28292c381cb9"},"package":"0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535"} \ No newline at end of file +{"files":{"CHANGELOG.md":"78962e256666b2ca4b3ad5393da51da94decbc465e4d283a882ffdb0400973b8","Cargo.toml":"480d1eff4ff1840deaedf5670ff0cec6d5cfad8e818545072942a0a72ddba8c0","LICENSE-APACHE":"a60eea817514531668d7e00765731449fe14d059d3249e0bc93b36de45f759f2","LICENSE-MIT":"65f94e99ddaf4f5d1782a6dae23f35d4293a9a01444a13135a6887017d353cee","README.md":"19edaf495926291be237a1d8f958dd736940a2bbb75181ffefaeca3d2ce81046","src/alloc.rs":"3a9645d9e8db1f2a8549ee928cafa5263a828f25c88ce4d2b07996ecc14bfa81","src/boxed.rs":"5fc935f8e1a7bc1b8f6a39b2bcc4355a2be4743f2308fe3ffd557455a3a27cb2","src/collections/collect_in.rs":"0588a4ff3967a4323abb4218bbd615af4b123639ab4fae9130c6590c258b3d15","src/collections/mod.rs":"d58dc46eb4f9fcdde574f09bc5b8646f53e42d49c169561d98e0c23e5b36848a","src/collections/raw_vec.rs":"a37069763ff1434bb12356318d0a00cc25a273f0c2fc0bfea35615785808d1c6","src/collections/str/lossy.rs":"c5d62b16e01071e2a574ae41ef6693ad12f1e6c786c5d38f7a13ebd6cb23c088","src/collections/str/mod.rs":"d82a8bd417fbf52a589d89a16ea2a0ac4f6ac920c3976ab1f5b6ac0c8493c4f2","src/collections/string.rs":"39b2a94b552a82066fa4996d65d1dea4073e2a6724b5c237d530ec46e16bc222","src/collections/vec.rs":"a224894cd743954a90f275a5f19e7127414694bfa1d49c4647ebf789aab1721a","src/lib.rs":"c71735f5eac817d378fa47d9013056cb9feb55d15eb8247e50607bfb4ea4cdbd"},"package":"7ff69b9dd49fd426c69a0db9fc04dd934cdb6645ff000864d98f7e2af8830eaa"} \ No newline at end of file diff --git a/third_party/rust/bumpalo/CHANGELOG.md b/third_party/rust/bumpalo/CHANGELOG.md index afc142eb90..3f9e366032 100644 --- a/third_party/rust/bumpalo/CHANGELOG.md +++ b/third_party/rust/bumpalo/CHANGELOG.md @@ -28,6 +28,126 @@ Released YYYY-MM-DD. -------------------------------------------------------------------------------- +## 3.15.4 + +Released 2024-03-07. + +### Added + +* Added the `bumpalo::collections::Vec::extend_from_slices_copy` method, which + is a faster way to extend a vec from multiple slices when the element is + `Copy` than calling `extend_from_slice_copy` N times. + +-------------------------------------------------------------------------------- + +## 3.15.3 + +Released 2024-02-22. + +### Added + +* Added additional performance improvements to `bumpalo::collections::Vec` + related to reserving capacity. + +-------------------------------------------------------------------------------- + +## 3.15.2 + +Released 2024-02-21. + +### Added + +* Add a `bumpalo::collections::Vec::extend_from_slice_copy` method. This doesn't + exist on the standard library's `Vec` but they have access to specialization, + so their regular `extend_from_slice` has a specialization for `Copy` + types. Using this new method for `Copy` types is a ~80x performance + improvement over the plain `extend_from_slice` method. + +-------------------------------------------------------------------------------- + +## 3.15.1 + +Released 2024-02-20. + +### Fixed + +* Fixed the MSRV listed in `Cargo.toml`, whose update was forgotten when the + MSRV bumped in release 3.15.0. + +-------------------------------------------------------------------------------- + +## 3.15.0 + +Released 2024-02-15. + +### Changed + +* The minimum supported Rust version (MSRV) is now 1.73.0. +* `bumpalo::collections::String::push_str` and + `bumpalo::collections::String::from_str_in` received significant performance + improvements. +* Allocator trait methods are now marked `#[inline]`, increasing performance for + some callers. + +### Fixed + +* Fixed an edge-case bug in the `Allocator::shrink` method. + +-------------------------------------------------------------------------------- + +## 3.14.0 + +Released 2023-09-14. + +### Added + +* Added the `std` cargo feature, which enables implementations of `std` traits + for various things. Right now that is just `std::io::Write` for + `bumpalo::collections::Vec`, but could be more in the future. + +-------------------------------------------------------------------------------- + +## 3.13.0 + +Released 2023-05-22. + +### Added + +* New `"allocator-api2"` feature enables the use of the allocator API on + stable. This feature uses a crate that mirrors the API of the unstable Rust + `allocator_api` feature. If the feature is enabled, references to `Bump` will + implement `allocator_api2::Allocator`. This allows `Bump` to be used as an + allocator for collection types from `allocator-api2` and any other crates that + support `allocator-api2`. + +### Changed + +* The minimum supported Rust version (MSRV) is now 1.63.0. + +-------------------------------------------------------------------------------- + +## 3.12.2 + +Released 2023-05-09. + +### Changed + +* Added `rust-version` metadata to `Cargo.toml` which helps `cargo` with version + resolution. + +-------------------------------------------------------------------------------- + +## 3.12.1 + +Released 2023-04-21. + +### Fixed + +* Fixed a bug where `Bump::try_with_capacity(n)` where `n > isize::MAX` could + lead to attempts to create invalid `Layout`s. + +-------------------------------------------------------------------------------- + ## 3.12.0 Released 2023-01-17. @@ -489,7 +609,7 @@ Released 2019-12-20. from the allocated chunks are slightly different from the old `each_allocated_chunk`: only up to 16-byte alignment is supported now. If you allocate anything with greater alignment than that into the bump arena, there - might be uninitilized padding inserted in the chunks, and therefore it is no + might be uninitialized padding inserted in the chunks, and therefore it is no longer safe to read them via `MaybeUninit::assume_init`. See also the note about bump direction in the "changed" section; if you're iterating chunks, you're likely affected by that change! @@ -528,7 +648,7 @@ Released 2019-05-20. * Fixed a bug where chunks were always deallocated with the default chunk layout, not the layout that the chunk was actually allocated with (i.e. if we - started growing largers chunks with larger layouts, we would deallocate those + started growing larger chunks with larger layouts, we would deallocate those chunks with an incorrect layout). -------------------------------------------------------------------------------- diff --git a/third_party/rust/bumpalo/Cargo.toml b/third_party/rust/bumpalo/Cargo.toml index 02ec679c2b..53fc06bb97 100644 --- a/third_party/rust/bumpalo/Cargo.toml +++ b/third_party/rust/bumpalo/Cargo.toml @@ -11,8 +11,9 @@ [package] edition = "2021" +rust-version = "1.73.0" name = "bumpalo" -version = "3.12.0" +version = "3.15.4" authors = ["Nick Fitzgerald "] exclude = [ "/.github/*", @@ -29,7 +30,7 @@ categories = [ "rust-patterns", "no-std", ] -license = "MIT/Apache-2.0" +license = "MIT OR Apache-2.0" repository = "https://github.com/fitzgen/bumpalo" [package.metadata.docs.rs] @@ -50,6 +51,11 @@ path = "benches/benches.rs" harness = false required-features = ["collections"] +[dependencies.allocator-api2] +version = "0.2.8" +optional = true +default-features = false + [dev-dependencies.criterion] version = "0.3.6" @@ -64,3 +70,4 @@ allocator_api = [] boxed = [] collections = [] default = [] +std = [] diff --git a/third_party/rust/bumpalo/README.md b/third_party/rust/bumpalo/README.md index 3d73e2967e..d0da14c87d 100644 --- a/third_party/rust/bumpalo/README.md +++ b/third_party/rust/bumpalo/README.md @@ -155,7 +155,14 @@ in its space itself. ### `#![no_std]` Support -Bumpalo is a `no_std` crate. It depends only on the `alloc` and `core` crates. +Bumpalo is a `no_std` crate by default. It depends only on the `alloc` and `core` crates. + +### `std` Support + +You can optionally decide to enable the `std` feature in order to enable some +std only trait implementations for some collections: + +* `std::io::Write` for `Vec<'bump, u8>` ### Thread support @@ -179,7 +186,7 @@ First, enable the `allocator_api` feature in your `Cargo.toml`: ```toml [dependencies] -bumpalo = { version = "3.9", features = ["allocator_api"] } +bumpalo = { version = "3", features = ["allocator_api"] } ``` Next, enable the `allocator_api` nightly Rust feature in your `src/lib.rs` or @@ -207,9 +214,17 @@ v.push(2); [`Allocator`]: https://doc.rust-lang.org/std/alloc/trait.Allocator.html -#### Minimum Supported Rust Version (MSRV) +### Using the `Allocator` API on Stable Rust + +You can enable the `allocator-api2` Cargo feature and `bumpalo` will use [the +`allocator-api2` crate](https://crates.io/crates/allocator-api2) to implement +the unstable nightly`Allocator` API on stable Rust. This means that +`bumpalo::Bump` will be usable with any collection that is generic over +`allocator_api2::Allocator`. + +### Minimum Supported Rust Version (MSRV) -This crate is guaranteed to compile on stable Rust **1.56** and up. It might +This crate is guaranteed to compile on stable Rust **1.73** and up. It might compile with older versions but that may change in any new patch release. We reserve the right to increment the MSRV on minor releases, however we will diff --git a/third_party/rust/bumpalo/src/alloc.rs b/third_party/rust/bumpalo/src/alloc.rs index 0bcc21f22c..6947e2a6cf 100644 --- a/third_party/rust/bumpalo/src/alloc.rs +++ b/third_party/rust/bumpalo/src/alloc.rs @@ -752,7 +752,7 @@ pub unsafe trait Alloc { match (Layout::array::(n_old), Layout::array::(n_new)) { (Ok(ref k_old), Ok(ref k_new)) if k_old.size() > 0 && k_new.size() > 0 => { debug_assert!(k_old.align() == k_new.align()); - self.realloc(ptr.cast(), k_old.clone(), k_new.size()) + self.realloc(ptr.cast(), *k_old, k_new.size()) .map(NonNull::cast) } _ => Err(AllocErr), diff --git a/third_party/rust/bumpalo/src/collections/raw_vec.rs b/third_party/rust/bumpalo/src/collections/raw_vec.rs index ac3bd0758c..456829d447 100644 --- a/third_party/rust/bumpalo/src/collections/raw_vec.rs +++ b/third_party/rust/bumpalo/src/collections/raw_vec.rs @@ -319,7 +319,7 @@ impl<'a, T> RawVec<'a, T> { used_cap: usize, needed_extra_cap: usize, ) -> Result<(), CollectionAllocErr> { - self.reserve_internal(used_cap, needed_extra_cap, Fallible, Exact) + self.fallible_reserve_internal(used_cap, needed_extra_cap, Exact) } /// Ensures that the buffer contains at least enough space to hold @@ -343,11 +343,7 @@ impl<'a, T> RawVec<'a, T> { /// /// Aborts on OOM pub fn reserve_exact(&mut self, used_cap: usize, needed_extra_cap: usize) { - match self.reserve_internal(used_cap, needed_extra_cap, Infallible, Exact) { - Err(CapacityOverflow) => capacity_overflow(), - Err(AllocErr) => unreachable!(), - Ok(()) => { /* yay */ } - } + self.infallible_reserve_internal(used_cap, needed_extra_cap, Exact) } /// Calculates the buffer's new size given that it'll hold `used_cap + @@ -374,7 +370,7 @@ impl<'a, T> RawVec<'a, T> { used_cap: usize, needed_extra_cap: usize, ) -> Result<(), CollectionAllocErr> { - self.reserve_internal(used_cap, needed_extra_cap, Fallible, Amortized) + self.fallible_reserve_internal(used_cap, needed_extra_cap, Amortized) } /// Ensures that the buffer contains at least enough space to hold @@ -429,13 +425,11 @@ impl<'a, T> RawVec<'a, T> { /// # vector.push_all(&[1, 3, 5, 7, 9]); /// # } /// ``` + #[inline(always)] pub fn reserve(&mut self, used_cap: usize, needed_extra_cap: usize) { - match self.reserve_internal(used_cap, needed_extra_cap, Infallible, Amortized) { - Err(CapacityOverflow) => capacity_overflow(), - Err(AllocErr) => unreachable!(), - Ok(()) => { /* yay */ } - } + self.infallible_reserve_internal(used_cap, needed_extra_cap, Amortized) } + /// Attempts to ensure that the buffer contains at least enough space to hold /// `used_cap + needed_extra_cap` elements. If it doesn't already have /// enough capacity, will reallocate in place enough space plus comfortable slack @@ -593,6 +587,68 @@ enum ReserveStrategy { use self::ReserveStrategy::*; impl<'a, T> RawVec<'a, T> { + #[inline(always)] + fn fallible_reserve_internal( + &mut self, + used_cap: usize, + needed_extra_cap: usize, + strategy: ReserveStrategy, + ) -> Result<(), CollectionAllocErr> { + // This portion of the method should always be inlined. + if self.cap().wrapping_sub(used_cap) >= needed_extra_cap { + return Ok(()); + } + // This portion of the method should never be inlined, and will only be called when + // the check above has confirmed that it is necessary. + self.reserve_internal_or_error(used_cap, needed_extra_cap, Fallible, strategy) + } + + #[inline(always)] + fn infallible_reserve_internal( + &mut self, + used_cap: usize, + needed_extra_cap: usize, + strategy: ReserveStrategy, + ) { + // This portion of the method should always be inlined. + if self.cap().wrapping_sub(used_cap) >= needed_extra_cap { + return; + } + // This portion of the method should never be inlined, and will only be called when + // the check above has confirmed that it is necessary. + self.reserve_internal_or_panic(used_cap, needed_extra_cap, strategy) + } + + #[inline(never)] + fn reserve_internal_or_panic( + &mut self, + used_cap: usize, + needed_extra_cap: usize, + strategy: ReserveStrategy, + ) { + // Delegates the call to `reserve_internal_or_error` and panics in the event of an error. + // This allows the method to have a return type of `()`, simplifying the assembly at the + // call site. + match self.reserve_internal(used_cap, needed_extra_cap, Infallible, strategy) { + Err(CapacityOverflow) => capacity_overflow(), + Err(AllocErr) => unreachable!(), + Ok(()) => { /* yay */ } + } + } + + #[inline(never)] + fn reserve_internal_or_error( + &mut self, + used_cap: usize, + needed_extra_cap: usize, + fallibility: Fallibility, + strategy: ReserveStrategy,)-> Result<(), CollectionAllocErr> { + // Delegates the call to `reserve_internal`, which can be inlined. + self.reserve_internal(used_cap, needed_extra_cap, fallibility, strategy) + } + + /// Helper method to reserve additional space, reallocating the backing memory. + /// The caller is responsible for confirming that there is not already enough space available. fn reserve_internal( &mut self, used_cap: usize, @@ -608,12 +664,6 @@ impl<'a, T> RawVec<'a, T> { // If we make it past the first branch then we are guaranteed to // panic. - // Don't actually need any more capacity. - // Wrapping in case they gave a bad `used_cap`. - if self.cap().wrapping_sub(used_cap) >= needed_extra_cap { - return Ok(()); - } - // Nothing we can really do about these checks :( let new_cap = match strategy { Exact => used_cap diff --git a/third_party/rust/bumpalo/src/collections/string.rs b/third_party/rust/bumpalo/src/collections/string.rs index ffd1db92de..e9fafbf204 100644 --- a/third_party/rust/bumpalo/src/collections/string.rs +++ b/third_party/rust/bumpalo/src/collections/string.rs @@ -680,8 +680,19 @@ impl<'bump> String<'bump> { /// assert_eq!(s, "hello"); /// ``` pub fn from_str_in(s: &str, bump: &'bump Bump) -> String<'bump> { - let mut t = String::with_capacity_in(s.len(), bump); - t.push_str(s); + let len = s.len(); + let mut t = String::with_capacity_in(len, bump); + // SAFETY: + // * `src` is valid for reads of `s.len()` bytes by virtue of being an allocated `&str`. + // * `dst` is valid for writes of `s.len()` bytes as `String::with_capacity_in(s.len(), bump)` + // above guarantees that. + // * Alignment is not relevant as `u8` has no alignment requirements. + // * Source and destination ranges cannot overlap as we just reserved the destination + // range from the bump. + unsafe { ptr::copy_nonoverlapping(s.as_ptr(), t.vec.as_mut_ptr(), len) }; + // SAFETY: We reserved sufficent capacity for the string above. + // The elements at `0..len` were initialized by `copy_nonoverlapping` above. + unsafe { t.vec.set_len(len) }; t } @@ -925,7 +936,7 @@ impl<'bump> String<'bump> { /// ``` #[inline] pub fn push_str(&mut self, string: &str) { - self.vec.extend_from_slice(string.as_bytes()) + self.vec.extend_from_slice_copy(string.as_bytes()) } /// Returns this `String`'s capacity, in bytes. diff --git a/third_party/rust/bumpalo/src/collections/vec.rs b/third_party/rust/bumpalo/src/collections/vec.rs index 312aa055b9..0dab700727 100644 --- a/third_party/rust/bumpalo/src/collections/vec.rs +++ b/third_party/rust/bumpalo/src/collections/vec.rs @@ -104,6 +104,8 @@ use core::ops::{Index, IndexMut, RangeBounds}; use core::ptr; use core::ptr::NonNull; use core::slice; +#[cfg(feature = "std")] +use std::io; unsafe fn arith_offset(p: *const T, offset: isize) -> *const T { p.offset(offset) @@ -1775,6 +1777,132 @@ impl<'bump, T: 'bump + Clone> Vec<'bump, T> { } } +impl<'bump, T: 'bump + Copy> Vec<'bump, T> { + /// Helper method to copy all of the items in `other` and append them to the end of `self`. + /// + /// SAFETY: + /// * The caller is responsible for: + /// * calling [`reserve`](Self::reserve) beforehand to guarantee that there is enough + /// capacity to store `other.len()` more items. + /// * guaranteeing that `self` and `other` do not overlap. + unsafe fn extend_from_slice_copy_unchecked(&mut self, other: &[T]) { + let old_len = self.len(); + debug_assert!(old_len + other.len() <= self.capacity()); + + // SAFETY: + // * `src` is valid for reads of `other.len()` values by virtue of being a `&[T]`. + // * `dst` is valid for writes of `other.len()` bytes because the caller of this + // method is required to `reserve` capacity to store at least `other.len()` items + // beforehand. + // * Because `src` is a `&[T]` and dst is a `&[T]` within the `Vec`, + // `copy_nonoverlapping`'s alignment requirements are met. + // * Caller is required to guarantee that the source and destination ranges cannot overlap + unsafe { + let src = other.as_ptr(); + let dst = self.as_mut_ptr().add(old_len); + ptr::copy_nonoverlapping(src, dst, other.len()); + self.set_len(old_len + other.len()); + } + } + + + /// Copies all elements in the slice `other` and appends them to the `Vec`. + /// + /// Note that this function is same as [`extend_from_slice`] except that it is optimized for + /// slices of types that implement the `Copy` trait. If and when Rust gets specialization + /// this function will likely be deprecated (but still available). + /// + /// To copy and append the data from multiple source slices at once, see + /// [`extend_from_slices_copy`]. + /// + /// # Examples + /// + /// ``` + /// use bumpalo::{Bump, collections::Vec}; + /// + /// let b = Bump::new(); + /// + /// let mut vec = bumpalo::vec![in &b; 1]; + /// vec.extend_from_slice_copy(&[2, 3, 4]); + /// assert_eq!(vec, [1, 2, 3, 4]); + /// ``` + /// + /// ``` + /// use bumpalo::{Bump, collections::Vec}; + /// + /// let b = Bump::new(); + /// + /// let mut vec = bumpalo::vec![in &b; 'H' as u8]; + /// vec.extend_from_slice_copy("ello, world!".as_bytes()); + /// assert_eq!(vec, "Hello, world!".as_bytes()); + /// ``` + /// + /// [`extend_from_slice`]: #method.extend_from_slice + /// [`extend_from_slices`]: #method.extend_from_slices + pub fn extend_from_slice_copy(&mut self, other: &[T]) { + // Reserve space in the Vec for the values to be added + self.reserve(other.len()); + + // Copy values into the space that was just reserved + // SAFETY: + // * `self` has enough capacity to store `other.len()` more items as `self.reserve(other.len())` + // above guarantees that. + // * Source and destination data ranges cannot overlap as we just reserved the destination + // range from the bump. + unsafe { + self.extend_from_slice_copy_unchecked(other); + } + } + + /// For each slice in `slices`, copies all elements in the slice and appends them to the `Vec`. + /// + /// This method is equivalent to calling [`extend_from_slice_copy`] in a loop, but is able + /// to precompute the total amount of space to reserve in advance. This reduces the potential + /// maximum number of reallocations needed from one-per-slice to just one. + /// + /// # Examples + /// + /// ``` + /// use bumpalo::{Bump, collections::Vec}; + /// + /// let b = Bump::new(); + /// + /// let mut vec = bumpalo::vec![in &b; 1]; + /// vec.extend_from_slices_copy(&[&[2, 3], &[], &[4]]); + /// assert_eq!(vec, [1, 2, 3, 4]); + /// ``` + /// + /// ``` + /// use bumpalo::{Bump, collections::Vec}; + /// + /// let b = Bump::new(); + /// + /// let mut vec = bumpalo::vec![in &b; 'H' as u8]; + /// vec.extend_from_slices_copy(&["ello,".as_bytes(), &[], " world!".as_bytes()]); + /// assert_eq!(vec, "Hello, world!".as_bytes()); + /// ``` + /// + /// [`extend_from_slice_copy`]: #method.extend_from_slice_copy + pub fn extend_from_slices_copy(&mut self, slices: &[&[T]]) { + // Reserve the total amount of capacity we'll need to safely append the aggregated contents + // of each slice in `slices`. + let capacity_to_reserve: usize = slices.iter().map(|slice| slice.len()).sum(); + self.reserve(capacity_to_reserve); + + // SAFETY: + // * `dst` is valid for writes of `capacity_to_reserve` items as + // `self.reserve(capacity_to_reserve)` above guarantees that. + // * Source and destination ranges cannot overlap as we just reserved the destination + // range from the bump. + unsafe { + // Copy the contents of each slice onto the end of `self` + slices.iter().for_each(|slice| { + self.extend_from_slice_copy_unchecked(slice); + }); + } + } +} + // This code generalises `extend_with_{element,default}`. trait ExtendWith { fn next(&mut self) -> T; @@ -2612,3 +2740,23 @@ where } } } + +#[cfg(feature = "std")] +impl<'bump> io::Write for Vec<'bump, u8> { + #[inline] + fn write(&mut self, buf: &[u8]) -> io::Result { + self.extend_from_slice_copy(buf); + Ok(buf.len()) + } + + #[inline] + fn write_all(&mut self, buf: &[u8]) -> io::Result<()> { + self.extend_from_slice_copy(buf); + Ok(()) + } + + #[inline] + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} diff --git a/third_party/rust/bumpalo/src/lib.rs b/third_party/rust/bumpalo/src/lib.rs old mode 100644 new mode 100755 index 74dfcd4361..b23cfeabc8 --- a/third_party/rust/bumpalo/src/lib.rs +++ b/third_party/rust/bumpalo/src/lib.rs @@ -1,11 +1,8 @@ #![doc = include_str!("../README.md")] #![deny(missing_debug_implementations)] #![deny(missing_docs)] -#![no_std] -#![cfg_attr( - feature = "allocator_api", - feature(allocator_api, nonnull_slice_from_raw_parts) -)] +#![cfg_attr(not(feature = "std"), no_std)] +#![cfg_attr(feature = "allocator_api", feature(allocator_api))] #[doc(hidden)] pub extern crate alloc as core_alloc; @@ -26,9 +23,13 @@ use core::ptr::{self, NonNull}; use core::slice; use core::str; use core_alloc::alloc::{alloc, dealloc, Layout}; + #[cfg(feature = "allocator_api")] use core_alloc::alloc::{AllocError, Allocator}; +#[cfg(all(feature = "allocator-api2", not(feature = "allocator_api")))] +use allocator_api2::alloc::{AllocError, Allocator}; + pub use alloc::AllocErr; /// An error returned from [`Bump::try_alloc_try_with`]. @@ -354,7 +355,7 @@ static EMPTY_CHUNK: EmptyChunkFooter = EmptyChunkFooter(ChunkFooter { impl EmptyChunkFooter { fn get(&'static self) -> NonNull { - unsafe { NonNull::new_unchecked(&self.0 as *const ChunkFooter as *mut ChunkFooter) } + NonNull::from(&self.0) } } @@ -405,6 +406,15 @@ unsafe fn dealloc_chunk_list(mut footer: NonNull) { // prevent sending the `Bump` across threads until the borrows end. unsafe impl Send for Bump {} +#[inline] +fn is_pointer_aligned_to(pointer: *mut T, align: usize) -> bool { + debug_assert!(align.is_power_of_two()); + + let pointer = pointer as usize; + let pointer_aligned = round_down_to(pointer, align); + pointer == pointer_aligned +} + #[inline] pub(crate) fn round_up_to(n: usize, divisor: usize) -> Option { debug_assert!(divisor > 0); @@ -419,6 +429,14 @@ pub(crate) fn round_down_to(n: usize, divisor: usize) -> usize { n & !(divisor - 1) } +/// Same as `round_down_to` but preserves pointer provenance. +#[inline] +pub(crate) fn round_mut_ptr_down_to(ptr: *mut u8, divisor: usize) -> *mut u8 { + debug_assert!(divisor > 0); + debug_assert!(divisor.is_power_of_two()); + ptr.wrapping_sub(ptr as usize & (divisor - 1)) +} + // After this point, we try to hit page boundaries instead of powers of 2 const PAGE_STRATEGY_CUTOFF: usize = 0x1000; @@ -463,12 +481,8 @@ struct NewChunkMemoryDetails { /// Wrapper around `Layout::from_size_align` that adds debug assertions. #[inline] -unsafe fn layout_from_size_align(size: usize, align: usize) -> Layout { - if cfg!(debug_assertions) { - Layout::from_size_align(size, align).unwrap() - } else { - Layout::from_size_align_unchecked(size, align) - } +fn layout_from_size_align(size: usize, align: usize) -> Result { + Layout::from_size_align(size, align).map_err(|_| AllocErr) } #[inline(never)] @@ -476,12 +490,6 @@ fn allocation_size_overflow() -> T { panic!("requested allocation size overflowed") } -// This can be migrated to directly use `usize::abs_diff` when the MSRV -// reaches `1.60` -fn abs_diff(a: usize, b: usize) -> usize { - usize::max(a, b) - usize::min(a, b) -} - impl Bump { /// Construct a new arena to bump allocate into. /// @@ -535,7 +543,7 @@ impl Bump { }); } - let layout = unsafe { layout_from_size_align(capacity, 1) }; + let layout = layout_from_size_align(capacity, 1)?; let chunk_footer = unsafe { Self::new_chunk( @@ -589,7 +597,7 @@ impl Bump { /// assert!(bump.try_alloc(5).is_err()); /// ``` pub fn set_allocation_limit(&self, limit: Option) { - self.allocation_limit.set(limit) + self.allocation_limit.set(limit); } /// How much headroom an arena has before it hits its allocation @@ -600,7 +608,7 @@ impl Bump { if allocated_bytes > allocation_limit { None } else { - Some(abs_diff(allocation_limit, allocated_bytes)) + Some(usize::abs_diff(allocation_limit, allocated_bytes)) } }) } @@ -682,7 +690,7 @@ impl Bump { size, } = new_chunk_memory_details; - let layout = layout_from_size_align(size, align); + let layout = layout_from_size_align(size, align).ok()?; debug_assert!(size >= requested_layout.size()); @@ -801,7 +809,6 @@ impl Bump { /// assert_eq!(*x, "hello"); /// ``` #[inline(always)] - #[allow(clippy::mut_from_ref)] pub fn alloc(&self, val: T) -> &mut T { self.alloc_with(|| val) } @@ -821,7 +828,6 @@ impl Bump { /// assert_eq!(x, Ok(&mut "hello")); /// ``` #[inline(always)] - #[allow(clippy::mut_from_ref)] pub fn try_alloc(&self, val: T) -> Result<&mut T, AllocErr> { self.try_alloc_with(|| val) } @@ -846,7 +852,6 @@ impl Bump { /// assert_eq!(*x, "hello"); /// ``` #[inline(always)] - #[allow(clippy::mut_from_ref)] pub fn alloc_with(&self, f: F) -> &mut T where F: FnOnce() -> T, @@ -866,7 +871,7 @@ impl Bump { // directly into the heap instead. It seems we get it to realize // this most consistently if we put this critical line into it's // own function instead of inlining it into the surrounding code. - ptr::write(ptr, f()) + ptr::write(ptr, f()); } let layout = Layout::new::(); @@ -899,7 +904,6 @@ impl Bump { /// assert_eq!(x, Ok(&mut "hello")); /// ``` #[inline(always)] - #[allow(clippy::mut_from_ref)] pub fn try_alloc_with(&self, f: F) -> Result<&mut T, AllocErr> where F: FnOnce() -> T, @@ -919,7 +923,7 @@ impl Bump { // directly into the heap instead. It seems we get it to realize // this most consistently if we put this critical line into it's // own function instead of inlining it into the surrounding code. - ptr::write(ptr, f()) + ptr::write(ptr, f()); } //SAFETY: Self-contained: @@ -971,7 +975,6 @@ impl Bump { /// # Result::<_, ()>::Ok(()) /// ``` #[inline(always)] - #[allow(clippy::mut_from_ref)] pub fn alloc_try_with(&self, f: F) -> Result<&mut T, E> where F: FnOnce() -> Result, @@ -1080,7 +1083,6 @@ impl Bump { /// # Result::<_, bumpalo::AllocOrInitError<()>>::Ok(()) /// ``` #[inline(always)] - #[allow(clippy::mut_from_ref)] pub fn try_alloc_try_with(&self, f: F) -> Result<&mut T, AllocOrInitError> where F: FnOnce() -> Result, @@ -1165,7 +1167,6 @@ impl Bump { /// assert_eq!(x, &[1, 2, 3]); /// ``` #[inline(always)] - #[allow(clippy::mut_from_ref)] pub fn alloc_slice_copy(&self, src: &[T]) -> &mut [T] where T: Copy, @@ -1205,7 +1206,6 @@ impl Bump { /// assert_eq!(originals, clones); /// ``` #[inline(always)] - #[allow(clippy::mut_from_ref)] pub fn alloc_slice_clone(&self, src: &[T]) -> &mut [T] where T: Clone, @@ -1236,7 +1236,6 @@ impl Bump { /// assert_eq!("hello world", hello); /// ``` #[inline(always)] - #[allow(clippy::mut_from_ref)] pub fn alloc_str(&self, src: &str) -> &mut str { let buffer = self.alloc_slice_copy(src.as_bytes()); unsafe { @@ -1263,7 +1262,6 @@ impl Bump { /// assert_eq!(x, &[5, 10, 15, 20, 25]); /// ``` #[inline(always)] - #[allow(clippy::mut_from_ref)] pub fn alloc_slice_fill_with(&self, len: usize, mut f: F) -> &mut [T] where F: FnMut(usize) -> T, @@ -1299,7 +1297,6 @@ impl Bump { /// assert_eq!(x, &[42, 42, 42, 42, 42]); /// ``` #[inline(always)] - #[allow(clippy::mut_from_ref)] pub fn alloc_slice_fill_copy(&self, len: usize, value: T) -> &mut [T] { self.alloc_slice_fill_with(len, |_| value) } @@ -1324,7 +1321,6 @@ impl Bump { /// assert_eq!(&x[1], &s); /// ``` #[inline(always)] - #[allow(clippy::mut_from_ref)] pub fn alloc_slice_fill_clone(&self, len: usize, value: &T) -> &mut [T] { self.alloc_slice_fill_with(len, |_| value.clone()) } @@ -1347,7 +1343,6 @@ impl Bump { /// assert_eq!(x, [4, 9, 25]); /// ``` #[inline(always)] - #[allow(clippy::mut_from_ref)] pub fn alloc_slice_fill_iter(&self, iter: I) -> &mut [T] where I: IntoIterator, @@ -1378,7 +1373,6 @@ impl Bump { /// assert_eq!(x, &[0, 0, 0, 0, 0]); /// ``` #[inline(always)] - #[allow(clippy::mut_from_ref)] pub fn alloc_slice_fill_default(&self, len: usize) -> &mut [T] { self.alloc_slice_fill_with(len, |_| T::default()) } @@ -1435,11 +1429,10 @@ impl Bump { } let ptr = ptr.wrapping_sub(layout.size()); - let rem = ptr as usize % layout.align(); - let aligned_ptr = ptr.wrapping_sub(rem); + let aligned_ptr = round_mut_ptr_down_to(ptr, layout.align()); if aligned_ptr >= start { - let aligned_ptr = NonNull::new_unchecked(aligned_ptr as *mut u8); + let aligned_ptr = NonNull::new_unchecked(aligned_ptr); footer.ptr.set(aligned_ptr); Some(aligned_ptr) } else { @@ -1464,12 +1457,13 @@ impl Bump { let current_footer = self.current_chunk_footer.get(); let current_footer = unsafe { current_footer.as_ref() }; - current_footer as *const _ as usize - current_footer.data.as_ptr() as usize + current_footer.ptr.get().as_ptr() as usize - current_footer.data.as_ptr() as usize } /// Slow path allocation for when we need to allocate a new chunk from the /// parent bump set because there isn't enough room in our current chunk. #[inline(never)] + #[cold] fn alloc_layout_slow(&self, layout: Layout) -> Option> { unsafe { let size = layout.size(); @@ -1488,21 +1482,14 @@ impl Bump { .checked_mul(2)? .max(min_new_chunk_size); let chunk_memory_details = iter::from_fn(|| { - let bypass_min_chunk_size_for_small_limits = match self.allocation_limit() { - Some(limit) - if layout.size() < limit + let bypass_min_chunk_size_for_small_limits = matches!(self.allocation_limit(), Some(limit) if layout.size() < limit && base_size >= layout.size() && limit < DEFAULT_CHUNK_SIZE_WITHOUT_FOOTER - && self.allocated_bytes() == 0 => - { - true - } - _ => false, - }; + && self.allocated_bytes() == 0); if base_size >= min_new_chunk_size || bypass_min_chunk_size_for_small_limits { let size = base_size; - base_size = base_size / 2; + base_size /= 2; Bump::new_chunk_memory_details(Some(size), layout) } else { None @@ -1537,14 +1524,14 @@ impl Bump { // at least the requested size. let mut ptr = new_footer.ptr.get().as_ptr().sub(size); // Round the pointer down to the requested alignment. - ptr = ptr.sub(ptr as usize % layout.align()); + ptr = round_mut_ptr_down_to(ptr, layout.align()); debug_assert!( ptr as *const _ <= new_footer, "{:p} <= {:p}", ptr, new_footer ); - let ptr = NonNull::new_unchecked(ptr as *mut u8); + let ptr = NonNull::new_unchecked(ptr); new_footer.ptr.set(ptr); // Return a pointer to the freshly allocated region in this chunk. @@ -1696,6 +1683,16 @@ impl Bump { unsafe { footer.as_ref().allocated_bytes } } + /// Calculates the number of bytes requested from the Rust allocator for this `Bump`. + /// + /// This number is equal to the [`allocated_bytes()`](Self::allocated_bytes) plus + /// the size of the bump metadata. + pub fn allocated_bytes_including_metadata(&self) -> usize { + let metadata_size = + unsafe { self.iter_allocated_chunks_raw().count() * mem::size_of::() }; + self.allocated_bytes() + metadata_size + } + #[inline] unsafe fn is_last_allocation(&self, ptr: NonNull) -> bool { let footer = self.current_chunk_footer.get(); @@ -1720,13 +1717,31 @@ impl Bump { old_layout: Layout, new_layout: Layout, ) -> Result, AllocErr> { + // If the new layout demands greater alignment than the old layout has, + // then either + // + // 1. the pointer happens to satisfy the new layout's alignment, so we + // got lucky and can return the pointer as-is, or + // + // 2. the pointer is not aligned to the new layout's demanded alignment, + // and we are unlucky. + // + // In the case of (2), to successfully "shrink" the allocation, we would + // have to allocate a whole new region for the new layout, without being + // able to free the old region. That is unacceptable, so simply return + // an allocation failure error instead. + if old_layout.align() < new_layout.align() { + if is_pointer_aligned_to(ptr.as_ptr(), new_layout.align()) { + return Ok(ptr); + } else { + return Err(AllocErr); + } + } + + debug_assert!(is_pointer_aligned_to(ptr.as_ptr(), new_layout.align())); + let old_size = old_layout.size(); let new_size = new_layout.size(); - let align_is_compatible = old_layout.align() >= new_layout.align(); - - if !align_is_compatible { - return Err(AllocErr); - } // This is how much space we would *actually* reclaim while satisfying // the requested alignment. @@ -1736,7 +1751,32 @@ impl Bump { // Only reclaim the excess space (which requires a copy) if it // is worth it: we are actually going to recover "enough" space // and we can do a non-overlapping copy. - && delta >= old_size / 2 + // + // We do `(old_size + 1) / 2` so division rounds up rather than + // down. Consider when: + // + // old_size = 5 + // new_size = 3 + // + // If we do not take care to round up, this will result in: + // + // delta = 2 + // (old_size / 2) = (5 / 2) = 2 + // + // And the the check will succeed even though we are have + // overlapping ranges: + // + // |--------old-allocation-------| + // |------from-------| + // |-------to--------| + // +-----+-----+-----+-----+-----+ + // | a | b | c | . | . | + // +-----+-----+-----+-----+-----+ + // + // But we MUST NOT have overlapping ranges because we use + // `copy_nonoverlapping` below! Therefore, we round the division + // up to avoid this issue. + && delta >= (old_size + 1) / 2 { let footer = self.current_chunk_footer.get(); let footer = footer.as_ref(); @@ -1751,9 +1791,11 @@ impl Bump { ptr::copy_nonoverlapping(ptr.as_ptr(), new_ptr.as_ptr(), new_size); return Ok(new_ptr); - } else { - return Ok(ptr); } + + // If this wasn't the last allocation, or shrinking wasn't worth it, + // simply return the old pointer as-is. + Ok(ptr) } #[inline] @@ -1772,7 +1814,7 @@ impl Bump { // reuse the currently allocated space. let delta = new_size - old_size; if let Some(p) = - self.try_alloc_layout_fast(layout_from_size_align(delta, old_layout.align())) + self.try_alloc_layout_fast(layout_from_size_align(delta, old_layout.align())?) { ptr::copy(ptr.as_ptr(), p.as_ptr(), old_size); return Ok(p); @@ -1867,7 +1909,7 @@ unsafe impl<'a> alloc::Alloc for &'a Bump { #[inline] unsafe fn dealloc(&mut self, ptr: NonNull, layout: Layout) { - Bump::dealloc(self, ptr, layout) + Bump::dealloc(self, ptr, layout); } #[inline] @@ -1883,7 +1925,7 @@ unsafe impl<'a> alloc::Alloc for &'a Bump { return self.try_alloc_layout(layout); } - let new_layout = layout_from_size_align(new_size, layout.align()); + let new_layout = layout_from_size_align(new_size, layout.align())?; if new_size <= old_size { self.shrink(ptr, layout, new_layout) } else { @@ -1892,18 +1934,23 @@ unsafe impl<'a> alloc::Alloc for &'a Bump { } } -#[cfg(feature = "allocator_api")] +#[cfg(any(feature = "allocator_api", feature = "allocator-api2"))] unsafe impl<'a> Allocator for &'a Bump { + #[inline] fn allocate(&self, layout: Layout) -> Result, AllocError> { self.try_alloc_layout(layout) - .map(|p| NonNull::slice_from_raw_parts(p, layout.size())) + .map(|p| unsafe { + NonNull::new_unchecked(ptr::slice_from_raw_parts_mut(p.as_ptr(), layout.size())) + }) .map_err(|_| AllocError) } + #[inline] unsafe fn deallocate(&self, ptr: NonNull, layout: Layout) { Bump::dealloc(self, ptr, layout) } + #[inline] unsafe fn shrink( &self, ptr: NonNull, @@ -1911,10 +1958,13 @@ unsafe impl<'a> Allocator for &'a Bump { new_layout: Layout, ) -> Result, AllocError> { Bump::shrink(self, ptr, old_layout, new_layout) - .map(|p| NonNull::slice_from_raw_parts(p, new_layout.size())) + .map(|p| unsafe { + NonNull::new_unchecked(ptr::slice_from_raw_parts_mut(p.as_ptr(), new_layout.size())) + }) .map_err(|_| AllocError) } + #[inline] unsafe fn grow( &self, ptr: NonNull, @@ -1922,10 +1972,13 @@ unsafe impl<'a> Allocator for &'a Bump { new_layout: Layout, ) -> Result, AllocError> { Bump::grow(self, ptr, old_layout, new_layout) - .map(|p| NonNull::slice_from_raw_parts(p, new_layout.size())) + .map(|p| unsafe { + NonNull::new_unchecked(ptr::slice_from_raw_parts_mut(p.as_ptr(), new_layout.size())) + }) .map_err(|_| AllocError) } + #[inline] unsafe fn grow_zeroed( &self, ptr: NonNull, @@ -1953,7 +2006,6 @@ mod tests { // Uses private `alloc` module. #[test] - #[allow(clippy::cognitive_complexity)] fn test_realloc() { use crate::alloc::Alloc; diff --git a/third_party/rust/coreaudio-sys-utils/.cargo-checksum.json b/third_party/rust/coreaudio-sys-utils/.cargo-checksum.json index 5459f53214..f22a44473c 100644 --- a/third_party/rust/coreaudio-sys-utils/.cargo-checksum.json +++ b/third_party/rust/coreaudio-sys-utils/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"Cargo.toml":"87292d055a2fc0f070f54abd549a5f79ec8ac33611ecde80ba394f256b88294c","src/aggregate_device.rs":"7d2bd5f5fd7f3d008ebb69ad81f522ca0cb73db6d7b3e50ed1a63ea26ff721f4","src/audio_device_extensions.rs":"2852f9ce65581cb5cf3f8e581f2652087eae0a569ed429be362e636db6441b6b","src/audio_object.rs":"5447179330a862659a25bceedfdc5d29a1296f63490908d1c868c6b21c5f95a1","src/audio_unit.rs":"d783878930df4923b57ad230138c0f3fd6b0b9bb80a39725092ff4c6615162d8","src/cf_mutable_dict.rs":"fc42edd270c6dfb02f123214d2d8e487bbd62b5bd923b71eec13190fd0104d2a","src/dispatch.rs":"f6267fe587217c3d3ad5fe7f3a35955221c936103bf853c477a2e44eba5f1e46","src/lib.rs":"c93ed1411dd6cc39db44f57e0d7683bbc54745f84a3c9f9533a088895ec97abe","src/string.rs":"28f88b816c768bcfcc674a60d962b93f1c94e5e0f4cc8ed2a1301138b91039e7"},"package":null} \ No newline at end of file +{"files":{"Cargo.toml":"87292d055a2fc0f070f54abd549a5f79ec8ac33611ecde80ba394f256b88294c","src/aggregate_device.rs":"7d2bd5f5fd7f3d008ebb69ad81f522ca0cb73db6d7b3e50ed1a63ea26ff721f4","src/audio_device_extensions.rs":"5c869d791947d15eec8bffe0bb302fe32d0578111ffe0049213e720eb60a34e1","src/audio_object.rs":"34f7e038c1ed30d503d669d89f01864ae90e009a2fa74ef50fac343a53113ff2","src/audio_unit.rs":"d38007faed2ce4d88efb70054a1fdfadf8249d0e55b900eb3ac8eae04355bf2b","src/cf_mutable_dict.rs":"fc42edd270c6dfb02f123214d2d8e487bbd62b5bd923b71eec13190fd0104d2a","src/dispatch.rs":"24b6bcf0dcaa6618e03039cd060a274c8f9ed48264e14de465ae3aacb2daad57","src/lib.rs":"c93ed1411dd6cc39db44f57e0d7683bbc54745f84a3c9f9533a088895ec97abe","src/string.rs":"28f88b816c768bcfcc674a60d962b93f1c94e5e0f4cc8ed2a1301138b91039e7"},"package":null} \ No newline at end of file diff --git a/third_party/rust/coreaudio-sys-utils/src/audio_device_extensions.rs b/third_party/rust/coreaudio-sys-utils/src/audio_device_extensions.rs index 04eeaaf566..8224d87c28 100644 --- a/third_party/rust/coreaudio-sys-utils/src/audio_device_extensions.rs +++ b/third_party/rust/coreaudio-sys-utils/src/audio_device_extensions.rs @@ -1,3 +1,4 @@ +use crate::dispatch::*; use coreaudio_sys::*; // See https://opensource.apple.com/source/WebCore/WebCore-7604.5.6/platform/spi/cf/CoreAudioSPI.h.auto.html @@ -18,5 +19,6 @@ pub fn audio_device_duck( in_start_time: *const AudioTimeStamp, in_ramp_duration: f32, ) -> OSStatus { + debug_assert_running_serially(); unsafe { AudioDeviceDuck(in_device, in_ducked_level, in_start_time, in_ramp_duration) } } diff --git a/third_party/rust/coreaudio-sys-utils/src/audio_object.rs b/third_party/rust/coreaudio-sys-utils/src/audio_object.rs index 368d6caadc..b8d74c80fc 100644 --- a/third_party/rust/coreaudio-sys-utils/src/audio_object.rs +++ b/third_party/rust/coreaudio-sys-utils/src/audio_object.rs @@ -1,3 +1,4 @@ +use crate::dispatch::*; use coreaudio_sys::*; use std::fmt; use std::os::raw::c_void; @@ -77,6 +78,7 @@ pub fn audio_object_set_property_data( size: usize, data: *const T, ) -> OSStatus { + debug_assert_running_serially(); unsafe { AudioObjectSetPropertyData( id, @@ -99,6 +101,7 @@ pub fn audio_object_add_property_listener( listener: audio_object_property_listener_proc, data: *mut T, ) -> OSStatus { + debug_assert_running_serially(); unsafe { AudioObjectAddPropertyListener(id, address, Some(listener), data as *mut c_void) } } @@ -108,6 +111,7 @@ pub fn audio_object_remove_property_listener( listener: audio_object_property_listener_proc, data: *mut T, ) -> OSStatus { + debug_assert_running_serially(); unsafe { AudioObjectRemovePropertyListener(id, address, Some(listener), data as *mut c_void) } } diff --git a/third_party/rust/coreaudio-sys-utils/src/audio_unit.rs b/third_party/rust/coreaudio-sys-utils/src/audio_unit.rs index 059a58f26b..30e5a3cf4b 100644 --- a/third_party/rust/coreaudio-sys-utils/src/audio_unit.rs +++ b/third_party/rust/coreaudio-sys-utils/src/audio_unit.rs @@ -1,3 +1,4 @@ +use crate::dispatch::debug_assert_running_serially; use coreaudio_sys::*; use std::convert::TryFrom; use std::os::raw::c_void; @@ -13,6 +14,7 @@ pub fn audio_unit_get_property_info( ) -> OSStatus { assert!(!unit.is_null()); assert!(UInt32::try_from(*size).is_ok()); // Check if `size` can be converted to a UInt32. + debug_assert_running_serially(); unsafe { AudioUnitGetPropertyInfo( unit, @@ -35,6 +37,7 @@ pub fn audio_unit_get_property( ) -> OSStatus { assert!(!unit.is_null()); assert!(UInt32::try_from(*size).is_ok()); // Check if `size` can be converted to a UInt32. + debug_assert_running_serially(); unsafe { AudioUnitGetProperty( unit, @@ -56,6 +59,7 @@ pub fn audio_unit_set_property( size: usize, ) -> OSStatus { assert!(!unit.is_null()); + debug_assert_running_serially(); unsafe { AudioUnitSetProperty( unit, @@ -76,6 +80,7 @@ pub fn audio_unit_get_parameter( value: &mut AudioUnitParameterValue, ) -> OSStatus { assert!(!unit.is_null()); + debug_assert_running_serially(); unsafe { AudioUnitGetParameter( unit, @@ -96,30 +101,36 @@ pub fn audio_unit_set_parameter( buffer_offset_in_frames: UInt32, ) -> OSStatus { assert!(!unit.is_null()); + debug_assert_running_serially(); unsafe { AudioUnitSetParameter(unit, id, scope, element, value, buffer_offset_in_frames) } } pub fn audio_unit_initialize(unit: AudioUnit) -> OSStatus { assert!(!unit.is_null()); + debug_assert_running_serially(); unsafe { AudioUnitInitialize(unit) } } pub fn audio_unit_uninitialize(unit: AudioUnit) -> OSStatus { assert!(!unit.is_null()); + debug_assert_running_serially(); unsafe { AudioUnitUninitialize(unit) } } pub fn dispose_audio_unit(unit: AudioUnit) -> OSStatus { + debug_assert_running_serially(); unsafe { AudioComponentInstanceDispose(unit) } } pub fn audio_output_unit_start(unit: AudioUnit) -> OSStatus { assert!(!unit.is_null()); + debug_assert_running_serially(); unsafe { AudioOutputUnitStart(unit) } } pub fn audio_output_unit_stop(unit: AudioUnit) -> OSStatus { assert!(!unit.is_null()); + debug_assert_running_serially(); unsafe { AudioOutputUnitStop(unit) } } @@ -155,6 +166,7 @@ pub fn audio_unit_add_property_listener( data: *mut T, ) -> OSStatus { assert!(!unit.is_null()); + debug_assert_running_serially(); unsafe { AudioUnitAddPropertyListener(unit, id, Some(listener), data as *mut c_void) } } @@ -165,6 +177,7 @@ pub fn audio_unit_remove_property_listener_with_user_data( data: *mut T, ) -> OSStatus { assert!(!unit.is_null()); + debug_assert_running_serially(); unsafe { AudioUnitRemovePropertyListenerWithUserData(unit, id, Some(listener), data as *mut c_void) } diff --git a/third_party/rust/coreaudio-sys-utils/src/dispatch.rs b/third_party/rust/coreaudio-sys-utils/src/dispatch.rs index 602304bde8..c5da137cab 100644 --- a/third_party/rust/coreaudio-sys-utils/src/dispatch.rs +++ b/third_party/rust/coreaudio-sys-utils/src/dispatch.rs @@ -3,40 +3,118 @@ use coreaudio_sys::*; use std::ffi::CString; use std::mem; use std::os::raw::c_void; +use std::panic; use std::ptr; use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Mutex, OnceLock}; +#[cfg(test)] +use std::thread; +#[cfg(test)] +use std::time::Duration; +use std::time::Instant; -// Queue: A wrapper around `dispatch_queue_t`. +pub const DISPATCH_QUEUE_LABEL: &str = "org.mozilla.cubeb"; + +pub fn get_serial_queue_singleton() -> &'static Queue { + static SERIAL_QUEUE: OnceLock = OnceLock::new(); + SERIAL_QUEUE.get_or_init(|| Queue::new(DISPATCH_QUEUE_LABEL)) +} + +pub fn debug_assert_running_serially() { + get_serial_queue_singleton().debug_assert_is_current(); +} + +pub fn debug_assert_not_running_serially() { + get_serial_queue_singleton().debug_assert_is_not_current(); +} + +pub fn run_serially(work: F) -> B +where + F: FnOnce() -> B, +{ + get_serial_queue_singleton().run_sync(|| work()).unwrap() +} + +pub fn run_serially_forward_panics(work: F) -> B +where + F: panic::UnwindSafe + FnOnce() -> B, +{ + match run_serially(|| panic::catch_unwind(|| work())) { + Ok(res) => res, + Err(e) => panic::resume_unwind(e), + } +} + +// Queue: A wrapper around `dispatch_queue_t` that is always serial. // ------------------------------------------------------------------------------------------------ #[derive(Debug)] -pub struct Queue(dispatch_queue_t); +pub struct Queue { + queue: Mutex, + owned: AtomicBool, +} impl Queue { - pub fn new(label: &str) -> Self { + pub fn new_with_target(label: &str, target: &Queue) -> Self { const DISPATCH_QUEUE_SERIAL: dispatch_queue_attr_t = ptr::null_mut::(); let label = CString::new(label).unwrap(); let c_string = label.as_ptr(); - let queue = Self(unsafe { dispatch_queue_create(c_string, DISPATCH_QUEUE_SERIAL) }); + let queue = { + let target_guard = target.queue.lock().unwrap(); + Self { + queue: Mutex::new(unsafe { + dispatch_queue_create_with_target( + c_string, + DISPATCH_QUEUE_SERIAL, + *target_guard, + ) + }), + owned: AtomicBool::new(true), + } + }; queue.set_should_cancel(Box::new(AtomicBool::new(false))); queue } + pub fn new(label: &str) -> Self { + Queue::new_with_target(label, &Queue::get_global_queue()) + } + + pub fn get_global_queue() -> Self { + Self { + queue: Mutex::new(unsafe { dispatch_get_global_queue(QOS_CLASS_DEFAULT as isize, 0) }), + owned: AtomicBool::new(false), + } + } + #[cfg(debug_assertions)] pub fn debug_assert_is_current(&self) { + let guard = self.queue.lock().unwrap(); unsafe { - dispatch_assert_queue(self.0); + dispatch_assert_queue(*guard); } } #[cfg(not(debug_assertions))] pub fn debug_assert_is_current(&self) {} + #[cfg(debug_assertions)] + pub fn debug_assert_is_not_current(&self) { + let guard = self.queue.lock().unwrap(); + unsafe { + dispatch_assert_queue_not(*guard); + } + } + + #[cfg(not(debug_assertions))] + pub fn debug_assert_is_not_current(&self) {} + pub fn run_async(&self, work: F) where F: Send + FnOnce(), { - let should_cancel = self.get_should_cancel(); + let guard = self.queue.lock().unwrap(); + let should_cancel = self.get_should_cancel(*guard); let (closure, executor) = Self::create_closure_and_executor(|| { if should_cancel.map_or(false, |v| v.load(Ordering::SeqCst)) { return; @@ -44,15 +122,22 @@ impl Queue { work(); }); unsafe { - dispatch_async_f(self.0, closure, executor); + dispatch_async_f(*guard, closure, executor); } } - pub fn run_sync(&self, work: F) + pub fn run_after(&self, when: Instant, work: F) where F: Send + FnOnce(), { - let should_cancel = self.get_should_cancel(); + let now = Instant::now(); + if when <= now { + return self.run_async(work); + } + let nanos = (when - now).as_nanos() as i64; + let when = unsafe { dispatch_time(DISPATCH_TIME_NOW.into(), nanos) }; + let guard = self.queue.lock().unwrap(); + let should_cancel = self.get_should_cancel(*guard); let (closure, executor) = Self::create_closure_and_executor(|| { if should_cancel.map_or(false, |v| v.load(Ordering::SeqCst)) { return; @@ -60,38 +145,85 @@ impl Queue { work(); }); unsafe { - dispatch_sync_f(self.0, closure, executor); + dispatch_after_f(when, *guard, closure, executor); } } - pub fn run_final(&self, work: F) + pub fn run_sync(&self, work: F) -> Option where - F: Send + FnOnce(), + F: FnOnce() -> B, { - let should_cancel = self.get_should_cancel(); - let (closure, executor) = Self::create_closure_and_executor(|| { - work(); - should_cancel - .expect("dispatch context should be allocated!") - .store(true, Ordering::SeqCst); - }); + let queue: Option; + let mut res: Option = None; + let cex: Option<(*mut c_void, dispatch_function_t)>; + { + let guard = self.queue.lock().unwrap(); + queue = Some(*guard); + let should_cancel = self.get_should_cancel(*guard); + cex = Some(Self::create_closure_and_executor(|| { + if should_cancel.map_or(false, |v| v.load(Ordering::SeqCst)) { + return; + } + res = Some(work()); + })); + } + let (closure, executor) = cex.unwrap(); + unsafe { + dispatch_sync_f(queue.unwrap(), closure, executor); + } + res + } + + pub fn run_final(&self, work: F) -> Option + where + F: FnOnce() -> B, + { + assert!( + self.owned.load(Ordering::SeqCst), + "Doesn't make sense to finalize global queue" + ); + let queue: Option; + let mut res: Option = None; + let cex: Option<(*mut c_void, dispatch_function_t)>; + { + let guard = self.queue.lock().unwrap(); + queue = Some(*guard); + let should_cancel = self.get_should_cancel(*guard); + debug_assert!( + should_cancel.is_some(), + "dispatch context should be allocated!" + ); + cex = Some(Self::create_closure_and_executor(|| { + res = Some(work()); + should_cancel + .expect("dispatch context should be allocated!") + .store(true, Ordering::SeqCst); + })); + } + let (closure, executor) = cex.unwrap(); unsafe { - dispatch_sync_f(self.0, closure, executor); + dispatch_sync_f(queue.unwrap(), closure, executor); } + res } - fn get_should_cancel(&self) -> Option<&mut AtomicBool> { + fn get_should_cancel(&self, queue: dispatch_queue_t) -> Option<&mut AtomicBool> { + if !self.owned.load(Ordering::SeqCst) { + return None; + } unsafe { - let context = dispatch_get_context( - mem::transmute::(self.0), - ) as *mut AtomicBool; + let context = + dispatch_get_context(mem::transmute::(queue)) + as *mut AtomicBool; context.as_mut() } } fn set_should_cancel(&self, context: Box) { + assert!(self.owned.load(Ordering::SeqCst)); unsafe { - let queue = mem::transmute::(self.0); + let guard = self.queue.lock().unwrap(); + let queue = mem::transmute::(*guard); // Leak the context from Box. dispatch_set_context(queue, Box::into_raw(context) as *mut c_void); @@ -106,13 +238,13 @@ impl Queue { } fn release(&self) { + let guard = self.queue.lock().unwrap(); + let queue = *guard; unsafe { // This will release the inner `dispatch_queue_t` asynchronously. // TODO: It's incredibly unsafe to call `transmute` directly. // Find another way to release the queue. - dispatch_release(mem::transmute::( - self.0, - )); + dispatch_release(mem::transmute::(queue)); } } @@ -143,23 +275,35 @@ impl Queue { impl Drop for Queue { fn drop(&mut self) { - self.release(); + if self.owned.load(Ordering::SeqCst) { + self.release(); + } } } impl Clone for Queue { fn clone(&self) -> Self { + assert!( + self.owned.load(Ordering::SeqCst), + "No need to clone a static queue" + ); + let guard = self.queue.lock().unwrap(); + let queue = *guard; // TODO: It's incredibly unsafe to call `transmute` directly. // Find another way to release the queue. unsafe { - dispatch_retain(mem::transmute::( - self.0, - )); + dispatch_retain(mem::transmute::(queue)); + } + Self { + queue: Mutex::new(queue), + owned: AtomicBool::new(true), } - Self(self.0) } } +unsafe impl Send for Queue {} +unsafe impl Sync for Queue {} + #[test] fn run_tasks_in_order() { let mut visited = Vec::::new(); @@ -176,12 +320,12 @@ fn run_tasks_in_order() { let queue = Queue::new("Run tasks in order"); - queue.run_sync(move || visit(1, ptr)); - queue.run_sync(move || visit(2, ptr)); - queue.run_async(move || visit(3, ptr)); - queue.run_async(move || visit(4, ptr)); + queue.run_sync(|| visit(1, ptr)); + queue.run_sync(|| visit(2, ptr)); + queue.run_async(|| visit(3, ptr)); + queue.run_async(|| visit(4, ptr)); // Call sync here to block the current thread and make sure all the tasks are done. - queue.run_sync(move || visit(5, ptr)); + queue.run_sync(|| visit(5, ptr)); assert_eq!(visited, vec![1, 2, 3, 4, 5]); } @@ -203,14 +347,52 @@ fn run_final_task() { let queue = Queue::new("Task after run_final will be cancelled"); - queue.run_sync(move || visit(1, ptr)); - queue.run_async(move || visit(2, ptr)); - queue.run_final(move || visit(3, ptr)); - queue.run_async(move || visit(4, ptr)); - queue.run_sync(move || visit(5, ptr)); + queue.run_sync(|| visit(1, ptr)); + queue.run_async(|| visit(2, ptr)); + queue.run_final(|| visit(3, ptr)); + queue.run_async(|| visit(4, ptr)); + queue.run_sync(|| visit(5, ptr)); } // `queue` will be dropped asynchronously and then the `finalizer` of the `queue` // should be fired to clean up the `context` set in the `queue`. assert_eq!(visited, vec![1, 2, 3]); } + +#[test] +fn sync_return_value() { + let q = Queue::new("Test queue"); + assert_eq!(q.run_sync(|| 42), Some(42)); + assert_eq!(q.run_final(|| "foo"), Some("foo")); + assert_eq!(q.run_sync(|| Ok::<(), u32>(())), None); +} + +#[test] +fn run_after() { + let mut visited = Vec::::new(); + + { + // Rust compilter doesn't allow a pointer to be passed across threads. + // A hacky way to do that is to cast the pointer into a value, then + // the value, which is actually an address, can be copied into threads. + let ptr = &mut visited as *mut Vec as usize; + + fn visit(v: u32, visited_ptr: usize) { + let visited = unsafe { &mut *(visited_ptr as *mut Vec) }; + visited.push(v); + } + + let queue = Queue::new("Task after run_final will be cancelled"); + + queue.run_async(|| visit(1, ptr)); + queue.run_after(Instant::now() + Duration::from_millis(10), || visit(2, ptr)); + queue.run_after(Instant::now() + Duration::from_secs(1), || visit(3, ptr)); + queue.run_async(|| visit(4, ptr)); + thread::sleep(Duration::from_millis(100)); + queue.run_final(|| visit(5, ptr)); + } + // `queue` will be dropped asynchronously and then the `finalizer` of the `queue` + // should be fired to clean up the `context` set in the `queue`. + + assert_eq!(visited, vec![1, 4, 2, 5]); +} diff --git a/third_party/rust/cubeb-coreaudio/.cargo-checksum.json b/third_party/rust/cubeb-coreaudio/.cargo-checksum.json index 5c8366f60a..fa6f229b4c 100644 --- a/third_party/rust/cubeb-coreaudio/.cargo-checksum.json +++ b/third_party/rust/cubeb-coreaudio/.cargo-checksum.json @@ -1 +1 @@ -{"files":{".circleci/config.yml":"7f3dc865105ca8f33965a7958b1fe2e627ae2d5a703f3b2a4ab6e2e796018597",".editorconfig":"4e53b182bcc78b83d7e1b5c03efa14d22d4955c4ed2514d1ba4e99c1eb1a50ba",".githooks/pre-push":"8b8b26544cd56f54c0c33812551f786bb25cb08c86dbfeb6bf3daad881c826a1",".github/workflows/test.yml":"aa1998a3b104ad131805ca3513832cef3f65300192824f8b1efc9a5a0cc108f6",".travis.yml":"dc07bac53f70f16c9bdf52264bdc58500ae6018c1b4c567bc7642f6b4ca3cc35","Cargo.toml":"d7e757e664c23fae52028f1dfc5917f92523c08702e3a1f95e1fd38ed714416c","LICENSE":"6e6f56aff5bbf3cbc60747e152fb1a719bd0716aaf6d711c554f57d92e96297c","README.md":"0007782a05a5330f739ad789c19c82562c82e32386b0447000fc72c0d48405bc","build-audiounit-rust-in-cubeb.sh":"d228a05985dcd02ec1ecac66a2b64dae5a530804a25a7054ccc95905aedfb7ef","install_git_hook.sh":"d38c8e51e636f6b90b489621ac34ccd1d1b1f40dccce3d178ed1da1c5068f16d","install_rustfmt_clippy.sh":"4ae90d8dcb9757cb3ae4ae142ef80e5377c0dde61c63f4a3c32418646e80ca7b","run_device_tests.sh":"d717e598c96e4911d9494b18382d6bd3a8d5038b7d68d3166ad4336e237a97d8","run_sanitizers.sh":"84e93a0da137803018f37403511e8c92760be730426bf6cea34419d93d1a7ff8","run_tests.sh":"916a7ae4a406d2274417d6eca939a878db5adcb6144e5680d9d148bf90178f1c","src/backend/aggregate_device.rs":"43511107ba2a75a19340ac663c981362ca1b75b679b6c295d88b5035bd7e3619","src/backend/auto_release.rs":"050fdcee74cf46b9a8a85a877e166d72a853d33220f59cf734cbb6ea09daa441","src/backend/buffer_manager.rs":"e9bcf964347daa8952f98caa2746e34a31ea8908375204896593f56e4b6147ca","src/backend/device_property.rs":"a7622feaa41db1cd76fd35a85a022e44f4894e396a104a59008d5b8757d2ab4e","src/backend/mixer.rs":"ed299d3954e2a823060c870a8244673a7d4bca530830cb66b964d047a80ee3af","src/backend/mod.rs":"1591669c30a3d07754bfb39c9cb042cdd101f0ab89be13f6cdf74d376e441cf8","src/backend/resampler.rs":"48bf8f56ae8d60dbabca6417b768000619abee8731ac3902164b45651ac08a4d","src/backend/tests/aggregate_device.rs":"e3f94e118e1dd47941fbba4417de40bddc4254d9f06b1e938f58d8f1aa566a5c","src/backend/tests/api.rs":"cd7e7551e2e82b19da883621a494d2a6779c373f3ff2d12ee52fae8efec1e7b8","src/backend/tests/backlog.rs":"3b189a7e036543c467cc242af0ed3332721179ee2b1c8847a6db563546f1ac52","src/backend/tests/device_change.rs":"f68c2eaa55c3ec2a58894832fbca1e2a2e79e740b145f76a0f45452af465a934","src/backend/tests/device_property.rs":"ea0be5f8834be494cb33f854ce9d334b5763dc5287f949bcb4bd025d8a8b2d3b","src/backend/tests/interfaces.rs":"af8e3fdeb58226621699b29f1a90621b2260e3f17292dac54860cd05fe4eec71","src/backend/tests/manual.rs":"4a1634e86beb145d2703722a8be057a762953241329c82ee09acf7dc0f0d9d0c","src/backend/tests/mod.rs":"8dba770023d7f9c4228f0e11915347f0e07da5fd818e3ee4478c4b197af9aa2a","src/backend/tests/parallel.rs":"59632744e70616ab7037facb0787db339b96800c8cc397d203241548c5cfb7f5","src/backend/tests/tone.rs":"779cc14fc2a362bf7f26ce66ad70c0639501176175655a99b7fefb3c59d56c7a","src/backend/tests/utils.rs":"efb8b3709aff7ed5e2923566084de3e0709f3bd9c18a04f3310d7a3b86fa4b71","src/backend/utils.rs":"6c3ffbcd602e6cc9f56deb9ecb07b2eef2e6f074ef924178e466f380aae5c595","src/capi.rs":"21b66b70545bf04ec719928004d1d9adb45b24ced51288f5b2993d79aaf78f5f","src/lib.rs":"5e586d45cd6b3722f0a6736d9252593299269817a153eef1930a5fb9bfbb56f5","todo.md":"efc1f012eb9a331a040cad4ac03aa79307f25885f71b6fb38f3ad7af8d7d515c"},"package":null} \ No newline at end of file +{"files":{".circleci/config.yml":"7f3dc865105ca8f33965a7958b1fe2e627ae2d5a703f3b2a4ab6e2e796018597",".editorconfig":"4e53b182bcc78b83d7e1b5c03efa14d22d4955c4ed2514d1ba4e99c1eb1a50ba",".githooks/pre-push":"8b8b26544cd56f54c0c33812551f786bb25cb08c86dbfeb6bf3daad881c826a1",".github/workflows/test.yml":"ac8f4cf5b7631b5c738d50c0cf78113bd395940b9e76593904bbaf2d02d16a70",".travis.yml":"dc07bac53f70f16c9bdf52264bdc58500ae6018c1b4c567bc7642f6b4ca3cc35","Cargo.toml":"0fb7c56c04e05dacffa5176f885cb8019ee6ab7f885479be501aba0eaac2148f","LICENSE":"6e6f56aff5bbf3cbc60747e152fb1a719bd0716aaf6d711c554f57d92e96297c","README.md":"0007782a05a5330f739ad789c19c82562c82e32386b0447000fc72c0d48405bc","build-audiounit-rust-in-cubeb.sh":"d228a05985dcd02ec1ecac66a2b64dae5a530804a25a7054ccc95905aedfb7ef","install_git_hook.sh":"d38c8e51e636f6b90b489621ac34ccd1d1b1f40dccce3d178ed1da1c5068f16d","install_rustfmt_clippy.sh":"4ae90d8dcb9757cb3ae4ae142ef80e5377c0dde61c63f4a3c32418646e80ca7b","run_device_tests.sh":"90c2542fa3ff8a35fed894fae3a1aa0157117b7f0e28df14b8e6f7b1f1f43797","run_sanitizers.sh":"84e93a0da137803018f37403511e8c92760be730426bf6cea34419d93d1a7ff8","run_tests.sh":"bae82f66dd47a060b6fdcc238520084aec1079d5b1b1d66d103baa1ffaa8773d","src/backend/aggregate_device.rs":"db7d644358090b1d65ff2d53ad854369790ae4ad7dfa12b79888c0002c1b4950","src/backend/auto_release.rs":"050fdcee74cf46b9a8a85a877e166d72a853d33220f59cf734cbb6ea09daa441","src/backend/buffer_manager.rs":"e9bcf964347daa8952f98caa2746e34a31ea8908375204896593f56e4b6147ca","src/backend/device_property.rs":"a7622feaa41db1cd76fd35a85a022e44f4894e396a104a59008d5b8757d2ab4e","src/backend/mixer.rs":"ed299d3954e2a823060c870a8244673a7d4bca530830cb66b964d047a80ee3af","src/backend/mod.rs":"e52b79a17dbf7faa072ec87cc3e4201b907772104c3be777498275733b9c334e","src/backend/resampler.rs":"48bf8f56ae8d60dbabca6417b768000619abee8731ac3902164b45651ac08a4d","src/backend/tests/aggregate_device.rs":"770cf90f32b5ab2203476031c1fbc8379b713baa97bec36f7fd0d77fef1efd60","src/backend/tests/api.rs":"d72d7c0de8d12e880966948be4686bcf8c789f0ef19cb435c242fd72f2d252f9","src/backend/tests/backlog.rs":"3b189a7e036543c467cc242af0ed3332721179ee2b1c8847a6db563546f1ac52","src/backend/tests/device_change.rs":"babf50326fb38db24fe80f24f546e1b6ad04319ae8835bb372d893fc9b3038a2","src/backend/tests/device_property.rs":"73c25f579a995e8a59c9b7d391813afb75f739b5e2f825480cba04499a1d46e8","src/backend/tests/interfaces.rs":"654333cd6d6023e72ba392d98872d33bc55f8f052205a9f701aec72069449e24","src/backend/tests/manual.rs":"e550cc8bb7619bb80b68e49bf7f475c029e0f1b34323d1d30edcbe322cf4efc7","src/backend/tests/mod.rs":"8dba770023d7f9c4228f0e11915347f0e07da5fd818e3ee4478c4b197af9aa2a","src/backend/tests/parallel.rs":"a7ebd579339c40ca64c0757cc9da6baec641e670f226e1b2ec5049894700bd7a","src/backend/tests/tone.rs":"b028c67777b6453a26190b6a49785dfe28556adcbe179cb10862ce0d47ee8509","src/backend/tests/utils.rs":"80d7e4ebc06b23c63a4d2867e0c80e0bfe05449fa55edd21e785ed2c089bf7d5","src/backend/utils.rs":"6c3ffbcd602e6cc9f56deb9ecb07b2eef2e6f074ef924178e466f380aae5c595","src/capi.rs":"21b66b70545bf04ec719928004d1d9adb45b24ced51288f5b2993d79aaf78f5f","src/lib.rs":"5e586d45cd6b3722f0a6736d9252593299269817a153eef1930a5fb9bfbb56f5","todo.md":"efc1f012eb9a331a040cad4ac03aa79307f25885f71b6fb38f3ad7af8d7d515c"},"package":null} \ No newline at end of file diff --git a/third_party/rust/cubeb-coreaudio/.github/workflows/test.yml b/third_party/rust/cubeb-coreaudio/.github/workflows/test.yml index 06fc86fa33..2bbb9eab5d 100644 --- a/third_party/rust/cubeb-coreaudio/.github/workflows/test.yml +++ b/third_party/rust/cubeb-coreaudio/.github/workflows/test.yml @@ -4,36 +4,52 @@ on: [push, pull_request] jobs: build: - runs-on: macOS-latest + runs-on: ${{ matrix.os }} continue-on-error: ${{ matrix.experimental }} strategy: fail-fast: false matrix: + os: [macos-12, macos-13, macos-14] rust: [stable] experimental: [false] include: - - rust: nightly + - os: macos-14 + rust: nightly experimental: true steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: submodules: recursive - name: Install Rust run: rustup toolchain install ${{ matrix.rust }} --profile minimal --component rustfmt clippy - - - name: Setup + + - name: Setup Rust run: | rustup default ${{ matrix.rust }} toolchain=$(rustup default) echo "Use Rust toolchain: $toolchain" rustc --version cargo --version - + + - name: Setup Audio + if: ${{ matrix.os == 'macos-13' || matrix.os == 'macos-14' }} + run: | + brew install switchaudio-osx + brew install blackhole-2ch + SwitchAudioSource -s "BlackHole 2ch" -t input + SwitchAudioSource -s "BlackHole 2ch" -t output + + - name: Grant microphone access + if: ${{ matrix.os == 'macos-13' || matrix.os == 'macos-14' }} + env: + tcc_extra_columns: ${{ matrix.os == 'macos-14' && ',NULL,NULL,''UNUSED'',1687786159' || '' }} + run: sqlite3 $HOME/Library/Application\ Support/com.apple.TCC/TCC.db "INSERT OR IGNORE INTO access VALUES ('kTCCServiceMicrophone','/usr/local/opt/runner/provisioner/provisioner',1,2,4,1,NULL,NULL,0,'UNUSED',NULL,0,1687786159${{ env.tcc_extra_columns }});" + - name: Build run: cargo build --verbose - + - name: Regular Test run: sh run_tests.sh diff --git a/third_party/rust/cubeb-coreaudio/Cargo.toml b/third_party/rust/cubeb-coreaudio/Cargo.toml index f9067d3c08..8a73548f57 100644 --- a/third_party/rust/cubeb-coreaudio/Cargo.toml +++ b/third_party/rust/cubeb-coreaudio/Cargo.toml @@ -36,6 +36,7 @@ libc = "0.2" mach = "0.3" ringbuf = "0.2.6" triple_buffer = "5.0.5" +whatsys = "0.3" [dependencies.coreaudio-sys-utils] path = "coreaudio-sys-utils" diff --git a/third_party/rust/cubeb-coreaudio/run_device_tests.sh b/third_party/rust/cubeb-coreaudio/run_device_tests.sh index ae6df49713..69bb3385f4 100755 --- a/third_party/rust/cubeb-coreaudio/run_device_tests.sh +++ b/third_party/rust/cubeb-coreaudio/run_device_tests.sh @@ -15,8 +15,6 @@ cargo test test_plug_and_unplug_device -- --ignored --nocapture cargo test test_register_device_changed_callback_to_check_default_device_changed_input -- --ignored --nocapture cargo test test_register_device_changed_callback_to_check_default_device_changed_output -- --ignored --nocapture cargo test test_register_device_changed_callback_to_check_default_device_changed_duplex -- --ignored --nocapture -cargo test test_register_device_changed_callback_to_check_input_alive_changed_input -- --ignored --nocapture -cargo test test_register_device_changed_callback_to_check_input_alive_changed_duplex -- --ignored --nocapture cargo test test_destroy_input_stream_after_unplugging_a_nondefault_input_device -- --ignored --nocapture cargo test test_suspend_input_stream_by_unplugging_a_nondefault_input_device -- --ignored --nocapture diff --git a/third_party/rust/cubeb-coreaudio/run_tests.sh b/third_party/rust/cubeb-coreaudio/run_tests.sh index e119da1f03..2b048f4790 100755 --- a/third_party/rust/cubeb-coreaudio/run_tests.sh +++ b/third_party/rust/cubeb-coreaudio/run_tests.sh @@ -39,8 +39,9 @@ cargo clippy -- -D warnings # Regular Tests cargo test --verbose -cargo test test_configure_output -- --ignored -cargo test test_aggregate -- --ignored --test-threads=1 + +# Timing sensitive tests must run serially so they cannot be impacted by other tasks on the queue +cargo test test_ops_timing_sensitive -- --ignored --test-threads=1 # Parallel Tests cargo test test_parallel -- --ignored --nocapture --test-threads=1 diff --git a/third_party/rust/cubeb-coreaudio/src/backend/aggregate_device.rs b/third_party/rust/cubeb-coreaudio/src/backend/aggregate_device.rs index 2738631b87..782e76de2f 100644 --- a/third_party/rust/cubeb-coreaudio/src/backend/aggregate_device.rs +++ b/third_party/rust/cubeb-coreaudio/src/backend/aggregate_device.rs @@ -69,6 +69,7 @@ impl AggregateDevice { input_id: AudioObjectID, output_id: AudioObjectID, ) -> std::result::Result { + debug_assert_running_serially(); let plugin_id = Self::get_system_plugin_id()?; let device_id = Self::create_blank_device_sync(plugin_id)?; @@ -399,12 +400,12 @@ impl AggregateDevice { let sub_devices = CFArrayCreateMutable(ptr::null(), 0, &kCFTypeArrayCallBacks); // The order of the items in the array is significant and is used to determine the order of the streams // of the AudioAggregateDevice. - for device in output_sub_devices { + for device in input_sub_devices { let uid = get_device_global_uid(device)?; CFArrayAppendValue(sub_devices, uid.get_raw() as *const c_void); } - for device in input_sub_devices { + for device in output_sub_devices { let uid = get_device_global_uid(device)?; CFArrayAppendValue(sub_devices, uid.get_raw() as *const c_void); } @@ -466,6 +467,28 @@ impl AggregateDevice { } } + pub fn get_master_device_uid(device_id: AudioDeviceID) -> std::result::Result { + let address = AudioObjectPropertyAddress { + mSelector: kAudioAggregateDevicePropertyMainSubDevice, + mScope: kAudioObjectPropertyScopeGlobal, + mElement: kAudioObjectPropertyElementMaster, + }; + + let mut master: CFStringRef = ptr::null_mut(); + let mut size = mem::size_of::(); + let status = audio_object_get_property_data(device_id, &address, &mut size, &mut master); + if status != NO_ERR { + return Err(Error::from(status)); + } + + if master.is_null() { + return Ok(String::default()); + } + + let master = StringRef::new(master as _); + Ok(master.into_string()) + } + pub fn set_master_device( device_id: AudioDeviceID, primary_id: AudioDeviceID, @@ -480,12 +503,12 @@ impl AggregateDevice { ); let address = AudioObjectPropertyAddress { - mSelector: kAudioAggregateDevicePropertyMasterSubDevice, + mSelector: kAudioAggregateDevicePropertyMainSubDevice, mScope: kAudioObjectPropertyScopeGlobal, mElement: kAudioObjectPropertyElementMaster, }; - // Master become the 1st sub device of the primary device + // The master device will be the 1st sub device of the primary device. let output_sub_devices = Self::get_sub_devices(primary_id)?; assert!(!output_sub_devices.is_empty()); let master_sub_device_uid = get_device_global_uid(output_sub_devices[0]).unwrap(); @@ -548,16 +571,23 @@ impl AggregateDevice { return Err(Error::from(status)); } + let master_sub_device_uid = Self::get_master_device_uid(device_id)?; + let address = AudioObjectPropertyAddress { mSelector: kAudioSubDevicePropertyDriftCompensation, mScope: kAudioObjectPropertyScopeGlobal, mElement: kAudioObjectPropertyElementMaster, }; - // Start from the second device since the first is the master clock - for device in &sub_devices[1..] { + for &device in &sub_devices { + let uid = get_device_global_uid(device) + .map(|sr| sr.into_string()) + .unwrap_or_default(); + if uid == master_sub_device_uid { + continue; + } let status = audio_object_set_property_data( - *device, + device, &address, mem::size_of::(), &DRIFT_COMPENSATION, @@ -671,6 +701,7 @@ impl Default for AggregateDevice { impl Drop for AggregateDevice { fn drop(&mut self) { + debug_assert_running_serially(); if self.plugin_id != kAudioObjectUnknown && self.device_id != kAudioObjectUnknown { if let Err(r) = Self::destroy_device(self.plugin_id, self.device_id) { cubeb_log!( diff --git a/third_party/rust/cubeb-coreaudio/src/backend/mod.rs b/third_party/rust/cubeb-coreaudio/src/backend/mod.rs index 61ae44fea1..e6be028a2e 100644 --- a/third_party/rust/cubeb-coreaudio/src/backend/mod.rs +++ b/third_party/rust/cubeb-coreaudio/src/backend/mod.rs @@ -46,15 +46,15 @@ use std::mem; use std::os::raw::{c_uint, c_void}; use std::ptr; use std::slice; +use std::str::FromStr; use std::sync::atomic::{AtomicBool, AtomicU32, AtomicUsize, Ordering}; -use std::sync::{Arc, Condvar, Mutex}; -use std::time::Duration; +use std::sync::{Arc, Condvar, Mutex, MutexGuard, Weak}; +use std::time::{Duration, Instant}; const NO_ERR: OSStatus = 0; const AU_OUT_BUS: AudioUnitElement = 0; const AU_IN_BUS: AudioUnitElement = 1; -const DISPATCH_QUEUE_LABEL: &str = "org.mozilla.cubeb"; const PRIVATE_AGGREGATE_DEVICE_NAME: &str = "CubebAggregateDevice"; const VOICEPROCESSING_AGGREGATE_DEVICE_NAME: &str = "VPAUAggregateAudioDevice"; @@ -65,6 +65,34 @@ const APPLE_STUDIO_DISPLAY_USB_ID: &str = "05AC:1114"; const SAFE_MIN_LATENCY_FRAMES: u32 = 128; const SAFE_MAX_LATENCY_FRAMES: u32 = 512; +const VPIO_IDLE_TIMEOUT: Duration = Duration::from_secs(10); + +const MACOS_KERNEL_MAJOR_VERSION_MONTEREY: u32 = 21; + +#[derive(Debug, PartialEq)] +enum ParseMacOSKernelVersionError { + SysCtl, + Malformed, + Parsing, +} + +fn macos_kernel_major_version() -> std::result::Result { + let ver = whatsys::kernel_version(); + if ver.is_none() { + return Err(ParseMacOSKernelVersionError::SysCtl); + } + let ver = ver.unwrap(); + let major = ver.split('.').next(); + if major.is_none() { + return Err(ParseMacOSKernelVersionError::Malformed); + } + let parsed_major = u32::from_str(major.unwrap()); + if parsed_major.is_err() { + return Err(ParseMacOSKernelVersionError::Parsing); + } + Ok(parsed_major.unwrap()) +} + bitflags! { #[allow(non_camel_case_types)] #[derive(Clone, Debug, PartialEq, Copy)] @@ -191,6 +219,7 @@ fn set_notification_runloop() { fn create_device_info(devid: AudioDeviceID, devtype: DeviceType) -> Option { assert_ne!(devid, kAudioObjectSystemObject); + debug_assert_running_serially(); let mut flags = match devtype { DeviceType::INPUT => device_flags::DEV_INPUT, @@ -457,7 +486,6 @@ extern "C" fn audiounit_input_callback( assert!(!user_ptr.is_null()); let stm = unsafe { &mut *(user_ptr as *mut AudioUnitStream) }; - let using_voice_processing_unit = stm.core_stream_data.using_voice_processing_unit(); if unsafe { *flags | kAudioTimeStampHostTimeValid } != 0 { let now = unsafe { mach_absolute_time() }; @@ -597,17 +625,20 @@ extern "C" fn audiounit_input_callback( ErrorHandle::Return(NO_ERR) }; - // If the input (input-only stream) or the output is drained (duplex stream), - // cancel this callback. Note that for voice processing cases (a single unit), - // the output callback handles stopping the unit and notifying of state. - if !using_voice_processing_unit && stm.draining.load(Ordering::SeqCst) { - let r = stop_audiounit(stm.core_stream_data.input_unit); - assert!(r.is_ok()); - // Only fire state-changed callback for input-only stream. - // The state-changed callback for the duplex stream is fired in the output callback. - if stm.core_stream_data.output_unit.is_null() { - stm.notify_state_changed(State::Drained); - } + // If the input (input-only stream) is drained, cancel this callback. Whenever an output + // is involved, the output callback handles stopping all units and notifying of state. + if stm.core_stream_data.output_unit.is_null() && stm.draining.load(Ordering::SeqCst) { + stm.stopped.store(true, Ordering::SeqCst); + cubeb_alog!("({:p}) Input-only drained.", stm as *const AudioUnitStream); + stm.notify_state_changed(State::Drained); + let queue = stm.queue.clone(); + // Use a new thread, through the queue, to avoid deadlock when calling + // AudioOutputUnitStop method from inside render callback + let stm_ptr = user_ptr as usize; + queue.run_async(move || { + let stm = unsafe { &mut *(stm_ptr as *mut AudioUnitStream) }; + stm.core_stream_data.stop_audiounits(); + }); } match handle { @@ -681,11 +712,17 @@ extern "C" fn audiounit_output_callback( } if stm.draining.load(Ordering::SeqCst) { - // Cancel the output callback only. For duplex stream, - // the input callback will be cancelled in its own callback. - let r = stop_audiounit(stm.core_stream_data.output_unit); - assert!(r.is_ok()); + // Cancel all callbacks. For input-only streams, the input callback handles + // cancelling itself. + stm.stopped.store(true, Ordering::SeqCst); + cubeb_alog!("({:p}) output drained.", stm as *const AudioUnitStream); stm.notify_state_changed(State::Drained); + let queue = stm.queue.clone(); + // Use a new thread, through the queue, to avoid deadlock when calling + // AudioOutputUnitStop method from inside render callback + queue.run_async(move || { + stm.core_stream_data.stop_audiounits(); + }); audiounit_make_silent(&buffers[0]); return NO_ERR; } @@ -1157,38 +1194,56 @@ fn create_audiounit(device: &device_info) -> Result { Ok(unit) } -fn create_voiceprocessing_audiounit( +fn get_voiceprocessing_audiounit( + shared_voice_processing_unit: &mut SharedVoiceProcessingUnitManager, in_device: &device_info, out_device: &device_info, -) -> Result { +) -> Result> { + debug_assert_running_serially(); assert!(in_device.flags.contains(device_flags::DEV_INPUT)); assert!(!in_device.flags.contains(device_flags::DEV_OUTPUT)); assert!(!out_device.flags.contains(device_flags::DEV_INPUT)); - assert!(out_device.flags.contains(device_flags::DEV_OUTPUT)); - - let unit = create_typed_audiounit(kAudioUnitSubType_VoiceProcessingIO)?; - if let Err(e) = set_device_to_audiounit(unit, in_device.id, AU_IN_BUS) { + let unit_handle = shared_voice_processing_unit.take_or_create(); + if let Err(e) = unit_handle { cubeb_log!( - "Failed to set in device {} to the created audiounit. Error: {}", - in_device.id, + "Failed to create shared voiceprocessing audiounit. Error: {}", e ); - dispose_audio_unit(unit); return Err(Error::error()); } + let mut unit_handle = unit_handle.unwrap(); - if let Err(e) = set_device_to_audiounit(unit, out_device.id, AU_OUT_BUS) { + if let Err(e) = set_device_to_audiounit(unit_handle.as_mut().unit, in_device.id, AU_IN_BUS) { cubeb_log!( - "Failed to set out device {} to the created audiounit. Error: {}", - out_device.id, + "Failed to set in device {} to the created audiounit. Error: {}", + in_device.id, e ); - dispose_audio_unit(unit); return Err(Error::error()); } - Ok(unit) + let has_output = out_device.id != kAudioObjectUnknown; + if let Err(e) = + enable_audiounit_scope(unit_handle.as_mut().unit, DeviceType::OUTPUT, has_output) + { + cubeb_log!("Failed to enable audiounit input scope. Error: {}", e); + return Err(Error::error()); + } + if has_output { + if let Err(e) = + set_device_to_audiounit(unit_handle.as_mut().unit, out_device.id, AU_OUT_BUS) + { + cubeb_log!( + "Failed to set out device {} to the created audiounit. Error: {}", + out_device.id, + e + ); + return Err(Error::error()); + } + } + + Ok(unit_handle) } fn enable_audiounit_scope( @@ -1276,6 +1331,31 @@ fn create_blank_audiounit() -> Result { return create_typed_audiounit(kAudioUnitSubType_RemoteIO); } +fn create_voiceprocessing_audiounit() -> Result { + let res = create_typed_audiounit(kAudioUnitSubType_VoiceProcessingIO); + if res.is_err() { + return Err(Error::error()); + } + + match get_default_device(DeviceType::OUTPUT) { + None => { + cubeb_log!("Could not get default output device in order to undo vpio ducking"); + } + Some(id) => { + let r = audio_device_duck(id, 1.0, ptr::null_mut(), 0.5); + if r != NO_ERR { + cubeb_log!( + "Failed to undo ducking of voiceprocessing on output device {}. Proceeding... Error: {}", + id, + r + ); + } + } + }; + + res.map(|unit| VoiceProcessingUnit { unit }) +} + fn get_buffer_size(unit: AudioUnit, devtype: DeviceType) -> std::result::Result { assert!(!unit.is_null()); let (scope, element) = match devtype { @@ -1558,7 +1638,7 @@ fn get_range_of_sample_rates( if rates.is_empty() { return Err(String::from("No data")); } - let (mut min, mut max) = (std::f64::MAX, std::f64::MIN); + let (mut min, mut max) = (f64::MAX, f64::MIN); for rate in rates { if rate.mMaximum > max { max = rate.mMaximum; @@ -2056,7 +2136,7 @@ struct LatencyController { } impl LatencyController { - fn add_stream(&mut self, latency: u32) -> Option { + fn add_stream(&mut self, latency: u32) -> u32 { self.streams += 1; // For the 1st stream set anything within safe min-max if self.streams == 1 { @@ -2065,16 +2145,322 @@ impl LatencyController { // synthetize the clock from the callbacks, and we want the clock to update often. self.latency = Some(latency.clamp(SAFE_MIN_LATENCY_FRAMES, SAFE_MAX_LATENCY_FRAMES)); } - self.latency + self.latency.unwrap_or(latency) } - fn subtract_stream(&mut self) -> Option { + fn subtract_stream(&mut self) { self.streams -= 1; if self.streams == 0 { assert!(self.latency.is_some()); self.latency = None; } - self.latency + } +} + +// SharedStorage below looks generic but has evolved to be pretty tailored +// the observed behavior of VoiceProcessingIO audio units on macOS 14. +// Some key points are: +// - Creating the first VoiceProcessingIO unit in a process takes a long time, often > 3s. +// - Creating a second VoiceProcessingIO unit in a process is significantly faster, < 1s. +// - Disposing of a VoiceProcessingIO unit when all other VoiceProcessingIO units are +// uninitialized will take significantly longer than disposing the remaining +// VoiceProcessingIO units, and will have other side effects: starting another +// VoiceProcessingIO unit after this is on par with creating the first one in the +// process, bluetooth devices will move away from the handsfree profile, etc. +// The takeaway is that there is something internal to the VoiceProcessingIO audio unit +// that is costly to create and dispose of and its creation is triggered by creation of +// the first VoiceProcessingIO unit, and its disposal is triggered by the disposal of +// the first VoiceProcessingIO unit when no other VoiceProcessingIO units are initialized. +// +// The intended behavior of SharedStorage and SharedVoiceProcessingUnitManager is therefore: +// - Retain ideally just one VoiceProcessingIO unit after stream destruction, so device +// switching is fast. The benefit of retaining more than one is unclear. +// - Dispose of either all VoiceProcessingIO units, or none at all, such that the retained +// VoiceProcessingIO unit really helps speed up creating and starting the next. In practice +// this means we retain all VoiceProcessingIO units until they can all be disposed of. + +#[derive(Debug)] +struct SharedStorageInternal { + // Storage for shared elements. + elements: Vec, + // Number of elements in use, i.e. all elements created/taken and not recycled. + outstanding_element_count: usize, + // Used for invalidation of in-flight tasks to clear elements. + // Incremented when something takes a shared element. + generation: usize, +} + +#[derive(Debug)] +struct SharedStorage { + queue: Queue, + idle_timeout: Duration, + storage: Mutex>, +} + +impl SharedStorage { + fn with_idle_timeout(queue: Queue, idle_timeout: Duration) -> Self { + Self { + queue, + idle_timeout, + storage: Mutex::new(SharedStorageInternal:: { + elements: Vec::default(), + outstanding_element_count: 0, + generation: 0, + }), + } + } + + fn take_locked(guard: &mut MutexGuard<'_, SharedStorageInternal>) -> Result { + if let Some(e) = guard.elements.pop() { + cubeb_log!("Taking shared element #{}.", guard.elements.len()); + guard.outstanding_element_count += 1; + guard.generation += 1; + return Ok(e); + } + + Err(Error::not_supported()) + } + + fn create_with_locked( + guard: &mut MutexGuard<'_, SharedStorageInternal>, + f: F, + ) -> Result + where + F: FnOnce() -> Result, + { + let start = Instant::now(); + match f() { + Ok(obj) => { + cubeb_log!( + "Just created shared element #{}. Took {}s.", + guard.outstanding_element_count, + (Instant::now() - start).as_secs_f32() + ); + guard.outstanding_element_count += 1; + guard.generation += 1; + Ok(obj) + } + Err(_) => { + cubeb_log!("Creating shared element failed"); + Err(Error::error()) + } + } + } + + #[cfg(test)] + fn take(&self) -> Result { + let mut guard = self.storage.lock().unwrap(); + SharedStorage::take_locked(&mut guard) + } + + fn take_or_create_with(&self, f: F) -> Result + where + F: FnOnce() -> Result, + { + let mut guard = self.storage.lock().unwrap(); + SharedStorage::take_locked(&mut guard) + .or_else(|_| SharedStorage::create_with_locked(&mut guard, f)) + } + + fn recycle(&self, obj: T) { + let mut guard = self.storage.lock().unwrap(); + guard.outstanding_element_count -= 1; + cubeb_log!( + "Recycling shared element #{}. Nr of live elements now {}.", + guard.elements.len(), + guard.outstanding_element_count + ); + guard.elements.push(obj); + } + + fn clear_locked(guard: &mut MutexGuard<'_, SharedStorageInternal>) { + let count = guard.elements.len(); + let start = Instant::now(); + guard.elements.clear(); + cubeb_log!( + "Cleared {} shared element{}. Took {}s.", + count, + if count == 1 { "" } else { "s" }, + (Instant::now() - start).as_secs_f32() + ); + } + + fn clear(&self) { + debug_assert_running_serially(); + let mut guard = self.storage.lock().unwrap(); + SharedStorage::clear_locked(&mut guard); + } + + fn clear_if_all_idle_async(storage: &Arc>) { + let (queue, outstanding_element_count, generation) = { + let guard = storage.storage.lock().unwrap(); + ( + storage.queue.clone(), + guard.outstanding_element_count, + guard.generation, + ) + }; + if outstanding_element_count > 0 { + cubeb_log!( + "Not clearing shared voiceprocessing unit storage because {} elements are in use. Generation={}.", + outstanding_element_count, + generation + ); + return; + } + cubeb_log!( + "Clearing shared voiceprocessing unit storage in {}s if still at generation {}.", + storage.idle_timeout.as_secs_f32(), + generation + ); + let storage = storage.clone(); + queue.run_after(Instant::now() + storage.idle_timeout, move || { + let mut guard = storage.storage.lock().unwrap(); + if generation != guard.generation { + cubeb_log!( + "Not clearing shared voiceprocessing unit storage for generation {} as we're now at {}.", + generation, + guard.generation + ); + return; + } + SharedStorage::clear_locked(&mut guard); + }); + } +} + +#[derive(Debug)] +struct OwningHandle +where + T: Send, +{ + storage: Weak>, + obj: Option, +} + +impl OwningHandle { + fn new(storage: Weak>, obj: T) -> Self { + Self { + storage, + obj: Some(obj), + } + } +} + +impl AsRef for OwningHandle { + fn as_ref(&self) -> &T { + self.obj.as_ref().unwrap() + } +} + +impl AsMut for OwningHandle { + fn as_mut(&mut self) -> &mut T { + self.obj.as_mut().unwrap() + } +} + +impl Drop for OwningHandle { + fn drop(&mut self) { + let storage = self.storage.upgrade(); + assert!( + storage.is_some(), + "Storage must outlive the handle, but didn't" + ); + let storage = storage.unwrap(); + if self.obj.is_none() { + return; + } + let obj = self.obj.take().unwrap(); + storage.recycle(obj); + SharedStorage::clear_if_all_idle_async(&storage); + } +} + +#[derive(Debug)] +struct VoiceProcessingUnit { + unit: AudioUnit, +} + +impl Drop for VoiceProcessingUnit { + fn drop(&mut self) { + assert!(!self.unit.is_null()); + dispose_audio_unit(self.unit); + } +} + +unsafe impl Send for VoiceProcessingUnit {} + +#[derive(Debug)] +struct SharedVoiceProcessingUnitManager { + sync_storage: Mutex>>>, + queue: Queue, + idle_timeout: Duration, +} + +impl SharedVoiceProcessingUnitManager { + fn with_idle_timeout(queue: Queue, idle_timeout: Duration) -> Self { + Self { + sync_storage: Mutex::new(None), + queue, + idle_timeout, + } + } + + fn new(queue: Queue) -> Self { + SharedVoiceProcessingUnitManager::with_idle_timeout(queue, VPIO_IDLE_TIMEOUT) + } + + fn ensure_storage_locked( + &self, + guard: &mut MutexGuard>>>, + ) { + if guard.is_some() { + return; + } + cubeb_log!("Creating shared voiceprocessing storage."); + let storage = SharedStorage::::with_idle_timeout( + self.queue.clone(), + self.idle_timeout, + ); + let old_storage = guard.replace(Arc::from(storage)); + assert!(old_storage.is_none()); + } + + // Take an already existing, shared, vpio unit, if one is available. + #[cfg(test)] + fn take(&mut self) -> Result> { + debug_assert_running_serially(); + let mut guard = self.sync_storage.lock().unwrap(); + self.ensure_storage_locked(&mut guard); + let storage = guard.as_mut().unwrap(); + let res = storage.take(); + res.map(|u| OwningHandle::new(Arc::downgrade(storage), u)) + } + + // Take an already existing, shared, vpio unit, or create one if none are available. + fn take_or_create(&mut self) -> Result> { + debug_assert_running_serially(); + let mut guard = self.sync_storage.lock().unwrap(); + self.ensure_storage_locked(&mut guard); + let storage = guard.as_mut().unwrap(); + let res = storage.take_or_create_with(create_voiceprocessing_audiounit); + res.map(|u| OwningHandle::new(Arc::downgrade(storage), u)) + } +} + +unsafe impl Send for SharedVoiceProcessingUnitManager {} +unsafe impl Sync for SharedVoiceProcessingUnitManager {} + +impl Drop for SharedVoiceProcessingUnitManager { + fn drop(&mut self) { + debug_assert_not_running_serially(); + self.queue.run_final(|| { + let mut guard = self.sync_storage.lock().unwrap(); + if guard.is_none() { + return; + } + guard.as_mut().unwrap().clear(); + }); } } @@ -2091,15 +2477,26 @@ pub struct AudioUnitContext { serial_queue: Queue, latency_controller: Mutex, devices: Mutex, + // Storage for a context-global vpio unit. Duplex streams that need one will take this + // and return it when done. + shared_voice_processing_unit: SharedVoiceProcessingUnitManager, } impl AudioUnitContext { fn new() -> Self { + let queue_label = format!("{}.context", DISPATCH_QUEUE_LABEL); + let serial_queue = + Queue::new_with_target(queue_label.as_str(), get_serial_queue_singleton()); + let shared_vp_queue = Queue::new_with_target( + format!("{}.context.shared_vpio", DISPATCH_QUEUE_LABEL).as_str(), + &serial_queue, + ); Self { _ops: &OPS as *const _, - serial_queue: Queue::new(DISPATCH_QUEUE_LABEL), + serial_queue, latency_controller: Mutex::new(LatencyController::default()), devices: Mutex::new(SharedDevices::default()), + shared_voice_processing_unit: SharedVoiceProcessingUnitManager::new(shared_vp_queue), } } @@ -2108,14 +2505,14 @@ impl AudioUnitContext { controller.streams } - fn update_latency_by_adding_stream(&self, latency_frames: u32) -> Option { + fn update_latency_by_adding_stream(&self, latency_frames: u32) -> u32 { let mut controller = self.latency_controller.lock().unwrap(); controller.add_stream(latency_frames) } - fn update_latency_by_removing_stream(&self) -> Option { + fn update_latency_by_removing_stream(&self) { let mut controller = self.latency_controller.lock().unwrap(); - controller.subtract_stream() + controller.subtract_stream(); } fn add_devices_changed_listener( @@ -2228,8 +2625,16 @@ impl AudioUnitContext { impl ContextOps for AudioUnitContext { fn init(_context_name: Option<&CStr>) -> Result { - set_notification_runloop(); - let ctx = Box::new(AudioUnitContext::new()); + run_serially(set_notification_runloop); + let mut ctx = Box::new(AudioUnitContext::new()); + let queue_label = format!("{}.context.{:p}", DISPATCH_QUEUE_LABEL, ctx.as_ref()); + ctx.serial_queue = + Queue::new_with_target(queue_label.as_str(), get_serial_queue_singleton()); + let shared_vp_queue = Queue::new_with_target( + format!("{}.shared_vpio", queue_label).as_str(), + &ctx.serial_queue, + ); + ctx.shared_voice_processing_unit = SharedVoiceProcessingUnitManager::new(shared_vp_queue); Ok(unsafe { Context::from_ptr(Box::into_raw(ctx) as *mut _) }) } @@ -2379,28 +2784,18 @@ impl ContextOps for AudioUnitContext { return Err(Error::invalid_parameter()); } - // Latency cannot change if another stream is operating in parallel. In this case - // latency is set to the other stream value. - let global_latency_frames = self - .update_latency_by_adding_stream(latency_frames) - .unwrap(); - if global_latency_frames != latency_frames { - cubeb_log!( - "Use global latency {} instead of the requested latency {}.", - global_latency_frames, - latency_frames - ); - } - let in_stm_settings = if let Some(params) = input_stream_params { - let in_device = - match create_device_info(input_device as AudioDeviceID, DeviceType::INPUT) { - None => { - cubeb_log!("Fail to create device info for input"); - return Err(Error::error()); - } - Some(d) => d, - }; + let in_device = match self + .serial_queue + .run_sync(|| create_device_info(input_device as AudioDeviceID, DeviceType::INPUT)) + .unwrap() + { + None => { + cubeb_log!("Fail to create device info for input"); + return Err(Error::error()); + } + Some(d) => d, + }; let stm_params = StreamParams::from(unsafe { *params.as_ptr() }); Some((stm_params, in_device)) } else { @@ -2408,20 +2803,34 @@ impl ContextOps for AudioUnitContext { }; let out_stm_settings = if let Some(params) = output_stream_params { - let out_device = - match create_device_info(output_device as AudioDeviceID, DeviceType::OUTPUT) { - None => { - cubeb_log!("Fail to create device info for output"); - return Err(Error::error()); - } - Some(d) => d, - }; + let out_device = match self + .serial_queue + .run_sync(|| create_device_info(output_device as AudioDeviceID, DeviceType::OUTPUT)) + .unwrap() + { + None => { + cubeb_log!("Fail to create device info for output"); + return Err(Error::error()); + } + Some(d) => d, + }; let stm_params = StreamParams::from(unsafe { *params.as_ptr() }); Some((stm_params, out_device)) } else { None }; + // Latency cannot change if another stream is operating in parallel. In this case + // latency is set to the other stream value. + let global_latency_frames = self.update_latency_by_adding_stream(latency_frames); + if global_latency_frames != latency_frames { + cubeb_log!( + "Use global latency {} instead of the requested latency {}.", + global_latency_frames, + latency_frames + ); + } + let mut boxed_stream = Box::new(AudioUnitStream::new( self, user_ptr, @@ -2431,16 +2840,25 @@ impl ContextOps for AudioUnitContext { )); // Rename the task queue to be an unique label. - let queue_label = format!("{}.{:p}", DISPATCH_QUEUE_LABEL, boxed_stream.as_ref()); - boxed_stream.queue = Queue::new(queue_label.as_str()); + let queue_label = format!( + "{}.stream.{:p}", + DISPATCH_QUEUE_LABEL, + boxed_stream.as_ref() + ); + boxed_stream.queue = Queue::new_with_target(queue_label.as_str(), &boxed_stream.queue); boxed_stream.core_stream_data = CoreStreamData::new(boxed_stream.as_ref(), in_stm_settings, out_stm_settings); - let mut result = Ok(()); - boxed_stream.queue.clone().run_sync(|| { - result = boxed_stream.core_stream_data.setup(); - }); + let result = boxed_stream + .queue + .clone() + .run_sync(|| { + boxed_stream + .core_stream_data + .setup(&mut boxed_stream.context.shared_voice_processing_unit) + }) + .unwrap(); if let Err(r) = result { cubeb_log!( "({:p}) Could not setup the audiounit stream.", @@ -2465,25 +2883,43 @@ impl ContextOps for AudioUnitContext { if devtype == DeviceType::UNKNOWN { return Err(Error::invalid_parameter()); } - if collection_changed_callback.is_some() { - self.add_devices_changed_listener(devtype, collection_changed_callback, user_ptr) - } else { - self.remove_devices_changed_listener(devtype) - } + self.serial_queue + .clone() + .run_sync(|| { + if collection_changed_callback.is_some() { + self.add_devices_changed_listener( + devtype, + collection_changed_callback, + user_ptr, + ) + } else { + self.remove_devices_changed_listener(devtype) + } + }) + .unwrap() } } impl Drop for AudioUnitContext { fn drop(&mut self) { - let devices = self.devices.lock().unwrap(); - assert!( + assert!({ + let devices = self.devices.lock().unwrap(); devices.input.changed_callback.is_none() && devices.output.changed_callback.is_none() - ); + }); + + self.shared_voice_processing_unit = + SharedVoiceProcessingUnitManager::new(self.serial_queue.clone()); + + // Make sure all the pending (device-collection-changed-callback) tasks + // in queue are done, and cancel all the tasks appended after `drop` is executed. + let queue = self.serial_queue.clone(); + queue.run_final(|| {}); { let controller = self.latency_controller.lock().unwrap(); - // Disabling this assert for bug 1083664 -- we seem to leak a stream + // Disabling this assert in release for bug 1083664 -- we seem to leak a stream // assert(controller.streams == 0); + debug_assert!(controller.streams == 0); if controller.streams > 0 { cubeb_log!( "({:p}) API misuse, {} streams active when context destroyed!", @@ -2492,10 +2928,6 @@ impl Drop for AudioUnitContext { ); } } - // Make sure all the pending (device-collection-changed-callback) tasks - // in queue are done, and cancel all the tasks appended after `drop` is executed. - let queue = self.serial_queue.clone(); - queue.run_final(|| {}); } } @@ -2562,6 +2994,8 @@ struct CoreStreamData<'ctx> { // I/O AudioUnits. input_unit: AudioUnit, output_unit: AudioUnit, + // Handle to shared voiceprocessing AudioUnit, if in use. + voiceprocessing_unit_handle: Option>, // Info of the I/O devices. input_device: device_info, output_device: device_info, @@ -2603,6 +3037,7 @@ impl<'ctx> Default for CoreStreamData<'ctx> { output_dev_desc: AudioStreamBasicDescription::default(), input_unit: ptr::null_mut(), output_unit: ptr::null_mut(), + voiceprocessing_unit_handle: None, input_device: device_info::default(), output_device: device_info::default(), input_processing_params: InputProcessingParams::NONE, @@ -2649,6 +3084,7 @@ impl<'ctx> CoreStreamData<'ctx> { output_dev_desc: AudioStreamBasicDescription::default(), input_unit: ptr::null_mut(), output_unit: ptr::null_mut(), + voiceprocessing_unit_handle: None, input_device: in_dev, output_device: out_dev, input_processing_params: InputProcessingParams::NONE, @@ -2716,7 +3152,7 @@ impl<'ctx> CoreStreamData<'ctx> { } fn using_voice_processing_unit(&self) -> bool { - !self.input_unit.is_null() && self.input_unit == self.output_unit + self.voiceprocessing_unit_handle.is_some() } fn same_clock_domain(&self) -> bool { @@ -2744,6 +3180,22 @@ impl<'ctx> CoreStreamData<'ctx> { input_domain == output_domain } + #[allow(non_upper_case_globals)] + fn should_force_vpio_for_input_device(&self, in_device: &device_info) -> bool { + assert!(in_device.id != kAudioObjectUnknown); + self.debug_assert_is_on_stream_queue(); + match get_device_transport_type(in_device.id, DeviceType::INPUT) { + Ok(kAudioDeviceTransportTypeBuiltIn) => { + cubeb_log!( + "Forcing VPIO because input device is built in, and its volume \ + is known to be very low without VPIO whenever VPIO is hooked up to it elsewhere." + ); + true + } + _ => false, + } + } + fn should_block_vpio_for_device_pair( &self, in_device: &device_info, @@ -2751,7 +3203,10 @@ impl<'ctx> CoreStreamData<'ctx> { ) -> bool { self.debug_assert_is_on_stream_queue(); cubeb_log!("Evaluating device pair against VPIO block list"); - let log_device = |id, devtype| -> std::result::Result<(), OSStatus> { + let log_device_and_get_model_uid = |id, devtype| -> String { + let device_model_uid = get_device_model_uid(id, devtype) + .map(|s| s.into_string()) + .unwrap_or_default(); cubeb_log!("{} uid=\"{}\", model_uid=\"{}\", transport_type={:?}, source={:?}, source_name=\"{}\", name=\"{}\", manufacturer=\"{}\"", if devtype == DeviceType::INPUT { "Input" @@ -2760,43 +3215,59 @@ impl<'ctx> CoreStreamData<'ctx> { "Output" }, get_device_uid(id, devtype).map(|s| s.into_string()).unwrap_or_default(), - get_device_model_uid(id, devtype).map(|s| s.into_string()).unwrap_or_default(), + device_model_uid, convert_uint32_into_string(get_device_transport_type(id, devtype).unwrap_or(0)), convert_uint32_into_string(get_device_source(id, devtype).unwrap_or(0)), get_device_source_name(id, devtype).map(|s| s.into_string()).unwrap_or_default(), get_device_name(id, devtype).map(|s| s.into_string()).unwrap_or_default(), get_device_manufacturer(id, devtype).map(|s| s.into_string()).unwrap_or_default()); - Ok(()) + device_model_uid }; - log_device(in_device.id, DeviceType::INPUT); - log_device(out_device.id, DeviceType::OUTPUT); - match ( - get_device_model_uid(in_device.id, DeviceType::INPUT).map(|s| s.to_string()), - get_device_model_uid(out_device.id, DeviceType::OUTPUT).map(|s| s.to_string()), - ) { - (Ok(in_model_uid), Ok(out_model_uid)) - if in_model_uid.contains(APPLE_STUDIO_DISPLAY_USB_ID) - && out_model_uid.contains(APPLE_STUDIO_DISPLAY_USB_ID) => - { - cubeb_log!("Both input and output device is an Apple Studio Display. BLOCKED"); - true - } - _ => { - cubeb_log!("Device pair is not blocked"); - false - } + + #[allow(non_upper_case_globals)] + let in_id = match in_device.id { + kAudioObjectUnknown => None, + id => Some(id), + }; + #[allow(non_upper_case_globals)] + let out_id = match out_device.id { + kAudioObjectUnknown => None, + id => Some(id), + }; + + let (in_model_uid, out_model_uid) = ( + in_id + .map(|id| log_device_and_get_model_uid(id, DeviceType::INPUT)) + .unwrap_or_default(), + out_id + .map(|id| log_device_and_get_model_uid(id, DeviceType::OUTPUT)) + .unwrap_or_default(), + ); + + if in_model_uid.contains(APPLE_STUDIO_DISPLAY_USB_ID) + && out_model_uid.contains(APPLE_STUDIO_DISPLAY_USB_ID) + { + cubeb_log!("Both input and output device is an Apple Studio Display. BLOCKED"); + return true; } + + cubeb_log!("Device pair is not blocked"); + false } - fn create_audiounits(&mut self) -> Result<(device_info, device_info)> { + fn create_audiounits( + &mut self, + shared_voice_processing_unit: &mut SharedVoiceProcessingUnitManager, + ) -> Result<(device_info, device_info)> { self.debug_assert_is_on_stream_queue(); let should_use_voice_processing_unit = self.has_input() - && self.has_output() - && self + && (self .input_stream_params .prefs() .contains(StreamPrefs::VOICE) - && !self.should_block_vpio_for_device_pair(&self.input_device, &self.output_device); + || self.should_force_vpio_for_input_device(&self.input_device)) + && !self.should_block_vpio_for_device_pair(&self.input_device, &self.output_device) + && macos_kernel_major_version() != Ok(MACOS_KERNEL_MAJOR_VERSION_MONTEREY); let should_use_aggregate_device = { // It's impossible to create an aggregate device from an aggregate device, and it's @@ -2843,16 +3314,20 @@ impl<'ctx> CoreStreamData<'ctx> { // - As last resort, create regular AudioUnits. This is also the normal non-duplex path. if should_use_voice_processing_unit { - if let Ok(au) = - create_voiceprocessing_audiounit(&self.input_device, &self.output_device) - { - cubeb_log!("({:p}) Using VoiceProcessingIO AudioUnit", self.stm_ptr); - self.input_unit = au; - self.output_unit = au; + if let Ok(mut au_handle) = get_voiceprocessing_audiounit( + shared_voice_processing_unit, + &self.input_device, + &self.output_device, + ) { + self.input_unit = au_handle.as_mut().unit; + if self.has_output() { + self.output_unit = au_handle.as_mut().unit; + } + self.voiceprocessing_unit_handle = Some(au_handle); return Ok((self.input_device.clone(), self.output_device.clone())); } cubeb_log!( - "({:p}) Failed to create VoiceProcessingIO AudioUnit. Trying a regular one.", + "({:p}) Failed to get VoiceProcessingIO AudioUnit. Trying a regular one.", self.stm_ptr ); } @@ -2954,7 +3429,10 @@ impl<'ctx> CoreStreamData<'ctx> { } #[allow(clippy::cognitive_complexity)] // TODO: Refactoring. - fn setup(&mut self) -> Result<()> { + fn setup( + &mut self, + shared_voice_processing_unit: &mut SharedVoiceProcessingUnitManager, + ) -> Result<()> { self.debug_assert_is_on_stream_queue(); if self .input_stream_params @@ -2970,7 +3448,7 @@ impl<'ctx> CoreStreamData<'ctx> { } let same_clock_domain = self.same_clock_domain(); - let (in_dev_info, out_dev_info) = self.create_audiounits()?; + let (in_dev_info, out_dev_info) = self.create_audiounits(shared_voice_processing_unit)?; let using_voice_processing_unit = self.using_voice_processing_unit(); assert!(!self.stm_ptr.is_null()); @@ -3155,6 +3633,25 @@ impl<'ctx> CoreStreamData<'ctx> { ); } + if self.has_input() && !self.has_output() && using_voice_processing_unit { + // We must configure the output side of VPIO to match the input side, even if we don't use it. + let r = audio_unit_set_property( + self.input_unit, + kAudioUnitProperty_StreamFormat, + kAudioUnitScope_Input, + AU_OUT_BUS, + &self.input_dev_desc, + mem::size_of::(), + ); + if r != NO_ERR { + cubeb_log!( + "AudioUnitSetProperty/output/kAudioUnitProperty_StreamFormat rv={}", + r + ); + return Err(Error::error()); + } + } + if self.has_output() { assert!(!self.output_unit.is_null()); @@ -3463,15 +3960,31 @@ impl<'ctx> CoreStreamData<'ctx> { // NOTE: On MacOS 14 the ducking happens on creation of the VPIO AudioUnit. // On MacOS 10.15 it happens on both creation and initialization, which // is why we defer the unducking until now. - let r = audio_device_duck(self.output_device.id, 1.0, ptr::null_mut(), 0.5); - if r != NO_ERR { - cubeb_log!( - "({:p}) Failed to undo ducking of voiceprocessing on output device {}. Proceeding... Error: {}", - self.stm_ptr, - self.output_device.id, - r - ); - } + #[allow(non_upper_case_globals)] + let mut device = match self.output_device.id { + kAudioObjectUnknown => None, + id => Some(id), + }; + device = device.or_else(|| get_default_device(DeviceType::OUTPUT)); + match device { + None => { + cubeb_log!( + "({:p}) No output device to undo vpio ducking on", + self.stm_ptr + ); + } + Some(id) => { + let r = audio_device_duck(id, 1.0, ptr::null_mut(), 0.5); + if r != NO_ERR { + cubeb_log!( + "({:p}) Failed to undo ducking of voiceprocessing on output device {}. Proceeding... Error: {}", + self.stm_ptr, + id, + r + ); + } + } + }; // Always try to remember the applied input mute state. If it cannot be applied // to the new device pair, we notify the client of an error and it will have to @@ -3558,10 +4071,16 @@ impl<'ctx> CoreStreamData<'ctx> { } if !self.input_unit.is_null() { - dispose_audio_unit(self.input_unit); + if !self.using_voice_processing_unit() { + // The VPIO unit is shared and must not be disposed. + dispose_audio_unit(self.input_unit); + } self.input_unit = ptr::null_mut(); } + // Return the VPIO unit if present. + self.voiceprocessing_unit_handle = None; + self.resampler.destroy(); self.mixer = None; self.aggregate_device = None; @@ -3873,8 +4392,6 @@ struct OutputCallbackTimingData { // #[repr(C)] is used to prevent any padding from being added in the beginning of the AudioUnitStream. #[repr(C)] #[derive(Debug)] -// Allow exposing this private struct in public interfaces when running tests. -#[cfg_attr(test, allow(private_in_public))] struct AudioUnitStream<'ctx> { context: &'ctx mut AudioUnitContext, user_ptr: *mut c_void, @@ -3927,10 +4444,11 @@ impl<'ctx> AudioUnitStream<'ctx> { }); let (output_callback_timing_data_write, output_callback_timing_data_read) = output_callback_timing_data.split(); + let queue = context.serial_queue.clone(); AudioUnitStream { context, user_ptr, - queue: Queue::new(DISPATCH_QUEUE_LABEL), + queue, data_callback, state_callback, device_changed_callback: Mutex::new(None), @@ -4046,10 +4564,12 @@ impl<'ctx> AudioUnitStream<'ctx> { } } - self.core_stream_data.setup().map_err(|e| { - cubeb_log!("({:p}) Setup failed.", self.core_stream_data.stm_ptr); - e - })?; + self.core_stream_data + .setup(&mut self.context.shared_voice_processing_unit) + .map_err(|e| { + cubeb_log!("({:p}) Setup failed.", self.core_stream_data.stm_ptr); + e + })?; if let Ok(volume) = vol_rv { set_volume(self.core_stream_data.output_unit, volume); @@ -4171,7 +4691,7 @@ impl<'ctx> AudioUnitStream<'ctx> { impl<'ctx> Drop for AudioUnitStream<'ctx> { fn drop(&mut self) { // Execute destroy in serial queue to avoid collision with reinit when un/plug devices - self.queue.clone().run_final(move || { + self.queue.clone().run_final(|| { self.destroy(); self.core_stream_data = CoreStreamData::default(); }); @@ -4184,12 +4704,10 @@ impl<'ctx> StreamOps for AudioUnitStream<'ctx> { self.draining.store(false, Ordering::SeqCst); // Execute start in serial queue to avoid racing with destroy or reinit. - let mut result = Err(Error::error()); - let started = &mut result; - let stream = &self; - self.queue.run_sync(move || { - *started = stream.core_stream_data.start_audiounits(); - }); + let result = self + .queue + .run_sync(|| self.core_stream_data.start_audiounits()) + .unwrap(); result?; @@ -4205,10 +4723,8 @@ impl<'ctx> StreamOps for AudioUnitStream<'ctx> { self.stopped.store(true, Ordering::SeqCst); // Execute stop in serial queue to avoid racing with destroy or reinit. - let stream = &self; - self.queue.run_sync(move || { - stream.core_stream_data.stop_audiounits(); - }); + self.queue + .run_sync(|| self.core_stream_data.stop_audiounits()); self.notify_state_changed(State::Stopped); @@ -4285,12 +4801,10 @@ impl<'ctx> StreamOps for AudioUnitStream<'ctx> { } fn set_volume(&mut self, volume: f32) -> Result<()> { // Execute set_volume in serial queue to avoid racing with destroy or reinit. - let mut result = Err(Error::error()); - let set = &mut result; - let stream = &self; - self.queue.run_sync(move || { - *set = set_volume(stream.core_stream_data.output_unit, volume); - }); + let result = self + .queue + .run_sync(|| set_volume(self.core_stream_data.output_unit, volume)) + .unwrap(); result?; diff --git a/third_party/rust/cubeb-coreaudio/src/backend/tests/aggregate_device.rs b/third_party/rust/cubeb-coreaudio/src/backend/tests/aggregate_device.rs index 1d3c341ae8..6c5e494059 100644 --- a/third_party/rust/cubeb-coreaudio/src/backend/tests/aggregate_device.rs +++ b/third_party/rust/cubeb-coreaudio/src/backend/tests/aggregate_device.rs @@ -3,6 +3,8 @@ use super::utils::{ test_get_drift_compensations, test_get_master_device, DeviceFilter, Scope, }; use super::*; +use std::iter::zip; +use std::panic; // AggregateDevice::set_sub_devices // ------------------------------------ @@ -19,21 +21,27 @@ fn test_aggregate_set_sub_devices_for_an_unknown_aggregate_device() { let default_input = default_input.unwrap(); let default_output = default_output.unwrap(); assert!( - AggregateDevice::set_sub_devices(kAudioObjectUnknown, default_input, default_output) - .is_err() + run_serially_forward_panics(|| AggregateDevice::set_sub_devices( + kAudioObjectUnknown, + default_input, + default_output + )) + .is_err() ); } #[test] #[should_panic] fn test_aggregate_set_sub_devices_for_unknown_devices() { - // If aggregate device id is kAudioObjectUnknown, we are unable to set device list. - assert!(AggregateDevice::set_sub_devices( - kAudioObjectUnknown, - kAudioObjectUnknown, - kAudioObjectUnknown - ) - .is_err()); + run_serially_forward_panics(|| { + // If aggregate device id is kAudioObjectUnknown, we are unable to set device list. + assert!(AggregateDevice::set_sub_devices( + kAudioObjectUnknown, + kAudioObjectUnknown, + kAudioObjectUnknown + ) + .is_err()); + }); } // AggregateDevice::get_sub_devices @@ -48,7 +56,13 @@ fn test_aggregate_get_sub_devices() { // containing `device` itself if it's not an aggregate device. This test assumes devices // is not an empty aggregate device (Test will panic when calling get_sub_devices with // an empty aggregate device). - let sub_devices = AggregateDevice::get_sub_devices(device).unwrap(); + println!( + "get_sub_devices({}={})", + device, + run_serially_forward_panics(|| get_device_uid(device)) + ); + let sub_devices = + run_serially_forward_panics(|| AggregateDevice::get_sub_devices(device).unwrap()); // TODO: If the device is a blank aggregate device, then the assertion fails! assert!(!sub_devices.is_empty()); } @@ -57,8 +71,10 @@ fn test_aggregate_get_sub_devices() { #[test] #[should_panic] fn test_aggregate_get_sub_devices_for_a_unknown_device() { - let devices = AggregateDevice::get_sub_devices(kAudioObjectUnknown).unwrap(); - assert!(devices.is_empty()); + run_serially_forward_panics(|| { + let devices = AggregateDevice::get_sub_devices(kAudioObjectUnknown).unwrap(); + assert!(devices.is_empty()); + }); } // AggregateDevice::set_master_device @@ -66,7 +82,11 @@ fn test_aggregate_get_sub_devices_for_a_unknown_device() { #[test] #[should_panic] fn test_aggregate_set_master_device_for_an_unknown_aggregate_device() { - assert!(AggregateDevice::set_master_device(kAudioObjectUnknown, kAudioObjectUnknown).is_err()); + run_serially_forward_panics(|| { + assert!( + AggregateDevice::set_master_device(kAudioObjectUnknown, kAudioObjectUnknown).is_err() + ); + }); } // AggregateDevice::activate_clock_drift_compensation @@ -74,7 +94,9 @@ fn test_aggregate_set_master_device_for_an_unknown_aggregate_device() { #[test] #[should_panic] fn test_aggregate_activate_clock_drift_compensation_for_an_unknown_aggregate_device() { - assert!(AggregateDevice::activate_clock_drift_compensation(kAudioObjectUnknown).is_err()); + run_serially_forward_panics(|| { + assert!(AggregateDevice::activate_clock_drift_compensation(kAudioObjectUnknown).is_err()); + }); } // AggregateDevice::destroy_device @@ -82,60 +104,55 @@ fn test_aggregate_activate_clock_drift_compensation_for_an_unknown_aggregate_dev #[test] #[should_panic] fn test_aggregate_destroy_device_for_unknown_plugin_and_aggregate_devices() { - assert!(AggregateDevice::destroy_device(kAudioObjectUnknown, kAudioObjectUnknown).is_err()) + run_serially_forward_panics(|| { + assert!(AggregateDevice::destroy_device(kAudioObjectUnknown, kAudioObjectUnknown).is_err()) + }); } #[test] #[should_panic] fn test_aggregate_destroy_aggregate_device_for_a_unknown_aggregate_device() { - let plugin = AggregateDevice::get_system_plugin_id().unwrap(); - assert!(AggregateDevice::destroy_device(plugin, kAudioObjectUnknown).is_err()); + run_serially_forward_panics(|| { + let plugin = AggregateDevice::get_system_plugin_id().unwrap(); + assert!(AggregateDevice::destroy_device(plugin, kAudioObjectUnknown).is_err()); + }); } -// Default Ignored Tests -// ================================================================================================ -// The following tests that calls `AggregateDevice::create_blank_device` are marked `ignore` by -// default since the device-collection-changed callbacks will be fired upon -// `AggregateDevice::create_blank_device` is called (it will plug a new device in system!). -// Some tests rely on the device-collection-changed callbacks in a certain way. The callbacks -// fired from a unexpected `AggregateDevice::create_blank_device` will break those tests. - // AggregateDevice::create_blank_device_sync // ------------------------------------ #[test] -#[ignore] fn test_aggregate_create_blank_device() { // TODO: Test this when there is no available devices. - let plugin = AggregateDevice::get_system_plugin_id().unwrap(); - let device = AggregateDevice::create_blank_device_sync(plugin).unwrap(); + let plugin = run_serially(|| AggregateDevice::get_system_plugin_id()).unwrap(); + let device = run_serially(|| AggregateDevice::create_blank_device_sync(plugin)).unwrap(); let devices = test_get_all_devices(DeviceFilter::IncludeAll); let device = devices.into_iter().find(|dev| dev == &device).unwrap(); - let uid = get_device_global_uid(device).unwrap().into_string(); + let uid = run_serially(|| get_device_global_uid(device).unwrap().into_string()); assert!(uid.contains(PRIVATE_AGGREGATE_DEVICE_NAME)); - assert!(AggregateDevice::destroy_device(plugin, device).is_ok()); + assert!(run_serially(|| AggregateDevice::destroy_device(plugin, device)).is_ok()); } // AggregateDevice::get_sub_devices // ------------------------------------ #[test] -#[ignore] #[should_panic] fn test_aggregate_get_sub_devices_for_blank_aggregate_devices() { - // TODO: Test this when there is no available devices. - let plugin = AggregateDevice::get_system_plugin_id().unwrap(); - let device = AggregateDevice::create_blank_device_sync(plugin).unwrap(); - // There is no sub device in a blank aggregate device! - // AggregateDevice::get_sub_devices guarantees returning a non-empty devices vector, so - // the following call will panic! - let sub_devices = AggregateDevice::get_sub_devices(device).unwrap(); - assert!(sub_devices.is_empty()); - assert!(AggregateDevice::destroy_device(plugin, device).is_ok()); + run_serially_forward_panics(|| { + // TODO: Test this when there is no available devices. + let plugin = AggregateDevice::get_system_plugin_id().unwrap(); + let device = AggregateDevice::create_blank_device_sync(plugin).unwrap(); + // There is no sub device in a blank aggregate device! + // AggregateDevice::get_sub_devices guarantees returning a non-empty devices vector, so + // the following call will panic! + let sub_devices = AggregateDevice::get_sub_devices(device).unwrap(); + assert!(sub_devices.is_empty()); + assert!(AggregateDevice::destroy_device(plugin, device).is_ok()); + }); } // AggregateDevice::set_sub_devices_sync // ------------------------------------ #[test] -#[ignore] fn test_aggregate_set_sub_devices() { let input_device = test_get_default_device(Scope::Input); let output_device = test_get_default_device(Scope::Output); @@ -147,13 +164,20 @@ fn test_aggregate_set_sub_devices() { let input_device = input_device.unwrap(); let output_device = output_device.unwrap(); - let plugin = AggregateDevice::get_system_plugin_id().unwrap(); - let device = AggregateDevice::create_blank_device_sync(plugin).unwrap(); - assert!(AggregateDevice::set_sub_devices_sync(device, input_device, output_device).is_ok()); - - let sub_devices = AggregateDevice::get_sub_devices(device).unwrap(); - let input_sub_devices = AggregateDevice::get_sub_devices(input_device).unwrap(); - let output_sub_devices = AggregateDevice::get_sub_devices(output_device).unwrap(); + let plugin = run_serially(|| AggregateDevice::get_system_plugin_id()).unwrap(); + let device = run_serially(|| AggregateDevice::create_blank_device_sync(plugin)).unwrap(); + assert!(run_serially(|| AggregateDevice::set_sub_devices_sync( + device, + input_device, + output_device + )) + .is_ok()); + + let sub_devices = run_serially(|| AggregateDevice::get_sub_devices(device)).unwrap(); + let input_sub_devices = + run_serially(|| AggregateDevice::get_sub_devices(input_device)).unwrap(); + let output_sub_devices = + run_serially(|| AggregateDevice::get_sub_devices(output_device)).unwrap(); // TODO: There may be overlapping devices between input_sub_devices and output_sub_devices, // but now AggregateDevice::set_sub_devices will add them directly. @@ -168,10 +192,10 @@ fn test_aggregate_set_sub_devices() { assert!(sub_devices.contains(dev)); } - let onwed_devices = test_get_all_onwed_devices(device); - let onwed_device_uids = get_device_uids(&onwed_devices); - let input_sub_device_uids = get_device_uids(&input_sub_devices); - let output_sub_device_uids = get_device_uids(&output_sub_devices); + let onwed_devices = run_serially(|| test_get_all_onwed_devices(device)); + let onwed_device_uids = run_serially(|| get_device_uids(&onwed_devices)); + let input_sub_device_uids = run_serially(|| get_device_uids(&input_sub_devices)); + let output_sub_device_uids = run_serially(|| get_device_uids(&output_sub_devices)); for uid in &input_sub_device_uids { assert!(onwed_device_uids.contains(uid)); } @@ -179,11 +203,10 @@ fn test_aggregate_set_sub_devices() { assert!(onwed_device_uids.contains(uid)); } - assert!(AggregateDevice::destroy_device(plugin, device).is_ok()); + assert!(run_serially(|| AggregateDevice::destroy_device(plugin, device)).is_ok()); } #[test] -#[ignore] #[should_panic] fn test_aggregate_set_sub_devices_for_unknown_input_devices() { let output_device = test_get_default_device(Scope::Output); @@ -192,16 +215,19 @@ fn test_aggregate_set_sub_devices_for_unknown_input_devices() { } let output_device = output_device.unwrap(); - let plugin = AggregateDevice::get_system_plugin_id().unwrap(); - let device = AggregateDevice::create_blank_device_sync(plugin).unwrap(); + run_serially_forward_panics(|| { + let plugin = AggregateDevice::get_system_plugin_id().unwrap(); + let device = AggregateDevice::create_blank_device_sync(plugin).unwrap(); - assert!(AggregateDevice::set_sub_devices(device, kAudioObjectUnknown, output_device).is_err()); + assert!( + AggregateDevice::set_sub_devices(device, kAudioObjectUnknown, output_device).is_err() + ); - assert!(AggregateDevice::destroy_device(plugin, device).is_ok()); + assert!(AggregateDevice::destroy_device(plugin, device).is_ok()); + }); } #[test] -#[ignore] #[should_panic] fn test_aggregate_set_sub_devices_for_unknown_output_devices() { let input_device = test_get_default_device(Scope::Input); @@ -210,12 +236,16 @@ fn test_aggregate_set_sub_devices_for_unknown_output_devices() { } let input_device = input_device.unwrap(); - let plugin = AggregateDevice::get_system_plugin_id().unwrap(); - let device = AggregateDevice::create_blank_device_sync(plugin).unwrap(); + run_serially_forward_panics(|| { + let plugin = AggregateDevice::get_system_plugin_id().unwrap(); + let device = AggregateDevice::create_blank_device_sync(plugin).unwrap(); - assert!(AggregateDevice::set_sub_devices(device, input_device, kAudioObjectUnknown).is_err()); + assert!( + AggregateDevice::set_sub_devices(device, input_device, kAudioObjectUnknown).is_err() + ); - assert!(AggregateDevice::destroy_device(plugin, device).is_ok()); + assert!(AggregateDevice::destroy_device(plugin, device).is_ok()); + }); } fn get_device_uids(devices: &Vec) -> Vec { @@ -228,7 +258,6 @@ fn get_device_uids(devices: &Vec) -> Vec { // AggregateDevice::set_master_device // ------------------------------------ #[test] -#[ignore] fn test_aggregate_set_master_device() { let input_device = test_get_default_device(Scope::Input); let output_device = test_get_default_device(Scope::Output); @@ -240,22 +269,28 @@ fn test_aggregate_set_master_device() { let input_device = input_device.unwrap(); let output_device = output_device.unwrap(); - let plugin = AggregateDevice::get_system_plugin_id().unwrap(); - let device = AggregateDevice::create_blank_device_sync(plugin).unwrap(); - assert!(AggregateDevice::set_sub_devices_sync(device, input_device, output_device).is_ok()); - assert!(AggregateDevice::set_master_device(device, output_device).is_ok()); - - // Check if master is set to the first sub device of the default output device. - let first_output_sub_device_uid = - get_device_uid(AggregateDevice::get_sub_devices(device).unwrap()[0]); - let master_device_uid = test_get_master_device(device); + let plugin = run_serially(|| AggregateDevice::get_system_plugin_id()).unwrap(); + let device = run_serially(|| AggregateDevice::create_blank_device_sync(plugin)).unwrap(); + assert!(run_serially(|| AggregateDevice::set_sub_devices_sync( + device, + input_device, + output_device + )) + .is_ok()); + assert!(run_serially(|| AggregateDevice::set_master_device(device, output_device)).is_ok()); + + let output_sub_devices = + run_serially(|| AggregateDevice::get_sub_devices(output_device)).unwrap(); + let first_output_sub_device_uid = run_serially(|| get_device_uid(output_sub_devices[0])); + + // Check that the first sub device of the output device is set as master device. + let master_device_uid = run_serially(|| test_get_master_device(device)); assert_eq!(first_output_sub_device_uid, master_device_uid); - assert!(AggregateDevice::destroy_device(plugin, device).is_ok()); + assert!(run_serially(|| AggregateDevice::destroy_device(plugin, device)).is_ok()); } #[test] -#[ignore] fn test_aggregate_set_master_device_for_a_blank_aggregate_device() { let output_device = test_get_default_device(Scope::Output); if output_device.is_none() { @@ -263,9 +298,11 @@ fn test_aggregate_set_master_device_for_a_blank_aggregate_device() { return; } - let plugin = AggregateDevice::get_system_plugin_id().unwrap(); - let device = AggregateDevice::create_blank_device_sync(plugin).unwrap(); - assert!(AggregateDevice::set_master_device(device, output_device.unwrap()).is_ok()); + let plugin = run_serially(|| AggregateDevice::get_system_plugin_id()).unwrap(); + let device = run_serially(|| AggregateDevice::create_blank_device_sync(plugin)).unwrap(); + assert!( + run_serially(|| AggregateDevice::set_master_device(device, output_device.unwrap())).is_ok() + ); // TODO: it's really weird the aggregate device actually own nothing // but its master device can be set successfully! @@ -275,17 +312,16 @@ fn test_aggregate_set_master_device_for_a_blank_aggregate_device() { // The CFStringRef of the master device returned from `test_get_master_device` is actually // non-null. - assert!(AggregateDevice::destroy_device(plugin, device).is_ok()); + assert!(run_serially(|| AggregateDevice::destroy_device(plugin, device)).is_ok()); } fn get_device_uid(id: AudioObjectID) -> String { - get_device_global_uid(id).unwrap().into_string() + get_device_global_uid(id).map_or(String::new(), |uid| uid.into_string()) } // AggregateDevice::activate_clock_drift_compensation // ------------------------------------ #[test] -#[ignore] fn test_aggregate_activate_clock_drift_compensation() { let input_device = test_get_default_device(Scope::Input); let output_device = test_get_default_device(Scope::Output); @@ -297,27 +333,40 @@ fn test_aggregate_activate_clock_drift_compensation() { let input_device = input_device.unwrap(); let output_device = output_device.unwrap(); - let plugin = AggregateDevice::get_system_plugin_id().unwrap(); - let device = AggregateDevice::create_blank_device_sync(plugin).unwrap(); - assert!(AggregateDevice::set_sub_devices_sync(device, input_device, output_device).is_ok()); - assert!(AggregateDevice::set_master_device(device, output_device).is_ok()); - assert!(AggregateDevice::activate_clock_drift_compensation(device).is_ok()); + let plugin = run_serially(|| AggregateDevice::get_system_plugin_id()).unwrap(); + let device = run_serially(|| AggregateDevice::create_blank_device_sync(plugin)).unwrap(); + assert!(run_serially(|| AggregateDevice::set_sub_devices_sync( + device, + input_device, + output_device + )) + .is_ok()); + assert!(run_serially(|| AggregateDevice::set_master_device(device, output_device)).is_ok()); + assert!(run_serially(|| AggregateDevice::activate_clock_drift_compensation(device)).is_ok()); // Check the compensations. - let devices = test_get_all_onwed_devices(device); - let compensations = get_drift_compensations(&devices); + let devices = run_serially(|| test_get_all_onwed_devices(device)); + let compensations = run_serially(|| get_drift_compensations(&devices)); + let master_device_uid = run_serially(|| test_get_master_device(device)); assert!(!compensations.is_empty()); assert_eq!(devices.len(), compensations.len()); - for (i, compensation) in compensations.iter().enumerate() { - assert_eq!(*compensation, if i == 0 { 0 } else { DRIFT_COMPENSATION }); + for (device, compensation) in zip(devices, compensations) { + let uid = get_device_uid(device); + assert_eq!( + compensation, + if uid == master_device_uid { + 0 + } else { + DRIFT_COMPENSATION + } + ); } - assert!(AggregateDevice::destroy_device(plugin, device).is_ok()); + assert!(run_serially(|| AggregateDevice::destroy_device(plugin, device)).is_ok()); } #[test] -#[ignore] fn test_aggregate_activate_clock_drift_compensation_for_an_aggregate_device_without_master_device() { let input_device = test_get_default_device(Scope::Input); @@ -330,25 +379,32 @@ fn test_aggregate_activate_clock_drift_compensation_for_an_aggregate_device_with let input_device = input_device.unwrap(); let output_device = output_device.unwrap(); - let plugin = AggregateDevice::get_system_plugin_id().unwrap(); - let device = AggregateDevice::create_blank_device_sync(plugin).unwrap(); - assert!(AggregateDevice::set_sub_devices_sync(device, input_device, output_device).is_ok()); - - // TODO: Is the master device the first output sub device by default if we - // don't set that ? Is it because we add the output sub device list - // before the input's one ? (See implementation of - // AggregateDevice::set_sub_devices). - let first_output_sub_device_uid = - get_device_uid(AggregateDevice::get_sub_devices(output_device).unwrap()[0]); - let master_device_uid = test_get_master_device(device); - assert_eq!(first_output_sub_device_uid, master_device_uid); + let plugin = run_serially(|| AggregateDevice::get_system_plugin_id()).unwrap(); + let device = run_serially(|| AggregateDevice::create_blank_device_sync(plugin)).unwrap(); + assert!(run_serially(|| AggregateDevice::set_sub_devices_sync( + device, + input_device, + output_device + )) + .is_ok()); + + // The master device is by default the first sub device in the list. + // This happens to be the first sub device of the input device, see implementation of + // AggregateDevice::set_sub_devices. + let first_input_sub_device_uid = + run_serially(|| get_device_uid(AggregateDevice::get_sub_devices(input_device).unwrap()[0])); + let first_sub_device_uid = + run_serially(|| get_device_uid(AggregateDevice::get_sub_devices(device).unwrap()[0])); + assert_eq!(first_input_sub_device_uid, first_sub_device_uid); + let master_device_uid = run_serially(|| test_get_master_device(device)); + assert_eq!(first_sub_device_uid, master_device_uid); // Compensate the drift directly without setting master device. - assert!(AggregateDevice::activate_clock_drift_compensation(device).is_ok()); + assert!(run_serially(|| AggregateDevice::activate_clock_drift_compensation(device)).is_ok()); // Check the compensations. - let devices = test_get_all_onwed_devices(device); - let compensations = get_drift_compensations(&devices); + let devices = run_serially(|| test_get_all_onwed_devices(device)); + let compensations = run_serially(|| get_drift_compensations(&devices)); assert!(!compensations.is_empty()); assert_eq!(devices.len(), compensations.len()); @@ -356,25 +412,26 @@ fn test_aggregate_activate_clock_drift_compensation_for_an_aggregate_device_with assert_eq!(*compensation, if i == 0 { 0 } else { DRIFT_COMPENSATION }); } - assert!(AggregateDevice::destroy_device(plugin, device).is_ok()); + assert!(run_serially(|| AggregateDevice::destroy_device(plugin, device)).is_ok()); } #[test] #[should_panic] -#[ignore] fn test_aggregate_activate_clock_drift_compensation_for_a_blank_aggregate_device() { - let plugin = AggregateDevice::get_system_plugin_id().unwrap(); - let device = AggregateDevice::create_blank_device_sync(plugin).unwrap(); + run_serially_forward_panics(|| { + let plugin = AggregateDevice::get_system_plugin_id().unwrap(); + let device = AggregateDevice::create_blank_device_sync(plugin).unwrap(); - let sub_devices = AggregateDevice::get_sub_devices(device).unwrap(); - assert!(sub_devices.is_empty()); - let onwed_devices = test_get_all_onwed_devices(device); - assert!(onwed_devices.is_empty()); + let sub_devices = AggregateDevice::get_sub_devices(device).unwrap(); + assert!(sub_devices.is_empty()); + let onwed_devices = test_get_all_onwed_devices(device); + assert!(onwed_devices.is_empty()); - // Get a panic since no sub devices to be set compensation. - assert!(AggregateDevice::activate_clock_drift_compensation(device).is_err()); + // Get a panic since no sub devices to be set compensation. + assert!(AggregateDevice::activate_clock_drift_compensation(device).is_err()); - assert!(AggregateDevice::destroy_device(plugin, device).is_ok()); + assert!(AggregateDevice::destroy_device(plugin, device).is_ok()); + }); } fn get_drift_compensations(devices: &Vec) -> Vec { @@ -391,10 +448,56 @@ fn get_drift_compensations(devices: &Vec) -> Vec { // AggregateDevice::destroy_device // ------------------------------------ #[test] -#[ignore] #[should_panic] fn test_aggregate_destroy_aggregate_device_for_a_unknown_plugin_device() { - let plugin = AggregateDevice::get_system_plugin_id().unwrap(); - let device = AggregateDevice::create_blank_device_sync(plugin).unwrap(); - assert!(AggregateDevice::destroy_device(kAudioObjectUnknown, device).is_err()); + run_serially_forward_panics(|| { + let plugin = AggregateDevice::get_system_plugin_id().unwrap(); + let device = AggregateDevice::create_blank_device_sync(plugin).unwrap(); + assert!(AggregateDevice::destroy_device(kAudioObjectUnknown, device).is_err()); + }); +} + +// AggregateDevice::new +// ------------------------------------ +#[test] +fn test_aggregate_new() { + let input_device = test_get_default_device(Scope::Input); + let output_device = test_get_default_device(Scope::Output); + if input_device.is_none() || output_device.is_none() || input_device == output_device { + println!("No input or output device to create an aggregate device."); + return; + } + + run_serially_forward_panics(|| { + let input_device = input_device.unwrap(); + let output_device = output_device.unwrap(); + + let aggr = AggregateDevice::new(input_device, output_device).unwrap(); + + // Check main device + let output_sub_devices = AggregateDevice::get_sub_devices(output_device).unwrap(); + let first_output_sub_device_uid = get_device_uid(output_sub_devices[0]); + let master_device_uid = test_get_master_device(aggr.get_device_id()); + assert_eq!(first_output_sub_device_uid, master_device_uid); + + // Check drift compensation + let devices = test_get_all_onwed_devices(aggr.get_device_id()); + let compensations = get_drift_compensations(&devices); + assert!(!compensations.is_empty()); + assert_eq!(devices.len(), compensations.len()); + + let device_uids = devices.iter().map(|&id| get_device_uid(id)); + for (uid, compensation) in zip(device_uids, compensations) { + assert_eq!( + compensation, + if uid == master_device_uid { + 0 + } else { + DRIFT_COMPENSATION + }, + "Unexpected drift value for device with uid {}", + uid + ); + } + }); } diff --git a/third_party/rust/cubeb-coreaudio/src/backend/tests/api.rs b/third_party/rust/cubeb-coreaudio/src/backend/tests/api.rs index 4cd86c094e..5ce2374a3e 100644 --- a/third_party/rust/cubeb-coreaudio/src/backend/tests/api.rs +++ b/third_party/rust/cubeb-coreaudio/src/backend/tests/api.rs @@ -56,7 +56,7 @@ fn test_increase_and_decrease_context_streams() { assert_eq!(context.active_streams(), STREAMS); check_streams(&context, STREAMS); - check_latency(&context, latencies[0]); + check_latency(&context, Some(latencies[0])); for i in 0..latencies.len() - 1 { assert_eq!(latencies[i], latencies[i + 1]); } @@ -149,7 +149,8 @@ fn test_minimum_resampling_input_frames_equal_input_output_rate() { #[test] fn test_create_device_info_from_unknown_input_device() { if let Some(default_device_id) = test_get_default_device(Scope::Input) { - let default_device = create_device_info(kAudioObjectUnknown, DeviceType::INPUT).unwrap(); + let default_device = + run_serially(|| create_device_info(kAudioObjectUnknown, DeviceType::INPUT).unwrap()); assert_eq!(default_device.id, default_device_id); assert_eq!( default_device.flags, @@ -163,7 +164,8 @@ fn test_create_device_info_from_unknown_input_device() { #[test] fn test_create_device_info_from_unknown_output_device() { if let Some(default_device_id) = test_get_default_device(Scope::Output) { - let default_device = create_device_info(kAudioObjectUnknown, DeviceType::OUTPUT).unwrap(); + let default_device = + run_serially(|| create_device_info(kAudioObjectUnknown, DeviceType::OUTPUT)).unwrap(); assert_eq!(default_device.id, default_device_id); assert_eq!( default_device.flags, @@ -177,13 +179,17 @@ fn test_create_device_info_from_unknown_output_device() { #[test] #[should_panic] fn test_set_device_info_to_system_input_device() { - let _device = create_device_info(kAudioObjectSystemObject, DeviceType::INPUT); + let _device = run_serially_forward_panics(|| { + create_device_info(kAudioObjectSystemObject, DeviceType::INPUT) + }); } #[test] #[should_panic] fn test_set_device_info_to_system_output_device() { - let _device = create_device_info(kAudioObjectSystemObject, DeviceType::OUTPUT); + let _device = run_serially_forward_panics(|| { + create_device_info(kAudioObjectSystemObject, DeviceType::OUTPUT) + }); } // FIXME: Is it ok to set input device to a nonexistent device ? @@ -192,7 +198,8 @@ fn test_set_device_info_to_system_output_device() { #[should_panic] fn test_set_device_info_to_nonexistent_input_device() { let nonexistent_id = std::u32::MAX; - let _device = create_device_info(nonexistent_id, DeviceType::INPUT); + let _device = + run_serially_forward_panics(|| create_device_info(nonexistent_id, DeviceType::INPUT)); } // FIXME: Is it ok to set output device to a nonexistent device ? @@ -201,7 +208,8 @@ fn test_set_device_info_to_nonexistent_input_device() { #[should_panic] fn test_set_device_info_to_nonexistent_output_device() { let nonexistent_id = std::u32::MAX; - let _device = create_device_info(nonexistent_id, DeviceType::OUTPUT); + let _device = + run_serially_forward_panics(|| create_device_info(nonexistent_id, DeviceType::OUTPUT)); } // add_listener (for default output device) @@ -227,10 +235,10 @@ fn test_add_listener_unknown_device() { ), callback, ); - let mut res: OSStatus = 0; - stream + let res = stream .queue - .run_sync(|| res = stream.add_device_listener(&listener)); + .run_sync(|| stream.add_device_listener(&listener)) + .unwrap(); assert_eq!(res, kAudioHardwareBadObjectError as OSStatus); }); } @@ -258,14 +266,15 @@ fn test_add_listener_then_remove_system_device() { ), callback, ); - let mut res: OSStatus = 0; - stream + let res = stream .queue - .run_sync(|| res = stream.add_device_listener(&listener)); + .run_sync(|| stream.add_device_listener(&listener)) + .unwrap(); assert_eq!(res, NO_ERR); - stream + let res = stream .queue - .run_sync(|| res = stream.remove_device_listener(&listener)); + .run_sync(|| stream.remove_device_listener(&listener)) + .unwrap(); assert_eq!(res, NO_ERR); }); } @@ -291,10 +300,10 @@ fn test_remove_listener_without_adding_any_listener_before_system_device() { ), callback, ); - let mut res: OSStatus = 0; - stream + let res = stream .queue - .run_sync(|| res = stream.remove_device_listener(&listener)); + .run_sync(|| stream.remove_device_listener(&listener)) + .unwrap(); assert_eq!(res, NO_ERR); }); } @@ -320,10 +329,10 @@ fn test_remove_listener_unknown_device() { ), callback, ); - let mut res: OSStatus = 0; - stream + let res = stream .queue - .run_sync(|| res = stream.remove_device_listener(&listener)); + .run_sync(|| stream.remove_device_listener(&listener)) + .unwrap(); assert_eq!(res, kAudioHardwareBadObjectError as OSStatus); }); } @@ -334,14 +343,14 @@ fn test_remove_listener_unknown_device() { fn test_get_default_device_id() { if test_get_default_device(Scope::Input).is_some() { assert_ne!( - get_default_device_id(DeviceType::INPUT).unwrap(), + run_serially(|| get_default_device_id(DeviceType::INPUT)).unwrap(), kAudioObjectUnknown, ); } if test_get_default_device(Scope::Output).is_some() { assert_ne!( - get_default_device_id(DeviceType::OUTPUT).unwrap(), + run_serially(|| get_default_device_id(DeviceType::OUTPUT)).unwrap(), kAudioObjectUnknown, ); } @@ -350,13 +359,16 @@ fn test_get_default_device_id() { #[test] #[should_panic] fn test_get_default_device_id_with_unknown_type() { - assert!(get_default_device_id(DeviceType::UNKNOWN).is_err()); + assert!(run_serially_forward_panics(|| get_default_device_id(DeviceType::UNKNOWN)).is_err()); } #[test] #[should_panic] fn test_get_default_device_id_with_inout_type() { - assert!(get_default_device_id(DeviceType::INPUT | DeviceType::OUTPUT).is_err()); + assert!(run_serially_forward_panics(|| get_default_device_id( + DeviceType::INPUT | DeviceType::OUTPUT + )) + .is_err()); } // convert_channel_layout @@ -724,9 +736,11 @@ fn test_convert_channel_layout() { #[test] fn test_get_preferred_channel_layout_output() { match test_get_default_audiounit(Scope::Output) { - Some(unit) => assert!(!audiounit_get_preferred_channel_layout(unit.get_inner()) - .unwrap() - .is_empty()), + Some(unit) => assert!(!run_serially(|| audiounit_get_preferred_channel_layout( + unit.get_inner() + )) + .unwrap() + .is_empty()), None => println!("No output audiounit for test."), } } @@ -736,9 +750,15 @@ fn test_get_preferred_channel_layout_output() { #[test] fn test_get_current_channel_layout_output() { match test_get_default_audiounit(Scope::Output) { - Some(unit) => assert!(!audiounit_get_current_channel_layout(unit.get_inner()) - .unwrap() - .is_empty()), + Some(unit) => { + assert!( + !run_serially_forward_panics(|| audiounit_get_current_channel_layout( + unit.get_inner() + )) + .unwrap() + .is_empty() + ) + } None => println!("No output audiounit for test."), } } @@ -814,10 +834,30 @@ fn test_enable_audiounit_scope() { // for the unit whose subtype is kAudioUnitSubType_HALOutput // even when there is no available input or output devices. if let Some(unit) = test_create_audiounit(ComponentSubType::HALOutput) { - assert!(enable_audiounit_scope(unit.get_inner(), DeviceType::OUTPUT, true).is_ok()); - assert!(enable_audiounit_scope(unit.get_inner(), DeviceType::OUTPUT, false).is_ok()); - assert!(enable_audiounit_scope(unit.get_inner(), DeviceType::INPUT, true).is_ok()); - assert!(enable_audiounit_scope(unit.get_inner(), DeviceType::INPUT, false).is_ok()); + assert!(run_serially_forward_panics(|| enable_audiounit_scope( + unit.get_inner(), + DeviceType::OUTPUT, + true + )) + .is_ok()); + assert!(run_serially_forward_panics(|| enable_audiounit_scope( + unit.get_inner(), + DeviceType::OUTPUT, + false + )) + .is_ok()); + assert!(run_serially_forward_panics(|| enable_audiounit_scope( + unit.get_inner(), + DeviceType::INPUT, + true + )) + .is_ok()); + assert!(run_serially_forward_panics(|| enable_audiounit_scope( + unit.get_inner(), + DeviceType::INPUT, + false + )) + .is_ok()); } else { println!("No audiounit to perform test."); } @@ -827,19 +867,23 @@ fn test_enable_audiounit_scope() { fn test_enable_audiounit_scope_for_default_output_unit() { if let Some(unit) = test_create_audiounit(ComponentSubType::DefaultOutput) { assert_eq!( - enable_audiounit_scope(unit.get_inner(), DeviceType::OUTPUT, true).unwrap_err(), + run_serially(|| enable_audiounit_scope(unit.get_inner(), DeviceType::OUTPUT, true)) + .unwrap_err(), kAudioUnitErr_InvalidProperty ); assert_eq!( - enable_audiounit_scope(unit.get_inner(), DeviceType::OUTPUT, false).unwrap_err(), + run_serially(|| enable_audiounit_scope(unit.get_inner(), DeviceType::OUTPUT, false)) + .unwrap_err(), kAudioUnitErr_InvalidProperty ); assert_eq!( - enable_audiounit_scope(unit.get_inner(), DeviceType::INPUT, true).unwrap_err(), + run_serially(|| enable_audiounit_scope(unit.get_inner(), DeviceType::INPUT, true)) + .unwrap_err(), kAudioUnitErr_InvalidProperty ); assert_eq!( - enable_audiounit_scope(unit.get_inner(), DeviceType::INPUT, false).unwrap_err(), + run_serially(|| enable_audiounit_scope(unit.get_inner(), DeviceType::INPUT, false)) + .unwrap_err(), kAudioUnitErr_InvalidProperty ); } @@ -849,7 +893,10 @@ fn test_enable_audiounit_scope_for_default_output_unit() { #[should_panic] fn test_enable_audiounit_scope_with_null_unit() { let unit: AudioUnit = ptr::null_mut(); - assert!(enable_audiounit_scope(unit, DeviceType::INPUT, false).is_err()); + assert!( + run_serially_forward_panics(|| enable_audiounit_scope(unit, DeviceType::INPUT, false)) + .is_err() + ); } // create_audiounit @@ -868,29 +915,29 @@ fn test_for_create_audiounit() { // Check the output scope is enabled. if device.flags.contains(device_flags::DEV_OUTPUT) && default_output.is_some() { device.id = default_output.unwrap(); - let unit = create_audiounit(&device).unwrap(); + let unit = run_serially(|| create_audiounit(&device).unwrap()); assert!(!unit.is_null()); assert!(test_audiounit_scope_is_enabled(unit, Scope::Output)); // Destroy the AudioUnit. - unsafe { + run_serially(|| unsafe { AudioUnitUninitialize(unit); AudioComponentInstanceDispose(unit); - } + }); } // Check the input scope is enabled. if device.flags.contains(device_flags::DEV_INPUT) && default_input.is_some() { let device_id = default_input.unwrap(); device.id = device_id; - let unit = create_audiounit(&device).unwrap(); + let unit = run_serially(|| create_audiounit(&device).unwrap()); assert!(!unit.is_null()); assert!(test_audiounit_scope_is_enabled(unit, Scope::Input)); // Destroy the AudioUnit. - unsafe { + run_serially(|| unsafe { AudioUnitUninitialize(unit); AudioComponentInstanceDispose(unit); - } + }); } } } @@ -899,7 +946,7 @@ fn test_for_create_audiounit() { #[should_panic] fn test_create_audiounit_with_unknown_scope() { let device = device_info::default(); - let _unit = create_audiounit(&device); + let _unit = run_serially_forward_panics(|| create_audiounit(&device)); } // set_buffer_size_sync @@ -927,9 +974,12 @@ fn test_set_buffer_size_sync() { .unwrap(); assert_ne!(buffer_frames, 0); buffer_frames *= 2; - assert!( - set_buffer_size_sync(unit.get_inner(), scope.clone().into(), buffer_frames).is_ok() - ); + assert!(run_serially(|| set_buffer_size_sync( + unit.get_inner(), + scope.clone().into(), + buffer_frames + )) + .is_ok()); let new_buffer_frames = test_audiounit_get_buffer_frame_size(unit.get_inner(), scope.clone(), prop_scope) .unwrap(); @@ -951,7 +1001,9 @@ fn test_set_buffer_size_sync_for_output_with_null_output_unit() { fn test_set_buffer_size_sync_by_scope_with_null_unit(scope: Scope) { let unit: AudioUnit = ptr::null_mut(); - assert!(set_buffer_size_sync(unit, scope.into(), 2048).is_err()); + assert!( + run_serially_forward_panics(|| set_buffer_size_sync(unit, scope.into(), 2048)).is_err() + ); } // get_volume, set_volume @@ -960,8 +1012,11 @@ fn test_set_buffer_size_sync_by_scope_with_null_unit(scope: Scope) { fn test_stream_get_volume() { if let Some(unit) = test_get_default_audiounit(Scope::Output) { let expected_volume: f32 = 0.5; - set_volume(unit.get_inner(), expected_volume); - assert_eq!(expected_volume, get_volume(unit.get_inner()).unwrap()); + run_serially(|| set_volume(unit.get_inner(), expected_volume)); + assert_eq!( + expected_volume, + run_serially(|| get_volume(unit.get_inner()).unwrap()) + ); } else { println!("No output audiounit."); } @@ -988,7 +1043,9 @@ fn test_get_channel_count() { fn test_channel_count(scope: Scope) { if let Some(device) = test_get_default_device(scope.clone()) { - let channels = get_channel_count(device, DeviceType::from(scope.clone())).unwrap(); + let channels = + run_serially(|| get_channel_count(device, DeviceType::from(scope.clone()))) + .unwrap(); assert!(channels > 0); assert_eq!( channels, @@ -1008,7 +1065,7 @@ fn test_get_channel_count_of_input_for_a_output_only_deivce() { if test_device_in_scope(device, Scope::Input) { continue; } - let count = get_channel_count(device, DeviceType::INPUT).unwrap(); + let count = run_serially(|| get_channel_count(device, DeviceType::INPUT)).unwrap(); assert_eq!(count, 0); } } @@ -1021,7 +1078,7 @@ fn test_get_channel_count_of_output_for_a_input_only_deivce() { if test_device_in_scope(device, Scope::Output) { continue; } - let count = get_channel_count(device, DeviceType::OUTPUT).unwrap(); + let count = run_serially(|| get_channel_count(device, DeviceType::OUTPUT)).unwrap(); assert_eq!(count, 0); } } @@ -1029,7 +1086,11 @@ fn test_get_channel_count_of_output_for_a_input_only_deivce() { #[test] #[should_panic] fn test_get_channel_count_of_unknown_device() { - assert!(get_channel_count(kAudioObjectUnknown, DeviceType::OUTPUT).is_err()); + assert!(run_serially_forward_panics(|| get_channel_count( + kAudioObjectUnknown, + DeviceType::OUTPUT + )) + .is_err()); } #[test] @@ -1039,14 +1100,16 @@ fn test_get_channel_count_of_inout_type() { fn test_channel_count(scope: Scope) { if let Some(device) = test_get_default_device(scope.clone()) { - assert_eq!( - get_channel_count(device, DeviceType::INPUT | DeviceType::OUTPUT), - get_channel_count(device, DeviceType::INPUT).map(|c| c + get_channel_count( - device, - DeviceType::OUTPUT - ) - .unwrap_or(0)) - ); + run_serially_forward_panics(|| { + assert_eq!( + get_channel_count(device, DeviceType::INPUT | DeviceType::OUTPUT), + get_channel_count(device, DeviceType::INPUT).map(|c| c + get_channel_count( + device, + DeviceType::OUTPUT + ) + .unwrap_or(0)) + ); + }); } else { println!("No device for {:?}.", scope); } @@ -1095,7 +1158,9 @@ fn test_get_range_of_sample_rates() { ]; let mut ranges = Vec::new(); for scope in scopes.iter() { - ranges.push(get_range_of_sample_rates(id, *scope).unwrap()); + ranges.push( + run_serially_forward_panics(|| get_range_of_sample_rates(id, *scope)).unwrap(), + ); } ranges } @@ -1117,7 +1182,7 @@ fn test_get_device_presentation_latency() { fn test_get_device_presentation_latencies_in_scope(scope: Scope) { if let Some(device) = test_get_default_device(scope.clone()) { // TODO: The latencies very from devices to devices. Check nothing here. - let latency = get_fixed_latency(device, scope.clone().into()); + let latency = run_serially(|| get_fixed_latency(device, scope.clone().into())); println!( "present latency on the device {} in scope {:?}: {}", device, scope, latency @@ -1133,7 +1198,7 @@ fn test_get_device_presentation_latency() { #[test] fn test_get_device_group_id() { if let Some(device) = test_get_default_device(Scope::Input) { - match get_device_group_id(device, DeviceType::INPUT) { + match run_serially(|| get_device_group_id(device, DeviceType::INPUT)) { Ok(id) => println!("input group id: {:?}", id), Err(e) => println!("No input group id. Error: {}", e), } @@ -1142,7 +1207,7 @@ fn test_get_device_group_id() { } if let Some(device) = test_get_default_device(Scope::Output) { - match get_device_group_id(device, DeviceType::OUTPUT) { + match run_serially(|| get_device_group_id(device, DeviceType::OUTPUT)) { Ok(id) => println!("output group id: {:?}", id), Err(e) => println!("No output group id. Error: {}", e), } @@ -1165,8 +1230,8 @@ fn test_get_same_group_id_for_builtin_device_pairs() { let mut input_group_ids = HashMap::::new(); let input_devices = test_get_devices_in_scope(Scope::Input); for device in input_devices.iter() { - match get_device_source(*device, DeviceType::INPUT) { - Ok(source) => match get_device_group_id(*device, DeviceType::INPUT) { + match run_serially(|| get_device_source(*device, DeviceType::INPUT)) { + Ok(source) => match run_serially(|| get_device_group_id(*device, DeviceType::INPUT)) { Ok(id) => assert!(input_group_ids .insert(source, id.into_string().unwrap()) .is_none()), @@ -1181,8 +1246,8 @@ fn test_get_same_group_id_for_builtin_device_pairs() { let mut output_group_ids = HashMap::::new(); let output_devices = test_get_devices_in_scope(Scope::Output); for device in output_devices.iter() { - match get_device_source(*device, DeviceType::OUTPUT) { - Ok(source) => match get_device_group_id(*device, DeviceType::OUTPUT) { + match run_serially(|| get_device_source(*device, DeviceType::OUTPUT)) { + Ok(source) => match run_serially(|| get_device_group_id(*device, DeviceType::OUTPUT)) { Ok(id) => assert!(output_group_ids .insert(source, id.into_string().unwrap()) .is_none()), @@ -1210,7 +1275,11 @@ fn test_get_same_group_id_for_builtin_device_pairs() { #[test] #[should_panic] fn test_get_device_group_id_by_unknown_device() { - assert!(get_device_group_id(kAudioObjectUnknown, DeviceType::INPUT).is_err()); + assert!(run_serially_forward_panics(|| get_device_group_id( + kAudioObjectUnknown, + DeviceType::INPUT + )) + .is_err()); } // get_device_label @@ -1218,14 +1287,14 @@ fn test_get_device_group_id_by_unknown_device() { #[test] fn test_get_device_label() { if let Some(device) = test_get_default_device(Scope::Input) { - let name = get_device_label(device, DeviceType::INPUT).unwrap(); + let name = run_serially(|| get_device_label(device, DeviceType::INPUT)).unwrap(); println!("input device label: {}", name.into_string()); } else { println!("No input device."); } if let Some(device) = test_get_default_device(Scope::Output) { - let name = get_device_label(device, DeviceType::OUTPUT).unwrap(); + let name = run_serially(|| get_device_label(device, DeviceType::OUTPUT)).unwrap(); println!("output device label: {}", name.into_string()); } else { println!("No output device."); @@ -1235,7 +1304,11 @@ fn test_get_device_label() { #[test] #[should_panic] fn test_get_device_label_by_unknown_device() { - assert!(get_device_label(kAudioObjectUnknown, DeviceType::INPUT).is_err()); + assert!(run_serially_forward_panics(|| get_device_label( + kAudioObjectUnknown, + DeviceType::INPUT + )) + .is_err()); } // get_device_global_uid @@ -1244,14 +1317,14 @@ fn test_get_device_label_by_unknown_device() { fn test_get_device_global_uid() { // Input device. if let Some(input) = test_get_default_device(Scope::Input) { - let uid = get_device_global_uid(input).unwrap(); + let uid = run_serially(|| get_device_global_uid(input)).unwrap(); let uid = uid.into_string(); assert!(!uid.is_empty()); } // Output device. if let Some(output) = test_get_default_device(Scope::Output) { - let uid = get_device_global_uid(output).unwrap(); + let uid = run_serially(|| get_device_global_uid(output)).unwrap(); let uid = uid.into_string(); assert!(!uid.is_empty()); } @@ -1285,7 +1358,7 @@ fn test_create_cubeb_device_info() { if is_input { let mut input_device_info = input_result.unwrap(); check_device_info_by_device(&input_device_info, device, Scope::Input); - destroy_cubeb_device_info(&mut input_device_info); + run_serially(|| destroy_cubeb_device_info(&mut input_device_info)); } else { assert_eq!(input_result.unwrap_err(), Error::error()); } @@ -1294,7 +1367,7 @@ fn test_create_cubeb_device_info() { if is_output { let mut output_device_info = output_result.unwrap(); check_device_info_by_device(&output_device_info, device, Scope::Output); - destroy_cubeb_device_info(&mut output_device_info); + run_serially(|| destroy_cubeb_device_info(&mut output_device_info)); } else { assert_eq!(output_result.unwrap_err(), Error::error()); } @@ -1309,7 +1382,7 @@ fn test_create_cubeb_device_info() { let dev_types = [DeviceType::INPUT, DeviceType::OUTPUT]; let mut results = VecDeque::new(); for dev_type in dev_types.iter() { - results.push_back(create_cubeb_device_info(id, *dev_type)); + results.push_back(run_serially(|| create_cubeb_device_info(id, *dev_type))); } results } @@ -1369,7 +1442,9 @@ fn test_create_device_info_with_unknown_type() { fn test_create_device_info_with_unknown_type_by_scope(scope: Scope) { if let Some(device) = test_get_default_device(scope.clone()) { - assert!(create_cubeb_device_info(device, DeviceType::UNKNOWN).is_err()); + assert!( + run_serially(|| create_cubeb_device_info(device, DeviceType::UNKNOWN)).is_err() + ); } } } @@ -1401,9 +1476,11 @@ fn test_create_device_from_hwdev_with_inout_type() { fn test_create_device_from_hwdev_with_inout_type_by_scope(scope: Scope) { if let Some(device) = test_get_default_device(scope.clone()) { // Get a kAudioHardwareUnknownPropertyError in get_channel_count actually. - assert!( - create_cubeb_device_info(device, DeviceType::INPUT | DeviceType::OUTPUT).is_err() - ); + assert!(run_serially(|| create_cubeb_device_info( + device, + DeviceType::INPUT | DeviceType::OUTPUT + )) + .is_err()); } else { println!("No device for {:?}.", scope); } @@ -1416,9 +1493,10 @@ fn test_create_device_from_hwdev_with_inout_type() { fn test_get_devices_of_type() { use std::collections::HashSet; - let all_devices = audiounit_get_devices_of_type(DeviceType::INPUT | DeviceType::OUTPUT); - let input_devices = audiounit_get_devices_of_type(DeviceType::INPUT); - let output_devices = audiounit_get_devices_of_type(DeviceType::OUTPUT); + let all_devices = + run_serially(|| audiounit_get_devices_of_type(DeviceType::INPUT | DeviceType::OUTPUT)); + let input_devices = run_serially(|| audiounit_get_devices_of_type(DeviceType::INPUT)); + let output_devices = run_serially(|| audiounit_get_devices_of_type(DeviceType::OUTPUT)); let mut expected_all = test_get_all_devices(DeviceFilter::ExcludeCubebAggregateAndVPIO); expected_all.sort(); @@ -1443,8 +1521,10 @@ fn test_get_devices_of_type() { #[test] #[should_panic] fn test_get_devices_of_type_unknown() { - let no_devs = audiounit_get_devices_of_type(DeviceType::UNKNOWN); - assert!(no_devs.is_empty()); + run_serially_forward_panics(|| { + let no_devs = audiounit_get_devices_of_type(DeviceType::UNKNOWN); + assert!(no_devs.is_empty()); + }); } // add_devices_changed_listener @@ -1468,9 +1548,10 @@ fn test_add_devices_changed_listener() { assert!(get_devices_changed_callback(context, Scope::Output).is_none()); // Register a callback within a specific scope. - assert!(context - .add_devices_changed_listener(*devtype, Some(*callback), ptr::null_mut()) - .is_ok()); + assert!(run_serially(|| { + context.add_devices_changed_listener(*devtype, Some(*callback), ptr::null_mut()) + }) + .is_ok()); if devtype.contains(DeviceType::INPUT) { let cb = get_devices_changed_callback(context, Scope::Input); @@ -1491,9 +1572,10 @@ fn test_add_devices_changed_listener() { } // Unregister the callbacks within all scopes. - assert!(context - .remove_devices_changed_listener(DeviceType::INPUT | DeviceType::OUTPUT) - .is_ok()); + assert!(run_serially(|| { + context.remove_devices_changed_listener(DeviceType::INPUT | DeviceType::OUTPUT) + }) + .is_ok()); assert!(get_devices_changed_callback(context, Scope::Input).is_none()); assert!(get_devices_changed_callback(context, Scope::Output).is_none()); @@ -1547,9 +1629,12 @@ fn test_remove_devices_changed_listener() { // Register callbacks within all scopes. for (scope, listener) in map.iter() { - assert!(context - .add_devices_changed_listener(*scope, Some(*listener), ptr::null_mut()) - .is_ok()); + assert!(run_serially(|| context.add_devices_changed_listener( + *scope, + Some(*listener), + ptr::null_mut() + )) + .is_ok()); } let input_callback = get_devices_changed_callback(context, Scope::Input); @@ -1566,7 +1651,7 @@ fn test_remove_devices_changed_listener() { ); // Unregister the callbacks within one specific scopes. - assert!(context.remove_devices_changed_listener(*devtype).is_ok()); + assert!(run_serially(|| context.remove_devices_changed_listener(*devtype)).is_ok()); if devtype.contains(DeviceType::INPUT) { let cb = get_devices_changed_callback(context, Scope::Input); @@ -1587,9 +1672,10 @@ fn test_remove_devices_changed_listener() { } // Unregister the callbacks within all scopes. - assert!(context - .remove_devices_changed_listener(DeviceType::INPUT | DeviceType::OUTPUT) - .is_ok()); + assert!(run_serially( + || context.remove_devices_changed_listener(DeviceType::INPUT | DeviceType::OUTPUT) + ) + .is_ok()); } }); } @@ -1602,7 +1688,7 @@ fn test_remove_devices_changed_listener_without_adding_listeners() { DeviceType::OUTPUT, DeviceType::INPUT | DeviceType::OUTPUT, ] { - assert!(context.remove_devices_changed_listener(*devtype).is_ok()); + assert!(run_serially(|| context.remove_devices_changed_listener(*devtype)).is_ok()); } }); } @@ -1625,9 +1711,12 @@ fn test_remove_devices_changed_listener_within_all_scopes() { assert!(get_devices_changed_callback(context, Scope::Input).is_none()); assert!(get_devices_changed_callback(context, Scope::Output).is_none()); - assert!(context - .add_devices_changed_listener(*devtype, Some(*callback), ptr::null_mut()) - .is_ok()); + assert!(run_serially(|| context.add_devices_changed_listener( + *devtype, + Some(*callback), + ptr::null_mut() + )) + .is_ok()); if devtype.contains(DeviceType::INPUT) { let cb = get_devices_changed_callback(context, Scope::Input); @@ -1641,9 +1730,10 @@ fn test_remove_devices_changed_listener_within_all_scopes() { assert_eq!(cb.unwrap(), *callback); } - assert!(context - .remove_devices_changed_listener(DeviceType::INPUT | DeviceType::OUTPUT) - .is_ok()); + assert!(run_serially( + || context.remove_devices_changed_listener(DeviceType::INPUT | DeviceType::OUTPUT) + ) + .is_ok()); assert!(get_devices_changed_callback(context, Scope::Input).is_none()); assert!(get_devices_changed_callback(context, Scope::Output).is_none()); @@ -1661,3 +1751,135 @@ fn get_devices_changed_callback( Scope::Output => devices_guard.output.changed_callback, } } + +// SharedVoiceProcessingUnitManager +// ------------------------------------ +#[test] +fn test_shared_voice_processing_unit() { + let queue = Queue::new_with_target( + "test_shared_voice_processing_unit", + get_serial_queue_singleton(), + ); + let mut shared = SharedVoiceProcessingUnitManager::new(queue.clone()); + let r1 = queue.run_sync(|| shared.take()).unwrap(); + assert!(r1.is_err()); + let r2 = queue.run_sync(|| shared.take_or_create()).unwrap(); + assert!(r2.is_ok()); + { + let _handle = r2.unwrap(); + let r3 = queue.run_sync(|| shared.take()).unwrap(); + assert!(r3.is_err()); + } + let r4 = queue.run_sync(|| shared.take()).unwrap(); + assert!(r4.is_ok()); +} + +#[test] +#[should_panic] +fn test_shared_voice_processing_unit_bad_release_order() { + let queue = Queue::new_with_target( + "test_shared_voice_processing_unit_bad_release_order", + get_serial_queue_singleton(), + ); + let mut shared = SharedVoiceProcessingUnitManager::new(queue.clone()); + let r1 = queue.run_sync(|| shared.take()).unwrap(); + assert!(r1.is_ok()); + drop(shared); + run_serially_forward_panics(|| drop(r1)); +} + +#[test] +fn test_shared_voice_processing_multiple_units() { + let queue = Queue::new_with_target( + "test_shared_voice_processing_multiple_units", + get_serial_queue_singleton(), + ); + let mut shared = SharedVoiceProcessingUnitManager::new(queue.clone()); + let r1 = queue.run_sync(|| shared.take_or_create()).unwrap(); + assert!(r1.is_ok()); + let r2 = queue.run_sync(|| shared.take_or_create()).unwrap(); + assert!(r2.is_ok()); + { + let _handle1 = r1.unwrap(); + let _handle2 = r2.unwrap(); + let r3 = queue.run_sync(|| shared.take()).unwrap(); + assert!(r3.is_err()); + } + let r1 = queue.run_sync(|| shared.take()).unwrap(); + assert!(r1.is_ok()); + let r2 = queue.run_sync(|| shared.take()).unwrap(); + assert!(r2.is_ok()); + let r3 = queue.run_sync(|| shared.take()).unwrap(); + assert!(r3.is_err()); +} + +#[test] +fn test_shared_voice_processing_release_on_idle() { + let queue = Queue::new_with_target( + "test_shared_voice_processing_release_on_idle", + get_serial_queue_singleton(), + ); + let mut shared = SharedVoiceProcessingUnitManager::with_idle_timeout( + queue.clone(), + Duration::from_millis(0), + ); + let r = queue.run_sync(|| shared.take_or_create()).unwrap(); + assert!(r.is_ok()); + { + let _handle = r.unwrap(); + } + queue.run_sync(|| {}); + let r = queue.run_sync(|| shared.take()).unwrap(); + assert!(r.is_err()); +} + +#[test] +fn test_shared_voice_processing_no_release_on_outstanding() { + let queue = Queue::new_with_target( + "test_shared_voice_processing_no_release_on_outstanding", + get_serial_queue_singleton(), + ); + let mut shared = SharedVoiceProcessingUnitManager::with_idle_timeout( + queue.clone(), + Duration::from_millis(0), + ); + let r1 = queue.run_sync(|| shared.take_or_create()).unwrap(); + assert!(r1.is_ok()); + let r2 = queue.run_sync(|| shared.take_or_create()).unwrap(); + assert!(r2.is_ok()); + { + let _handle1 = r1.unwrap(); + } + queue.run_sync(|| {}); + let r1 = queue.run_sync(|| shared.take()).unwrap(); + assert!(r1.is_ok()); +} + +#[test] +fn test_shared_voice_processing_release_on_idle_cancel_on_take() { + let queue = Queue::new_with_target( + "test_shared_voice_processing_release_on_idle_cancel_on_take", + get_serial_queue_singleton(), + ); + let mut shared = SharedVoiceProcessingUnitManager::with_idle_timeout( + queue.clone(), + Duration::from_millis(0), + ); + let r1 = queue.run_sync(|| shared.take_or_create()).unwrap(); + assert!(r1.is_ok()); + let r2 = queue.run_sync(|| shared.take_or_create()).unwrap(); + assert!(r2.is_ok()); + let r1 = queue + .run_sync(|| { + { + let _handle1 = r1.unwrap(); + let _handle2 = r2.unwrap(); + } + shared.take() + }) + .unwrap(); + assert!(r1.is_ok()); + queue.run_sync(|| {}); + let r2 = queue.run_sync(|| shared.take()).unwrap(); + assert!(r2.is_ok()); +} diff --git a/third_party/rust/cubeb-coreaudio/src/backend/tests/device_change.rs b/third_party/rust/cubeb-coreaudio/src/backend/tests/device_change.rs index c27dada7ad..1201c446ea 100644 --- a/third_party/rust/cubeb-coreaudio/src/backend/tests/device_change.rs +++ b/third_party/rust/cubeb-coreaudio/src/backend/tests/device_change.rs @@ -21,7 +21,7 @@ use super::utils::{ test_set_default_device, Scope, StreamType, TestDevicePlugger, TestDeviceSwitcher, }; use super::*; -use std::sync::{LockResult, MutexGuard, WaitTimeoutResult}; +use std::sync::{LockResult, WaitTimeoutResult}; // Switch default devices used by the active streams, to test stream reinitialization // ================================================================================================ @@ -49,19 +49,21 @@ fn test_switch_device_in_scope(scope: Scope) { let notifier = Arc::new(Notifier::new(0)); let also_notifier = notifier.clone(); - let listener = test_create_device_change_listener(scope.clone(), move |_addresses| { - let mut cnt = notifier.lock().unwrap(); - *cnt += 1; - notifier.notify(cnt); - NO_ERR + let listener = run_serially(|| { + test_create_device_change_listener(scope.clone(), move |_addresses| { + let mut cnt = notifier.lock().unwrap(); + *cnt += 1; + notifier.notify(cnt); + NO_ERR + }) }); - listener.start(); + run_serially(|| listener.start()); let changed_watcher = Watcher::new(&also_notifier); test_get_started_stream_in_scope(scope.clone(), move |_stream| loop { - let mut guard = changed_watcher.lock().unwrap(); - let start_cnt = guard.clone(); + let start_cnt = changed_watcher.lock().unwrap().clone(); device_switcher.next(); + let mut guard = changed_watcher.lock().unwrap(); guard = changed_watcher .wait_while(guard, |cnt| *cnt == start_cnt) .unwrap(); @@ -709,11 +711,7 @@ fn test_unplug_a_device_on_an_active_stream( state_callback, device_changed_callback, |stream| { - stream.start(); - - let changed_watcher = Watcher::new(¬ifier); - let mut data_guard = notifier.lock().unwrap(); - assert_eq!(data_guard.states.last().unwrap(), &ffi::CUBEB_STATE_STARTED); + assert_eq!(stream.start(), Ok(())); println!( "Stream runs on the device {} for {:?}", @@ -722,13 +720,20 @@ fn test_unplug_a_device_on_an_active_stream( ); let dev = plugger.get_device_id(); - let start_changed_count = data_guard.changed_count.clone(); + let start_changed_count = { + let guard = notifier.lock().unwrap(); + assert_eq!(guard.states.last().unwrap(), &ffi::CUBEB_STATE_STARTED); + guard.changed_count.clone() + }; assert!(plugger.unplug().is_ok()); + let changed_watcher = Watcher::new(¬ifier); + if set_device_to_default { // The stream will be reinitialized if it follows the default input or output device. println!("Waiting for default device to change and reinit"); + let mut data_guard = notifier.lock().unwrap(); data_guard = changed_watcher .wait_while(data_guard, |data| { data.changed_count == start_changed_count @@ -740,6 +745,7 @@ fn test_unplug_a_device_on_an_active_stream( // stream can be dropped immediately before device-changed callback // so we only check the states if we wait for it explicitly. println!("Waiting for non-default device to enter error state"); + let mut data_guard = notifier.lock().unwrap(); let (new_guard, timeout_res) = changed_watcher .wait_timeout_while(data_guard, Duration::from_millis(wait_up_to_ms), |data| { data.states.last().unwrap_or(&ffi::CUBEB_STATE_STARTED) diff --git a/third_party/rust/cubeb-coreaudio/src/backend/tests/device_property.rs b/third_party/rust/cubeb-coreaudio/src/backend/tests/device_property.rs index 8277a7642d..a974aee64b 100644 --- a/third_party/rust/cubeb-coreaudio/src/backend/tests/device_property.rs +++ b/third_party/rust/cubeb-coreaudio/src/backend/tests/device_property.rs @@ -7,14 +7,14 @@ use super::*; fn test_get_device_uid() { // Input device. if let Some(input) = test_get_default_device(Scope::Input) { - let uid = get_device_uid(input, DeviceType::INPUT).unwrap(); + let uid = run_serially(|| get_device_uid(input, DeviceType::INPUT)).unwrap(); let uid = uid.into_string(); assert!(!uid.is_empty()); } // Output device. if let Some(output) = test_get_default_device(Scope::Output) { - let uid = get_device_uid(output, DeviceType::OUTPUT).unwrap(); + let uid = run_serially(|| get_device_uid(output, DeviceType::OUTPUT)).unwrap(); let uid = uid.into_string(); assert!(!uid.is_empty()); } @@ -24,7 +24,10 @@ fn test_get_device_uid() { #[should_panic] fn test_get_device_uid_by_unknwon_device() { // Unknown device. - assert!(get_device_uid(kAudioObjectUnknown, DeviceType::INPUT).is_err()); + assert!( + run_serially_forward_panics(|| get_device_uid(kAudioObjectUnknown, DeviceType::INPUT)) + .is_err() + ); } // get_device_model_uid @@ -33,7 +36,7 @@ fn test_get_device_uid_by_unknwon_device() { #[test] fn test_get_device_model_uid() { if let Some(device) = test_get_default_device(Scope::Input) { - match get_device_model_uid(device, DeviceType::INPUT) { + match run_serially(|| get_device_model_uid(device, DeviceType::INPUT)) { Ok(uid) => println!("input model uid: {}", uid.into_string()), Err(e) => println!("No input model uid. Error: {}", e), } @@ -42,7 +45,7 @@ fn test_get_device_model_uid() { } if let Some(device) = test_get_default_device(Scope::Output) { - match get_device_model_uid(device, DeviceType::OUTPUT) { + match run_serially(|| get_device_model_uid(device, DeviceType::OUTPUT)) { Ok(uid) => println!("output model uid: {}", uid.into_string()), Err(e) => println!("No output model uid. Error: {}", e), } @@ -54,7 +57,11 @@ fn test_get_device_model_uid() { #[test] #[should_panic] fn test_get_device_model_uid_by_unknown_device() { - assert!(get_device_model_uid(kAudioObjectUnknown, DeviceType::INPUT).is_err()); + assert!(run_serially_forward_panics(|| get_device_model_uid( + kAudioObjectUnknown, + DeviceType::INPUT + )) + .is_err()); } // get_device_transport_type @@ -62,7 +69,7 @@ fn test_get_device_model_uid_by_unknown_device() { #[test] fn test_get_device_transport_type() { if let Some(device) = test_get_default_device(Scope::Input) { - match get_device_transport_type(device, DeviceType::INPUT) { + match run_serially(|| get_device_transport_type(device, DeviceType::INPUT)) { Ok(trans_type) => println!( "input transport type: {:X}, {:?}", trans_type, @@ -75,7 +82,7 @@ fn test_get_device_transport_type() { } if let Some(device) = test_get_default_device(Scope::Output) { - match get_device_transport_type(device, DeviceType::OUTPUT) { + match run_serially(|| get_device_transport_type(device, DeviceType::OUTPUT)) { Ok(trans_type) => println!( "output transport type: {:X}, {:?}", trans_type, @@ -91,7 +98,11 @@ fn test_get_device_transport_type() { #[test] #[should_panic] fn test_get_device_transport_type_by_unknown_device() { - assert!(get_device_transport_type(kAudioObjectUnknown, DeviceType::INPUT).is_err()); + assert!(run_serially_forward_panics(|| get_device_transport_type( + kAudioObjectUnknown, + DeviceType::INPUT + )) + .is_err()); } // get_device_source @@ -100,7 +111,7 @@ fn test_get_device_transport_type_by_unknown_device() { #[test] fn test_get_device_source() { if let Some(device) = test_get_default_device(Scope::Input) { - match get_device_source(device, DeviceType::INPUT) { + match run_serially(|| get_device_source(device, DeviceType::INPUT)) { Ok(source) => println!( "input source: {:X}, {:?}", source, @@ -113,7 +124,7 @@ fn test_get_device_source() { } if let Some(device) = test_get_default_device(Scope::Output) { - match get_device_source(device, DeviceType::OUTPUT) { + match run_serially(|| get_device_source(device, DeviceType::OUTPUT)) { Ok(source) => println!( "output source: {:X}, {:?}", source, @@ -129,7 +140,11 @@ fn test_get_device_source() { #[test] #[should_panic] fn test_get_device_source_by_unknown_device() { - assert!(get_device_source(kAudioObjectUnknown, DeviceType::INPUT).is_err()); + assert!(run_serially_forward_panics(|| get_device_source( + kAudioObjectUnknown, + DeviceType::INPUT + )) + .is_err()); } // get_device_source_name @@ -137,7 +152,7 @@ fn test_get_device_source_by_unknown_device() { #[test] fn test_get_device_source_name() { if let Some(device) = test_get_default_device(Scope::Input) { - match get_device_source_name(device, DeviceType::INPUT) { + match run_serially(|| get_device_source_name(device, DeviceType::INPUT)) { Ok(name) => println!("input: {}", name.into_string()), Err(e) => println!("No input data source name. Error: {}", e), } @@ -146,7 +161,7 @@ fn test_get_device_source_name() { } if let Some(device) = test_get_default_device(Scope::Output) { - match get_device_source_name(device, DeviceType::OUTPUT) { + match run_serially(|| get_device_source_name(device, DeviceType::OUTPUT)) { Ok(name) => println!("output: {}", name.into_string()), Err(e) => println!("No output data source name. Error: {}", e), } @@ -158,7 +173,11 @@ fn test_get_device_source_name() { #[test] #[should_panic] fn test_get_device_source_name_by_unknown_device() { - assert!(get_device_source_name(kAudioObjectUnknown, DeviceType::INPUT).is_err()); + assert!(run_serially_forward_panics(|| get_device_source_name( + kAudioObjectUnknown, + DeviceType::INPUT + )) + .is_err()); } // get_device_name @@ -166,14 +185,14 @@ fn test_get_device_source_name_by_unknown_device() { #[test] fn test_get_device_name() { if let Some(device) = test_get_default_device(Scope::Input) { - let name = get_device_name(device, DeviceType::INPUT).unwrap(); + let name = run_serially(|| get_device_name(device, DeviceType::INPUT)).unwrap(); println!("input device name: {}", name.into_string()); } else { println!("No input device."); } if let Some(device) = test_get_default_device(Scope::Output) { - let name = get_device_name(device, DeviceType::OUTPUT).unwrap(); + let name = run_serially(|| get_device_name(device, DeviceType::OUTPUT).unwrap()); println!("output device name: {}", name.into_string()); } else { println!("No output device."); @@ -183,7 +202,11 @@ fn test_get_device_name() { #[test] #[should_panic] fn test_get_device_name_by_unknown_device() { - assert!(get_device_name(kAudioObjectUnknown, DeviceType::INPUT).is_err()); + assert!(run_serially_forward_panics(|| get_device_name( + kAudioObjectUnknown, + DeviceType::INPUT + )) + .is_err()); } // get_device_manufacturer @@ -193,7 +216,7 @@ fn test_get_device_manufacturer() { if let Some(device) = test_get_default_device(Scope::Input) { // Some devices like AirPods cannot get the vendor info so we print the error directly. // TODO: Replace `map` and `unwrap_or_else` by `map_or_else` - let name = get_device_manufacturer(device, DeviceType::INPUT) + let name = run_serially(|| get_device_manufacturer(device, DeviceType::INPUT)) .map(|name| name.into_string()) .unwrap_or_else(|e| format!("Error: {}", e)); println!("input device vendor: {}", name); @@ -204,9 +227,10 @@ fn test_get_device_manufacturer() { if let Some(device) = test_get_default_device(Scope::Output) { // Some devices like AirPods cannot get the vendor info so we print the error directly. // TODO: Replace `map` and `unwrap_or_else` by `map_or_else` - let name = get_device_manufacturer(device, DeviceType::OUTPUT) - .map(|name| name.into_string()) - .unwrap_or_else(|e| format!("Error: {}", e)); + let name = + run_serially_forward_panics(|| get_device_manufacturer(device, DeviceType::OUTPUT)) + .map(|name| name.into_string()) + .unwrap_or_else(|e| format!("Error: {}", e)); println!("output device vendor: {}", name); } else { println!("No output device."); @@ -216,7 +240,11 @@ fn test_get_device_manufacturer() { #[test] #[should_panic] fn test_get_device_manufacturer_by_unknown_device() { - assert!(get_device_manufacturer(kAudioObjectUnknown, DeviceType::INPUT).is_err()); + assert!(run_serially_forward_panics(|| get_device_manufacturer( + kAudioObjectUnknown, + DeviceType::INPUT + )) + .is_err()); } // get_device_buffer_frame_size_range @@ -224,7 +252,8 @@ fn test_get_device_manufacturer_by_unknown_device() { #[test] fn test_get_device_buffer_frame_size_range() { if let Some(device) = test_get_default_device(Scope::Input) { - let range = get_device_buffer_frame_size_range(device, DeviceType::INPUT).unwrap(); + let range = + run_serially(|| get_device_buffer_frame_size_range(device, DeviceType::INPUT)).unwrap(); println!( "range of input buffer frame size: {}-{}", range.mMinimum, range.mMaximum @@ -234,7 +263,8 @@ fn test_get_device_buffer_frame_size_range() { } if let Some(device) = test_get_default_device(Scope::Output) { - let range = get_device_buffer_frame_size_range(device, DeviceType::OUTPUT).unwrap(); + let range = run_serially(|| get_device_buffer_frame_size_range(device, DeviceType::OUTPUT)) + .unwrap(); println!( "range of output buffer frame size: {}-{}", range.mMinimum, range.mMaximum @@ -247,7 +277,13 @@ fn test_get_device_buffer_frame_size_range() { #[test] #[should_panic] fn test_get_device_buffer_frame_size_range_by_unknown_device() { - assert!(get_device_buffer_frame_size_range(kAudioObjectUnknown, DeviceType::INPUT).is_err()); + assert!( + run_serially_forward_panics(|| get_device_buffer_frame_size_range( + kAudioObjectUnknown, + DeviceType::INPUT + )) + .is_err() + ); } // get_device_latency @@ -255,14 +291,14 @@ fn test_get_device_buffer_frame_size_range_by_unknown_device() { #[test] fn test_get_device_latency() { if let Some(device) = test_get_default_device(Scope::Input) { - let latency = get_device_latency(device, DeviceType::INPUT).unwrap(); + let latency = run_serially(|| get_device_latency(device, DeviceType::INPUT)).unwrap(); println!("latency of input device: {}", latency); } else { println!("No input device."); } if let Some(device) = test_get_default_device(Scope::Output) { - let latency = get_device_latency(device, DeviceType::OUTPUT).unwrap(); + let latency = run_serially(|| get_device_latency(device, DeviceType::OUTPUT)).unwrap(); println!("latency of output device: {}", latency); } else { println!("No output device."); @@ -272,7 +308,11 @@ fn test_get_device_latency() { #[test] #[should_panic] fn test_get_device_latency_by_unknown_device() { - assert!(get_device_latency(kAudioObjectUnknown, DeviceType::INPUT).is_err()); + assert!(run_serially_forward_panics(|| get_device_latency( + kAudioObjectUnknown, + DeviceType::INPUT + )) + .is_err()); } // get_device_streams @@ -280,7 +320,7 @@ fn test_get_device_latency_by_unknown_device() { #[test] fn test_get_device_streams() { if let Some(device) = test_get_default_device(Scope::Input) { - let streams = get_device_streams(device, DeviceType::INPUT).unwrap(); + let streams = run_serially(|| get_device_streams(device, DeviceType::INPUT)).unwrap(); println!("streams on the input device: {:?}", streams); assert!(!streams.is_empty()); } else { @@ -288,7 +328,7 @@ fn test_get_device_streams() { } if let Some(device) = test_get_default_device(Scope::Output) { - let streams = get_device_streams(device, DeviceType::OUTPUT).unwrap(); + let streams = run_serially(|| get_device_streams(device, DeviceType::OUTPUT)).unwrap(); println!("streams on the output device: {:?}", streams); assert!(!streams.is_empty()); } else { @@ -299,7 +339,11 @@ fn test_get_device_streams() { #[test] #[should_panic] fn test_get_device_streams_by_unknown_device() { - assert!(get_device_streams(kAudioObjectUnknown, DeviceType::INPUT).is_err()); + assert!(run_serially_forward_panics(|| get_device_streams( + kAudioObjectUnknown, + DeviceType::INPUT + )) + .is_err()); } // get_device_sample_rate @@ -307,14 +351,14 @@ fn test_get_device_streams_by_unknown_device() { #[test] fn test_get_device_sample_rate() { if let Some(device) = test_get_default_device(Scope::Input) { - let rate = get_device_sample_rate(device, DeviceType::INPUT).unwrap(); + let rate = run_serially(|| get_device_sample_rate(device, DeviceType::INPUT)).unwrap(); println!("input sample rate: {}", rate); } else { println!("No input device."); } if let Some(device) = test_get_default_device(Scope::Output) { - let rate = get_device_sample_rate(device, DeviceType::OUTPUT).unwrap(); + let rate = run_serially(|| get_device_sample_rate(device, DeviceType::OUTPUT).unwrap()); println!("output sample rate: {}", rate); } else { println!("No output device."); @@ -324,7 +368,11 @@ fn test_get_device_sample_rate() { #[test] #[should_panic] fn test_get_device_sample_rate_by_unknown_device() { - assert!(get_device_sample_rate(kAudioObjectUnknown, DeviceType::INPUT).is_err()); + assert!(run_serially_forward_panics(|| get_device_sample_rate( + kAudioObjectUnknown, + DeviceType::INPUT + )) + .is_err()); } // get_ranges_of_device_sample_rate @@ -332,14 +380,16 @@ fn test_get_device_sample_rate_by_unknown_device() { #[test] fn test_get_ranges_of_device_sample_rate() { if let Some(device) = test_get_default_device(Scope::Input) { - let ranges = get_ranges_of_device_sample_rate(device, DeviceType::INPUT).unwrap(); + let ranges = + run_serially(|| get_ranges_of_device_sample_rate(device, DeviceType::INPUT)).unwrap(); println!("ranges of input sample rate: {:?}", ranges); } else { println!("No input device."); } if let Some(device) = test_get_default_device(Scope::Output) { - let ranges = get_ranges_of_device_sample_rate(device, DeviceType::OUTPUT).unwrap(); + let ranges = + run_serially(|| get_ranges_of_device_sample_rate(device, DeviceType::OUTPUT)).unwrap(); println!("ranges of output sample rate: {:?}", ranges); } else { println!("No output device."); @@ -349,7 +399,13 @@ fn test_get_ranges_of_device_sample_rate() { #[test] #[should_panic] fn test_get_ranges_of_device_sample_rate_by_unknown_device() { - assert!(get_ranges_of_device_sample_rate(kAudioObjectUnknown, DeviceType::INPUT).is_err()); + assert!( + run_serially_forward_panics(|| get_ranges_of_device_sample_rate( + kAudioObjectUnknown, + DeviceType::INPUT + )) + .is_err() + ); } // get_stream_latency @@ -357,9 +413,9 @@ fn test_get_ranges_of_device_sample_rate_by_unknown_device() { #[test] fn test_get_stream_latency() { if let Some(device) = test_get_default_device(Scope::Input) { - let streams = get_device_streams(device, DeviceType::INPUT).unwrap(); + let streams = run_serially(|| get_device_streams(device, DeviceType::INPUT)).unwrap(); for stream in streams { - let latency = get_stream_latency(stream).unwrap(); + let latency = run_serially(|| get_stream_latency(stream)).unwrap(); println!("latency of the input stream {} is {}", stream, latency); } } else { @@ -367,9 +423,9 @@ fn test_get_stream_latency() { } if let Some(device) = test_get_default_device(Scope::Output) { - let streams = get_device_streams(device, DeviceType::OUTPUT).unwrap(); + let streams = run_serially(|| get_device_streams(device, DeviceType::OUTPUT)).unwrap(); for stream in streams { - let latency = get_stream_latency(stream).unwrap(); + let latency = run_serially(|| get_stream_latency(stream)).unwrap(); println!("latency of the output stream {} is {}", stream, latency); } } else { @@ -388,10 +444,10 @@ fn test_get_stream_latency_by_unknown_device() { #[test] fn test_get_stream_virtual_format() { if let Some(device) = test_get_default_device(Scope::Input) { - let streams = get_device_streams(device, DeviceType::INPUT).unwrap(); + let streams = run_serially(|| get_device_streams(device, DeviceType::INPUT)).unwrap(); let formats = streams .iter() - .map(|s| get_stream_virtual_format(*s)) + .map(|s| run_serially(|| get_stream_virtual_format(*s))) .collect::>>(); println!("input stream formats: {:?}", formats); assert!(!formats.is_empty()); @@ -400,10 +456,10 @@ fn test_get_stream_virtual_format() { } if let Some(device) = test_get_default_device(Scope::Output) { - let streams = get_device_streams(device, DeviceType::OUTPUT).unwrap(); + let streams = run_serially(|| get_device_streams(device, DeviceType::OUTPUT)).unwrap(); let formats = streams .iter() - .map(|s| get_stream_virtual_format(*s)) + .map(|s| run_serially(|| get_stream_virtual_format(*s))) .collect::>>(); println!("output stream formats: {:?}", formats); assert!(!formats.is_empty()); @@ -415,7 +471,9 @@ fn test_get_stream_virtual_format() { #[test] #[should_panic] fn test_get_stream_virtual_format_by_unknown_stream() { - assert!(get_stream_virtual_format(kAudioObjectUnknown).is_err()); + assert!( + run_serially_forward_panics(|| get_stream_virtual_format(kAudioObjectUnknown)).is_err() + ); } // get_stream_terminal_type @@ -442,25 +500,21 @@ fn test_get_stream_terminal_type() { } } if let Some(device) = test_get_default_device(Scope::Input) { - let streams = get_device_streams(device, DeviceType::INPUT).unwrap(); - for stream in streams { - assert_eq!( - terminal_type_to_device_type(get_stream_terminal_type(stream).unwrap()), - Some(DeviceType::INPUT) - ); - } + let streams = run_serially(|| get_device_streams(device, DeviceType::INPUT)).unwrap(); + assert!(streams.iter().any(|&s| { + terminal_type_to_device_type(run_serially(|| get_stream_terminal_type(s)).unwrap()) + == Some(DeviceType::INPUT) + })); } else { println!("No input device."); } if let Some(device) = test_get_default_device(Scope::Output) { - let streams = get_device_streams(device, DeviceType::OUTPUT).unwrap(); - for stream in streams { - assert_eq!( - terminal_type_to_device_type(get_stream_terminal_type(stream).unwrap()), - Some(DeviceType::OUTPUT) - ); - } + let streams = run_serially(|| get_device_streams(device, DeviceType::OUTPUT)).unwrap(); + assert!(streams.iter().any(|&s| { + terminal_type_to_device_type(run_serially(|| get_stream_terminal_type(s)).unwrap()) + == Some(DeviceType::OUTPUT) + })); } else { println!("No output device."); } diff --git a/third_party/rust/cubeb-coreaudio/src/backend/tests/interfaces.rs b/third_party/rust/cubeb-coreaudio/src/backend/tests/interfaces.rs index 340fec002d..aa15b7428b 100644 --- a/third_party/rust/cubeb-coreaudio/src/backend/tests/interfaces.rs +++ b/third_party/rust/cubeb-coreaudio/src/backend/tests/interfaces.rs @@ -2,10 +2,12 @@ extern crate itertools; use self::itertools::iproduct; use super::utils::{ - get_devices_info_in_scope, noop_data_callback, test_device_channels_in_scope, - test_get_default_device, test_ops_context_operation, test_ops_stream_operation, Scope, + draining_data_callback, get_devices_info_in_scope, noop_data_callback, + test_device_channels_in_scope, test_get_default_device, test_ops_context_operation, + test_ops_stream_operation, test_ops_stream_operation_on_context, Scope, }; use super::*; +use std::thread; // Context Operations // ------------------------------------------------------------------------------------------------ @@ -368,9 +370,6 @@ fn test_ops_context_register_device_collection_changed() { #[test] fn test_ops_context_register_device_collection_changed_with_a_duplex_stream() { - use std::thread; - use std::time::Duration; - extern "C" fn callback(_: *mut ffi::cubeb, got_called_ptr: *mut c_void) { let got_called = unsafe { &mut *(got_called_ptr as *mut bool) }; *got_called = true; @@ -667,14 +666,20 @@ fn test_ops_context_stream_init_channel_rate_combinations() { ffi::CUBEB_OK ); assert!(!stream.is_null()); + + unsafe { OPS.stream_destroy.unwrap()(stream) }; } }); } // Stream Operations // ------------------------------------------------------------------------------------------------ -fn test_default_output_stream_operation(name: &'static str, operation: F) -where +fn test_default_output_stream_operation_on_context_with_callback( + name: &'static str, + context_ptr: *mut ffi::cubeb, + data_callback: ffi::cubeb_data_callback, + operation: F, +) where F: FnOnce(*mut ffi::cubeb_stream), { // Make sure the parameters meet the requirements of AudioUnitContext::stream_init @@ -686,23 +691,52 @@ where output_params.layout = ffi::CUBEB_LAYOUT_UNDEFINED; output_params.prefs = ffi::CUBEB_STREAM_PREF_NONE; - test_ops_stream_operation( + test_ops_stream_operation_on_context( name, + context_ptr, ptr::null_mut(), // Use default input device. ptr::null_mut(), // No input parameters. ptr::null_mut(), // Use default output device. &mut output_params, 4096, // TODO: Get latency by get_min_latency instead ? - Some(noop_data_callback), + data_callback, None, // No state callback. ptr::null_mut(), // No user data pointer. operation, ); } -fn test_default_duplex_stream_operation(name: &'static str, operation: F) +fn test_default_output_stream_operation_with_callback( + name: &'static str, + data_callback: ffi::cubeb_data_callback, + operation: F, +) where + F: FnOnce(*mut ffi::cubeb_stream), +{ + test_ops_context_operation("context: default output stream operation", |context_ptr| { + test_default_output_stream_operation_on_context_with_callback( + name, + context_ptr, + data_callback, + operation, + ); + }); +} + +fn test_default_output_stream_operation(name: &'static str, operation: F) where F: FnOnce(*mut ffi::cubeb_stream), +{ + test_default_output_stream_operation_with_callback(name, Some(noop_data_callback), operation); +} + +fn test_default_duplex_stream_operation_on_context_with_callback( + name: &'static str, + context_ptr: *mut ffi::cubeb, + data_callback: ffi::cubeb_data_callback, + operation: F, +) where + F: FnOnce(*mut ffi::cubeb_stream), { // Make sure the parameters meet the requirements of AudioUnitContext::stream_init // (in the comments). @@ -720,23 +754,52 @@ where output_params.layout = ffi::CUBEB_LAYOUT_UNDEFINED; output_params.prefs = ffi::CUBEB_STREAM_PREF_NONE; - test_ops_stream_operation( + test_ops_stream_operation_on_context( name, + context_ptr, ptr::null_mut(), // Use default input device. &mut input_params, ptr::null_mut(), // Use default output device. &mut output_params, 4096, // TODO: Get latency by get_min_latency instead ? - Some(noop_data_callback), + data_callback, None, // No state callback. ptr::null_mut(), // No user data pointer. operation, ); } -fn test_stereo_input_duplex_stream_operation(name: &'static str, operation: F) +fn test_default_duplex_stream_operation_with_callback( + name: &'static str, + data_callback: ffi::cubeb_data_callback, + operation: F, +) where + F: FnOnce(*mut ffi::cubeb_stream), +{ + test_ops_context_operation("context: default duplex stream operation", |context_ptr| { + test_default_duplex_stream_operation_on_context_with_callback( + name, + context_ptr, + data_callback, + operation, + ); + }); +} + +fn test_default_duplex_stream_operation(name: &'static str, operation: F) where F: FnOnce(*mut ffi::cubeb_stream), +{ + test_default_duplex_stream_operation_with_callback(name, Some(noop_data_callback), operation); +} + +fn test_stereo_input_duplex_stream_operation_on_context_with_callback( + name: &'static str, + context_ptr: *mut ffi::cubeb, + data_callback: ffi::cubeb_data_callback, + operation: F, +) where + F: FnOnce(*mut ffi::cubeb_stream), { let mut input_devices = get_devices_info_in_scope(Scope::Input); input_devices.retain(|d| test_device_channels_in_scope(d.id, Scope::Input).unwrap_or(0) >= 2); @@ -759,23 +822,137 @@ where output_params.layout = ffi::CUBEB_LAYOUT_UNDEFINED; output_params.prefs = ffi::CUBEB_STREAM_PREF_NONE; - test_ops_stream_operation( + test_ops_stream_operation_on_context( name, + context_ptr, input_devices[0].id as ffi::cubeb_devid, &mut input_params, ptr::null_mut(), // Use default output device. &mut output_params, 4096, // TODO: Get latency by get_min_latency instead ? + data_callback, + None, // No state callback. + ptr::null_mut(), // No user data pointer. + operation, + ); +} + +fn test_stereo_input_duplex_stream_operation_with_callback( + name: &'static str, + data_callback: ffi::cubeb_data_callback, + operation: F, +) where + F: FnOnce(*mut ffi::cubeb_stream), +{ + test_ops_context_operation( + "context: stereo input duplex stream operation", + |context_ptr| { + test_stereo_input_duplex_stream_operation_on_context_with_callback( + name, + context_ptr, + data_callback, + operation, + ); + }, + ); +} + +fn test_stereo_input_duplex_stream_operation(name: &'static str, operation: F) +where + F: FnOnce(*mut ffi::cubeb_stream), +{ + test_stereo_input_duplex_stream_operation_with_callback( + name, Some(noop_data_callback), + operation, + ); +} + +fn test_default_input_voice_stream_operation_on_context_with_callback( + name: &'static str, + context_ptr: *mut ffi::cubeb, + data_callback: ffi::cubeb_data_callback, + operation: F, +) where + F: FnOnce(*mut ffi::cubeb_stream), +{ + // Make sure the parameters meet the requirements of AudioUnitContext::stream_init + // (in the comments). + let mut input_params = ffi::cubeb_stream_params::default(); + input_params.format = ffi::CUBEB_SAMPLE_FLOAT32NE; + input_params.rate = 44100; + input_params.channels = 1; + input_params.layout = ffi::CUBEB_LAYOUT_UNDEFINED; + input_params.prefs = ffi::CUBEB_STREAM_PREF_VOICE; + + test_ops_stream_operation_on_context( + name, + context_ptr, + ptr::null_mut(), // Use default input device. + &mut input_params, + ptr::null_mut(), // Use default output device. + ptr::null_mut(), // No output parameters. + 4096, // TODO: Get latency by get_min_latency instead ? + data_callback, None, // No state callback. ptr::null_mut(), // No user data pointer. operation, ); } -fn test_default_duplex_voice_stream_operation(name: &'static str, operation: F) +fn test_default_input_voice_stream_operation_on_context( + name: &'static str, + context_ptr: *mut ffi::cubeb, + operation: F, +) where + F: FnOnce(*mut ffi::cubeb_stream), +{ + test_default_input_voice_stream_operation_on_context_with_callback( + name, + context_ptr, + Some(noop_data_callback), + operation, + ); +} + +fn test_default_input_voice_stream_operation_with_callback( + name: &'static str, + data_callback: ffi::cubeb_data_callback, + operation: F, +) where + F: FnOnce(*mut ffi::cubeb_stream), +{ + test_ops_context_operation( + "context: default input voice stream operation", + |context_ptr| { + test_default_input_voice_stream_operation_on_context_with_callback( + name, + context_ptr, + data_callback, + operation, + ); + }, + ); +} + +fn test_default_input_voice_stream_operation(name: &'static str, operation: F) where F: FnOnce(*mut ffi::cubeb_stream), +{ + test_default_input_voice_stream_operation_with_callback( + name, + Some(noop_data_callback), + operation, + ); +} + +fn test_default_duplex_voice_stream_operation_on_context_with_callback( + name: &'static str, + context_ptr: *mut ffi::cubeb, + data_callback: ffi::cubeb_data_callback, + operation: F, +) where + F: FnOnce(*mut ffi::cubeb_stream), { // Make sure the parameters meet the requirements of AudioUnitContext::stream_init // (in the comments). @@ -793,20 +970,64 @@ where output_params.layout = ffi::CUBEB_LAYOUT_UNDEFINED; output_params.prefs = ffi::CUBEB_STREAM_PREF_VOICE; - test_ops_stream_operation( + test_ops_stream_operation_on_context( name, + context_ptr, ptr::null_mut(), // Use default input device. &mut input_params, ptr::null_mut(), // Use default output device. &mut output_params, 4096, // TODO: Get latency by get_min_latency instead ? - Some(noop_data_callback), + data_callback, None, // No state callback. ptr::null_mut(), // No user data pointer. operation, ); } +fn test_default_duplex_voice_stream_operation_on_context( + name: &'static str, + context_ptr: *mut ffi::cubeb, + operation: F, +) where + F: FnOnce(*mut ffi::cubeb_stream), +{ + test_default_duplex_voice_stream_operation_on_context_with_callback( + name, + context_ptr, + Some(noop_data_callback), + operation, + ); +} + +fn test_default_duplex_voice_stream_operation_with_callback( + name: &'static str, + data_callback: ffi::cubeb_data_callback, + operation: F, +) where + F: FnOnce(*mut ffi::cubeb_stream), +{ + test_ops_context_operation("context: duplex voice stream operation", |context_ptr| { + test_default_duplex_voice_stream_operation_on_context_with_callback( + name, + context_ptr, + data_callback, + operation, + ); + }); +} + +fn test_default_duplex_voice_stream_operation(name: &'static str, operation: F) +where + F: FnOnce(*mut ffi::cubeb_stream), +{ + test_default_duplex_voice_stream_operation_with_callback( + name, + Some(noop_data_callback), + operation, + ); +} + fn test_stereo_input_duplex_voice_stream_operation(name: &'static str, operation: F) where F: FnOnce(*mut ffi::cubeb_stream), @@ -865,6 +1086,18 @@ fn test_ops_stream_stop() { }); } +#[test] +fn test_ops_stream_drain() { + test_default_output_stream_operation_with_callback( + "stream: drain", + Some(draining_data_callback), + |stream| { + assert_eq!(unsafe { OPS.stream_start.unwrap()(stream) }, ffi::CUBEB_OK); + thread::sleep(Duration::from_millis(10)); + }, + ); +} + #[test] fn test_ops_stream_position() { test_default_output_stream_operation("stream: position", |stream| { @@ -987,17 +1220,72 @@ fn test_ops_stereo_input_duplex_stream_stop() { } #[test] -fn test_ops_duplex_voice_stream_init_and_destroy() { - test_default_duplex_voice_stream_operation( - "duplex voice stream: init and destroy", - |_stream| {}, +fn test_ops_stereo_input_duplex_stream_drain() { + test_stereo_input_duplex_stream_operation_with_callback( + "stereo-input duplex stream: drain", + Some(draining_data_callback), + |stream| { + assert_eq!(unsafe { OPS.stream_start.unwrap()(stream) }, ffi::CUBEB_OK); + thread::sleep(Duration::from_millis(10)); + }, ); } +#[test] +fn test_ops_input_voice_stream_init_and_destroy() { + test_default_input_voice_stream_operation("input voice stream: init and destroy", |stream| { + let stm = unsafe { &mut *(stream as *mut AudioUnitStream) }; + assert_eq!( + stm.core_stream_data.using_voice_processing_unit(), + macos_kernel_major_version().unwrap() != MACOS_KERNEL_MAJOR_VERSION_MONTEREY + ); + }); +} + +#[test] +fn test_ops_input_voice_stream_start() { + test_default_input_voice_stream_operation("input voice stream: start", |stream| { + assert_eq!(unsafe { OPS.stream_start.unwrap()(stream) }, ffi::CUBEB_OK); + let stm = unsafe { &mut *(stream as *mut AudioUnitStream) }; + assert_eq!( + stm.core_stream_data.using_voice_processing_unit(), + macos_kernel_major_version().unwrap() != MACOS_KERNEL_MAJOR_VERSION_MONTEREY + ); + }); +} + +#[test] +fn test_ops_input_voice_stream_stop() { + test_default_input_voice_stream_operation("input voice stream: stop", |stream| { + assert_eq!(unsafe { OPS.stream_stop.unwrap()(stream) }, ffi::CUBEB_OK); + let stm = unsafe { &mut *(stream as *mut AudioUnitStream) }; + assert_eq!( + stm.core_stream_data.using_voice_processing_unit(), + macos_kernel_major_version().unwrap() != MACOS_KERNEL_MAJOR_VERSION_MONTEREY + ); + }); +} + +#[test] +fn test_ops_duplex_voice_stream_init_and_destroy() { + test_default_duplex_voice_stream_operation("duplex voice stream: init and destroy", |stream| { + let stm = unsafe { &mut *(stream as *mut AudioUnitStream) }; + assert_eq!( + stm.core_stream_data.using_voice_processing_unit(), + macos_kernel_major_version().unwrap() != MACOS_KERNEL_MAJOR_VERSION_MONTEREY + ); + }); +} + #[test] fn test_ops_duplex_voice_stream_start() { test_default_duplex_voice_stream_operation("duplex voice stream: start", |stream| { assert_eq!(unsafe { OPS.stream_start.unwrap()(stream) }, ffi::CUBEB_OK); + let stm = unsafe { &mut *(stream as *mut AudioUnitStream) }; + assert_eq!( + stm.core_stream_data.using_voice_processing_unit(), + macos_kernel_major_version().unwrap() != MACOS_KERNEL_MAJOR_VERSION_MONTEREY + ); }); } @@ -1005,21 +1293,334 @@ fn test_ops_duplex_voice_stream_start() { fn test_ops_duplex_voice_stream_stop() { test_default_duplex_voice_stream_operation("duplex voice stream: stop", |stream| { assert_eq!(unsafe { OPS.stream_stop.unwrap()(stream) }, ffi::CUBEB_OK); + let stm = unsafe { &mut *(stream as *mut AudioUnitStream) }; + assert_eq!( + stm.core_stream_data.using_voice_processing_unit(), + macos_kernel_major_version().unwrap() != MACOS_KERNEL_MAJOR_VERSION_MONTEREY + ); + }); +} + +#[test] +fn test_ops_duplex_voice_stream_drain() { + test_default_duplex_voice_stream_operation_with_callback( + "duplex voice stream: drain", + Some(draining_data_callback), + |stream| { + assert_eq!(unsafe { OPS.stream_start.unwrap()(stream) }, ffi::CUBEB_OK); + let stm = unsafe { &mut *(stream as *mut AudioUnitStream) }; + assert_eq!( + stm.core_stream_data.using_voice_processing_unit(), + macos_kernel_major_version().unwrap() != MACOS_KERNEL_MAJOR_VERSION_MONTEREY + ); + thread::sleep(Duration::from_millis(10)); + }, + ); +} + +#[test] +#[ignore] +fn test_ops_timing_sensitive_multiple_voice_stream_init_and_destroy() { + if macos_kernel_major_version().unwrap() == MACOS_KERNEL_MAJOR_VERSION_MONTEREY { + // We disable VPIO on Monterey. + return; + } + let start = Instant::now(); + let mut t1 = start; + let mut t2 = start; + let mut t3 = start; + let mut t4 = start; + let mut t5 = start; + let mut t6 = start; + let mut t7 = start; + let mut t8 = start; + let mut t9 = start; + let mut t10 = start; + test_ops_context_operation("multiple duplex voice streams", |context_ptr| { + // First stream uses vpio, creates the shared vpio unit. + test_default_duplex_voice_stream_operation_on_context( + "multiple voice streams: stream 1, duplex", + context_ptr, + |stream| { + let stm = unsafe { &mut *(stream as *mut AudioUnitStream) }; + assert!(stm.core_stream_data.using_voice_processing_unit()); + + // Two concurrent vpio streams are supported. + test_default_input_voice_stream_operation_on_context( + "multiple voice streams: stream 2, input-only", + context_ptr, + |stream| { + let stm = unsafe { &mut *(stream as *mut AudioUnitStream) }; + assert!(stm.core_stream_data.using_voice_processing_unit()); + + // Three concurrent vpio streams are supported. + test_default_duplex_voice_stream_operation_on_context( + "multiple voice streams: stream 3, duplex", + context_ptr, + |stream| { + let stm = unsafe { &mut *(stream as *mut AudioUnitStream) }; + assert!(stm.core_stream_data.using_voice_processing_unit()); + }, + ); + }, + ); + }, + ); + t1 = Instant::now(); + // Fourth stream uses vpio, allows reuse of one already created. + test_default_duplex_voice_stream_operation_on_context( + "multiple voice streams: stream 4, duplex", + context_ptr, + |stream| { + let stm = unsafe { &mut *(stream as *mut AudioUnitStream) }; + assert!(stm.core_stream_data.using_voice_processing_unit()); + t2 = Instant::now(); + + // Fifth stream uses vpio, allows reuse of one already created. + test_default_duplex_voice_stream_operation_on_context( + "multiple voice streams: stream 5, duplex", + context_ptr, + |stream| { + let stm = unsafe { &mut *(stream as *mut AudioUnitStream) }; + assert!(stm.core_stream_data.using_voice_processing_unit()); + t3 = Instant::now(); + + // Sixth stream uses vpio, allows reuse of one already created. + test_default_input_voice_stream_operation_on_context( + "multiple voice streams: stream 6, input-only", + context_ptr, + |stream| { + let stm = unsafe { &mut *(stream as *mut AudioUnitStream) }; + assert!(stm.core_stream_data.using_voice_processing_unit()); + t4 = Instant::now(); + + // Seventh stream uses vpio, but is created anew. + test_default_input_voice_stream_operation_on_context( + "multiple voice streams: stream 7, input-only", + context_ptr, + |stream| { + let stm = unsafe { &mut *(stream as *mut AudioUnitStream) }; + assert!(stm.core_stream_data.using_voice_processing_unit()); + t5 = Instant::now(); + }, + ); + t6 = Instant::now(); + }, + ); + t7 = Instant::now(); + }, + ); + t8 = Instant::now(); + }, + ); + t9 = Instant::now(); + }); + t10 = Instant::now(); + + let reuse_vpio_1 = t2 - t1; + let reuse_vpio_2 = t3 - t2; + let reuse_vpio_3 = t4 - t3; + let create_standalone_vpio = t5 - t4; + assert!( + create_standalone_vpio > reuse_vpio_1 * 2, + "Failed create_standalone_vpio={}s > reuse_vpio_1={}s * 2", + create_standalone_vpio.as_secs_f32(), + reuse_vpio_1.as_secs_f32() + ); + assert!( + create_standalone_vpio > reuse_vpio_2 * 2, + "Failed create_standalone_vpio={}s > reuse_vpio_2={}s * 2", + create_standalone_vpio.as_secs_f32(), + reuse_vpio_2.as_secs_f32() + ); + assert!( + create_standalone_vpio > reuse_vpio_3 * 2, + "Failed create_standalone_vpio={}s > reuse_vpio_3={}s * 2", + create_standalone_vpio.as_secs_f32(), + reuse_vpio_3.as_secs_f32() + ); + + let recycle_vpio_1 = t6 - t5; + let recycle_vpio_2 = t7 - t6; + let recycle_vpio_3 = t8 - t7; + let recycle_vpio_4 = t9 - t8; + let dispose_vpios = t10 - t9; + assert!( + dispose_vpios > recycle_vpio_1 * 2, + "Failed dispose_vpios={}s > recycle_vpio_1 ={}s * 2", + dispose_vpios.as_secs_f32(), + recycle_vpio_1.as_secs_f32() + ); + assert!( + dispose_vpios > recycle_vpio_2 * 2, + "Failed dispose_vpios={}s > recycle_vpio_2 ={}s * 2", + dispose_vpios.as_secs_f32(), + recycle_vpio_2.as_secs_f32() + ); + assert!( + dispose_vpios > recycle_vpio_3 * 2, + "Failed dispose_vpios={}s > recycle_vpio_3 ={}s * 2", + dispose_vpios.as_secs_f32(), + recycle_vpio_3.as_secs_f32() + ); + assert!( + dispose_vpios > recycle_vpio_4 * 2, + "Failed dispose_vpios={}s > recycle_vpio_4 ={}s * 2", + dispose_vpios.as_secs_f32(), + recycle_vpio_4.as_secs_f32() + ); +} + +#[test] +#[ignore] +fn test_ops_timing_sensitive_multiple_duplex_voice_stream_start() { + if macos_kernel_major_version().unwrap() == MACOS_KERNEL_MAJOR_VERSION_MONTEREY { + // We disable VPIO on Monterey. + return; + } + test_ops_context_operation("multiple duplex voice streams", |context_ptr| { + let start = Instant::now(); + // First stream uses vpio, creates the shared vpio unit. + test_default_duplex_voice_stream_operation_on_context( + "multiple duplex voice streams: stream 1", + context_ptr, + |stream| { + let stm = unsafe { &mut *(stream as *mut AudioUnitStream) }; + assert!(stm.core_stream_data.using_voice_processing_unit()); + assert_eq!(unsafe { OPS.stream_start.unwrap()(stream) }, ffi::CUBEB_OK); + }, + ); + let d1 = start.elapsed(); + // Second stream uses vpio, allows reuse of the one already created. + test_default_duplex_voice_stream_operation_on_context( + "multiple duplex voice streams: stream 2", + context_ptr, + |stream| { + let stm = unsafe { &mut *(stream as *mut AudioUnitStream) }; + assert!(stm.core_stream_data.using_voice_processing_unit()); + assert_eq!(unsafe { OPS.stream_start.unwrap()(stream) }, ffi::CUBEB_OK); + }, + ); + let d2 = start.elapsed() - d1; + // d1 being significantly longer than d2 is proof we reuse vpio. + assert!( + d1 > d2 * 2, + "Failed d1={}s > d2={}s * s", + d1.as_secs_f32(), + d2.as_secs_f32() + ); + }); +} + +#[test] +#[ignore] +fn test_ops_timing_sensitive_multiple_duplex_voice_stream_params() { + if macos_kernel_major_version().unwrap() == MACOS_KERNEL_MAJOR_VERSION_MONTEREY { + // We disable VPIO on Monterey. + return; + } + test_ops_context_operation("multiple duplex voice streams with params", |context_ptr| { + let start = Instant::now(); + // First stream uses vpio, creates the shared vpio unit. + test_default_duplex_voice_stream_operation_on_context( + "multiple duplex voice streams: stream 1", + context_ptr, + |stream| { + let stm = unsafe { &mut *(stream as *mut AudioUnitStream) }; + assert!(stm.core_stream_data.using_voice_processing_unit()); + assert_eq!( + unsafe { + OPS.stream_set_input_processing_params.unwrap()( + stream, + ffi::CUBEB_INPUT_PROCESSING_PARAM_ECHO_CANCELLATION + | ffi::CUBEB_INPUT_PROCESSING_PARAM_NOISE_SUPPRESSION, + ) + }, + ffi::CUBEB_OK + ); + assert_eq!( + unsafe { OPS.stream_set_input_mute.unwrap()(stream, 1) }, + ffi::CUBEB_OK + ); + }, + ); + let d1 = start.elapsed(); + // Second stream uses vpio, allows reuse of the one already created. + test_default_duplex_voice_stream_operation_on_context( + "multiple duplex voice streams: stream 2", + context_ptr, + |stream| { + let stm = unsafe { &mut *(stream as *mut AudioUnitStream) }; + assert!(stm.core_stream_data.using_voice_processing_unit()); + let queue = stm.queue.clone(); + // Test that input processing params does not carry over when reusing vpio. + let mut bypass: u32 = 0; + let r = queue + .run_sync(|| { + audio_unit_get_property( + stm.core_stream_data.input_unit, + kAUVoiceIOProperty_BypassVoiceProcessing, + kAudioUnitScope_Global, + AU_IN_BUS, + &mut bypass, + &mut mem::size_of::(), + ) + }) + .unwrap(); + assert_eq!(r, NO_ERR); + assert_eq!(bypass, 1); + + // Test that input mute state does not carry over when reusing vpio. + let mut mute: u32 = 0; + let r = queue + .run_sync(|| { + audio_unit_get_property( + stm.core_stream_data.input_unit, + kAUVoiceIOProperty_MuteOutput, + kAudioUnitScope_Global, + AU_IN_BUS, + &mut mute, + &mut mem::size_of::(), + ) + }) + .unwrap(); + assert_eq!(r, NO_ERR); + assert_eq!(mute, 0); + }, + ); + let d2 = start.elapsed() - d1; + // d1 being significantly longer than d2 is proof we reuse vpio. + assert!( + d1 > d2 * 2, + "Failed d1={}s > d2={}s * 2", + d1.as_secs_f32(), + d2.as_secs_f32() + ); }); } #[test] fn test_ops_duplex_voice_stream_set_input_mute() { + if macos_kernel_major_version().unwrap() == MACOS_KERNEL_MAJOR_VERSION_MONTEREY { + // We disable VPIO on Monterey. + return; + } test_default_duplex_voice_stream_operation("duplex voice stream: mute", |stream| { assert_eq!( unsafe { OPS.stream_set_input_mute.unwrap()(stream, 1) }, ffi::CUBEB_OK ); + let stm = unsafe { &mut *(stream as *mut AudioUnitStream) }; + assert!(stm.core_stream_data.using_voice_processing_unit()); }); } #[test] fn test_ops_duplex_voice_stream_set_input_mute_before_start() { + if macos_kernel_major_version().unwrap() == MACOS_KERNEL_MAJOR_VERSION_MONTEREY { + // We disable VPIO on Monterey. + return; + } test_default_duplex_voice_stream_operation( "duplex voice stream: mute before start", |stream| { @@ -1028,12 +1629,18 @@ fn test_ops_duplex_voice_stream_set_input_mute_before_start() { ffi::CUBEB_OK ); assert_eq!(unsafe { OPS.stream_start.unwrap()(stream) }, ffi::CUBEB_OK); + let stm = unsafe { &mut *(stream as *mut AudioUnitStream) }; + assert!(stm.core_stream_data.using_voice_processing_unit()); }, ); } #[test] fn test_ops_duplex_voice_stream_set_input_mute_before_start_with_reinit() { + if macos_kernel_major_version().unwrap() == MACOS_KERNEL_MAJOR_VERSION_MONTEREY { + // We disable VPIO on Monterey. + return; + } test_default_duplex_voice_stream_operation( "duplex voice stream: mute before start with reinit", |stream| { @@ -1045,6 +1652,7 @@ fn test_ops_duplex_voice_stream_set_input_mute_before_start_with_reinit() { // Hacky cast, but testing this here was simplest for now. let stm = unsafe { &mut *(stream as *mut AudioUnitStream) }; + assert!(stm.core_stream_data.using_voice_processing_unit()); stm.reinit_async(); let queue = stm.queue.clone(); let mut mute_after_reinit = false; @@ -1068,17 +1676,27 @@ fn test_ops_duplex_voice_stream_set_input_mute_before_start_with_reinit() { #[test] fn test_ops_duplex_voice_stream_set_input_mute_after_start() { + if macos_kernel_major_version().unwrap() == MACOS_KERNEL_MAJOR_VERSION_MONTEREY { + // We disable VPIO on Monterey. + return; + } test_default_duplex_voice_stream_operation("duplex voice stream: mute after start", |stream| { assert_eq!(unsafe { OPS.stream_start.unwrap()(stream) }, ffi::CUBEB_OK); assert_eq!( unsafe { OPS.stream_set_input_mute.unwrap()(stream, 1) }, ffi::CUBEB_OK ); + let stm = unsafe { &mut *(stream as *mut AudioUnitStream) }; + assert!(stm.core_stream_data.using_voice_processing_unit()); }); } #[test] fn test_ops_duplex_voice_stream_set_input_processing_params() { + if macos_kernel_major_version().unwrap() == MACOS_KERNEL_MAJOR_VERSION_MONTEREY { + // We disable VPIO on Monterey. + return; + } test_default_duplex_voice_stream_operation("duplex voice stream: processing", |stream| { let params: ffi::cubeb_input_processing_params = ffi::CUBEB_INPUT_PROCESSING_PARAM_ECHO_CANCELLATION @@ -1088,11 +1706,17 @@ fn test_ops_duplex_voice_stream_set_input_processing_params() { unsafe { OPS.stream_set_input_processing_params.unwrap()(stream, params) }, ffi::CUBEB_OK ); + let stm = unsafe { &mut *(stream as *mut AudioUnitStream) }; + assert!(stm.core_stream_data.using_voice_processing_unit()); }); } #[test] fn test_ops_duplex_voice_stream_set_input_processing_params_before_start() { + if macos_kernel_major_version().unwrap() == MACOS_KERNEL_MAJOR_VERSION_MONTEREY { + // We disable VPIO on Monterey. + return; + } test_default_duplex_voice_stream_operation( "duplex voice stream: processing before start", |stream| { @@ -1105,12 +1729,18 @@ fn test_ops_duplex_voice_stream_set_input_processing_params_before_start() { ffi::CUBEB_OK ); assert_eq!(unsafe { OPS.stream_start.unwrap()(stream) }, ffi::CUBEB_OK); + let stm = unsafe { &mut *(stream as *mut AudioUnitStream) }; + assert!(stm.core_stream_data.using_voice_processing_unit()); }, ); } #[test] fn test_ops_duplex_voice_stream_set_input_processing_params_before_start_with_reinit() { + if macos_kernel_major_version().unwrap() == MACOS_KERNEL_MAJOR_VERSION_MONTEREY { + // We disable VPIO on Monterey. + return; + } test_default_duplex_voice_stream_operation( "duplex voice stream: processing before start with reinit", |stream| { @@ -1126,6 +1756,7 @@ fn test_ops_duplex_voice_stream_set_input_processing_params_before_start_with_re // Hacky cast, but testing this here was simplest for now. let stm = unsafe { &mut *(stream as *mut AudioUnitStream) }; + assert!(stm.core_stream_data.using_voice_processing_unit()); stm.reinit_async(); let queue = stm.queue.clone(); let mut params_after_reinit: ffi::cubeb_input_processing_params = @@ -1170,9 +1801,15 @@ fn test_ops_duplex_voice_stream_set_input_processing_params_before_start_with_re #[test] fn test_ops_duplex_voice_stream_set_input_processing_params_after_start() { + if macos_kernel_major_version().unwrap() == MACOS_KERNEL_MAJOR_VERSION_MONTEREY { + // We disable VPIO on Monterey. + return; + } test_default_duplex_voice_stream_operation( "duplex voice stream: processing after start", |stream| { + let stm = unsafe { &mut *(stream as *mut AudioUnitStream) }; + assert!(stm.core_stream_data.using_voice_processing_unit()); assert_eq!(unsafe { OPS.stream_start.unwrap()(stream) }, ffi::CUBEB_OK); let params: ffi::cubeb_input_processing_params = ffi::CUBEB_INPUT_PROCESSING_PARAM_ECHO_CANCELLATION @@ -1190,7 +1827,13 @@ fn test_ops_duplex_voice_stream_set_input_processing_params_after_start() { fn test_ops_stereo_input_duplex_voice_stream_init_and_destroy() { test_stereo_input_duplex_voice_stream_operation( "stereo-input duplex voice stream: init and destroy", - |_stream| {}, + |stream| { + let stm = unsafe { &mut *(stream as *mut AudioUnitStream) }; + assert_eq!( + stm.core_stream_data.using_voice_processing_unit(), + macos_kernel_major_version().unwrap() != MACOS_KERNEL_MAJOR_VERSION_MONTEREY + ); + }, ); } @@ -1199,6 +1842,11 @@ fn test_ops_stereo_input_duplex_voice_stream_start() { test_stereo_input_duplex_voice_stream_operation( "stereo-input duplex voice stream: start", |stream| { + let stm = unsafe { &mut *(stream as *mut AudioUnitStream) }; + assert_eq!( + stm.core_stream_data.using_voice_processing_unit(), + macos_kernel_major_version().unwrap() != MACOS_KERNEL_MAJOR_VERSION_MONTEREY + ); assert_eq!(unsafe { OPS.stream_start.unwrap()(stream) }, ffi::CUBEB_OK); }, ); @@ -1209,6 +1857,11 @@ fn test_ops_stereo_input_duplex_voice_stream_stop() { test_stereo_input_duplex_voice_stream_operation( "stereo-input duplex voice stream: stop", |stream| { + let stm = unsafe { &mut *(stream as *mut AudioUnitStream) }; + assert_eq!( + stm.core_stream_data.using_voice_processing_unit(), + macos_kernel_major_version().unwrap() != MACOS_KERNEL_MAJOR_VERSION_MONTEREY + ); assert_eq!(unsafe { OPS.stream_stop.unwrap()(stream) }, ffi::CUBEB_OK); }, ); diff --git a/third_party/rust/cubeb-coreaudio/src/backend/tests/manual.rs b/third_party/rust/cubeb-coreaudio/src/backend/tests/manual.rs index b2b2241cc9..3bf96c65c6 100644 --- a/third_party/rust/cubeb-coreaudio/src/backend/tests/manual.rs +++ b/third_party/rust/cubeb-coreaudio/src/backend/tests/manual.rs @@ -4,7 +4,6 @@ use super::utils::{ }; use super::*; use std::io; -use std::sync::atomic::AtomicBool; #[ignore] #[test] @@ -153,25 +152,84 @@ fn test_device_collection_change() { let _ = std::io::stdin().read_line(&mut input); } +struct StreamData { + stream_ptr: *mut ffi::cubeb_stream, + enable_loopback: AtomicBool, +} + +impl StreamData { + fn new() -> Self { + Self { + stream_ptr: ptr::null_mut(), + enable_loopback: AtomicBool::new(false), + } + } +} + +struct StreamsData { + streams: Vec, + current_idx: Option, +} + +impl StreamsData { + fn new() -> Self { + Self { + streams: Vec::new(), + current_idx: None, + } + } + + fn len(&self) -> usize { + self.streams.len() + } + + fn current_mut(&mut self) -> &mut StreamData { + &mut self.streams[self.current_idx.unwrap()] + } + + fn current(&self) -> &StreamData { + &self.streams[self.current_idx.unwrap()] + } + + fn select(&mut self, idx: usize) { + assert!(idx < self.len()); + self.current_idx = Some(idx); + } + + fn push(&mut self, stream: StreamData) { + self.streams.push(stream) + } +} + #[ignore] #[test] fn test_stream_tester() { test_ops_context_operation("context: stream tester", |context_ptr| { - let mut stream_ptr: *mut ffi::cubeb_stream = ptr::null_mut(); - let enable_loopback = AtomicBool::new(false); + let mut input_prefs = StreamPrefs::NONE; + let mut output_prefs = StreamPrefs::NONE; + let mut streams = StreamsData::new(); loop { println!( - "commands:\n\ + "Current stream: {} (of {}). Commands:\n\ \t'q': quit\n\ + \t'b': change current stream\n\ + \t'i': set input stream prefs to be used for creating streams\n\ + \t'o': set output stream prefs to be used for creating streams\n\ \t'c': create a stream\n\ - \t'd': destroy a stream\n\ - \t's': start the created stream\n\ - \t't': stop the created stream\n\ + Commands on the current stream:\n\ + \t'd': destroy\n\ + \t's': start\n\ + \t't': stop\n\ \t'r': register a device changed callback\n\ \t'l': set loopback (DUPLEX-only)\n\ \t'v': set volume\n\ \t'm': set input mute\n\ - \t'p': set input processing" + \t'p': set input processing", + streams + .current_idx + .map(|i| format!("{}", i + 1 as usize)) + .unwrap_or(String::from("N/A")), + streams.len(), ); let mut command = String::new(); @@ -181,66 +239,130 @@ fn test_stream_tester() { match command.as_str() { "q" => { println!("Quit."); - destroy_stream(&mut stream_ptr); + for mut stream in streams.streams { + if !stream.stream_ptr.is_null() { + destroy_stream(&mut stream); + } + } break; } - "c" => create_stream(&mut stream_ptr, context_ptr, &enable_loopback), - "d" => destroy_stream(&mut stream_ptr), - "s" => start_stream(stream_ptr), - "t" => stop_stream(stream_ptr), - "r" => register_device_change_callback(stream_ptr), - "l" => set_loopback(stream_ptr, &enable_loopback), - "v" => set_volume(stream_ptr), - "m" => set_input_mute(stream_ptr), - "p" => set_input_processing(stream_ptr), - x => println!("Unknown command: {}", x), + "i" => set_prefs(&mut input_prefs), + "o" => set_prefs(&mut output_prefs), + "c" => create_stream(context_ptr, &mut streams, input_prefs, output_prefs), + _ if streams.current_idx.is_none() => { + println!("There are no streams! Create a stream first.") + } + cmd => match cmd { + "b" => select_stream(&mut streams), + "d" => destroy_stream(streams.current_mut()), + "s" => start_stream(streams.current()), + "t" => stop_stream(streams.current()), + "r" => register_device_change_callback(streams.current()), + "l" => set_loopback(streams.current()), + "v" => set_volume(streams.current()), + "m" => set_input_mute(streams.current()), + "p" => set_input_processing(streams.current()), + x => println!("Unknown command: {}", x), + }, } } }); - fn start_stream(stream_ptr: *mut ffi::cubeb_stream) { - if stream_ptr.is_null() { + fn set_prefs(prefs: &mut StreamPrefs) { + let mut done = false; + while !done { + println!( + "Current prefs: {:?}\nSelect action:\n\ + \t1) Set None\n\ + \t2) Toggle Loopback\n\ + \t3) Toggle Disable Device Switching\n\ + \t4) Toggle Voice\n\ + \t5) Set All\n\ + \t0) Done", + prefs + ); + let mut input = String::new(); + let _ = io::stdin().read_line(&mut input); + assert_eq!(input.pop().unwrap(), '\n'); + match input.as_str() { + "1" => *prefs = StreamPrefs::NONE, + "2" => prefs.toggle(StreamPrefs::LOOPBACK), + "3" => prefs.toggle(StreamPrefs::DISABLE_DEVICE_SWITCHING), + "4" => prefs.toggle(StreamPrefs::VOICE), + "5" => *prefs = StreamPrefs::all(), + "0" => done = true, + _ => println!("Invalid action. Select again.\n"), + } + } + } + + fn select_stream(streams: &mut StreamsData) { + let num_streams = streams.len(); + let current_idx = streams.current_idx.unwrap(); + println!( + "Current stream is {}. Select stream 1 to {} on which to apply commands:", + current_idx + 1 as usize, + num_streams + ); + let mut selection: Option = None; + while selection.is_none() { + let mut input = String::new(); + let _ = io::stdin().read_line(&mut input); + assert_eq!(input.pop().unwrap(), '\n'); + selection = match input.parse::() { + Ok(i) if (1..=num_streams).contains((&i).into()) => Some(i), + _ => { + println!("Invalid stream. Select again.\n"); + None + } + } + } + streams.select(selection.unwrap() - 1) + } + + fn start_stream(stream: &StreamData) { + if stream.stream_ptr.is_null() { println!("No stream can start."); return; } assert_eq!( - unsafe { OPS.stream_start.unwrap()(stream_ptr) }, + unsafe { OPS.stream_start.unwrap()(stream.stream_ptr) }, ffi::CUBEB_OK ); - println!("Stream {:p} started.", stream_ptr); + println!("Stream {:p} started.", stream.stream_ptr); } - fn stop_stream(stream_ptr: *mut ffi::cubeb_stream) { - if stream_ptr.is_null() { + fn stop_stream(stream: &StreamData) { + if stream.stream_ptr.is_null() { println!("No stream can stop."); return; } assert_eq!( - unsafe { OPS.stream_stop.unwrap()(stream_ptr) }, + unsafe { OPS.stream_stop.unwrap()(stream.stream_ptr) }, ffi::CUBEB_OK ); - println!("Stream {:p} stopped.", stream_ptr); + println!("Stream {:p} stopped.", stream.stream_ptr); } - fn set_volume(stream_ptr: *mut ffi::cubeb_stream) { - if stream_ptr.is_null() { + fn set_volume(stream: &StreamData) { + if stream.stream_ptr.is_null() { println!("No stream can set volume."); return; } const VOL: f32 = 0.5; assert_eq!( - unsafe { OPS.stream_set_volume.unwrap()(stream_ptr, VOL) }, + unsafe { OPS.stream_set_volume.unwrap()(stream.stream_ptr, VOL) }, ffi::CUBEB_OK ); - println!("Set stream {:p} volume to {}", stream_ptr, VOL); + println!("Set stream {:p} volume to {}", stream.stream_ptr, VOL); } - fn set_loopback(stream_ptr: *mut ffi::cubeb_stream, enable_loopback: &AtomicBool) { - if stream_ptr.is_null() { + fn set_loopback(stream: &StreamData) { + if stream.stream_ptr.is_null() { println!("No stream can set loopback."); return; } - let stm = unsafe { &mut *(stream_ptr as *mut AudioUnitStream) }; + let stm = unsafe { &mut *(stream.stream_ptr as *mut AudioUnitStream) }; if !stm.core_stream_data.has_input() || !stm.core_stream_data.has_output() { println!("Duplex stream needed to set loopback"); return; @@ -261,20 +383,20 @@ fn test_stream_tester() { } } let loopback = loopback.unwrap(); - enable_loopback.store(loopback, Ordering::SeqCst); + stream.enable_loopback.store(loopback, Ordering::SeqCst); println!( "Loopback {} for stream {:p}", if loopback { "enabled" } else { "disabled" }, - stream_ptr + stream.stream_ptr ); } - fn set_input_mute(stream_ptr: *mut ffi::cubeb_stream) { - if stream_ptr.is_null() { + fn set_input_mute(stream: &StreamData) { + if stream.stream_ptr.is_null() { println!("No stream can set input mute."); return; } - let stm = unsafe { &mut *(stream_ptr as *mut AudioUnitStream) }; + let stm = unsafe { &mut *(stream.stream_ptr as *mut AudioUnitStream) }; if !stm.core_stream_data.has_input() { println!("Input stream needed to set loopback"); return; @@ -295,7 +417,7 @@ fn test_stream_tester() { } } let mute = mute.unwrap(); - let res = unsafe { OPS.stream_set_input_mute.unwrap()(stream_ptr, mute.into()) }; + let res = unsafe { OPS.stream_set_input_mute.unwrap()(stream.stream_ptr, mute.into()) }; println!( "{} set stream {:p} input {}", if res == ffi::CUBEB_OK { @@ -303,17 +425,17 @@ fn test_stream_tester() { } else { "Failed to" }, - stream_ptr, + stream.stream_ptr, if mute { "mute" } else { "unmute" } ); } - fn set_input_processing(stream_ptr: *mut ffi::cubeb_stream) { - if stream_ptr.is_null() { + fn set_input_processing(stream: &StreamData) { + if stream.stream_ptr.is_null() { println!("No stream can set input processing."); return; } - let stm = unsafe { &mut *(stream_ptr as *mut AudioUnitStream) }; + let stm = unsafe { &mut *(stream.stream_ptr as *mut AudioUnitStream) }; if !stm.core_stream_data.using_voice_processing_unit() { println!("Duplex stream with voice processing needed to set input processing params"); return; @@ -338,6 +460,23 @@ fn test_stream_tester() { params.set(InputProcessingParams::ECHO_CANCELLATION, true); params.set(InputProcessingParams::NOISE_SUPPRESSION, true); } + let mut agc = u32::from(false); + let mut size: usize = mem::size_of::(); + assert_eq!( + audio_unit_get_property( + stm.core_stream_data.input_unit, + kAUVoiceIOProperty_VoiceProcessingEnableAGC, + kAudioUnitScope_Global, + AU_IN_BUS, + &mut agc, + &mut size, + ), + NO_ERR + ); + assert_eq!(size, mem::size_of::()); + if agc == 1 { + params.set(InputProcessingParams::AUTOMATIC_GAIN_CONTROL, true); + } } let mut done = false; while !done { @@ -367,8 +506,9 @@ fn test_stream_tester() { _ => println!("Invalid action. Select again.\n"), } } - let res = - unsafe { OPS.stream_set_input_processing_params.unwrap()(stream_ptr, params.bits()) }; + let res = unsafe { + OPS.stream_set_input_processing_params.unwrap()(stream.stream_ptr, params.bits()) + }; println!( "{} set stream {:p} input processing params to {:?}", if res == ffi::CUBEB_OK { @@ -376,48 +516,62 @@ fn test_stream_tester() { } else { "Failed to" }, - stream_ptr, + stream.stream_ptr, params, ); } - fn register_device_change_callback(stream_ptr: *mut ffi::cubeb_stream) { + fn register_device_change_callback(stream: &StreamData) { extern "C" fn callback(user_ptr: *mut c_void) { println!("user pointer @ {:p}", user_ptr); assert!(user_ptr.is_null()); } - if stream_ptr.is_null() { + if stream.stream_ptr.is_null() { println!("No stream for registering the callback."); return; } assert_eq!( unsafe { - OPS.stream_register_device_changed_callback.unwrap()(stream_ptr, Some(callback)) + OPS.stream_register_device_changed_callback.unwrap()( + stream.stream_ptr, + Some(callback), + ) }, ffi::CUBEB_OK ); - println!("Stream {:p} now has a device change callback.", stream_ptr); + println!( + "Stream {:p} now has a device change callback.", + stream.stream_ptr + ); } - fn destroy_stream(stream_ptr: &mut *mut ffi::cubeb_stream) { - if stream_ptr.is_null() { + fn destroy_stream(stream: &mut StreamData) { + if stream.stream_ptr.is_null() { println!("No need to destroy stream."); return; } unsafe { - OPS.stream_destroy.unwrap()(*stream_ptr); + OPS.stream_destroy.unwrap()((*stream).stream_ptr); } - println!("Stream {:p} destroyed.", *stream_ptr); - *stream_ptr = ptr::null_mut(); + println!("Stream {:p} destroyed.", stream.stream_ptr); + stream.stream_ptr = ptr::null_mut(); } fn create_stream( - stream_ptr: &mut *mut ffi::cubeb_stream, context_ptr: *mut ffi::cubeb, - enable_loopback: &AtomicBool, + streams: &mut StreamsData, + input_prefs: StreamPrefs, + output_prefs: StreamPrefs, ) { - if !stream_ptr.is_null() { + if streams.len() == 0 || !streams.current().stream_ptr.is_null() { + println!("Allocating stream {}.", streams.len() + 1); + streams.push(StreamData::new()); + streams.select(streams.len() - 1); + } + + let stream = streams.current_mut(); + if !stream.stream_ptr.is_null() { println!("Stream has been created."); return; } @@ -486,8 +640,8 @@ fn test_stream_tester() { } }; - let mut input_params = get_dummy_stream_params(Scope::Input); - let mut output_params = get_dummy_stream_params(Scope::Output); + let mut input_params = get_dummy_stream_params(Scope::Input, input_prefs); + let mut output_params = get_dummy_stream_params(Scope::Output, output_prefs); let (input_device, input_stream_params) = if stream_type.contains(StreamType::INPUT) { ( @@ -519,7 +673,7 @@ fn test_stream_tester() { unsafe { OPS.stream_init.unwrap()( context_ptr, - stream_ptr, + &mut stream.stream_ptr, stream_name.as_ptr(), input_device as ffi::cubeb_devid, input_stream_params, @@ -528,13 +682,13 @@ fn test_stream_tester() { 4096, // latency Some(data_callback), Some(state_callback), - enable_loopback as *const AtomicBool as *mut c_void, // user pointer + &stream.enable_loopback as *const AtomicBool as *mut c_void, // user pointer ) }, ffi::CUBEB_OK ); - assert!(!stream_ptr.is_null()); - println!("Stream {:p} created.", *stream_ptr); + assert!(!stream.stream_ptr.is_null()); + println!("Stream {:p} created.", stream.stream_ptr); extern "C" fn state_callback( stream: *mut ffi::cubeb_stream, @@ -592,14 +746,14 @@ fn test_stream_tester() { nframes } - fn get_dummy_stream_params(scope: Scope) -> ffi::cubeb_stream_params { + fn get_dummy_stream_params(scope: Scope, prefs: StreamPrefs) -> ffi::cubeb_stream_params { // The stream format for input and output must be same. const STREAM_FORMAT: u32 = ffi::CUBEB_SAMPLE_FLOAT32NE; // Make sure the parameters meet the requirements of AudioUnitContext::stream_init // (in the comments). let mut stream_params = ffi::cubeb_stream_params::default(); - stream_params.prefs = ffi::CUBEB_STREAM_PREF_VOICE; + stream_params.prefs = prefs.bits(); let (format, rate, channels, layout) = match scope { Scope::Input => (STREAM_FORMAT, 48000, 1, ffi::CUBEB_LAYOUT_MONO), Scope::Output => (STREAM_FORMAT, 44100, 2, ffi::CUBEB_LAYOUT_STEREO), diff --git a/third_party/rust/cubeb-coreaudio/src/backend/tests/parallel.rs b/third_party/rust/cubeb-coreaudio/src/backend/tests/parallel.rs index 16063d0011..be5edb1919 100644 --- a/third_party/rust/cubeb-coreaudio/src/backend/tests/parallel.rs +++ b/third_party/rust/cubeb-coreaudio/src/backend/tests/parallel.rs @@ -537,14 +537,16 @@ fn test_set_buffer_frame_size_in_parallel_in_scope(scope: Scope) { units.push(test_get_default_audiounit(scope.clone()).unwrap()); let unit_value = units.last().unwrap().get_inner() as usize; join_handles.push(thread::spawn(move || { - let status = audio_unit_set_property( - unit_value as AudioUnit, - kAudioDevicePropertyBufferFrameSize, - unit_scope, - unit_element, - &latency_frames, - mem::size_of::(), - ); + let status = run_serially(|| { + audio_unit_set_property( + unit_value as AudioUnit, + kAudioDevicePropertyBufferFrameSize, + unit_scope, + unit_element, + &latency_frames, + mem::size_of::(), + ) + }); (latency_frames, status) })); } diff --git a/third_party/rust/cubeb-coreaudio/src/backend/tests/tone.rs b/third_party/rust/cubeb-coreaudio/src/backend/tests/tone.rs index 42cb9ee997..5f54286153 100644 --- a/third_party/rust/cubeb-coreaudio/src/backend/tests/tone.rs +++ b/third_party/rust/cubeb-coreaudio/src/backend/tests/tone.rs @@ -1,6 +1,6 @@ use super::utils::{test_get_default_device, test_ops_stream_operation, Scope}; use super::*; -use std::sync::atomic::{AtomicI64, Ordering}; +use std::sync::atomic::AtomicI64; #[test] fn test_dial_tone() { @@ -202,7 +202,7 @@ fn test_dial_tone() { for data in buffer.iter_mut() { let t1 = (2.0 * PI * 350.0 * (closure.phase) as f32 / SAMPLE_FREQUENCY as f32).sin(); let t2 = (2.0 * PI * 440.0 * (closure.phase) as f32 / SAMPLE_FREQUENCY as f32).sin(); - *data = f32_to_i16_sample(0.5 * (t1 + t2)); + *data = f32_to_i16_sample(0.45 * (t1 + t2)); closure.phase += 1; } diff --git a/third_party/rust/cubeb-coreaudio/src/backend/tests/utils.rs b/third_party/rust/cubeb-coreaudio/src/backend/tests/utils.rs index ef07aeeeb4..6fb6d38fb3 100644 --- a/third_party/rust/cubeb-coreaudio/src/backend/tests/utils.rs +++ b/third_party/rust/cubeb-coreaudio/src/backend/tests/utils.rs @@ -25,6 +25,29 @@ pub extern "C" fn noop_data_callback( nframes } +pub extern "C" fn draining_data_callback( + stream: *mut ffi::cubeb_stream, + _user_ptr: *mut c_void, + _input_buffer: *const c_void, + output_buffer: *mut c_void, + nframes: i64, +) -> i64 { + assert!(!stream.is_null()); + + // Feed silence data to output buffer + if !output_buffer.is_null() { + let stm = unsafe { &mut *(stream as *mut AudioUnitStream) }; + let channels = stm.core_stream_data.output_stream_params.channels(); + let samples = nframes as usize * channels as usize; + let sample_size = cubeb_sample_size(stm.core_stream_data.output_stream_params.format()); + unsafe { + ptr::write_bytes(output_buffer, 0, samples * sample_size); + } + } + + nframes - 1 +} + #[derive(Clone, Debug, PartialEq)] pub enum Scope { Input, @@ -47,31 +70,35 @@ pub enum PropertyScope { } pub fn test_get_default_device(scope: Scope) -> Option { - let address = AudioObjectPropertyAddress { - mSelector: match scope { - Scope::Input => kAudioHardwarePropertyDefaultInputDevice, - Scope::Output => kAudioHardwarePropertyDefaultOutputDevice, - }, - mScope: kAudioObjectPropertyScopeGlobal, - mElement: kAudioObjectPropertyElementMaster, - }; + debug_assert_not_running_serially(); + run_serially_forward_panics(|| { + let address = AudioObjectPropertyAddress { + mSelector: match scope { + Scope::Input => kAudioHardwarePropertyDefaultInputDevice, + Scope::Output => kAudioHardwarePropertyDefaultOutputDevice, + }, + mScope: kAudioObjectPropertyScopeGlobal, + mElement: kAudioObjectPropertyElementMaster, + }; - let mut devid: AudioObjectID = kAudioObjectUnknown; - let mut size = mem::size_of::(); - let status = unsafe { - AudioObjectGetPropertyData( - kAudioObjectSystemObject, - &address, - 0, - ptr::null(), - &mut size as *mut usize as *mut UInt32, - &mut devid as *mut AudioObjectID as *mut c_void, - ) - }; - if status != NO_ERR || devid == kAudioObjectUnknown { - return None; - } - Some(devid) + let mut devid: AudioObjectID = kAudioObjectUnknown; + let mut size = mem::size_of::(); + let status = unsafe { + AudioObjectGetPropertyData( + kAudioObjectSystemObject, + &address, + 0, + ptr::null(), + &mut size as *mut usize as *mut UInt32, + &mut devid as *mut AudioObjectID as *mut c_void, + ) + }; + + if status != NO_ERR || devid == kAudioObjectUnknown { + return None; + } + Some(devid) + }) } // TODO: Create a GetProperty trait and add a default implementation for it, then implement it @@ -99,16 +126,17 @@ impl TestAudioUnit { impl Drop for TestAudioUnit { fn drop(&mut self) { - unsafe { + run_serially_forward_panics(|| unsafe { AudioUnitUninitialize(self.0); AudioComponentInstanceDispose(self.0); - } + }); } } // TODO: 1. Return Result with custom errors. // 2. Allow to create a in-out unit. pub fn test_get_default_audiounit(scope: Scope) -> Option { + debug_assert_not_running_serially(); let device = test_get_default_device(scope.clone()); let unit = test_create_audiounit(ComponentSubType::HALOutput); if device.is_none() || unit.is_none() { @@ -133,7 +161,7 @@ pub fn test_get_default_audiounit(scope: Scope) -> Option { } } - let status = unsafe { + let status = run_serially(|| unsafe { AudioUnitSetProperty( unit.get_inner(), kAudioOutputUnitProperty_CurrentDevice, @@ -142,7 +170,7 @@ pub fn test_get_default_audiounit(scope: Scope) -> Option { &device as *const AudioObjectID as *const c_void, mem::size_of::() as u32, ) - }; + }); if status == NO_ERR { Some(unit) } else { @@ -159,6 +187,7 @@ pub enum ComponentSubType { // Surprisingly the AudioUnit can be created even when there is no any device on the platform, // no matter its subtype is HALOutput or DefaultOutput. pub fn test_create_audiounit(unit_type: ComponentSubType) -> Option { + debug_assert_not_running_serially(); let desc = AudioComponentDescription { componentType: kAudioUnitType_Output, componentSubType: match unit_type { @@ -169,12 +198,12 @@ pub fn test_create_audiounit(unit_type: ComponentSubType) -> Option std::result::Result<(), OSStatus> { + debug_assert_not_running_serially(); assert!(!unit.is_null()); let (scope, element) = match scope { Scope::Input => (kAudioUnitScope_Input, AU_IN_BUS), Scope::Output => (kAudioUnitScope_Output, AU_OUT_BUS), }; let on_off: u32 = if enable { 1 } else { 0 }; - let status = unsafe { + let status = run_serially(|| unsafe { AudioUnitSetProperty( unit, kAudioOutputUnitProperty_EnableIO, @@ -203,7 +233,7 @@ fn test_enable_audiounit_in_scope( &on_off as *const u32 as *const c_void, mem::size_of::() as u32, ) - }; + }); if status == NO_ERR { Ok(()) } else { @@ -216,72 +246,80 @@ pub enum DeviceFilter { IncludeAll, } pub fn test_get_all_devices(filter: DeviceFilter) -> Vec { - let mut devices = Vec::new(); - let address = AudioObjectPropertyAddress { - mSelector: kAudioHardwarePropertyDevices, - mScope: kAudioObjectPropertyScopeGlobal, - mElement: kAudioObjectPropertyElementMaster, - }; - let mut size: usize = 0; - let status = unsafe { - AudioObjectGetPropertyDataSize( - kAudioObjectSystemObject, - &address, - 0, - ptr::null(), - &mut size as *mut usize as *mut u32, - ) - }; - // size will be 0 if there is no device at all. - if status != NO_ERR || size == 0 { - return devices; - } - assert_eq!(size % mem::size_of::(), 0); - let elements = size / mem::size_of::(); - devices.resize(elements, kAudioObjectUnknown); - let status = unsafe { - AudioObjectGetPropertyData( - kAudioObjectSystemObject, - &address, - 0, - ptr::null(), - &mut size as *mut usize as *mut u32, - devices.as_mut_ptr() as *mut c_void, - ) - }; - if status != NO_ERR { - devices.clear(); - return devices; - } - for device in devices.iter() { - assert_ne!(*device, kAudioObjectUnknown); - } + debug_assert_not_running_serially(); + // To avoid races, the devices getter and the device name filtering have + // to run in the same serial task. If not, a device may exist when the + // getter runs but not when getting its uid. + run_serially_forward_panics(|| { + let mut devices = Vec::new(); + let address = AudioObjectPropertyAddress { + mSelector: kAudioHardwarePropertyDevices, + mScope: kAudioObjectPropertyScopeGlobal, + mElement: kAudioObjectPropertyElementMaster, + }; + let mut size: usize = 0; + let status = unsafe { + AudioObjectGetPropertyDataSize( + kAudioObjectSystemObject, + &address, + 0, + ptr::null(), + &mut size as *mut usize as *mut u32, + ) + }; + // size will be 0 if there is no device at all. + if status != NO_ERR || size == 0 { + return devices; + } + assert_eq!(size % mem::size_of::(), 0); + let elements = size / mem::size_of::(); + devices.resize(elements, kAudioObjectUnknown); + let status = unsafe { + AudioObjectGetPropertyData( + kAudioObjectSystemObject, + &address, + 0, + ptr::null(), + &mut size as *mut usize as *mut u32, + devices.as_mut_ptr() as *mut c_void, + ) + }; + if status != NO_ERR { + devices.clear(); + return devices; + } + for device in devices.iter() { + assert_ne!(*device, kAudioObjectUnknown); + } - match filter { - DeviceFilter::ExcludeCubebAggregateAndVPIO => { - devices.retain(|&device| { - if let Ok(uid) = get_device_global_uid(device) { - let uid = uid.into_string(); - !uid.contains(PRIVATE_AGGREGATE_DEVICE_NAME) - && !uid.contains(VOICEPROCESSING_AGGREGATE_DEVICE_NAME) - } else { - true - } - }); + match filter { + DeviceFilter::ExcludeCubebAggregateAndVPIO => { + devices.retain(|&device| { + if let Ok(uid) = get_device_global_uid(device) { + let uid = uid.into_string(); + !uid.contains(PRIVATE_AGGREGATE_DEVICE_NAME) + && !uid.contains(VOICEPROCESSING_AGGREGATE_DEVICE_NAME) + } else { + true + } + }); + } + _ => {} } - _ => {} - } - devices + devices + }) } pub fn test_get_devices_in_scope(scope: Scope) -> Vec { + debug_assert_not_running_serially(); let mut devices = test_get_all_devices(DeviceFilter::ExcludeCubebAggregateAndVPIO); devices.retain(|device| test_device_in_scope(*device, scope.clone())); devices } pub fn get_devices_info_in_scope(scope: Scope) -> Vec { + debug_assert_not_running_serially(); fn print_info(info: &TestDeviceInfo) { println!("{:>4}: {}\n\tuid: {}", info.id, info.label, info.uid); } @@ -337,8 +375,9 @@ pub fn test_device_channels_in_scope( id: AudioObjectID, scope: Scope, ) -> std::result::Result { + debug_assert_not_running_serially(); let address = AudioObjectPropertyAddress { - mSelector: kAudioDevicePropertyStreamConfiguration, + mSelector: kAudioDevicePropertyStreams, mScope: match scope { Scope::Input => kAudioDevicePropertyScopeInput, Scope::Output => kAudioDevicePropertyScopeOutput, @@ -346,7 +385,7 @@ pub fn test_device_channels_in_scope( mElement: kAudioObjectPropertyElementMaster, }; let mut size: usize = 0; - let status = unsafe { + let status = run_serially(|| unsafe { AudioObjectGetPropertyDataSize( id, &address, @@ -354,49 +393,90 @@ pub fn test_device_channels_in_scope( ptr::null(), &mut size as *mut usize as *mut u32, ) - }; + }); if status != NO_ERR { return Err(status); } if size == 0 { return Ok(0); } - let byte_len = size / mem::size_of::(); - let mut bytes = vec![0u8; byte_len]; - let status = unsafe { + let mut stream_list = vec![0, (size / mem::size_of::()) as u32]; + let status = run_serially(|| unsafe { AudioObjectGetPropertyData( id, &address, 0, ptr::null(), &mut size as *mut usize as *mut u32, - bytes.as_mut_ptr() as *mut c_void, + stream_list.as_mut_ptr() as *mut c_void, ) - }; + }); if status != NO_ERR { return Err(status); } - let buf_list = unsafe { &*(bytes.as_mut_ptr() as *mut AudioBufferList) }; - let buf_len = buf_list.mNumberBuffers as usize; - if buf_len == 0 { - return Ok(0); - } - let buf_ptr = buf_list.mBuffers.as_ptr() as *const AudioBuffer; - let buffers = unsafe { slice::from_raw_parts(buf_ptr, buf_len) }; - let mut channels: u32 = 0; - for buffer in buffers { - channels += buffer.mNumberChannels; - } + let channels = stream_list + .iter() + .filter(|s: &&AudioObjectID| { + if scope != Scope::Input { + return true; + } + let address = AudioObjectPropertyAddress { + mSelector: kAudioStreamPropertyTerminalType, + mScope: kAudioObjectPropertyScopeGlobal, + mElement: kAudioObjectPropertyElementMaster, + }; + let mut ttype: u32 = 0; + let status = unsafe { + AudioObjectGetPropertyData( + **s, + &address, + 0, + ptr::null(), + &mut mem::size_of::() as *mut usize as *mut u32, + &mut ttype as *mut u32 as *mut c_void, + ) + }; + if status != NO_ERR { + return false; + } + ttype == kAudioStreamTerminalTypeMicrophone + || (INPUT_MICROPHONE..OUTPUT_UNDEFINED).contains(&ttype) + }) + .map(|s: &AudioObjectID| { + let address = AudioObjectPropertyAddress { + mSelector: kAudioStreamPropertyVirtualFormat, + mScope: kAudioObjectPropertyScopeGlobal, + mElement: kAudioObjectPropertyElementMaster, + }; + let mut format = AudioStreamBasicDescription::default(); + let status = unsafe { + AudioObjectGetPropertyData( + *s, + &address, + 0, + ptr::null(), + &mut mem::size_of::() as *mut usize as *mut u32, + &mut format as *mut AudioStreamBasicDescription as *mut c_void, + ) + }; + if status != NO_ERR { + return 0; + } + format.mChannelsPerFrame + }) + .sum(); Ok(channels) } pub fn test_device_in_scope(id: AudioObjectID, scope: Scope) -> bool { + debug_assert_not_running_serially(); let channels = test_device_channels_in_scope(id, scope); channels.is_ok() && channels.unwrap() > 0 } pub fn test_get_all_onwed_devices(id: AudioDeviceID) -> Vec { assert_ne!(id, kAudioObjectUnknown); + debug_assert_running_serially(); let address = AudioObjectPropertyAddress { mSelector: kAudioObjectPropertyOwnedObjects, @@ -409,45 +489,46 @@ pub fn test_get_all_onwed_devices(id: AudioDeviceID) -> Vec { let qualifier_data = &class_id; let mut size: usize = 0; - unsafe { - assert_eq!( + assert_eq!( + unsafe { AudioObjectGetPropertyDataSize( id, &address, qualifier_data_size as u32, qualifier_data as *const u32 as *const c_void, - &mut size as *mut usize as *mut u32 - ), - NO_ERR - ); - } + &mut size as *mut usize as *mut u32, + ) + }, + NO_ERR + ); assert_ne!(size, 0); let elements = size / mem::size_of::(); let mut devices: Vec = allocate_array(elements); - unsafe { - assert_eq!( + assert_eq!( + unsafe { AudioObjectGetPropertyData( id, &address, qualifier_data_size as u32, qualifier_data as *const u32 as *const c_void, &mut size as *mut usize as *mut u32, - devices.as_mut_ptr() as *mut c_void - ), - NO_ERR - ); - } + devices.as_mut_ptr() as *mut c_void, + ) + }, + NO_ERR + ); devices } pub fn test_get_master_device(id: AudioObjectID) -> String { assert_ne!(id, kAudioObjectUnknown); + debug_assert_running_serially(); let address = AudioObjectPropertyAddress { - mSelector: kAudioAggregateDevicePropertyMasterSubDevice, + mSelector: kAudioAggregateDevicePropertyMainSubDevice, mScope: kAudioObjectPropertyScopeGlobal, mElement: kAudioObjectPropertyElementMaster, }; @@ -465,6 +546,7 @@ pub fn test_get_master_device(id: AudioObjectID) -> String { } pub fn test_get_drift_compensations(id: AudioObjectID) -> std::result::Result { + debug_assert_running_serially(); let address = AudioObjectPropertyAddress { mSelector: kAudioSubDevicePropertyDriftCompensation, mScope: kAudioObjectPropertyScopeGlobal, @@ -491,6 +573,7 @@ pub fn test_get_drift_compensations(id: AudioObjectID) -> std::result::Result bool { assert!(!unit.is_null()); + debug_assert_not_running_serially(); let mut has_io: UInt32 = 0; let (scope, element) = match scope { Scope::Input => (kAudioUnitScope_Input, AU_IN_BUS), @@ -498,14 +581,14 @@ pub fn test_audiounit_scope_is_enabled(unit: AudioUnit, scope: Scope) -> bool { }; let mut size = mem::size_of::(); assert_eq!( - audio_unit_get_property( + run_serially(|| audio_unit_get_property( unit, kAudioOutputUnitProperty_HasIO, scope, element, &mut has_io, &mut size - ), + )), NO_ERR ); has_io != 0 @@ -516,6 +599,7 @@ pub fn test_audiounit_get_buffer_frame_size( scope: Scope, prop_scope: PropertyScope, ) -> std::result::Result { + debug_assert_not_running_serially(); let element = match scope { Scope::Input => AU_IN_BUS, Scope::Output => AU_OUT_BUS, @@ -526,7 +610,7 @@ pub fn test_audiounit_get_buffer_frame_size( }; let mut buffer_frames: u32 = 0; let mut size = mem::size_of::(); - let status = unsafe { + let status = run_serially(|| unsafe { AudioUnitGetProperty( unit, kAudioDevicePropertyBufferFrameSize, @@ -535,7 +619,7 @@ pub fn test_audiounit_get_buffer_frame_size( &mut buffer_frames as *mut u32 as *mut c_void, &mut size as *mut usize as *mut u32, ) - }; + }); if status == NO_ERR { Ok(buffer_frames) } else { @@ -554,6 +638,7 @@ pub fn test_set_default_device( device: AudioObjectID, scope: Scope, ) -> std::result::Result { + debug_assert_not_running_serially(); assert!(test_device_in_scope(device, scope.clone())); let default = test_get_default_device(scope.clone()).unwrap(); if default == device { @@ -570,7 +655,7 @@ pub fn test_set_default_device( mElement: kAudioObjectPropertyElementMaster, }; let size = mem::size_of::(); - let status = unsafe { + let status = run_serially(|| unsafe { AudioObjectSetPropertyData( kAudioObjectSystemObject, &address, @@ -579,7 +664,7 @@ pub fn test_set_default_device( size as u32, &device as *const AudioObjectID as *const c_void, ) - }; + }); let new_default = test_get_default_device(scope.clone()).unwrap(); if new_default == default { Err(-1) @@ -644,6 +729,7 @@ pub fn test_create_device_change_listener(scope: Scope, listener: F) -> TestP where F: Fn(&[AudioObjectPropertyAddress]) -> OSStatus, { + debug_assert_running_serially(); let address = AudioObjectPropertyAddress { mSelector: match scope { Scope::Input => kAudioHardwarePropertyDefaultInputDevice, @@ -652,6 +738,7 @@ where mScope: kAudioObjectPropertyScopeGlobal, mElement: kAudioObjectPropertyElementMaster, }; + TestPropertyListener::new(kAudioObjectSystemObject, address, listener) } @@ -677,6 +764,7 @@ where } pub fn start(&self) -> std::result::Result<(), OSStatus> { + debug_assert_running_serially(); let status = unsafe { AudioObjectAddPropertyListener( self.device, @@ -693,6 +781,7 @@ where } pub fn stop(&self) -> std::result::Result<(), OSStatus> { + debug_assert_running_serially(); let status = unsafe { AudioObjectRemovePropertyListener( self.device, @@ -726,7 +815,7 @@ where F: Fn(&[AudioObjectPropertyAddress]) -> OSStatus, { fn drop(&mut self) { - self.stop(); + run_serially(|| self.stop()); } } @@ -767,6 +856,7 @@ impl TestDevicePlugger { } fn destroy_aggregate_device(&mut self) -> std::result::Result<(), OSStatus> { + debug_assert_not_running_serially(); assert_ne!(self.plugin_id, kAudioObjectUnknown); assert_ne!(self.device_id, kAudioObjectUnknown); @@ -777,7 +867,7 @@ impl TestDevicePlugger { }; let mut size: usize = 0; - let status = unsafe { + let status = run_serially(|| unsafe { AudioObjectGetPropertyDataSize( self.plugin_id, &address, @@ -785,13 +875,13 @@ impl TestDevicePlugger { ptr::null(), &mut size as *mut usize as *mut u32, ) - }; + }); if status != NO_ERR { return Err(status); } assert_ne!(size, 0); - let status = unsafe { + let status = run_serially(|| unsafe { // This call can simulate removing a device. AudioObjectGetPropertyData( self.plugin_id, @@ -801,7 +891,7 @@ impl TestDevicePlugger { &mut size as *mut usize as *mut u32, &mut self.device_id as *mut AudioDeviceID as *mut c_void, ) - }; + }); if status == NO_ERR { self.device_id = kAudioObjectUnknown; Ok(()) @@ -811,6 +901,7 @@ impl TestDevicePlugger { } fn create_aggregate_device(&self) -> std::result::Result { + debug_assert_not_running_serially(); use std::time::{SystemTime, UNIX_EPOCH}; const TEST_AGGREGATE_DEVICE_NAME: &str = "TestAggregateDevice"; @@ -830,7 +921,7 @@ impl TestDevicePlugger { }; let mut size: usize = 0; - let status = unsafe { + let status = run_serially(|| unsafe { AudioObjectGetPropertyDataSize( self.plugin_id, &address, @@ -838,7 +929,7 @@ impl TestDevicePlugger { ptr::null(), &mut size as *mut usize as *mut u32, ) - }; + }); if status != NO_ERR { return Err(status); } @@ -915,14 +1006,18 @@ impl TestDevicePlugger { CFRelease(sub_devices as *const c_void); // This call can simulate adding a device. - let status = AudioObjectGetPropertyData( - self.plugin_id, - &address, - mem::size_of_val(&device_dict) as u32, - &device_dict as *const CFMutableDictionaryRef as *const c_void, - &mut size as *mut usize as *mut u32, - &mut device_id as *mut AudioDeviceID as *mut c_void, - ); + let status = { + run_serially(|| { + AudioObjectGetPropertyData( + self.plugin_id, + &address, + mem::size_of_val(&device_dict) as u32, + &device_dict as *const CFMutableDictionaryRef as *const c_void, + &mut size as *mut usize as *mut u32, + &mut device_id as *mut AudioDeviceID as *mut c_void, + ) + }) + }; CFRelease(device_dict as *const c_void); status }; @@ -935,6 +1030,7 @@ impl TestDevicePlugger { } fn get_system_plugin_id() -> std::result::Result { + debug_assert_not_running_serially(); let address = AudioObjectPropertyAddress { mSelector: kAudioHardwarePropertyPlugInForBundleID, mScope: kAudioObjectPropertyScopeGlobal, @@ -942,7 +1038,7 @@ impl TestDevicePlugger { }; let mut size: usize = 0; - let status = unsafe { + let status = run_serially(|| unsafe { AudioObjectGetPropertyDataSize( kAudioObjectSystemObject, &address, @@ -950,7 +1046,7 @@ impl TestDevicePlugger { ptr::null(), &mut size as *mut usize as *mut u32, ) - }; + }); if status != NO_ERR { return Err(status); } @@ -967,14 +1063,16 @@ impl TestDevicePlugger { assert_eq!(size, mem::size_of_val(&translation_value)); let status = unsafe { - let status = AudioObjectGetPropertyData( - kAudioObjectSystemObject, - &address, - 0, - ptr::null(), - &mut size as *mut usize as *mut u32, - &mut translation_value as *mut AudioValueTranslation as *mut c_void, - ); + let status = run_serially(|| { + AudioObjectGetPropertyData( + kAudioObjectSystemObject, + &address, + 0, + ptr::null(), + &mut size as *mut usize as *mut u32, + &mut translation_value as *mut AudioValueTranslation as *mut c_void, + ) + }); CFRelease(in_bundle_ref as *const c_void); status }; @@ -991,10 +1089,11 @@ impl TestDevicePlugger { // them into the array, if the device is an aggregate device. See the code in // AggregateDevice::get_sub_devices and audiounit_set_aggregate_sub_device_list. fn get_sub_devices(scope: Scope) -> Option { + debug_assert_not_running_serially(); let device = test_get_default_device(scope); device?; let device = device.unwrap(); - let uid = get_device_global_uid(device); + let uid = run_serially(|| get_device_global_uid(device)); if uid.is_err() { return None; } @@ -1033,6 +1132,7 @@ pub fn test_ops_context_operation(name: &'static str, operation: F) where F: FnOnce(*mut ffi::cubeb), { + debug_assert_not_running_serially(); let name_c_string = CString::new(name).expect("Failed to create context name"); let mut context = ptr::null_mut::(); assert_eq!( @@ -1047,8 +1147,9 @@ where // The in-out stream initializeed with different device will create an aggregate_device and // result in firing device-collection-changed callbacks. Run in-out streams with tests // capturing device-collection-changed callbacks may cause troubles. -pub fn test_ops_stream_operation( +pub fn test_ops_stream_operation_on_context( name: &'static str, + context_ptr: *mut ffi::cubeb, input_device: ffi::cubeb_devid, input_stream_params: *mut ffi::cubeb_stream_params, output_device: ffi::cubeb_devid, @@ -1061,43 +1162,72 @@ pub fn test_ops_stream_operation( ) where F: FnOnce(*mut ffi::cubeb_stream), { - test_ops_context_operation("context: stream operation", |context_ptr| { - // Do nothing if there is no input/output device to perform input/output tests. - if !input_stream_params.is_null() && test_get_default_device(Scope::Input).is_none() { - println!("No input device to perform input tests for \"{}\".", name); - return; - } + // Do nothing if there is no input/output device to perform input/output tests. + if !input_stream_params.is_null() && test_get_default_device(Scope::Input).is_none() { + println!("No input device to perform input tests for \"{}\".", name); + return; + } - if !output_stream_params.is_null() && test_get_default_device(Scope::Output).is_none() { - println!("No output device to perform output tests for \"{}\".", name); - return; - } + if !output_stream_params.is_null() && test_get_default_device(Scope::Output).is_none() { + println!("No output device to perform output tests for \"{}\".", name); + return; + } - let mut stream: *mut ffi::cubeb_stream = ptr::null_mut(); - let stream_name = CString::new(name).expect("Failed to create stream name"); - assert_eq!( - unsafe { - OPS.stream_init.unwrap()( - context_ptr, - &mut stream, - stream_name.as_ptr(), - input_device, - input_stream_params, - output_device, - output_stream_params, - latency_frames, - data_callback, - state_callback, - user_ptr, - ) - }, - ffi::CUBEB_OK - ); - assert!(!stream.is_null()); - operation(stream); + let mut stream: *mut ffi::cubeb_stream = ptr::null_mut(); + let stream_name = CString::new(name).expect("Failed to create stream name"); + assert_eq!( unsafe { - OPS.stream_destroy.unwrap()(stream); - } + OPS.stream_init.unwrap()( + context_ptr, + &mut stream, + stream_name.as_ptr(), + input_device, + input_stream_params, + output_device, + output_stream_params, + latency_frames, + data_callback, + state_callback, + user_ptr, + ) + }, + ffi::CUBEB_OK + ); + assert!(!stream.is_null()); + operation(stream); + unsafe { + OPS.stream_destroy.unwrap()(stream); + } +} + +pub fn test_ops_stream_operation( + name: &'static str, + input_device: ffi::cubeb_devid, + input_stream_params: *mut ffi::cubeb_stream_params, + output_device: ffi::cubeb_devid, + output_stream_params: *mut ffi::cubeb_stream_params, + latency_frames: u32, + data_callback: ffi::cubeb_data_callback, + state_callback: ffi::cubeb_state_callback, + user_ptr: *mut c_void, + operation: F, +) where + F: FnOnce(*mut ffi::cubeb_stream), +{ + test_ops_context_operation("context: stream operation", |context_ptr| { + test_ops_stream_operation_on_context( + name, + context_ptr, + input_device, + input_stream_params, + output_device, + output_stream_params, + latency_frames, + data_callback, + state_callback, + user_ptr, + operation, + ); }); } @@ -1136,7 +1266,7 @@ fn test_get_raw_stream( user_ptr, data_callback, state_callback, - global_latency_frames.unwrap(), + global_latency_frames, ); stream.core_stream_data = CoreStreamData::new(&stream, None, None); @@ -1154,6 +1284,7 @@ pub fn test_get_stream_with_default_data_callback_by_type( ) where F: FnOnce(&mut AudioUnitStream), { + debug_assert_not_running_serially(); let mut input_params = get_dummy_stream_params(Scope::Input); let mut output_params = get_dummy_stream_params(Scope::Output); diff --git a/third_party/rust/d3d12/.cargo-checksum.json b/third_party/rust/d3d12/.cargo-checksum.json index 8b8a944bf0..8b27c96861 100644 --- a/third_party/rust/d3d12/.cargo-checksum.json +++ b/third_party/rust/d3d12/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"CHANGELOG.md":"45fa76b0e5bc51721887147000e9e78a5934cb04d1ad628e501ef2082763d353","Cargo.toml":"1108efc3deca25c270c3f8631abca07e7e696aace92d324ef8b83503cb2edbc3","README.md":"76cee3209f773a62535de6c9724b53f158406359f35b4d48b17ac3747b6c102e","src/com.rs":"cfd6556a7abf38cba57559038f9f2cf86274418448fb2745436c251a99575e05","src/command_allocator.rs":"ef01059a661749470f3772d188fe0fab0f002e1d154facdab4b9b2932f4b2d93","src/command_list.rs":"8723f3b755b721e0dbb234bd604956c1b7922a2368231197495daa3fa6548e63","src/debug.rs":"aa33b98f7c3e71cba75fc42c6ca9af72d96b45122422c16e48525e24590c57bf","src/descriptor.rs":"fea0b820de1566b54d17d8d0c67e6f5a2126eda19526397eb710ff7d6db9db9e","src/device.rs":"c1dd479aabd22bced0d407523d60629ad1da439fb47ad89fe7b48bae1c4b23e5","src/dxgi.rs":"1516186845b91bf3df813a29b4a0e00a85ca5649fb7a2755da43fba984c41a42","src/heap.rs":"dae2380684896c97e97ed022929f79ce2cc4f5418a3ec34883086f7c88f423d0","src/lib.rs":"612e2f471b84502d219da3fb86ee13f3cbd6faf17d77407bab6c84e51ec424d0","src/pso.rs":"ff819c321536695e34a3be9a6051cf3e57765049a4a2035db6ab27add5a7978a","src/query.rs":"ff61a2b76a108afc1f082724bb9b07ac8b52afbe97356e0fcf6df0ff7e53e07d","src/queue.rs":"bd32813d0b8a3bedf3223b69ade9f9c799a138a9e27d970f86435d9ce32d1557","src/resource.rs":"8989cdb7c3ee0687c826047f39f85148459d9219754f20a970bf8aaa09b96e27","src/sync.rs":"5c287fb7498242a397eb1f08887be9cff9b48dc7cb13af5792cce5f7182b55f8"},"package":null} \ No newline at end of file +{"files":{"CHANGELOG.md":"45fa76b0e5bc51721887147000e9e78a5934cb04d1ad628e501ef2082763d353","Cargo.toml":"a3135c67216ba021525ebc8d18dc3de5d779f1a1ddde5f25f4439acabd45824a","README.md":"76cee3209f773a62535de6c9724b53f158406359f35b4d48b17ac3747b6c102e","src/com.rs":"cfd6556a7abf38cba57559038f9f2cf86274418448fb2745436c251a99575e05","src/command_allocator.rs":"ef01059a661749470f3772d188fe0fab0f002e1d154facdab4b9b2932f4b2d93","src/command_list.rs":"8723f3b755b721e0dbb234bd604956c1b7922a2368231197495daa3fa6548e63","src/debug.rs":"aa33b98f7c3e71cba75fc42c6ca9af72d96b45122422c16e48525e24590c57bf","src/descriptor.rs":"fea0b820de1566b54d17d8d0c67e6f5a2126eda19526397eb710ff7d6db9db9e","src/device.rs":"c1dd479aabd22bced0d407523d60629ad1da439fb47ad89fe7b48bae1c4b23e5","src/dxgi.rs":"1516186845b91bf3df813a29b4a0e00a85ca5649fb7a2755da43fba984c41a42","src/heap.rs":"dae2380684896c97e97ed022929f79ce2cc4f5418a3ec34883086f7c88f423d0","src/lib.rs":"612e2f471b84502d219da3fb86ee13f3cbd6faf17d77407bab6c84e51ec424d0","src/pso.rs":"ff819c321536695e34a3be9a6051cf3e57765049a4a2035db6ab27add5a7978a","src/query.rs":"ff61a2b76a108afc1f082724bb9b07ac8b52afbe97356e0fcf6df0ff7e53e07d","src/queue.rs":"bd32813d0b8a3bedf3223b69ade9f9c799a138a9e27d970f86435d9ce32d1557","src/resource.rs":"8989cdb7c3ee0687c826047f39f85148459d9219754f20a970bf8aaa09b96e27","src/sync.rs":"5c287fb7498242a397eb1f08887be9cff9b48dc7cb13af5792cce5f7182b55f8"},"package":null} \ No newline at end of file diff --git a/third_party/rust/d3d12/Cargo.toml b/third_party/rust/d3d12/Cargo.toml index 88aedd92f7..1425e10b80 100644 --- a/third_party/rust/d3d12/Cargo.toml +++ b/third_party/rust/d3d12/Cargo.toml @@ -40,7 +40,7 @@ implicit-link = [] bitflags = "2" [target."cfg(windows)".dependencies.libloading] -version = ">=0.7,<0.9" +version = ">=0.7, <0.9" optional = true [target."cfg(windows)".dependencies.winapi] diff --git a/third_party/rust/embed-manifest/.cargo-checksum.json b/third_party/rust/embed-manifest/.cargo-checksum.json new file mode 100644 index 0000000000..bb64810ff2 --- /dev/null +++ b/third_party/rust/embed-manifest/.cargo-checksum.json @@ -0,0 +1 @@ +{"files":{"CHANGELOG.md":"a1cbcc712504ce6658dc209cab886c724654e6e0ef7b26315fd86d27e04a6e09","Cargo.toml":"4f091eabd120a0c1beefc14b92728e312f0c4f57ec8d85e2b3c5008e1ab01fd0","LICENSE":"d041a5a86caaf9cc7720f3637d689252083a664d9ccd946a36d1c7c708d204cb","README.md":"8cf8a7b20cfb7fa43c4ee9585bf92ea6c5a5c4ca3ef700974edb02025d5146c8","rustfmt.toml":"84f2508ad6e506e71f2005a75ef20ba32eedb3bd6547af9ea04596517be883bb","src/embed/coff.rs":"55ba01eea1a5e16ef7ae4438f5cc51ec60b6eca45f5cabd706ee5b82da4f010b","src/embed/error.rs":"aecb4928e70b02b784557352608f6d4fb9b88b44ae3297a33969a0f2219c416c","src/embed/mod.rs":"a3f02a410c7b681f87ecfed415c97c4252569dee231b864c2b12e5d80ed4065e","src/embed/test.rs":"969dc4e2faf9ef9e9164b0e1ad01082a0a5e6ef5eb963526cb207a0ec380d809","src/lib.rs":"2786336ef4e787b0e3ccec9ab02043ae064f50df73a63ffe8c374217adfb353a","src/manifest/mod.rs":"c82936859d2ef795836972e08323c9698cc0513ba125cfed43414cd05d27e0b1","src/manifest/test.rs":"f4ab58ed4fc99b087ba30ab595026e980573aad980b8fa6f59a5b748404012ff","src/manifest/xml.rs":"1bce12120e17a49da43eabbd1b04f712b3f6ece7fcbca9186319e301890e20c5","testdata/sample.exe.manifest":"01e80ef76de2b628d452c7e80022654b9e0c8aee72ec64ee522c7083d835f4df"},"package":"41cd446c890d6bed1d8b53acef5f240069ebef91d6fae7c5f52efe61fe8b5eae"} \ No newline at end of file diff --git a/third_party/rust/embed-manifest/CHANGELOG.md b/third_party/rust/embed-manifest/CHANGELOG.md new file mode 100644 index 0000000000..8bc510bbf0 --- /dev/null +++ b/third_party/rust/embed-manifest/CHANGELOG.md @@ -0,0 +1,59 @@ +# Changelog +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [1.4.0] - 2023-06-22 +### Added +- Use `empty_manifest()` to start from a manifest with no default values. + Fixes [issue #6](https://gitlab.com/careyevans/embed-manifest/-/issues/6). +### Fixed +- Generate an object file with a single `.rsrc` section on GNU targets. + This lets it replace the default manifest from MinGW build tools. + Fixes [issue #5](https://gitlab.com/careyevans/embed-manifest/-/issues/5). + +## [1.3.1] - 2022-08-07 +### Added +- Format the code with Rustfmt. +- Assume `gnullvm` target environment should work like `gnu`. +- Add Windows 11 22H2 SDK version for maxversiontested. +### Changed +- Update `object` dependency and simplify unit tests. + +## [1.3.0] - 2022-05-01 +### Changed +- Use our own code again to generate COFF object files for GNU targets, but with + better knowledge of how such files are structured, reducing dependencies and + compile time. +- Link directly to the COFF object file instead of an archive file with one member. +### Fixed +- Make the custom `Error` type public. + +## [1.2.1] - 2022-04-18 +### Added +- Add checks for Windows builds to the documentation, for programs that + should still build for non-Windows targets. + +## [1.2.0] - 2022-04-17 +### Added +- Generate the manifest XML from Rust code rather than requiring the developer + to supply a correct manifest file. + +## [1.1.0] - 2022-03-24 +### Changed +- Use [Gimli Object](https://crates.io/crates/object) crate to build COFF + objects containing resources for GNU targets, removing a lot of magic numbers + and generating output more like LLVM `windres`. + +## [1.0.0] - 2021-12-18 +### Added +- Initial version. + +[1.0.0]: https://gitlab.com/careyevans/embed-manifest/-/releases/v1.0.0 +[1.1.0]: https://gitlab.com/careyevans/embed-manifest/-/releases/v1.1.0 +[1.2.0]: https://gitlab.com/careyevans/embed-manifest/-/releases/v1.2.0 +[1.2.1]: https://gitlab.com/careyevans/embed-manifest/-/releases/v1.2.1 +[1.3.0]: https://gitlab.com/careyevans/embed-manifest/-/releases/v1.3.0 +[1.3.1]: https://gitlab.com/careyevans/embed-manifest/-/releases/v1.3.1 +[1.4.0]: https://gitlab.com/careyevans/embed-manifest/-/releases/v1.4.0 diff --git a/third_party/rust/embed-manifest/Cargo.toml b/third_party/rust/embed-manifest/Cargo.toml new file mode 100644 index 0000000000..aeedb3598d --- /dev/null +++ b/third_party/rust/embed-manifest/Cargo.toml @@ -0,0 +1,38 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2021" +rust-version = "1.56" +name = "embed-manifest" +version = "1.4.0" +authors = ["Carey Evans "] +description = "Build script library to easily embed a Windows manifest." +readme = "README.md" +keywords = [ + "build", + "manifest", + "windows", +] +categories = ["development-tools::build-utils"] +license = "MIT" +repository = "https://gitlab.com/careyevans/embed-manifest" + +[dev-dependencies.object] +version = "0.31.1" +features = [ + "read_core", + "coff", +] +default-features = false + +[dev-dependencies.tempfile] +version = "3.5.0" diff --git a/third_party/rust/embed-manifest/LICENSE b/third_party/rust/embed-manifest/LICENSE new file mode 100644 index 0000000000..e724c68d3c --- /dev/null +++ b/third_party/rust/embed-manifest/LICENSE @@ -0,0 +1,23 @@ +MIT License + +Copyright (c) 2021, 2022 Carey Evans + +All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/third_party/rust/embed-manifest/README.md b/third_party/rust/embed-manifest/README.md new file mode 100644 index 0000000000..7752af0579 --- /dev/null +++ b/third_party/rust/embed-manifest/README.md @@ -0,0 +1,62 @@ +Windows Manifest Embedding for Rust +=================================== + +The `embed-manifest` crate provides a straightforward way to embed +a Windows manifest in an executable, whatever the build environment, +without dependencies on external tools from LLVM or MinGW. + +If you need to embed more resources than just a manifest, you may +find the [winres](https://crates.io/crates/winres) or +[embed-resource](https://crates.io/crates/embed-resource) +crates more suitable. They have additional dependencies and setup +requirements that may make them a little more difficult to use, though. + + +Why use it? +----------- + +The Rust compiler doesn’t add a manifest to a Windows executable, which +means that it runs with a few compatibility options and settings that +make it look like the program is running on an older version of Windows. +By adding an application manifest using this crate, even when cross-compiling: + +- 32-bit programs with names that look like installers don’t try to run + as an administrator. +- 32-bit programs aren’t shown a virtualised view of the filesystem and + registry. +- Many non-Unicode APIs accept UTF-8 strings, the same as Rust uses. +- The program sees the real Windows version, not Windows Vista. +- Built-in message boxes and other UI elements can display without blurring + on high-DPI monitors. +- Other features like long path names in APIs can be enabled. + + +Usage +----- + +To embed a default manifest, include this code in a `build.rs` build +script: + +```rust +use embed_manifest::{embed_manifest, new_manifest}; + +fn main() { + if std::env::var_os("CARGO_CFG_WINDOWS").is_some() { + embed_manifest(new_manifest("Contoso.Sample")) + .expect("unable to embed manifest file"); + } + println!("cargo:rerun-if-changed=build.rs"); +} +``` + +See the [crate documentation](https://docs.rs/embed-manifest) for +information about how to customise the embedded manifest. + + +License +------- + +For the avoidance of doubt, while this crate itself is licensed to +you under the MIT License, this does not affect the copyright status +and licensing of your own code when this is used from a Cargo build +script. diff --git a/third_party/rust/embed-manifest/rustfmt.toml b/third_party/rust/embed-manifest/rustfmt.toml new file mode 100644 index 0000000000..3aa392f175 --- /dev/null +++ b/third_party/rust/embed-manifest/rustfmt.toml @@ -0,0 +1,2 @@ +max_width = 132 +newline_style = "Unix" diff --git a/third_party/rust/embed-manifest/src/embed/coff.rs b/third_party/rust/embed-manifest/src/embed/coff.rs new file mode 100644 index 0000000000..bf751c3bf5 --- /dev/null +++ b/third_party/rust/embed-manifest/src/embed/coff.rs @@ -0,0 +1,192 @@ +//! Just as much COFF object file support as is needed to write a resource data segment +//! for GNU Windows targets. Inspired by the `write::Object` code from the `object` crate. +//! +//! Integers are converted from u64 to u32 and added without checking because the manifest +//! data cannot get anywhere close to overflowing unless the supplied application name or +//! number of dependencies was extremely long. If this code was used more generally or if +//! the input was less trustworthy then more checked conversions and checked arithmetic +//! would be needed. + +use std::io::{self, Seek, SeekFrom, Write}; +use std::time::SystemTime; + +#[derive(Debug, Clone, Copy)] +pub enum MachineType { + I386, + X86_64, + Aarch64, +} + +impl MachineType { + pub fn machine(&self) -> u16 { + match self { + Self::I386 => 0x014c, + Self::X86_64 => 0x8664, + Self::Aarch64 => 0xaa64, + } + } + + pub fn relocation_type(&self) -> u16 { + match self { + Self::I386 => 7, + Self::X86_64 => 3, + Self::Aarch64 => 2, + } + } +} + +pub struct CoffWriter { + /// wrapped writer or buffer + writer: W, + /// machine type for file header + machine_type: MachineType, + // size in bytes of resource section data + size_of_raw_data: u32, + // number of relocations at the end of the section + number_of_relocations: u16, +} + +impl CoffWriter { + /// Create a new instance wrapping a writer. + pub fn new(mut writer: W, machine_type: MachineType) -> io::Result { + // Add space for file header and section table. + writer.write_all(&[0; 60])?; + Ok(Self { + writer, + machine_type, + size_of_raw_data: 0, + number_of_relocations: 0, + }) + } + + /// Add data to a section and return its offset within the section. + pub fn add_data(&mut self, data: &[u8]) -> io::Result { + let start = self.size_of_raw_data; + self.writer.write_all(data)?; + self.size_of_raw_data = start + data.len() as u32; + Ok(start) + } + + // Pad the resource data to a multiple of `n` bytes. + pub fn align_to(&mut self, n: u32) -> io::Result<()> { + let offset = self.size_of_raw_data % n; + if offset != 0 { + let padding = n - offset; + for _ in 0..padding { + self.writer.write_all(&[0])?; + } + self.size_of_raw_data += padding; + } + Ok(()) + } + + /// Write a relocation for a symbol at the end of the section. + pub fn add_relocation(&mut self, address: u32) -> io::Result<()> { + self.number_of_relocations += 1; + self.writer.write_all(&address.to_le_bytes())?; + self.writer.write_all(&[0, 0, 0, 0])?; + self.writer.write_all(&self.machine_type.relocation_type().to_le_bytes()) + } + + /// Write the object and section headers and write the symbol table. + pub fn finish(mut self) -> io::Result { + // Get the timestamp for the header. `as` is correct here, as the low 32 bits + // should be used. + let timestamp = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .map_or(0, |d| d.as_secs() as u32); + + // Copy file location of the symbol table. + let pointer_to_symbol_table = self.writer.stream_position()? as u32; + + // Write the symbols and auxiliary data for the section. + self.writer.write_all(b".rsrc\0\0\0")?; // name + self.writer.write_all(&[0, 0, 0, 0])?; // address + self.writer.write_all(&[1, 0])?; // section number (1-based) + self.writer.write_all(&[0, 0, 3, 1])?; // type = 0, class = static, aux symbols = 1 + self.writer.write_all(&self.size_of_raw_data.to_le_bytes())?; + self.writer.write_all(&self.number_of_relocations.to_le_bytes())?; + self.writer.write_all(&[0; 12])?; + + // Write the empty string table. + self.writer.write_all(&[0; 4])?; + + // Write the object header. + let end_of_file = self.writer.seek(SeekFrom::Start(0))?; + self.writer.write_all(&self.machine_type.machine().to_le_bytes())?; + self.writer.write_all(&[1, 0])?; // number of sections + self.writer.write_all(×tamp.to_le_bytes())?; + self.writer.write_all(&pointer_to_symbol_table.to_le_bytes())?; + self.writer.write_all(&[2, 0, 0, 0])?; // number of symbol table entries + self.writer.write_all(&[0; 4])?; // optional header size = 0, characteristics = 0 + + // Write the section header. + self.writer.write_all(b".rsrc\0\0\0")?; + self.writer.write_all(&[0; 8])?; // virtual size = 0 and virtual address = 0 + self.writer.write_all(&self.size_of_raw_data.to_le_bytes())?; + self.writer.write_all(&[60, 0, 0, 0])?; // pointer to raw data + self.writer.write_all(&(self.size_of_raw_data + 60).to_le_bytes())?; // pointer to relocations + self.writer.write_all(&[0; 4])?; // pointer to line numbers + self.writer.write_all(&self.number_of_relocations.to_le_bytes())?; + self.writer.write_all(&[0; 2])?; // number of line numbers + self.writer.write_all(&[0x40, 0, 0x30, 0xc0])?; // characteristics + + // Return the inner writer and dispose of this object. + self.writer.seek(SeekFrom::Start(end_of_file))?; + Ok(self.writer) + } +} + +/// Returns the bytes for a resource directory table. +/// +/// Most of the fields are set to zero, including the timestamp, to aid +/// with making builds reproducible. +/// +/// ```c +/// typedef struct { +/// DWORD Characteristics, +/// DWORD TimeDateStamp, +/// WORD MajorVersion, +/// WORD MinorVersion, +/// WORD NumberOfNamedEntries, +/// WORD NumberOfIdEntries +/// } IMAGE_RESOURCE_DIRECTORY; +/// ``` +pub fn resource_directory_table(number_of_id_entries: u16) -> [u8; 16] { + let mut table = [0; 16]; + table[14..16].copy_from_slice(&number_of_id_entries.to_le_bytes()); + table +} + +/// Returns the bytes for a resource directory entry for an ID. +/// +/// ```c +/// typedef struct { +/// DWORD Name, +/// DWORD OffsetToData +/// } IMAGE_RESOURCE_DIRECTORY_ENTRY; +/// ``` +pub fn resource_directory_id_entry(id: u32, offset: u32, subdirectory: bool) -> [u8; 8] { + let mut entry = [0; 8]; + entry[0..4].copy_from_slice(&id.to_le_bytes()); + let flag: u32 = if subdirectory { 0x80000000 } else { 0 }; + entry[4..8].copy_from_slice(&((offset & 0x7fffffff) | flag).to_le_bytes()); + entry +} + +/// Returns the bytes for a resource data entry. +/// +/// ```c +/// typedef struct { +/// DWORD OffsetToData, +/// DWORD Size, +/// DWORD CodePage, +/// DWORD Reserved +/// } IMAGE_RESOURCE_DATA_ENTRY; +/// ``` +pub fn resource_data_entry(rva: u32, size: u32) -> [u8; 16] { + let mut entry = [0; 16]; + entry[0..4].copy_from_slice(&rva.to_le_bytes()); + entry[4..8].copy_from_slice(&size.to_le_bytes()); + entry +} diff --git a/third_party/rust/embed-manifest/src/embed/error.rs b/third_party/rust/embed-manifest/src/embed/error.rs new file mode 100644 index 0000000000..365d0525ed --- /dev/null +++ b/third_party/rust/embed-manifest/src/embed/error.rs @@ -0,0 +1,57 @@ +//! Error handling for application manifest embedding. + +use std::fmt::{self, Display, Formatter}; +use std::io::{self, ErrorKind}; + +/// The error type which is returned when application manifest embedding fails. +#[derive(Debug)] +pub struct Error { + repr: Repr, +} + +#[derive(Debug)] +enum Repr { + IoError(io::Error), + UnknownTarget, +} + +impl Error { + pub(crate) fn unknown_target() -> Error { + Error { + repr: Repr::UnknownTarget, + } + } +} + +impl Display for Error { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + match self.repr { + Repr::IoError(ref e) => write!(f, "I/O error: {}", e), + Repr::UnknownTarget => f.write_str("unknown target"), + } + } +} + +impl std::error::Error for Error { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self.repr { + Repr::IoError(ref e) => Some(e), + _ => None, + } + } +} + +impl From for Error { + fn from(e: io::Error) -> Self { + Error { repr: Repr::IoError(e) } + } +} + +impl From for io::Error { + fn from(e: Error) -> Self { + match e.repr { + Repr::IoError(ioe) => ioe, + _ => io::Error::new(ErrorKind::Other, e), + } + } +} diff --git a/third_party/rust/embed-manifest/src/embed/mod.rs b/third_party/rust/embed-manifest/src/embed/mod.rs new file mode 100644 index 0000000000..b054878ac6 --- /dev/null +++ b/third_party/rust/embed-manifest/src/embed/mod.rs @@ -0,0 +1,139 @@ +use std::env; +use std::fs::{self, File}; +use std::io::{self, stdout, BufWriter, Cursor, Write}; +use std::path::{Path, PathBuf}; + +use crate::manifest::ManifestBuilder; + +use self::coff::{resource_data_entry, resource_directory_id_entry, resource_directory_table, CoffWriter, MachineType}; +use self::error::Error; + +mod coff; +pub mod error; + +#[cfg(test)] +mod test; + +/// Embeds the manifest described by `manifest` by converting it to XML, +/// then saving it to a file and passing the correct options to the linker +/// on MSVC targets, or by building a static library and instructing Cargo +/// to link the executable against it on GNU targets. +pub fn embed_manifest(manifest: ManifestBuilder) -> Result<(), Error> { + let out_dir = get_out_dir()?; + let target = get_target()?; + if matches!(target.os, TargetOs::WindowsMsvc) { + let manifest_file = out_dir.join("manifest.xml"); + write!(BufWriter::new(File::create(&manifest_file)?), "{}", manifest)?; + link_manifest_msvc(&manifest_file, &mut stdout().lock()) + } else { + let manifest_data = manifest.to_string(); + link_manifest_gnu(manifest_data.as_bytes(), &out_dir, target.arch, &mut stdout().lock()) + } +} + +/// Directly embeds the manifest in the provided `file` by passing the correct +/// options to the linker on MSVC targets, or by building a static library +/// and instructing Cargo to link the executable against it on GNU targets. +pub fn embed_manifest_file>(file: P) -> Result<(), io::Error> { + let out_dir = get_out_dir()?; + let target = get_target()?; + if matches!(target.os, TargetOs::WindowsMsvc) { + Ok(link_manifest_msvc(file.as_ref(), &mut stdout().lock())?) + } else { + let manifest = fs::read(file.as_ref())?; + Ok(link_manifest_gnu(&manifest, &out_dir, target.arch, &mut stdout().lock())?) + } +} + +fn get_out_dir() -> Result { + match env::var_os("OUT_DIR") { + Some(dir) => Ok(PathBuf::from(dir)), + None => env::current_dir(), + } +} + +enum TargetOs { + WindowsGnu, + WindowsMsvc, +} + +struct Target { + arch: MachineType, + os: TargetOs, +} + +fn get_target() -> Result { + match env::var("TARGET") { + Ok(target) => parse_target(&target), + _ => Err(Error::unknown_target()), + } +} + +fn parse_target(target: &str) -> Result { + let mut iter = target.splitn(3, '-'); + let arch = match iter.next() { + Some("i686") => MachineType::I386, + Some("aarch64") => MachineType::Aarch64, + Some("x86_64") => MachineType::X86_64, + _ => return Err(Error::unknown_target()), + }; + if iter.next() != Some("pc") { + return Err(Error::unknown_target()); + } + let os = match iter.next() { + Some("windows-gnu") => TargetOs::WindowsGnu, + Some("windows-gnullvm") => TargetOs::WindowsGnu, + Some("windows-msvc") => TargetOs::WindowsMsvc, + _ => return Err(Error::unknown_target()), + }; + Ok(Target { arch, os }) +} + +fn link_manifest_msvc(manifest_path: &Path, out: &mut W) -> Result<(), Error> { + writeln!(out, "cargo:rustc-link-arg-bins=/MANIFEST:EMBED")?; + writeln!( + out, + "cargo:rustc-link-arg-bins=/MANIFESTINPUT:{}", + manifest_path.canonicalize()?.display() + )?; + writeln!(out, "cargo:rustc-link-arg-bins=/MANIFESTUAC:NO")?; + Ok(()) +} + +fn link_manifest_gnu(manifest: &[u8], out_dir: &Path, arch: MachineType, out: &mut W) -> Result<(), Error> { + // Generate a COFF object file containing the manifest in a .rsrc section. + let object_data = create_object_file(manifest, arch)?; + let path = out_dir.join("embed-manifest.o"); + fs::create_dir_all(out_dir)?; + fs::write(&path, object_data)?; + + // Link the manifest with the executable. + writeln!(out, "cargo:rustc-link-arg-bins={}", path.display())?; + Ok(()) +} + +fn create_object_file(manifest: &[u8], arch: MachineType) -> Result, io::Error> { + // Start object file with .rsrc section. + let mut obj = CoffWriter::new(Cursor::new(Vec::with_capacity(4096)), arch)?; + + // Create resource directories for type ID 24, name ID 1, language ID 1033. + obj.add_data(&resource_directory_table(1))?; + obj.add_data(&resource_directory_id_entry(24, 24, true))?; + obj.add_data(&resource_directory_table(1))?; + obj.add_data(&resource_directory_id_entry(1, 48, true))?; + obj.add_data(&resource_directory_table(1))?; + obj.add_data(&resource_directory_id_entry(1033, 72, false))?; + + // Add resource data entry with relocated address. + let address = obj.add_data(&resource_data_entry(88, manifest.len() as u32))?; + + // Add the manifest data. + obj.add_data(manifest)?; + obj.align_to(8)?; + + // Write manifest data relocation at the end of the section. + obj.add_relocation(address)?; + + // Finish writing file and return the populated object data. + Ok(obj.finish()?.into_inner()) +} diff --git a/third_party/rust/embed-manifest/src/embed/test.rs b/third_party/rust/embed-manifest/src/embed/test.rs new file mode 100644 index 0000000000..c464c6044c --- /dev/null +++ b/third_party/rust/embed-manifest/src/embed/test.rs @@ -0,0 +1,173 @@ +use std::fs; +use std::path::{Path, PathBuf}; + +use object::coff::CoffFile; +use object::pe::{ImageResourceDataEntry, ImageResourceDirectory, ImageResourceDirectoryEntry, IMAGE_RESOURCE_DATA_IS_DIRECTORY}; +use object::{ + pod, Architecture, LittleEndian, Object, ObjectSection, ObjectSymbol, RelocationEncoding, RelocationKind, SectionKind, +}; +use tempfile::{tempdir, TempDir}; + +use crate::embed::coff::MachineType; +use crate::embed::{create_object_file, link_manifest_gnu, link_manifest_msvc, TargetOs}; +use crate::new_manifest; + +#[test] +fn create_obj() { + let res = do_embed_file(MachineType::X86_64, TargetOs::WindowsGnu); + let data = fs::read(&res.object_file()).unwrap(); + let obj = CoffFile::parse(&data[..]).unwrap(); + assert_eq!(obj.architecture(), Architecture::X86_64); + let expected_manifest = fs::read(&sample_manifest_path()).unwrap(); + check_object_file(obj, &expected_manifest); +} + +#[test] +fn link_lib_gnu() { + let res = do_embed_file(MachineType::X86_64, TargetOs::WindowsGnu); + assert!(res.object_file().exists()); + let object_option = format!("cargo:rustc-link-arg-bins={}", res.object_file().display()); + assert_eq!(res.lines(), &[object_option.as_str()]); +} + +#[test] +fn link_file_msvc() { + let res = do_embed_file(MachineType::X86_64, TargetOs::WindowsMsvc); + assert!(!res.object_file().exists()); + let mut input_option = String::from("cargo:rustc-link-arg-bins=/MANIFESTINPUT:"); + input_option.push_str(res.manifest_path.canonicalize().unwrap().to_str().unwrap()); + assert_eq!( + res.lines(), + &[ + "cargo:rustc-link-arg-bins=/MANIFEST:EMBED", + input_option.as_str(), + "cargo:rustc-link-arg-bins=/MANIFESTUAC:NO" + ] + ); +} + +struct EmbedResult { + manifest_path: PathBuf, + out_dir: TempDir, + output: String, +} + +impl EmbedResult { + fn object_file(&self) -> PathBuf { + self.out_dir.path().join("embed-manifest.o") + } + + fn lines(&self) -> Vec<&str> { + self.output.lines().collect() + } +} + +fn sample_manifest_path() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("testdata/sample.exe.manifest") +} + +fn do_embed_file(arch: MachineType, os: TargetOs) -> EmbedResult { + let manifest_path = sample_manifest_path(); + let out_dir = tempdir().unwrap(); + let mut buf: Vec = Vec::new(); + if matches!(os, TargetOs::WindowsMsvc) { + link_manifest_msvc(&manifest_path, &mut buf).unwrap(); + } else { + link_manifest_gnu(&fs::read(&manifest_path).unwrap(), out_dir.path(), arch, &mut buf).unwrap(); + } + EmbedResult { + manifest_path, + out_dir, + output: String::from_utf8(buf).unwrap(), + } +} + +#[test] +fn object_file_x86() { + let manifest = new_manifest("Test.X86").to_string().into_bytes(); + let file = create_object_file(&manifest, MachineType::I386).unwrap(); + let obj = CoffFile::parse(&file[..]).unwrap(); + assert_eq!(obj.architecture(), Architecture::I386); + check_object_file(obj, &manifest); +} + +#[test] +fn object_file_x86_64() { + let manifest = new_manifest("Test.X86_64").to_string().into_bytes(); + let file = create_object_file(&manifest, MachineType::X86_64).unwrap(); + let obj = CoffFile::parse(&file[..]).unwrap(); + assert_eq!(obj.architecture(), Architecture::X86_64); + check_object_file(obj, &manifest); +} + +#[test] +fn object_file_aarch64() { + let manifest = new_manifest("Test.AARCH64").to_string().into_bytes(); + let file = create_object_file(&manifest, MachineType::Aarch64).unwrap(); + let obj = CoffFile::parse(&file[..]).unwrap(); + assert_eq!(obj.architecture(), Architecture::Aarch64); + check_object_file(obj, &manifest); +} + +fn check_object_file(obj: CoffFile, expected_manifest: &[u8]) { + // There should be one sections `.rsrc`. + assert_eq!( + obj.sections().map(|s| s.name().unwrap().to_string()).collect::>(), + &[".rsrc"] + ); + + // There should be one section symbol. + assert_eq!( + obj.symbols().map(|s| s.name().unwrap().to_string()).collect::>(), + &[".rsrc"] + ); + + // The resource section must be a data section. + let rsrc = obj.section_by_name(".rsrc").unwrap(); + assert_eq!(rsrc.address(), 0); + assert_eq!(rsrc.kind(), SectionKind::Data); + + // The data RVA in the resource data entry must be relocatable. + let (addr, reloc) = rsrc.relocations().next().unwrap(); + assert_eq!(reloc.kind(), RelocationKind::ImageOffset); + assert_eq!(reloc.encoding(), RelocationEncoding::Generic); + assert_eq!(addr, 0x48); // size of the directory table, three directories, and no strings + assert_eq!(reloc.addend(), 0); + + // The resource directory contains one manifest resource type subdirectory. + let data = rsrc.data().unwrap(); + let (dir, rest) = pod::from_bytes::(data).unwrap(); + assert_eq!(0, dir.number_of_named_entries.get(LittleEndian)); + assert_eq!(1, dir.number_of_id_entries.get(LittleEndian)); + let (entries, _) = pod::slice_from_bytes::(rest, 1).unwrap(); + assert_eq!(24, entries[0].name_or_id.get(LittleEndian)); + let offset = entries[0].offset_to_data_or_directory.get(LittleEndian); + assert_eq!(IMAGE_RESOURCE_DATA_IS_DIRECTORY, offset & IMAGE_RESOURCE_DATA_IS_DIRECTORY); + let offset = (offset & !IMAGE_RESOURCE_DATA_IS_DIRECTORY) as usize; + + // The manifest subdirectory contains one image (not DLL) manifest subdirectory. + let (dir, rest) = pod::from_bytes::(&data[offset..]).unwrap(); + assert_eq!(0, dir.number_of_named_entries.get(LittleEndian)); + assert_eq!(1, dir.number_of_id_entries.get(LittleEndian)); + let (entries, _) = pod::slice_from_bytes::(rest, 1).unwrap(); + assert_eq!(1, entries[0].name_or_id.get(LittleEndian)); + let offset = entries[0].offset_to_data_or_directory.get(LittleEndian); + assert_eq!(IMAGE_RESOURCE_DATA_IS_DIRECTORY, offset & IMAGE_RESOURCE_DATA_IS_DIRECTORY); + let offset = (offset & !IMAGE_RESOURCE_DATA_IS_DIRECTORY) as usize; + + // The image manifest subdirectory contains one US English manifest data entry. + let (dir, rest) = pod::from_bytes::(&data[offset..]).unwrap(); + assert_eq!(0, dir.number_of_named_entries.get(LittleEndian)); + assert_eq!(1, dir.number_of_id_entries.get(LittleEndian)); + let (entries, _) = pod::slice_from_bytes::(rest, 1).unwrap(); + assert_eq!(0x0409, entries[0].name_or_id.get(LittleEndian)); + let offset = entries[0].offset_to_data_or_directory.get(LittleEndian); + assert_eq!(0, offset & IMAGE_RESOURCE_DATA_IS_DIRECTORY); + let offset = offset as usize; + + // The manifest data matches what was added. + let (entry, resource_data) = pod::from_bytes::(&data[offset..]).unwrap(); + let end = entry.size.get(LittleEndian) as usize; + let manifest = &resource_data[..end]; + assert_eq!(manifest, expected_manifest); +} diff --git a/third_party/rust/embed-manifest/src/lib.rs b/third_party/rust/embed-manifest/src/lib.rs new file mode 100644 index 0000000000..6c60cdf541 --- /dev/null +++ b/third_party/rust/embed-manifest/src/lib.rs @@ -0,0 +1,134 @@ +//! The `embed-manifest` crate provides a straightforward way to embed +//! a Windows manifest in an executable, whatever the build environment +//! and even when cross-compiling, without dependencies on external +//! tools from LLVM or MinGW. +//! +//! This should be called from a [build script][1], as shown below. +//! +//! [1]: https://doc.rust-lang.org/cargo/reference/build-scripts.html +//! +//! On MSVC targets, the manifest file is embedded in the executable by +//! instructing Cargo to pass `/MANIFEST` options to `LINK.EXE`. This +//! requires Cargo from Rust 1.56. +//! +//! On GNU targets, the manifest file is added as a resource in a COFF +//! object file, and Cargo is instructed to link this file into the +//! executable, also using functionality from Rust 1.56. +//! +//! # Usage +//! +//! This crate should be added to the `[build-dependencies]` section in +//! your executable’s `Cargo.toml`: +//! +//! ```toml +//! [build-dependencies] +//! embed-manifest = "1.3.1" +//! ``` +//! +//! In the same directory, create a `build.rs` file to call this crate’s +//! code when building for Windows, and to only run once: +//! +//! ``` +//! use embed_manifest::{embed_manifest, new_manifest}; +//! +//! fn main() { +//! # let tempdir = tempfile::tempdir().unwrap(); +//! # std::env::set_var("OUT_DIR", tempdir.path()); +//! # std::env::set_var("TARGET", "x86_64-pc-windows-gnu"); +//! # std::env::set_var("CARGO_CFG_WINDOWS", ""); +//! if std::env::var_os("CARGO_CFG_WINDOWS").is_some() { +//! embed_manifest(new_manifest("Contoso.Sample")).expect("unable to embed manifest file"); +//! } +//! println!("cargo:rerun-if-changed=build.rs"); +//! } +//! ``` +//! +//! To customise the application manifest, use the methods on it to change things like +//! enabling the segment heap: +//! +//! ``` +//! use embed_manifest::{embed_manifest, new_manifest, manifest::HeapType}; +//! +//! fn main() { +//! # let tempdir = tempfile::tempdir().unwrap(); +//! # std::env::set_var("OUT_DIR", tempdir.path()); +//! # std::env::set_var("TARGET", "x86_64-pc-windows-gnu"); +//! # std::env::set_var("CARGO_CFG_WINDOWS", ""); +//! if std::env::var_os("CARGO_CFG_WINDOWS").is_some() { +//! embed_manifest(new_manifest("Contoso.Sample").heap_type(HeapType::SegmentHeap)) +//! .expect("unable to embed manifest file"); +//! } +//! println!("cargo:rerun-if-changed=build.rs"); +//! } +//! ``` +//! +//! or making it always use legacy single-byte API encoding and only declaring compatibility +//! up to Windows 8.1, without checking whether this is a Windows build: +//! +//! ``` +//! use embed_manifest::{embed_manifest, new_manifest}; +//! use embed_manifest::manifest::{ActiveCodePage::Legacy, SupportedOS::*}; +//! +//! fn main() { +//! # let tempdir = tempfile::tempdir().unwrap(); +//! # std::env::set_var("OUT_DIR", tempdir.path()); +//! # std::env::set_var("TARGET", "x86_64-pc-windows-gnu"); +//! let manifest = new_manifest("Contoso.Sample") +//! .active_code_page(Legacy) +//! .supported_os(Windows7..=Windows81); +//! embed_manifest(manifest).expect("unable to embed manifest file"); +//! println!("cargo:rerun-if-changed=build.rs"); +//! } +//! ``` + +#![allow(clippy::needless_doctest_main)] + +pub use embed::error::Error; +pub use embed::{embed_manifest, embed_manifest_file}; + +use crate::manifest::ManifestBuilder; + +mod embed; +pub mod manifest; + +/// Creates a new [`ManifestBuilder`] with sensible defaults, allowing customisation +/// before the Windows application manifest XML is generated. +/// +/// The initial values used by the manifest are: +/// - Version number from the `CARGO_PKG_VERSION_MAJOR`, `CARGO_PKG_VERSION_MINOR` and +/// `CARGO_PKG_VERSION_PATCH` environment variables. This can then be changed with +/// [`version()`][ManifestBuilder::version]. +/// - A dependency on version 6 of the Common Controls so that message boxes and dialogs +/// will use the latest design, and have the best available support for high DPI displays. +/// This can be removed with +/// [`remove_dependency`][ManifestBuilder::remove_dependency]. +/// - [Compatible with Windows from 7 to 11][ManifestBuilder::supported_os], +/// matching the Rust compiler [tier 1 targets][tier1]. +/// - A “[maximum version tested][ManifestBuilder::max_version_tested]” of Windows 10 +/// version 1903. +/// - An [active code page][ManifestBuilder::active_code_page] of UTF-8, so that +/// single-byte Windows APIs will generally interpret Rust strings correctly, starting +/// from Windows 10 version 1903. +/// - [Version 2 of per-monitor high DPI awareness][manifest::DpiAwareness::PerMonitorV2Only], +/// so that user interface elements can be scaled correctly by the application and +/// Common Controls from Windows 10 version 1703. +/// - [Long path awareness][ManifestBuilder::long_path_aware] from Windows 10 version +/// 1607 [when separately enabled][longpaths]. +/// - [Printer driver isolation][ManifestBuilder::printer_driver_isolation] enabled +/// to improve reliability and security. +/// - An [execution level][ManifestBuilder::requested_execution_level] of “as invoker” +/// so that the UAC elevation dialog will never be displayed, regardless of the name +/// of the program, and [UAC Virtualisation][uac] is disabled in 32-bit programs. +/// +/// [tier1]: https://doc.rust-lang.org/nightly/rustc/platform-support.html +/// [longpaths]: https://docs.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation?tabs=cmd#enable-long-paths-in-windows-10-version-1607-and-later +/// [uac]: https://docs.microsoft.com/en-us/windows/security/identity-protection/user-account-control/how-user-account-control-works#virtualization +pub fn new_manifest(name: &str) -> ManifestBuilder { + ManifestBuilder::new(name) +} + +/// Creates a new [`ManifestBuilder`] without any settings, allowing creation of +/// a manifest with only desired content. +pub fn empty_manifest() -> ManifestBuilder { + ManifestBuilder::empty() +} diff --git a/third_party/rust/embed-manifest/src/manifest/mod.rs b/third_party/rust/embed-manifest/src/manifest/mod.rs new file mode 100644 index 0000000000..716a3559f8 --- /dev/null +++ b/third_party/rust/embed-manifest/src/manifest/mod.rs @@ -0,0 +1,882 @@ +//! A builder for Windows application manifest XML files. +//! +//! This module allows the construction of application manifests from code with the +//! [`ManifestBuilder`], for use from a Cargo build script. Once configured, the builder +//! should be passed to [`embed_manifest()`][crate::embed_manifest] to generate the +//! correct instructions for Cargo. For any other use, the builder will output the XML +//! when formatted for [`Display`], or with [`to_string()`][ToString]. For more +//! information about the different elements of an application manifest, see +//! [Application Manifests][1] in the Microsoft Windows App Development documentation. +//! +//! [1]: https://docs.microsoft.com/en-us/windows/win32/sbscs/application-manifests +//! +//! To generate the manifest XML separately, the XML can be output with `write!` or +//! copied to a string with [`to_string()`][ToString]. To produce the manifest XML with +//! extra whitespace for formatting, format it with the ‘alternate’ flag: +//! +//! ``` +//! # use embed_manifest::new_manifest; +//! let builder = new_manifest("Company.OrgUnit.Program"); +//! assert_eq!(format!("{:#}", builder), r#" +//! +//! +//! +//! +//! +//! +//! +//! +//! +//! +//! +//! +//! +//! +//! +//! +//! +//! +//! UTF-8 +//! permonitorv2 +//! true +//! true +//! +//! +//! +//! +//! +//! +//! +//! +//! +//! "#.replace("\n", "\r\n")) +//! ``` + +use std::fmt::{Display, Formatter}; +use std::ops::RangeBounds; +use std::{env, fmt}; + +use crate::manifest::xml::XmlFormatter; + +mod xml; + +#[cfg(test)] +mod test; + +/// An opaque container to describe the Windows application manifest for the +/// executable. A new instance with reasonable defaults is created with +/// [`new_manifest()`][crate::new_manifest]. +#[derive(Debug)] +pub struct ManifestBuilder { + identity: Option, + dependent_assemblies: Vec, + compatibility: ApplicationCompatibility, + windows_settings: WindowsSettings, + requested_execution_level: Option, +} + +impl ManifestBuilder { + pub(crate) fn new(name: &str) -> Self { + ManifestBuilder { + identity: Some(AssemblyIdentity::application(name)), + dependent_assemblies: vec![AssemblyIdentity::new( + "Microsoft.Windows.Common-Controls", + [6, 0, 0, 0], + 0x6595b64144ccf1df, + )], + compatibility: ApplicationCompatibility { + max_version_tested: Some(MaxVersionTested::Windows10Version1903), + supported_os: vec![ + SupportedOS::Windows7, + SupportedOS::Windows8, + SupportedOS::Windows81, + SupportedOS::Windows10, + ], + }, + windows_settings: WindowsSettings::new(), + requested_execution_level: Some(RequestedExecutionLevel { + level: ExecutionLevel::AsInvoker, + ui_access: false, + }), + } + } + + pub(crate) fn empty() -> Self { + ManifestBuilder { + identity: None, + dependent_assemblies: Vec::new(), + compatibility: ApplicationCompatibility { + max_version_tested: None, + supported_os: Vec::new(), + }, + windows_settings: WindowsSettings::empty(), + requested_execution_level: None, + } + } + + // Set the dot-separated [application name][identity] in the manifest. + // + // [identity]: https://learn.microsoft.com/en-us/windows/win32/sbscs/application-manifests#assemblyIdentity + pub fn name(mut self, name: &str) -> Self { + match self.identity { + Some(ref mut identity) => identity.name = name.to_string(), + None => self.identity = Some(AssemblyIdentity::application_version(name, 0, 0, 0, 0)), + } + self + } + + /// Set the four-part application version number in the manifest. + pub fn version(mut self, major: u16, minor: u16, build: u16, revision: u16) -> Self { + match self.identity { + Some(ref mut identity) => identity.version = Version(major, minor, build, revision), + None => { + self.identity = Some(AssemblyIdentity::application_version("", major, minor, build, revision)); + } + } + self + } + + /// Add a dependency on a specific version of a side-by-side assembly + /// to the application manifest. For more information on side-by-side + /// assemblies, see [Using Side-by-side Assemblies][sxs]. + /// + /// [sxs]: https://docs.microsoft.com/en-us/windows/win32/sbscs/using-side-by-side-assemblies + pub fn dependency(mut self, identity: AssemblyIdentity) -> Self { + self.dependent_assemblies.push(identity); + self + } + + /// Remove a dependency on a side-by-side assembly. This can be used to + /// remove the default dependency on Common Controls version 6: + /// + /// ``` + /// # use embed_manifest::new_manifest; + /// new_manifest("Company.OrgUnit.Program") + /// .remove_dependency("Microsoft.Windows.Common-Controls") + /// # ; + /// ``` + pub fn remove_dependency(mut self, name: &str) -> Self { + self.dependent_assemblies.retain(|d| d.name != name); + self + } + + /// Set the “maximum version tested” based on a Windows SDK version. + /// This compatibility setting enables the use of XAML Islands, as described in + /// [Host a standard WinRT XAML control in a C++ desktop (Win32) app][xaml]. + /// + /// [xaml]: https://docs.microsoft.com/en-us/windows/apps/desktop/modernize/host-standard-control-with-xaml-islands-cpp + pub fn max_version_tested(mut self, version: MaxVersionTested) -> Self { + self.compatibility.max_version_tested = Some(version); + self + } + + /// Remove the “maximum version tested” from the application compatibility. + pub fn remove_max_version_tested(mut self) -> Self { + self.compatibility.max_version_tested = None; + self + } + + /// Set the range of supported versions of Windows for application compatibility. + /// The default value declares compatibility with every version from + /// [Windows 7][SupportedOS::Windows7] to [Windows 10 and 11][SupportedOS::Windows10]. + pub fn supported_os>(mut self, os_range: R) -> Self { + use SupportedOS::*; + + self.compatibility.supported_os.clear(); + for os in [WindowsVista, Windows7, Windows8, Windows81, Windows10] { + if os_range.contains(&os) { + self.compatibility.supported_os.push(os); + } + } + self + } + + /// Set the code page used for single-byte Windows API, starting from Windows 10 + /// version 1903. The default setting of [UTF-8][`ActiveCodePage::Utf8`] makes Rust + /// strings work directly with APIs like `MessageBoxA`. + pub fn active_code_page(mut self, code_page: ActiveCodePage) -> Self { + self.windows_settings.active_code_page = code_page; + self + } + + /// Configures how Windows should display this program on monitors where the + /// graphics need scaling, whether by the application drawing its user + /// interface at different sizes or by the Windows system rendering the graphics + /// to a bitmap then resizing that for display. + pub fn dpi_awareness(mut self, setting: DpiAwareness) -> Self { + self.windows_settings.dpi_awareness = setting; + self + } + + /// Attempts to scale GDI primitives by the per-monitor scaling values, + /// from Windows 10 version 1703. It seems to be best to leave this disabled. + pub fn gdi_scaling(mut self, setting: Setting) -> Self { + self.windows_settings.gdi_scaling = setting.enabled(); + self + } + + /// Select the memory allocator use by the standard heap allocation APIs, + /// including the default Rust allocator. Selecting a different algorithm + /// may make performance and memory use better or worse, so any changes + /// should be carefully tested. + pub fn heap_type(mut self, setting: HeapType) -> Self { + self.windows_settings.heap_type = setting; + self + } + + /// Enable paths longer than 260 characters with some wide-character Win32 APIs, + /// when also enabled in the Windows registry. For more details, see + /// [Maximum Path Length Limitation][1] in the Windows App Development + /// documentation. + /// + /// As of Rust 1.58, the [Rust standard library bypasses this limitation][2] itself + /// by using Unicode paths beginning with `\\?\`. + /// + /// [1]: https://docs.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation + /// [2]: https://github.com/rust-lang/rust/pull/89174 + pub fn long_path_aware(mut self, setting: Setting) -> Self { + self.windows_settings.long_path_aware = setting.enabled(); + self + } + + /// Enable printer driver isolation for the application, improving security and + /// stability when printing by loading the printer driver in a separate + /// application. This is poorly documented, but is described in a blog post, + /// “[Application-level Printer Driver Isolation][post]”. + /// + /// [post]: https://peteronprogramming.wordpress.com/2018/01/22/application-level-printer-driver-isolation/ + pub fn printer_driver_isolation(mut self, setting: Setting) -> Self { + self.windows_settings.printer_driver_isolation = setting.enabled(); + self + } + + /// Configure whether the application should receive mouse wheel scroll events + /// with a minimum delta of 1, 40 or 120, as described in + /// [Windows precision touchpad devices][touchpad]. + /// + /// [touchpad]: https://docs.microsoft.com/en-us/windows/win32/w8cookbook/windows-precision-touchpad-devices + pub fn scrolling_awareness(mut self, setting: ScrollingAwareness) -> Self { + self.windows_settings.scrolling_awareness = setting; + self + } + + /// Allows the application to disable the filtering that normally + /// removes UWP windows from the results of the `EnumWindows` API. + pub fn window_filtering(mut self, setting: Setting) -> Self { + self.windows_settings.disable_window_filtering = setting.disabled(); + self + } + + /// Selects the authorities to execute the program with, rather than + /// [guessing based on a filename][installer] like `setup.exe`, + /// `update.exe` or `patch.exe`. + /// + /// [installer]: https://docs.microsoft.com/en-us/windows/security/identity-protection/user-account-control/how-user-account-control-works#installer-detection-technology + pub fn requested_execution_level(mut self, level: ExecutionLevel) -> Self { + match self.requested_execution_level { + Some(ref mut requested_execution_level) => requested_execution_level.level = level, + None => self.requested_execution_level = Some(RequestedExecutionLevel { level, ui_access: false }), + } + self + } + + /// Allows the application to access the user interface of applications + /// running with elevated permissions when this program does not, typically + /// for accessibility. The program must additionally be correctly signed + /// and installed in a trusted location like the Program Files directory. + pub fn ui_access(mut self, access: bool) -> Self { + match self.requested_execution_level { + Some(ref mut requested_execution_level) => requested_execution_level.ui_access = access, + None => { + self.requested_execution_level = Some(RequestedExecutionLevel { + level: ExecutionLevel::AsInvoker, + ui_access: access, + }) + } + } + self + } +} + +impl Display for ManifestBuilder { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + let mut w = XmlFormatter::new(f); + w.start_document()?; + let mut attrs = vec![("xmlns", "urn:schemas-microsoft-com:asm.v1")]; + if !self.windows_settings.is_empty() || self.requested_execution_level.is_some() { + attrs.push(("xmlns:asmv3", "urn:schemas-microsoft-com:asm.v3")); + } + attrs.push(("manifestVersion", "1.0")); + w.start_element("assembly", &attrs)?; + if let Some(ref identity) = self.identity { + identity.xml_to(&mut w)?; + } + if !self.dependent_assemblies.is_empty() { + w.element("dependency", &[], |w| { + for d in self.dependent_assemblies.as_slice() { + w.element("dependentAssembly", &[], |w| d.xml_to(w))?; + } + Ok(()) + })?; + } + if !self.compatibility.is_empty() { + self.compatibility.xml_to(&mut w)?; + } + if !self.windows_settings.is_empty() { + self.windows_settings.xml_to(&mut w)?; + } + if let Some(ref requested_execution_level) = self.requested_execution_level { + requested_execution_level.xml_to(&mut w)?; + } + w.end_element("assembly") + } +} + +/// Identity of a side-by-side assembly dependency for the application. +#[derive(Debug)] +pub struct AssemblyIdentity { + r#type: AssemblyType, + name: String, + language: Option, + processor_architecture: Option, + version: Version, + public_key_token: Option, +} + +impl AssemblyIdentity { + fn application(name: &str) -> AssemblyIdentity { + let major = env::var("CARGO_PKG_VERSION_MAJOR").map_or(0, |s| s.parse().unwrap_or(0)); + let minor = env::var("CARGO_PKG_VERSION_MINOR").map_or(0, |s| s.parse().unwrap_or(0)); + let patch = env::var("CARGO_PKG_VERSION_PATCH").map_or(0, |s| s.parse().unwrap_or(0)); + AssemblyIdentity { + r#type: AssemblyType::Win32, + name: name.to_string(), + language: None, + processor_architecture: None, + version: Version(major, minor, patch, 0), + public_key_token: None, + } + } + + fn application_version(name: &str, major: u16, minor: u16, build: u16, revision: u16) -> AssemblyIdentity { + AssemblyIdentity { + r#type: AssemblyType::Win32, + name: name.to_string(), + language: None, + processor_architecture: None, + version: Version(major, minor, build, revision), + public_key_token: None, + } + } + + /// Creates a new value for a [manifest dependency][ManifestBuilder::dependency], + /// with the `version` as an array of numbers like `[1, 0, 0, 0]` for 1.0.0.0, + /// and the public key token as a 64-bit number like `0x6595b64144ccf1df`. + pub fn new(name: &str, version: [u16; 4], public_key_token: u64) -> AssemblyIdentity { + AssemblyIdentity { + r#type: AssemblyType::Win32, + name: name.to_string(), + language: Some("*".to_string()), + processor_architecture: Some(AssemblyProcessorArchitecture::All), + version: Version(version[0], version[1], version[2], version[3]), + public_key_token: Some(PublicKeyToken(public_key_token)), + } + } + + /// Change the language from `"*"` to the language code in `language`. + pub fn language(mut self, language: &str) -> Self { + self.language = Some(language.to_string()); + self + } + + /// Change the processor architecture from `"*"` to the architecture in `arch`. + pub fn processor_architecture(mut self, arch: AssemblyProcessorArchitecture) -> Self { + self.processor_architecture = Some(arch); + self + } + + fn xml_to(&self, w: &mut XmlFormatter) -> fmt::Result { + let version = self.version.to_string(); + let public_key_token = self.public_key_token.as_ref().map(|token| token.to_string()); + + let mut attrs: Vec<(&str, &str)> = Vec::with_capacity(6); + if let Some(ref language) = self.language { + attrs.push(("language", language)); + } + attrs.push(("name", &self.name)); + if let Some(ref arch) = self.processor_architecture { + attrs.push(("processorArchitecture", arch.as_str())) + } + if let Some(ref token) = public_key_token { + attrs.push(("publicKeyToken", token)); + } + attrs.push(("type", self.r#type.as_str())); + attrs.push(("version", &version)); + w.empty_element("assemblyIdentity", &attrs) + } +} + +#[derive(Debug)] +struct Version(u16, u16, u16, u16); + +impl fmt::Display for Version { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!(f, "{}.{}.{}.{}", self.0, self.1, self.2, self.3) + } +} + +#[derive(Debug)] +struct PublicKeyToken(u64); + +impl fmt::Display for PublicKeyToken { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!(f, "{:016x}", self.0) + } +} + +/// Processor architecture for an assembly identity. +#[derive(Debug)] +#[non_exhaustive] +pub enum AssemblyProcessorArchitecture { + /// Any processor architecture, as `"*"`. + All, + /// 32-bit x86 processors, as `"x86"`. + X86, + /// 64-bit x64 processors, as `"x64"`. + Amd64, + /// 32-bit ARM processors, as `"arm"`. + Arm, + /// 64-bit ARM processors, as `"arm64"`. + Arm64, +} + +impl AssemblyProcessorArchitecture { + pub fn as_str(&self) -> &'static str { + match self { + Self::All => "*", + Self::X86 => "x86", + Self::Amd64 => "amd64", + Self::Arm => "arm", + Self::Arm64 => "arm64", + } + } +} + +impl fmt::Display for AssemblyProcessorArchitecture { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + f.pad(self.as_str()) + } +} + +#[derive(Debug)] +#[non_exhaustive] +enum AssemblyType { + Win32, +} + +impl AssemblyType { + fn as_str(&self) -> &'static str { + "win32" + } +} + +#[derive(Debug)] +struct ApplicationCompatibility { + max_version_tested: Option, + supported_os: Vec, +} + +impl ApplicationCompatibility { + fn is_empty(&self) -> bool { + self.supported_os.is_empty() + } + + fn xml_to(&self, w: &mut XmlFormatter) -> fmt::Result { + w.element( + "compatibility", + &[("xmlns", "urn:schemas-microsoft-com:compatibility.v1")], + |w| { + w.element("application", &[], |w| { + if self.supported_os.contains(&SupportedOS::Windows10) { + if let Some(ref version) = self.max_version_tested { + w.empty_element("maxversiontested", &[("Id", version.as_str())])?; + } + } + for os in self.supported_os.iter() { + w.empty_element("supportedOS", &[("Id", os.as_str())])? + } + Ok(()) + }) + }, + ) + } +} + +/// Windows build versions for [`max_version_tested()`][ManifestBuilder::max_version_tested] +/// from the [Windows SDK archive](https://developer.microsoft.com/en-us/windows/downloads/sdk-archive/). +#[derive(Debug)] +#[non_exhaustive] +pub enum MaxVersionTested { + /// Windows 10 version 1903, with build version 10.0.18362.0. + Windows10Version1903, + /// Windows 10 version 2004, with build version 10.0.19041.0. + Windows10Version2004, + /// Windows 10 version 2104, with build version 10.0.20348.0. + Windows10Version2104, + /// Windows 11, with build version 10.0.22000.194. + Windows11, + /// Windows 11 version 22H2, with build version 10.0.22621.1. + Windows11Version22H2, +} + +impl MaxVersionTested { + /// Return the Windows version as a string. + pub fn as_str(&self) -> &'static str { + match self { + Self::Windows10Version1903 => "10.0.18362.1", + Self::Windows10Version2004 => "10.0.19041.0", + Self::Windows10Version2104 => "10.0.20348.0", + Self::Windows11 => "10.0.22000.194", + Self::Windows11Version22H2 => "10.0.22621.1", + } + } +} + +impl Display for MaxVersionTested { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + f.pad(self.as_str()) + } +} + +/// Operating system versions for Windows compatibility. +#[derive(Debug, Ord, PartialOrd, Eq, PartialEq)] +#[non_exhaustive] +pub enum SupportedOS { + /// Windows Vista and Windows Server 2008. + WindowsVista, + /// Windows 7 and Windows Server 2008 R2. + Windows7, + /// Windows 8 and Windows Server 2012. + Windows8, + /// Windows 8.1 and Windows Server 2012 R2. + Windows81, + /// Windows 10 and 11, and Windows Server 2016, 2019 and 2022. + Windows10, +} + +impl SupportedOS { + /// Returns the GUID string for the Windows version. + pub fn as_str(&self) -> &'static str { + match self { + Self::WindowsVista => "{e2011457-1546-43c5-a5fe-008deee3d3f0}", + Self::Windows7 => "{35138b9a-5d96-4fbd-8e2d-a2440225f93a}", + Self::Windows8 => "{4a2f28e3-53b9-4441-ba9c-d69d4a4a6e38}", + Self::Windows81 => "{1f676c76-80e1-4239-95bb-83d0f6d0da78}", + Self::Windows10 => "{8e0f7a12-bfb3-4fe8-b9a5-48fd50a15a9a}", + } + } +} + +impl Display for SupportedOS { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + f.pad(self.as_str()) + } +} + +static WS2005: (&str, &str) = ("xmlns", "http://schemas.microsoft.com/SMI/2005/WindowsSettings"); +static WS2011: (&str, &str) = ("xmlns", "http://schemas.microsoft.com/SMI/2011/WindowsSettings"); +static WS2013: (&str, &str) = ("xmlns", "http://schemas.microsoft.com/SMI/2013/WindowsSettings"); +static WS2016: (&str, &str) = ("xmlns", "http://schemas.microsoft.com/SMI/2016/WindowsSettings"); +static WS2017: (&str, &str) = ("xmlns", "http://schemas.microsoft.com/SMI/2017/WindowsSettings"); +static WS2019: (&str, &str) = ("xmlns", "http://schemas.microsoft.com/SMI/2019/WindowsSettings"); +static WS2020: (&str, &str) = ("xmlns", "http://schemas.microsoft.com/SMI/2020/WindowsSettings"); + +#[derive(Debug)] +struct WindowsSettings { + active_code_page: ActiveCodePage, + disable_window_filtering: bool, + dpi_awareness: DpiAwareness, + gdi_scaling: bool, + heap_type: HeapType, + long_path_aware: bool, + printer_driver_isolation: bool, + scrolling_awareness: ScrollingAwareness, +} + +impl WindowsSettings { + fn new() -> Self { + Self { + active_code_page: ActiveCodePage::Utf8, + disable_window_filtering: false, + dpi_awareness: DpiAwareness::PerMonitorV2Only, + gdi_scaling: false, + heap_type: HeapType::LowFragmentationHeap, + long_path_aware: true, + printer_driver_isolation: true, + scrolling_awareness: ScrollingAwareness::UltraHighResolution, + } + } + + fn empty() -> Self { + Self { + active_code_page: ActiveCodePage::System, + disable_window_filtering: false, + dpi_awareness: DpiAwareness::UnawareByDefault, + gdi_scaling: false, + heap_type: HeapType::LowFragmentationHeap, + long_path_aware: false, + printer_driver_isolation: false, + scrolling_awareness: ScrollingAwareness::UltraHighResolution, + } + } + + fn is_empty(&self) -> bool { + matches!( + self, + Self { + active_code_page: ActiveCodePage::System, + disable_window_filtering: false, + dpi_awareness: DpiAwareness::UnawareByDefault, + gdi_scaling: false, + heap_type: HeapType::LowFragmentationHeap, + long_path_aware: false, + printer_driver_isolation: false, + scrolling_awareness: ScrollingAwareness::UltraHighResolution, + } + ) + } + + fn xml_to(&self, w: &mut XmlFormatter) -> fmt::Result { + w.element("asmv3:application", &[], |w| { + w.element("asmv3:windowsSettings", &[], |w| { + self.active_code_page.xml_to(w)?; + if self.disable_window_filtering { + w.element("disableWindowFiltering", &[WS2011], |w| w.text("true"))?; + } + self.dpi_awareness.xml_to(w)?; + if self.gdi_scaling { + w.element("gdiScaling", &[WS2017], |w| w.text("true"))?; + } + if matches!(self.heap_type, HeapType::SegmentHeap) { + w.element("heapType", &[WS2020], |w| w.text("SegmentHeap"))?; + } + if self.long_path_aware { + w.element("longPathAware", &[WS2016], |w| w.text("true"))?; + } + if self.printer_driver_isolation { + w.element("printerDriverIsolation", &[WS2011], |w| w.text("true"))?; + } + self.scrolling_awareness.xml_to(w) + }) + }) + } +} + +/// Configure whether a Windows setting is enabled or disabled, avoiding confusion +/// over which of these options `true` and `false` represent. +#[derive(Debug)] +pub enum Setting { + Disabled = 0, + Enabled = 1, +} + +impl Setting { + /// Returns `true` if the setting should be disabled. + fn disabled(&self) -> bool { + matches!(self, Setting::Disabled) + } + + /// Returns `true` if the setting should be enabled. + fn enabled(&self) -> bool { + matches!(self, Setting::Enabled) + } +} + +/// The code page used by single-byte APIs in the program. +#[derive(Debug)] +#[non_exhaustive] +pub enum ActiveCodePage { + /// Use the code page from the configured system locale, or UTF-8 if “Use Unicode UTF-8 + /// for worldwide language support” is configured. + System, + /// Use UTF-8 from Windows 10 version 1903 and on Windows 11. + Utf8, + /// Use the code page from the configured system locale, even when “Use Unicode UTF-8 + /// for worldwide language support” is configured. + Legacy, + /// Use the code page from the configured system locale on Windows 10, or from this + /// locale on Windows 11. + Locale(String), +} + +impl ActiveCodePage { + pub fn as_str(&self) -> &str { + match self { + Self::System => "", + Self::Utf8 => "UTF-8", + Self::Legacy => "Legacy", + Self::Locale(s) => s, + } + } + + fn xml_to(&self, w: &mut XmlFormatter) -> fmt::Result { + match self { + Self::System => Ok(()), + _ => w.element("activeCodePage", &[WS2019], |w| w.text(self.as_str())), + } + } +} + +impl Display for ActiveCodePage { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + f.pad(self.as_str()) + } +} + +/// Options for how Windows will handle drawing on monitors when the graphics +/// need scaling to display at the correct size. +/// +/// See [High DPI Desktop Application Development on Windows][dpi] for more details +/// about the impact of these options. +/// +/// [dpi]: https://docs.microsoft.com/en-us/windows/win32/hidpi/high-dpi-desktop-application-development-on-windows +#[derive(Debug)] +#[non_exhaustive] +pub enum DpiAwareness { + /// DPI awareness is not configured, so Windows will scale the application unless + /// changed via the `SetProcessDpiAware` or `SetProcessDpiAwareness` API. + UnawareByDefault, + /// DPI awareness is not configured, with Windows 8.1, 10 and 11 not able + /// to change the setting via API. + Unaware, + /// The program uses the system DPI, or the DPI of the monitor they start on if + /// “Fix scaling for apps” is enabled. If the DPI does not match the current + /// monitor then Windows will scale the application. + System, + /// The program uses the system DPI on Windows Vista, 7 and 8, and version 1 of + /// per-monitor DPI awareness on Windows 8.1, 10 and 11. Using version 1 of the + /// API is not recommended. + PerMonitor, + /// The program uses the system DPI on Windows Vista, 7 and 8, version 1 of + /// per-monitor DPI awareness on Windows 8.1 and Windows 10 version 1507 and 1511, + /// and version 2 of per-monitor DPI awareness from Windows 10 version 1607. + PerMonitorV2, + /// The program uses the system DPI on Windows Vista, 7, 8, 8.1 and Windows 10 + /// version 1507 and 1511, and version 2 of per-monitor DPI awareness from + /// Windows 10 version 1607. + PerMonitorV2Only, +} + +impl DpiAwareness { + fn xml_to(&self, w: &mut XmlFormatter) -> fmt::Result { + let settings = match self { + Self::UnawareByDefault => (None, None), + Self::Unaware => (Some("false"), None), + Self::System => (Some("true"), None), + Self::PerMonitor => (Some("true/pm"), None), + Self::PerMonitorV2 => (Some("true/pm"), Some("permonitorv2,permonitor")), + Self::PerMonitorV2Only => (None, Some("permonitorv2")), + }; + if let Some(dpi_aware) = settings.0 { + w.element("dpiAware", &[WS2005], |w| w.text(dpi_aware))?; + } + if let Some(dpi_awareness) = settings.1 { + w.element("dpiAwareness", &[WS2016], |w| w.text(dpi_awareness))?; + } + Ok(()) + } +} + +/// The heap type for the default memory allocator. +#[derive(Debug)] +#[non_exhaustive] +pub enum HeapType { + /// The default heap type since Windows Vista. + LowFragmentationHeap, + /// The modern segment heap, which may reduce total memory allocation in some programs. + /// This is supported since Windows 10 version 2004. See + /// [Improving Memory Usage in Microsoft Edge][edge]. + /// + /// [edge]: https://blogs.windows.com/msedgedev/2020/06/17/improving-memory-usage/ + SegmentHeap, +} + +/// Whether the application supports scroll wheel events with a minimum delta +/// of 1, 40 or 120. +#[derive(Debug)] +#[non_exhaustive] +pub enum ScrollingAwareness { + /// The application can only handle scroll wheel events with the original delta of 120. + LowResolution, + /// The application can handle high precision scroll wheel events with a delta of 40. + HighResolution, + /// The application can handle ultra high precision scroll wheel events with a delta as low as 1. + UltraHighResolution, +} + +impl ScrollingAwareness { + fn xml_to(&self, w: &mut XmlFormatter) -> fmt::Result { + match self { + Self::LowResolution => w.element("ultraHighResolutionScrollingAware", &[WS2013], |w| w.text("false")), + Self::HighResolution => w.element("highResolutionScrollingAware", &[WS2013], |w| w.text("true")), + Self::UltraHighResolution => Ok(()), + } + } +} + +#[derive(Debug)] +struct RequestedExecutionLevel { + level: ExecutionLevel, + ui_access: bool, +} + +impl RequestedExecutionLevel { + fn xml_to(&self, w: &mut XmlFormatter) -> fmt::Result { + w.element("asmv3:trustInfo", &[], |w| { + w.element("asmv3:security", &[], |w| { + w.element("asmv3:requestedPrivileges", &[], |w| { + w.empty_element( + "asmv3:requestedExecutionLevel", + &[ + ("level", self.level.as_str()), + ("uiAccess", if self.ui_access { "true" } else { "false" }), + ], + ) + }) + }) + }) + } +} + +/// The requested execution level for the application when started. +/// +/// The behaviour of each option is described in +/// [Designing UAC Applications for Windows Vista Step 6: Create and Embed an Application Manifest][step6]. +/// +/// [step6]: https://msdn.microsoft.com/en-us/library/bb756929.aspx +#[derive(Debug)] +pub enum ExecutionLevel { + /// The application will always run with the same authorities as the program invoking it. + AsInvoker, + /// The program will run without special authorities for a standard user, but will try to + /// run with administrator authority if the user is an administrator. This is rarely used. + HighestAvailable, + /// The application will run as an administrator, prompting for elevation if necessary. + RequireAdministrator, +} + +impl ExecutionLevel { + pub fn as_str(&self) -> &'static str { + match self { + Self::AsInvoker => "asInvoker", + Self::HighestAvailable => "highestAvailable", + Self::RequireAdministrator => "requireAdministrator", + } + } +} + +impl Display for ExecutionLevel { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + f.pad(self.as_str()) + } +} diff --git a/third_party/rust/embed-manifest/src/manifest/test.rs b/third_party/rust/embed-manifest/src/manifest/test.rs new file mode 100644 index 0000000000..9290f7661d --- /dev/null +++ b/third_party/rust/embed-manifest/src/manifest/test.rs @@ -0,0 +1,117 @@ +use super::{ExecutionLevel, SupportedOS}; +use crate::{empty_manifest, new_manifest}; + +fn enc(s: &str) -> String { + let mut buf = String::with_capacity(1024); + buf.push('\u{feff}'); + for l in s.lines() { + buf.push_str(l); + buf.push_str("\r\n"); + } + buf.truncate(buf.len() - 2); + buf +} + +fn encp(s: &str) -> String { + s.replace("\n", "\r\n") +} + +#[test] +fn empty_manifest_canonical() { + let builder = empty_manifest(); + assert_eq!( + format!("{}", builder), + enc(r#" +"#) + ); +} + +#[test] +fn empty_manifest_pretty() { + let builder = empty_manifest(); + assert_eq!( + format!("{:#}", builder), + encp( + r#" +"# + ) + ); +} + +#[test] +fn only_execution_level() { + let builder = empty_manifest().requested_execution_level(ExecutionLevel::AsInvoker); + assert_eq!( + format!("{:#}", builder), + encp( + r#" + + + + + + + + +"# + ) + ); +} + +#[test] +fn only_windows10() { + let builder = empty_manifest().supported_os(SupportedOS::Windows10..); + assert_eq!( + format!("{:#}", builder), + encp( + r#" + + + + + + +"# + ) + ); +} + +#[test] +fn no_comctl32_6() { + let builder = new_manifest("Company.OrgUnit.Program") + .version(1, 0, 0, 2) + .remove_dependency("Microsoft.Windows.Common-Controls"); + assert_eq!( + format!("{:#}", builder), + encp( + r#" + + + + + + + + + + + + + + UTF-8 + permonitorv2 + true + true + + + + + + + + + +"# + ) + ); +} diff --git a/third_party/rust/embed-manifest/src/manifest/xml.rs b/third_party/rust/embed-manifest/src/manifest/xml.rs new file mode 100644 index 0000000000..a247f42a61 --- /dev/null +++ b/third_party/rust/embed-manifest/src/manifest/xml.rs @@ -0,0 +1,140 @@ +use std::fmt::{self, Display, Formatter, Write}; + +/// Simple but still over-engineered XML generator for a [`Formatter`], for generating +/// Windows Manifest XML. This can easily generate invalid XML. +/// +/// When used correctly, this should generate the same output as MT’s `-canonicalize` +/// option. +pub struct XmlFormatter<'a, 'f> { + f: &'f mut Formatter<'a>, + state: State, + depth: usize, +} + +#[derive(Eq, PartialEq)] +enum State { + Init, + StartTag, + Text, +} + +impl<'a, 'f> XmlFormatter<'a, 'f> { + pub fn new(f: &'f mut Formatter<'a>) -> Self { + Self { + f, + state: State::Init, + depth: 0, + } + } + + fn pretty(&mut self) -> fmt::Result { + if self.f.alternate() { + self.f.write_str("\r\n")?; + for _ in 0..self.depth { + self.f.write_str(" ")?; + } + } + Ok(()) + } + + pub fn start_document(&mut self) -> fmt::Result { + if !self.f.alternate() { + self.f.write_char('\u{FEFF}')?; + } + self.f + .write_str("\r\n") + } + + pub fn element fmt::Result>(&mut self, name: &str, attrs: &[(&str, &str)], f: F) -> fmt::Result { + self.start_element(name, attrs)?; + f(self)?; + self.end_element(name) + } + + pub fn empty_element(&mut self, name: &str, attrs: &[(&str, &str)]) -> fmt::Result { + self.start_element(name, attrs)?; + self.end_element(name) + } + + pub fn start_element(&mut self, name: &str, attrs: &[(&str, &str)]) -> fmt::Result { + if self.state == State::StartTag { + self.f.write_char('>')?; + } + if self.depth != 0 { + self.pretty()?; + } + write!(self.f, "<{}", name)?; + for (name, value) in attrs { + write!(self.f, " {}=\"{}\"", name, Xml(value))?; + } + self.depth += 1; + self.state = State::StartTag; + Ok(()) + } + + pub fn end_element(&mut self, name: &str) -> fmt::Result { + self.depth -= 1; + match self.state { + State::Init => { + self.pretty()?; + write!(self.f, "", name) + } + State::Text => { + self.state = State::Init; + write!(self.f, "", name) + } + State::StartTag => { + self.state = State::Init; + if self.f.alternate() { + self.f.write_str("/>") + } else { + write!(self.f, ">", name) + } + } + } + } + + pub fn text(&mut self, s: &str) -> fmt::Result { + if self.state == State::StartTag { + self.state = State::Text; + self.f.write_char('>')?; + } + Xml(s).fmt(self.f) + } +} + +/// Temporary wrapper for outputting a string with XML attribute encoding. +/// This does not do anything with the control characters which are not +/// valid in XML, encoded or not. +struct Xml<'a>(&'a str); + +impl<'a> Display for Xml<'a> { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + // Process the string in blocks separated by special characters, so that the parts that + // don't need encoding can be written all at once, not character by character, and with + // no checks for whether string slices are aligned on character boundaries. + for s in self.0.split_inclusive(&['<', '&', '>', '"', '\r'][..]) { + // Check whether the last character in the substring needs encoding. This will be + // `None` at the end of the input string. + let mut iter = s.chars(); + let ch = match iter.next_back() { + Some('<') => Some("<"), + Some('&') => Some("&"), + Some('>') => Some(">"), + Some('"') => Some("""), + Some('\r') => Some(" "), + _ => None, + }; + // Write the substring except the last character, then the encoded character; + // or the entire substring if it is not terminated by a special character. + match ch { + Some(enc) => { + f.write_str(iter.as_str())?; + f.write_str(enc)?; + } + None => f.write_str(s)?, + } + } + Ok(()) + } +} diff --git a/third_party/rust/embed-manifest/testdata/sample.exe.manifest b/third_party/rust/embed-manifest/testdata/sample.exe.manifest new file mode 100644 index 0000000000..b7e12866f0 --- /dev/null +++ b/third_party/rust/embed-manifest/testdata/sample.exe.manifest @@ -0,0 +1,11 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/third_party/rust/error-support/.cargo-checksum.json b/third_party/rust/error-support/.cargo-checksum.json index 475bff7665..e81ad51911 100644 --- a/third_party/rust/error-support/.cargo-checksum.json +++ b/third_party/rust/error-support/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"Cargo.toml":"49ef90bd388b59229db34b35fe06eb769183431c88b5712e6e9992851aef605d","README.md":"8030b4a314b1be31ba018ac12c3b586bb736db5307c3c395f2857fffe0130322","build.rs":"c8d3c38c1208eea36224662b284d8daf3e7ad1b07d22d750524f3da1cc66ccca","src/errorsupport.udl":"e793034d01a2608298528051757f38405e006ee1abc4cf65dc6f18c53590ace8","src/handling.rs":"6e0568b18d426531cb2ae9967c8dd0d51ece5a065f68b15eeb308b995edaa167","src/lib.rs":"96ae3cc2c1077ae45442ace6b5b5311b86267d0b9067f3ff58396af30ccbbc07","src/macros.rs":"0d03f82fab20c96a182f941baf3fcf2a286b00fea871ee7fd8e339abc14f9522","src/redact.rs":"c9a4df1a87be68b15d583587bda941d4c60a1d0449e2d43ff99f3611a290a863","src/reporting.rs":"38efd24d86ba8facfb181cb27e8b698d2831db0afab85691ffda034a4dc68dfa","uniffi.toml":"644fe81c12fe3c01ee81e017ca3c00d0e611f014b7eade51aadaf208179a3450"},"package":null} \ No newline at end of file +{"files":{"Cargo.toml":"bd2f0908b3576a3ad9a416ecb0e4f8441a48a95036cf0439a65e37d836178142","README.md":"8030b4a314b1be31ba018ac12c3b586bb736db5307c3c395f2857fffe0130322","build.rs":"c8d3c38c1208eea36224662b284d8daf3e7ad1b07d22d750524f3da1cc66ccca","src/errorsupport.udl":"e793034d01a2608298528051757f38405e006ee1abc4cf65dc6f18c53590ace8","src/handling.rs":"6e0568b18d426531cb2ae9967c8dd0d51ece5a065f68b15eeb308b995edaa167","src/lib.rs":"96ae3cc2c1077ae45442ace6b5b5311b86267d0b9067f3ff58396af30ccbbc07","src/macros.rs":"0d03f82fab20c96a182f941baf3fcf2a286b00fea871ee7fd8e339abc14f9522","src/redact.rs":"c9a4df1a87be68b15d583587bda941d4c60a1d0449e2d43ff99f3611a290a863","src/reporting.rs":"38efd24d86ba8facfb181cb27e8b698d2831db0afab85691ffda034a4dc68dfa","uniffi.toml":"644fe81c12fe3c01ee81e017ca3c00d0e611f014b7eade51aadaf208179a3450"},"package":null} \ No newline at end of file diff --git a/third_party/rust/error-support/Cargo.toml b/third_party/rust/error-support/Cargo.toml index e4c39618a6..074ad4a32d 100644 --- a/third_party/rust/error-support/Cargo.toml +++ b/third_party/rust/error-support/Cargo.toml @@ -21,7 +21,7 @@ license = "MPL-2.0" [dependencies] log = "0.4" -uniffi = "0.25.2" +uniffi = "0.27.1" [dependencies.backtrace] version = "0.3" @@ -37,5 +37,5 @@ version = "1.4" version = ">=0.11,<=0.12" [build-dependencies.uniffi] -version = "0.25.2" +version = "0.27.1" features = ["build"] diff --git a/third_party/rust/glean-core/.cargo-checksum.json b/third_party/rust/glean-core/.cargo-checksum.json index 54674fc768..663b3141c3 100644 --- a/third_party/rust/glean-core/.cargo-checksum.json +++ b/third_party/rust/glean-core/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"Cargo.toml":"2dde200f0e0e4e523634f8c2c8c1c2ca75af83163ac7b0ba8f62f3096fd0c97d","LICENSE":"1f256ecad192880510e84ad60474eab7589218784b9a50bc7ceee34c2b91f1d5","README.md":"026495898699b54608eb4ec16074ffafc57920d80ccb59961c501a1ea28c9985","build.rs":"4857bea99c6b8c08db8818efa9d3738716f52d3acb68159323957ae52892a3eb","src/common_metric_data.rs":"72051c4349885d4a94fc41bb4edda88d31551f2f9ebcdb4e868a98161bc76233","src/core/mod.rs":"8f5e98a108ec5d1849402af1de90b5f53ba839240743c2c5283a49a4045e1293","src/core_metrics.rs":"a877e42e0f8b932adb52a5681ad76fd977808cb48c7eeb29b1e4bbe804f1ea96","src/coverage.rs":"49613fd310bd24d779472720975fbe6c97ec370a95eb55f10afa43f67539c942","src/database/mod.rs":"3917bad7773696a43ab58e7860d5a8f1d63dca7c27920343aa8786acc5a586cd","src/debug.rs":"90158cc5d488ba67b60d06647e54e59a1d7bdeb906087e4fe4cfab4373c1cc6c","src/dispatcher/global.rs":"f69cd81a90a37c306d4e0ce8177ea5a3ae2ffda5b431ae46b9a22c9e38891271","src/dispatcher/mod.rs":"440a331a7beeaa6e9824c2fd4306c09ce2a115a358d5beb830dba4d69aec3563","src/error.rs":"b93c7d3e243b21bb9eafc95f17860aba1a942b2f2b0a7f43307690f05fece516","src/error_recording.rs":"d7858647808d81173558e12de88f3fbe4e589969e3bd246bfb388f90f8ff3814","src/event_database/mod.rs":"9d4d3e4b075dc585c00317304401b2c9115f39db8fdbd9f1c93d3fc6fd350fd5","src/fd_logger.rs":"0c9def6fa53db1a2ab93c85795f8a7df57797bcfd3978146923e151752e291a6","src/glean.udl":"24d9e431f95d79dc4254feff68f19a4ea4e6e76c33b110e10c5e5dbd5bc64ff2","src/glean_metrics.rs":"9414fb1453d19f6832df33e4c6ef7383d62203e47026bf5bc9552b083101ddd1","src/histogram/exponential.rs":"58bb1770bae45770d92995515d328eb50a7e78726224f779446ae7d1632a6a3e","src/histogram/functional.rs":"1a63a305b48bcef7bc38136b40d916df4bb8f098dc602514ada54a9b091f6951","src/histogram/linear.rs":"4342a1733175d7f97b2b41adb18100537c206100c9fccb5bd13bd782c9cb3c9a","src/histogram/mod.rs":"eeb7aff80806ab76cdce101dc08887b5552f8b4bdf64683f64f767e0f06a889d","src/internal_metrics.rs":"263779535963a804c8c7fa6f8e284ac8ec7f415ceeadbb6a8f913a1e7073ae18","src/internal_pings.rs":"7267166a8e357053526c545cf62bb502a7b6f07aed1de48d43041228d8835366","src/lib.rs":"367ea21f9d3f1c808b258011821d8505cd47d29eff8e8e6d938623e6e9997b73","src/lib_unit_tests.rs":"46897c6bb4003c5e00152d7b55c00d3176b5bffb28d8669a3fb0d10e5233e3a5","src/metrics/boolean.rs":"2b9ef57e3582c9bd8b2cca8ab94c962a4871ecc00e837b913c9b0349ba9dff08","src/metrics/counter.rs":"b4a52a8167fb0edd6354f952525e59f3eadb4261de3483374f03c94449d30b92","src/metrics/custom_distribution.rs":"e1f2edfefb67da4bf369bab3d3047f4ff6539a1fea0eee81c78d96626e5b4bb0","src/metrics/datetime.rs":"e4405762fc71718299fa1b208e3d5fda654bd1b82fe908c884c284e3530de2ec","src/metrics/denominator.rs":"95e8442f90bad97f80fc74b146782f215344b52c5f3825ae0a8baffdc001a714","src/metrics/event.rs":"7281d8b63f34758a47abd7ae3956f44701d1fd48433ccba7a4302526a9912255","src/metrics/experiment.rs":"5f9278cca4e133eb8df33bbfe36d1fe0ef3eade8c09f1b46db3c4d0790515412","src/metrics/labeled.rs":"8d6e76a07064d132cd617c7901f2bc11ff6ba31e3483ba3b96354a4a3736b58d","src/metrics/memory_distribution.rs":"7f6ca51acb470df277ff14427c0e7bb07d921c0a0087d0cc56aebe038d198ccc","src/metrics/memory_unit.rs":"d7a678e5242febd021283b30c0099a9e62729944816a3f17d2d91e2808bc0570","src/metrics/metrics_enabled_config.rs":"87fed12219c756ecf1e5c8cd6a21f26999b6bbcf3ffc1b5467b0a58ca5ad35d8","src/metrics/mod.rs":"8f8958b8cedfe01df6c97ec26b63f14fd7516f9de7ba62984062db96b5708720","src/metrics/numerator.rs":"937dfd583b797ac798a525cedca95c5a36262356760a89670d8113983c263154","src/metrics/object.rs":"89ce5190ed681b26b74a06a4ecaf9f96c36f96be1276f1fdb40f4406648e08c1","src/metrics/ping.rs":"4ccdf0ae2ac6f3e5a352334797d2805f1a3d932e92f08447285dd9bec4e7d724","src/metrics/quantity.rs":"aa13a8f8cf8e5e0281668fbbafc2998411df2a499479423558fd91b9bd7f8702","src/metrics/rate.rs":"603cc45c149c7a27c93b6a80146bf43f8ce70d9655f905bb5be6bc2c15bcb22b","src/metrics/recorded_experiment.rs":"33958abee79d8b55dec4cb5d20742640423713010f76314075cefde18b5c118a","src/metrics/string.rs":"2418632c492463970c3eca533d5318f519698bb361d73dd8781db108d7d1fbd8","src/metrics/string_list.rs":"ed53a095184c3e8224d0511809b5d7601ba3166505a39b0570f24ebeb0a5b97c","src/metrics/text.rs":"5c994a282b16b9dde6d6dc4922475457a72c82f64248778811b84db70ed4c116","src/metrics/time_unit.rs":"b7578010c6270a45b30342b59189a862b2ede9dd24e9afae3e90fa6b970b3d24","src/metrics/timespan.rs":"b0fda3a45597c8306a0d1928dcf0837538859e66ebd9db113ebb6efbea721d4c","src/metrics/timing_distribution.rs":"5da04272dd8b44502ffd0b60b12c84239a7fe359a51754b6c0cd96388a4e8a3c","src/metrics/url.rs":"f6b27a60d13a1268f0115c5d292c9b16b6bc370055961368cb2648283b7140a0","src/metrics/uuid.rs":"cacffd95ab30ed327ec2fa5feaf1359e667706746401f1e2c1195ad9553c4b54","src/ping/mod.rs":"fcadd52d2d536c9ace01f8a3812c3fb3c39b8094915db1b3656839fb87f771b5","src/scheduler.rs":"129863e31205404a3d1708627a62583324c347d143f976216f769893ec541ea0","src/storage/mod.rs":"04dc1a94be1d59097cd87b14386952a6ec8b9115bc06397ae389a323f6f55dcc","src/system.rs":"e3d1b54e1d39cafe6f4dc7ff5021b08c879733f909951b0e1332b3efa9ed97bd","src/traits/boolean.rs":"be0e130f8043215705becc956d45b126c340568f1b24a396c0af9b4334a41ced","src/traits/counter.rs":"c686d26e131d854cd7a7df83c900ca7c17a03c663a30cf58ab48c7259476ce85","src/traits/custom_distribution.rs":"0bd1d425e4c059cca6af2dfb13c78e5e4c6c07fb46c7e31489ad0c5959854833","src/traits/datetime.rs":"636ac1456b1b042e38cf5ae6193c5b232ea0b80df62f583a2097891baef9641b","src/traits/event.rs":"3f48aa336854141784d121f7fa9e283f6ff708a9214f9c0aade3a68cc38dda99","src/traits/labeled.rs":"c633c68e70a44e73f8aff88aaab1029c0faded3cad08d822590ed8838f24b4fd","src/traits/memory_distribution.rs":"55bb8f45e948319fbba9d28a50d8742da134b066a42e480887db7c7e435f4096","src/traits/mod.rs":"d14b69d0946848c1f92cc8977cbc3fc9338ff1b53b7acc31ea0fe2f1122beecb","src/traits/numerator.rs":"6e4f236bdc448f1bde7a8c249dcd086204c2c69990d3f444e746290929226ed3","src/traits/object.rs":"c03bad670ec7affbc578247f9e1904e898c1870b9bf25750c5094113f995623f","src/traits/ping.rs":"8831c106c03afeb458b0b028fa1ce61f056ebf8e82bc0a171a1bff255d920748","src/traits/quantity.rs":"6ffe25c913bef4315573d747308c182de740b2a4e02ba22cd21d0c33ba521f31","src/traits/rate.rs":"f000790440e0f389f0b160526a9a9a266e58d1405915ae56ac550f482858222c","src/traits/string.rs":"0c3c88382ff2e8eba89c7cfe129c4b84e31140af717819533c14919541ad790c","src/traits/string_list.rs":"14e56b62c2c2be1dd8013f12001f235b084abd2a0d5aa2f7932843877af49ac0","src/traits/text.rs":"8af7d3a0c87cfd8c6d33d6ad47532b431055bbdd395f9110da5630222c23cf93","src/traits/timespan.rs":"52be325a9c061916f34c5b638a07a93b4a14aa89fe365783103d2e06b998f547","src/traits/timing_distribution.rs":"00ebdef647a7a208c01d13ba7b3996750e36de98d1f63859b609c80c8df25b6f","src/traits/url.rs":"c27f7add23214ff051078b65b88120b620560d2841a1056c7214d5237e86b9e4","src/traits/uuid.rs":"81322e71c7e847bacaf827a2cd58f6193bdc208355524207f7f38db039da6aa8","src/upload/directory.rs":"6359220db9d85ee0f3931ca518f95ffb2020c1c03bd632f17ed5c16ddd00343b","src/upload/mod.rs":"a388563d5e2940c5c28b48fc7b67ca507512efccae95fd1c2f04b15ec21aa08c","src/upload/policy.rs":"c250957a37783e74af8002cd80ba06ef9780a389fb0f61b8b665b79688f0a360","src/upload/request.rs":"0b7e215f61499a681d1cebc9cf4a0efbaae2f543a5d44e5db40cbe61ed90549e","src/upload/result.rs":"7efbbe50e8d36beb3f23e7bfd172d22e1c003472d2dd8055b06f6050c36437c5","src/util.rs":"ee7500434d9758a320dd410f18d7e18da956591e19d2555db87eef9623e4b916","tests/boolean.rs":"76d6014ff108cb6514d9bceb1b2b14749a55b09921f4595a5e30f1bd3546e9f0","tests/common/mod.rs":"c1d980a9cff0b64f452ebbe43f24d70aa685b80b48db08fc4338a60466b07a5e","tests/counter.rs":"3663a3f5ec5c0bd2b758a9920cd20cc619a12566b445e4421ec7c98232bf5a32","tests/custom_distribution.rs":"41c593a0b4561e21f29d1a5b948de964a866253c58ca76ffefebe370fca150e0","tests/datetime.rs":"ec3c9760e70bb2cbc61ab23281c891bc1ec493c5c545466c29fd13e4f05c2c96","tests/event.rs":"67291cbcc4d1cba56ada6ba733fb1dc4c6327680059e8d7637add2ae45cd344b","tests/labeled.rs":"e9ea6dba17059d68114efce0c23373be9ceed922bf5e638a2158a6422c75a1c1","tests/memory_distribution.rs":"a5a7aa955e60823ea29a6f4bc96c61e41f1e41f08958aa4854668cf8fe04cde6","tests/object.rs":"8c35676e04f6ccf54a28764700915e753fc0355bfa5d7804d72caba66fd564cd","tests/ping.rs":"eb9f6be1aba21acc5dc670622bf622976718a706df1cc2095efa56a8e8b3fe1a","tests/ping_maker.rs":"b267ecf7c714ff27512424b743da0ea4f05a87755c1b96355bfca3e173e3f62e","tests/quantity.rs":"55e7dca346fd1d27f0974b78ca3fb12427cb5da2ee637afc08a54f360f947361","tests/rate.rs":"1de571b9f0ee9a9006cbc8a31f91352d3ff1190b50840f0f668b470a7cd2a3a5","tests/storage.rs":"f0c8312bd789d7bda502cd45f35fef6b8591652bd194d07da4d81935ebe69b48","tests/string.rs":"7ece988a4b8efe6932ccb90bfe2f3c8aaea983777e99d7de6028bf6a29459ee6","tests/string_list.rs":"77188a2b90663c3f8dac5da89a6cb6b1d16a9f8c66ccd032d02966dfd14a3486","tests/text.rs":"1d43f6b90a43124311cacf0a6ee16f9e1e9263bcd11fee8b996d6efd81633638","tests/timespan.rs":"d50d75c7d75da3a878d67331cb0df8ae5e6a099ffab474361f71a408e02528d7","tests/timing_distribution.rs":"20860a7baccdcee6aed40c9cc8202b94f3b2e61164fbaf8f2af96b0f404a895a","tests/uuid.rs":"052ad26a6927c56272219340211cf4a059d200f14287b482fe8621d7bce3cc54","uniffi.toml":"6ddc98b686b0925a81abd9d1c769e5c98ac29771b210a1c535931a46dec9a8e3"},"package":"ed9acc46fd38c5c995a0537e76364496addace660839dc279079e5957e3c1093"} \ No newline at end of file +{"files":{"Cargo.toml":"c590a29d01f2ccad65fdbed80578177ae3c02522d6c6c60eef9644d71f04a0e3","LICENSE":"1f256ecad192880510e84ad60474eab7589218784b9a50bc7ceee34c2b91f1d5","README.md":"026495898699b54608eb4ec16074ffafc57920d80ccb59961c501a1ea28c9985","build.rs":"4857bea99c6b8c08db8818efa9d3738716f52d3acb68159323957ae52892a3eb","src/common_metric_data.rs":"864990a1e5770d5d5fdebcd2c36b58c3442334030fb60f53811395b56baac94b","src/core/mod.rs":"9880520967e9da0b475d280c17cd70debf9a1d15912018cbba775e5fde0ff588","src/core_metrics.rs":"a877e42e0f8b932adb52a5681ad76fd977808cb48c7eeb29b1e4bbe804f1ea96","src/coverage.rs":"49613fd310bd24d779472720975fbe6c97ec370a95eb55f10afa43f67539c942","src/database/mod.rs":"b3684bb6a11e0aa2a51306a53feddbc89bc21879d4930d5e9995869950af3413","src/debug.rs":"d0dfc0932a0953bbbe029f723bf2613c8d691f34b017e858030c46b02a46a17d","src/dispatcher/global.rs":"f69cd81a90a37c306d4e0ce8177ea5a3ae2ffda5b431ae46b9a22c9e38891271","src/dispatcher/mod.rs":"391310269947452d7e0de24c848c183110c60149d75e345ba6d5d146f222dace","src/error.rs":"b93c7d3e243b21bb9eafc95f17860aba1a942b2f2b0a7f43307690f05fece516","src/error_recording.rs":"1aba34e9d3c741755055f5b76415114b25b146b0aa90049c3457cfe12066deda","src/event_database/mod.rs":"78633293e1f3c9e9d51705615a7a4b603d7f85567bfdc2b0bad35ccda6a12d44","src/fd_logger.rs":"0c9def6fa53db1a2ab93c85795f8a7df57797bcfd3978146923e151752e291a6","src/glean.udl":"0fcf72a8e3304d98e896dd3dfd9787208776c2b21b59f1c241029978ee37a925","src/glean_metrics.rs":"9414fb1453d19f6832df33e4c6ef7383d62203e47026bf5bc9552b083101ddd1","src/histogram/exponential.rs":"58bb1770bae45770d92995515d328eb50a7e78726224f779446ae7d1632a6a3e","src/histogram/functional.rs":"1a63a305b48bcef7bc38136b40d916df4bb8f098dc602514ada54a9b091f6951","src/histogram/linear.rs":"4342a1733175d7f97b2b41adb18100537c206100c9fccb5bd13bd782c9cb3c9a","src/histogram/mod.rs":"bbb9535a633b5a85b6b11c6e4eed3314ab797950355a9bb8ccf3a22000f1e093","src/internal_metrics.rs":"263779535963a804c8c7fa6f8e284ac8ec7f415ceeadbb6a8f913a1e7073ae18","src/internal_pings.rs":"0e3b8ce673cf92bd085fd4f07aa43876c97472dbd921a2d0dc0f10c9fe6b1c6b","src/lib.rs":"fb50a72a7221358c5daa1a8c0e59c92d5d358adc10b1ba9b831d2f07c07d90e2","src/lib_unit_tests.rs":"76d1997f7608b735cc4e905cfa94f79dd71a4a2ed1eccaa89d3d72ccd8d348e2","src/metrics/boolean.rs":"2b9ef57e3582c9bd8b2cca8ab94c962a4871ecc00e837b913c9b0349ba9dff08","src/metrics/counter.rs":"b4a52a8167fb0edd6354f952525e59f3eadb4261de3483374f03c94449d30b92","src/metrics/custom_distribution.rs":"e1f2edfefb67da4bf369bab3d3047f4ff6539a1fea0eee81c78d96626e5b4bb0","src/metrics/datetime.rs":"e4405762fc71718299fa1b208e3d5fda654bd1b82fe908c884c284e3530de2ec","src/metrics/denominator.rs":"95e8442f90bad97f80fc74b146782f215344b52c5f3825ae0a8baffdc001a714","src/metrics/event.rs":"cd52e200d313e2e6f31707419d4a7fe1cab34916ee145f8136440d6da34aaad4","src/metrics/experiment.rs":"5f9278cca4e133eb8df33bbfe36d1fe0ef3eade8c09f1b46db3c4d0790515412","src/metrics/labeled.rs":"8d6e76a07064d132cd617c7901f2bc11ff6ba31e3483ba3b96354a4a3736b58d","src/metrics/memory_distribution.rs":"7f6ca51acb470df277ff14427c0e7bb07d921c0a0087d0cc56aebe038d198ccc","src/metrics/memory_unit.rs":"ee32e020cb303dd631457374048a3ed53a2e7cbacc29c54d17d836fb15507538","src/metrics/metrics_enabled_config.rs":"c45f2cd48b36f8706e0e1d402d6fc375f5bab50f7d0840e0fbbbeacb6f2732af","src/metrics/mod.rs":"8f8958b8cedfe01df6c97ec26b63f14fd7516f9de7ba62984062db96b5708720","src/metrics/numerator.rs":"937dfd583b797ac798a525cedca95c5a36262356760a89670d8113983c263154","src/metrics/object.rs":"89ce5190ed681b26b74a06a4ecaf9f96c36f96be1276f1fdb40f4406648e08c1","src/metrics/ping.rs":"86dc577422075c759edb998acbd890c239569d72b30a994e7777d6d0f7676c5a","src/metrics/quantity.rs":"aa13a8f8cf8e5e0281668fbbafc2998411df2a499479423558fd91b9bd7f8702","src/metrics/rate.rs":"603cc45c149c7a27c93b6a80146bf43f8ce70d9655f905bb5be6bc2c15bcb22b","src/metrics/recorded_experiment.rs":"33958abee79d8b55dec4cb5d20742640423713010f76314075cefde18b5c118a","src/metrics/string.rs":"0906b4d5ec1ec10b7a56fd6eb39dc30500531658df2c8bc3f55c9579e15c88db","src/metrics/string_list.rs":"ed53a095184c3e8224d0511809b5d7601ba3166505a39b0570f24ebeb0a5b97c","src/metrics/text.rs":"757f6919124d74e0512faa5bb9751a729b6bbc63ebe4d16ca81e9087f5595eaf","src/metrics/time_unit.rs":"4704703e19e799933aec3f39e3d3a125058756d7c7ba04f8729885c7843df447","src/metrics/timespan.rs":"1ad5233c7522cab70b4c095fb24cace66ace9741731f97bc001ede071f10d1ef","src/metrics/timing_distribution.rs":"261f971d012e80e93180caea69da549498597d47771264c9bb0667a9573f47ed","src/metrics/url.rs":"589ae1f8047367ad8c19b57a48ca8130d5f36cf3ce5954124150f0eb89c620ea","src/metrics/uuid.rs":"cacffd95ab30ed327ec2fa5feaf1359e667706746401f1e2c1195ad9553c4b54","src/ping/mod.rs":"fcadd52d2d536c9ace01f8a3812c3fb3c39b8094915db1b3656839fb87f771b5","src/scheduler.rs":"129863e31205404a3d1708627a62583324c347d143f976216f769893ec541ea0","src/storage/mod.rs":"91f02556f113799e0d88d732ab342bda443f43461369e8b41c424c074d742591","src/system.rs":"e3d1b54e1d39cafe6f4dc7ff5021b08c879733f909951b0e1332b3efa9ed97bd","src/traits/boolean.rs":"be0e130f8043215705becc956d45b126c340568f1b24a396c0af9b4334a41ced","src/traits/counter.rs":"c686d26e131d854cd7a7df83c900ca7c17a03c663a30cf58ab48c7259476ce85","src/traits/custom_distribution.rs":"0bd1d425e4c059cca6af2dfb13c78e5e4c6c07fb46c7e31489ad0c5959854833","src/traits/datetime.rs":"636ac1456b1b042e38cf5ae6193c5b232ea0b80df62f583a2097891baef9641b","src/traits/event.rs":"a02235aae630aba7a45a3166b756927252b397af3ecdfab7236931e62725ac49","src/traits/labeled.rs":"c633c68e70a44e73f8aff88aaab1029c0faded3cad08d822590ed8838f24b4fd","src/traits/memory_distribution.rs":"55bb8f45e948319fbba9d28a50d8742da134b066a42e480887db7c7e435f4096","src/traits/mod.rs":"d14b69d0946848c1f92cc8977cbc3fc9338ff1b53b7acc31ea0fe2f1122beecb","src/traits/numerator.rs":"6e4f236bdc448f1bde7a8c249dcd086204c2c69990d3f444e746290929226ed3","src/traits/object.rs":"c03bad670ec7affbc578247f9e1904e898c1870b9bf25750c5094113f995623f","src/traits/ping.rs":"8831c106c03afeb458b0b028fa1ce61f056ebf8e82bc0a171a1bff255d920748","src/traits/quantity.rs":"6ffe25c913bef4315573d747308c182de740b2a4e02ba22cd21d0c33ba521f31","src/traits/rate.rs":"f000790440e0f389f0b160526a9a9a266e58d1405915ae56ac550f482858222c","src/traits/string.rs":"0c3c88382ff2e8eba89c7cfe129c4b84e31140af717819533c14919541ad790c","src/traits/string_list.rs":"14e56b62c2c2be1dd8013f12001f235b084abd2a0d5aa2f7932843877af49ac0","src/traits/text.rs":"8af7d3a0c87cfd8c6d33d6ad47532b431055bbdd395f9110da5630222c23cf93","src/traits/timespan.rs":"52be325a9c061916f34c5b638a07a93b4a14aa89fe365783103d2e06b998f547","src/traits/timing_distribution.rs":"00ebdef647a7a208c01d13ba7b3996750e36de98d1f63859b609c80c8df25b6f","src/traits/url.rs":"c27f7add23214ff051078b65b88120b620560d2841a1056c7214d5237e86b9e4","src/traits/uuid.rs":"81322e71c7e847bacaf827a2cd58f6193bdc208355524207f7f38db039da6aa8","src/upload/directory.rs":"e42c62f27ace5c6504cc7703a4c1d9ffd0e6ac7c4fba7d7dee231430fb67f8f8","src/upload/mod.rs":"6151a6d3b4fccb3df7ef03207e2f77bf34dbf04b3b705e2af55dd02a731f99f8","src/upload/policy.rs":"c250957a37783e74af8002cd80ba06ef9780a389fb0f61b8b665b79688f0a360","src/upload/request.rs":"5891364d4254aafdb43751f476b0b908b681544793ac98802fe103de321ec326","src/upload/result.rs":"7efbbe50e8d36beb3f23e7bfd172d22e1c003472d2dd8055b06f6050c36437c5","src/util.rs":"ee7500434d9758a320dd410f18d7e18da956591e19d2555db87eef9623e4b916","tests/boolean.rs":"76d6014ff108cb6514d9bceb1b2b14749a55b09921f4595a5e30f1bd3546e9f0","tests/common/mod.rs":"c5bf5a9f3660ae1a1c1dbb659ab6be60438c58bc7c459f2f96dca467d05d4ab3","tests/counter.rs":"3663a3f5ec5c0bd2b758a9920cd20cc619a12566b445e4421ec7c98232bf5a32","tests/custom_distribution.rs":"41c593a0b4561e21f29d1a5b948de964a866253c58ca76ffefebe370fca150e0","tests/datetime.rs":"ec3c9760e70bb2cbc61ab23281c891bc1ec493c5c545466c29fd13e4f05c2c96","tests/event.rs":"0fbec0e8929c99603b79c62a1f57f8cabe614451fdafb6eb9d47f22116303245","tests/labeled.rs":"e9ea6dba17059d68114efce0c23373be9ceed922bf5e638a2158a6422c75a1c1","tests/memory_distribution.rs":"a5a7aa955e60823ea29a6f4bc96c61e41f1e41f08958aa4854668cf8fe04cde6","tests/object.rs":"8c35676e04f6ccf54a28764700915e753fc0355bfa5d7804d72caba66fd564cd","tests/ping.rs":"eb9f6be1aba21acc5dc670622bf622976718a706df1cc2095efa56a8e8b3fe1a","tests/ping_maker.rs":"7ad1f76a1eda2dabf0422fff74d9c2c1a39b9d1d315a4dbe6057dff44efcfae0","tests/quantity.rs":"55e7dca346fd1d27f0974b78ca3fb12427cb5da2ee637afc08a54f360f947361","tests/rate.rs":"1de571b9f0ee9a9006cbc8a31f91352d3ff1190b50840f0f668b470a7cd2a3a5","tests/storage.rs":"f0c8312bd789d7bda502cd45f35fef6b8591652bd194d07da4d81935ebe69b48","tests/string.rs":"7ece988a4b8efe6932ccb90bfe2f3c8aaea983777e99d7de6028bf6a29459ee6","tests/string_list.rs":"77188a2b90663c3f8dac5da89a6cb6b1d16a9f8c66ccd032d02966dfd14a3486","tests/text.rs":"1d43f6b90a43124311cacf0a6ee16f9e1e9263bcd11fee8b996d6efd81633638","tests/timespan.rs":"d50d75c7d75da3a878d67331cb0df8ae5e6a099ffab474361f71a408e02528d7","tests/timing_distribution.rs":"20860a7baccdcee6aed40c9cc8202b94f3b2e61164fbaf8f2af96b0f404a895a","tests/uuid.rs":"052ad26a6927c56272219340211cf4a059d200f14287b482fe8621d7bce3cc54","uniffi.toml":"6ddc98b686b0925a81abd9d1c769e5c98ac29771b210a1c535931a46dec9a8e3"},"package":"ea06a592b1395e0a16a5f4d6872f009ca7c98acc5127a8119088f1b435b5aaae"} \ No newline at end of file diff --git a/third_party/rust/glean-core/Cargo.toml b/third_party/rust/glean-core/Cargo.toml index 9d33444fbd..932b16a4a7 100644 --- a/third_party/rust/glean-core/Cargo.toml +++ b/third_party/rust/glean-core/Cargo.toml @@ -13,7 +13,7 @@ edition = "2021" rust-version = "1.66" name = "glean-core" -version = "58.1.0" +version = "59.0.0" authors = [ "Jan-Erik Rediger ", "The Glean Team ", @@ -80,7 +80,7 @@ version = "1.0.4" version = "0.1.40" [dependencies.uniffi] -version = "0.25.2" +version = "0.27.0" default-features = false [dependencies.uuid] @@ -105,7 +105,7 @@ version = "0.4" version = "3.8.0" [build-dependencies.uniffi] -version = "0.25.2" +version = "0.27.0" features = ["build"] default-features = false diff --git a/third_party/rust/glean-core/src/common_metric_data.rs b/third_party/rust/glean-core/src/common_metric_data.rs index 033cbe1472..9bda9bb462 100644 --- a/third_party/rust/glean-core/src/common_metric_data.rs +++ b/third_party/rust/glean-core/src/common_metric_data.rs @@ -2,7 +2,6 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use std::convert::TryFrom; use std::sync::atomic::{AtomicU8, Ordering}; use crate::error::{Error, ErrorKind}; diff --git a/third_party/rust/glean-core/src/core/mod.rs b/third_party/rust/glean-core/src/core/mod.rs index 30f9a34f11..f69f0c3868 100644 --- a/third_party/rust/glean-core/src/core/mod.rs +++ b/third_party/rust/glean-core/src/core/mod.rs @@ -120,6 +120,7 @@ where /// rate_limit: None, /// enable_event_timestamps: true, /// experimentation_id: None, +/// enable_internal_pings: true, /// }; /// let mut glean = Glean::new(cfg).unwrap(); /// let ping = PingType::new("sample", true, false, true, true, vec![]); @@ -208,7 +209,7 @@ impl Glean { core_metrics: CoreMetrics::new(), additional_metrics: AdditionalMetrics::new(), database_metrics: DatabaseMetrics::new(), - internal_pings: InternalPings::new(), + internal_pings: InternalPings::new(cfg.enable_internal_pings), upload_manager, data_path: PathBuf::from(&cfg.data_path), application_id, @@ -288,7 +289,9 @@ impl Glean { } // We set this only for non-subprocess situations. - glean.schedule_metrics_pings = cfg.use_core_mps; + // If internal pings are disabled, we don't set up the MPS either, + // it wouldn't send any data anyway. + glean.schedule_metrics_pings = cfg.enable_internal_pings && cfg.use_core_mps; // We only scan the pendings pings directories **after** dealing with the upload state. // If upload is disabled, we delete all pending pings files @@ -305,6 +308,7 @@ impl Glean { data_path: &str, application_id: &str, upload_enabled: bool, + enable_internal_pings: bool, ) -> Self { let cfg = InternalConfiguration { data_path: data_path.into(), @@ -320,6 +324,7 @@ impl Glean { rate_limit: None, enable_event_timestamps: true, experimentation_id: None, + enable_internal_pings, }; let mut glean = Self::new(cfg).unwrap(); diff --git a/third_party/rust/glean-core/src/database/mod.rs b/third_party/rust/glean-core/src/database/mod.rs index af473c98d9..0dbf0220bc 100644 --- a/third_party/rust/glean-core/src/database/mod.rs +++ b/third_party/rust/glean-core/src/database/mod.rs @@ -824,7 +824,6 @@ mod test { use super::*; use crate::tests::new_glean; use std::collections::HashMap; - use std::path::Path; use tempfile::tempdir; #[test] diff --git a/third_party/rust/glean-core/src/debug.rs b/third_party/rust/glean-core/src/debug.rs index a572a02b8f..88f807bd88 100644 --- a/third_party/rust/glean-core/src/debug.rs +++ b/third_party/rust/glean-core/src/debug.rs @@ -240,7 +240,6 @@ fn validate_source_tags(tags: &Vec) -> bool { #[cfg(test)] mod test { use super::*; - use std::env; #[test] fn debug_option_is_correctly_loaded_from_env() { diff --git a/third_party/rust/glean-core/src/dispatcher/mod.rs b/third_party/rust/glean-core/src/dispatcher/mod.rs index 48efa4ef96..ead58fb867 100644 --- a/third_party/rust/glean-core/src/dispatcher/mod.rs +++ b/third_party/rust/glean-core/src/dispatcher/mod.rs @@ -360,9 +360,8 @@ impl Dispatcher { #[cfg(test)] mod test { use super::*; - use std::sync::atomic::{AtomicBool, AtomicU8, Ordering}; - use std::sync::{Arc, Mutex}; - use std::{thread, time::Duration}; + use std::sync::atomic::AtomicU8; + use std::sync::Mutex; fn enable_test_logging() { // When testing we want all logs to go to stdout/stderr by default, diff --git a/third_party/rust/glean-core/src/error_recording.rs b/third_party/rust/glean-core/src/error_recording.rs index aaf850d019..fa828242f1 100644 --- a/third_party/rust/glean-core/src/error_recording.rs +++ b/third_party/rust/glean-core/src/error_recording.rs @@ -12,7 +12,6 @@ //! but are not actually used directly, since the `send_in_pings` value needs to match the pings of the metric that is erroring (plus the "metrics" ping), //! not some constant value that we could define in `metrics.yaml`. -use std::convert::TryFrom; use std::fmt::Display; use crate::common_metric_data::CommonMetricDataInternal; diff --git a/third_party/rust/glean-core/src/event_database/mod.rs b/third_party/rust/glean-core/src/event_database/mod.rs index d83e56fbec..50b2488a4c 100644 --- a/third_party/rust/glean-core/src/event_database/mod.rs +++ b/third_party/rust/glean-core/src/event_database/mod.rs @@ -4,7 +4,6 @@ use std::cmp::Ordering; use std::collections::HashMap; -use std::convert::TryFrom; use std::fs; use std::fs::{create_dir_all, File, OpenOptions}; use std::io::BufRead; @@ -638,8 +637,8 @@ impl EventDatabase { #[cfg(test)] mod test { use super::*; + use crate::test_get_num_recorded_errors; use crate::tests::new_glean; - use crate::{test_get_num_recorded_errors, CommonMetricData}; use chrono::{TimeZone, Timelike}; #[test] diff --git a/third_party/rust/glean-core/src/glean.udl b/third_party/rust/glean-core/src/glean.udl index e68a57ea4c..dc71fea594 100644 --- a/third_party/rust/glean-core/src/glean.udl +++ b/third_party/rust/glean-core/src/glean.udl @@ -90,6 +90,7 @@ dictionary InternalConfiguration { PingRateLimit? rate_limit; boolean enable_event_timestamps; string? experimentation_id; + boolean enable_internal_pings; }; // How to specify the rate pings may be uploaded before they are throttled. diff --git a/third_party/rust/glean-core/src/histogram/mod.rs b/third_party/rust/glean-core/src/histogram/mod.rs index 282b02e0ab..6e2880dffa 100644 --- a/third_party/rust/glean-core/src/histogram/mod.rs +++ b/third_party/rust/glean-core/src/histogram/mod.rs @@ -5,7 +5,6 @@ //! A simple histogram implementation for exponential histograms. use std::collections::HashMap; -use std::convert::TryFrom; use serde::{Deserialize, Serialize}; diff --git a/third_party/rust/glean-core/src/internal_pings.rs b/third_party/rust/glean-core/src/internal_pings.rs index 07c3849006..1cf32feb60 100644 --- a/third_party/rust/glean-core/src/internal_pings.rs +++ b/third_party/rust/glean-core/src/internal_pings.rs @@ -19,9 +19,9 @@ pub struct InternalPings { } impl InternalPings { - pub fn new() -> InternalPings { + pub fn new(enabled: bool) -> InternalPings { InternalPings { - baseline: PingType::new( + baseline: PingType::new_internal( "baseline", true, true, @@ -32,8 +32,9 @@ impl InternalPings { "dirty_startup".to_string(), "inactive".to_string(), ], + enabled, ), - metrics: PingType::new( + metrics: PingType::new_internal( "metrics", true, false, @@ -46,8 +47,9 @@ impl InternalPings { "tomorrow".to_string(), "upgrade".to_string(), ], + enabled, ), - events: PingType::new( + events: PingType::new_internal( "events", true, false, @@ -58,6 +60,7 @@ impl InternalPings { "inactive".to_string(), "max_capacity".to_string(), ], + enabled, ), deletion_request: PingType::new( "deletion-request", diff --git a/third_party/rust/glean-core/src/lib.rs b/third_party/rust/glean-core/src/lib.rs index b7f9d73beb..af68fde264 100644 --- a/third_party/rust/glean-core/src/lib.rs +++ b/third_party/rust/glean-core/src/lib.rs @@ -17,7 +17,6 @@ use std::borrow::Cow; use std::collections::HashMap; -use std::convert::TryFrom; use std::fmt; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; @@ -25,7 +24,7 @@ use std::thread; use std::time::Duration; use crossbeam_channel::unbounded; -use log::{self, LevelFilter}; +use log::LevelFilter; use once_cell::sync::{Lazy, OnceCell}; use uuid::Uuid; @@ -136,6 +135,8 @@ pub struct InternalConfiguration { /// be noted that this has an underlying StringMetric and so should conform to the limitations that /// StringMetric places on length, etc. pub experimentation_id: Option, + /// Whether to enable internal pings. Default: true + pub enable_internal_pings: bool, } /// How to specify the rate at which pings may be uploaded before they are throttled. diff --git a/third_party/rust/glean-core/src/lib_unit_tests.rs b/third_party/rust/glean-core/src/lib_unit_tests.rs index cb1e4129d8..14d3b98417 100644 --- a/third_party/rust/glean-core/src/lib_unit_tests.rs +++ b/third_party/rust/glean-core/src/lib_unit_tests.rs @@ -6,12 +6,10 @@ // the lib.rs file. use std::collections::HashSet; -use std::iter::FromIterator; use serde_json::json; use super::*; -use crate::metrics::{StringMetric, TimeUnit, TimespanMetric, TimingDistributionMetric}; const GLOBAL_APPLICATION_ID: &str = "org.mozilla.glean.test.app"; pub fn new_glean(tempdir: Option) -> (Glean, tempfile::TempDir) { @@ -21,7 +19,7 @@ pub fn new_glean(tempdir: Option) -> (Glean, tempfile::TempDi None => tempfile::tempdir().unwrap(), }; let tmpname = dir.path().display().to_string(); - let glean = Glean::with_options(&tmpname, GLOBAL_APPLICATION_ID, true); + let glean = Glean::with_options(&tmpname, GLOBAL_APPLICATION_ID, true, true); (glean, dir) } @@ -41,7 +39,7 @@ fn path_is_constructed_from_data() { fn experiment_id_and_branch_get_truncated_if_too_long() { let t = tempfile::tempdir().unwrap(); let name = t.path().display().to_string(); - let glean = Glean::with_options(&name, "org.mozilla.glean.tests", true); + let glean = Glean::with_options(&name, "org.mozilla.glean.tests", true, true); // Generate long strings for the used ids. let very_long_id = "test-experiment-id".repeat(10); @@ -82,7 +80,7 @@ fn experiment_id_and_branch_get_truncated_if_too_long() { fn limits_on_experiments_extras_are_applied_correctly() { let t = tempfile::tempdir().unwrap(); let name = t.path().display().to_string(); - let glean = Glean::with_options(&name, "org.mozilla.glean.tests", true); + let glean = Glean::with_options(&name, "org.mozilla.glean.tests", true, true); let experiment_id = "test-experiment_id".to_string(); let branch_id = "test-branch-id".to_string(); @@ -138,7 +136,7 @@ fn limits_on_experiments_extras_are_applied_correctly() { fn experiments_status_is_correctly_toggled() { let t = tempfile::tempdir().unwrap(); let name = t.path().display().to_string(); - let glean = Glean::with_options(&name, "org.mozilla.glean.tests", true); + let glean = Glean::with_options(&name, "org.mozilla.glean.tests", true, true); // Define the experiment's data. let experiment_id: String = "test-toggle-experiment".into(); @@ -199,6 +197,7 @@ fn experimentation_id_is_set_correctly() { rate_limit: None, enable_event_timestamps: true, experimentation_id: Some(experimentation_id.to_string()), + enable_internal_pings: true, }) .unwrap(); @@ -219,7 +218,7 @@ fn client_id_and_first_run_date_must_be_regenerated() { let dir = tempfile::tempdir().unwrap(); let tmpname = dir.path().display().to_string(); { - let glean = Glean::with_options(&tmpname, GLOBAL_APPLICATION_ID, true); + let glean = Glean::with_options(&tmpname, GLOBAL_APPLICATION_ID, true, true); glean.data_store.as_ref().unwrap().clear_all(); @@ -236,7 +235,7 @@ fn client_id_and_first_run_date_must_be_regenerated() { } { - let glean = Glean::with_options(&tmpname, GLOBAL_APPLICATION_ID, true); + let glean = Glean::with_options(&tmpname, GLOBAL_APPLICATION_ID, true, true); assert!(glean .core_metrics .client_id @@ -339,7 +338,7 @@ fn client_id_is_managed_correctly_when_toggling_uploading() { fn client_id_is_set_to_known_value_when_uploading_disabled_at_start() { let dir = tempfile::tempdir().unwrap(); let tmpname = dir.path().display().to_string(); - let glean = Glean::with_options(&tmpname, GLOBAL_APPLICATION_ID, false); + let glean = Glean::with_options(&tmpname, GLOBAL_APPLICATION_ID, false, true); assert_eq!( *KNOWN_CLIENT_ID, @@ -355,7 +354,7 @@ fn client_id_is_set_to_known_value_when_uploading_disabled_at_start() { fn client_id_is_set_to_random_value_when_uploading_enabled_at_start() { let dir = tempfile::tempdir().unwrap(); let tmpname = dir.path().display().to_string(); - let glean = Glean::with_options(&tmpname, GLOBAL_APPLICATION_ID, true); + let glean = Glean::with_options(&tmpname, GLOBAL_APPLICATION_ID, true, true); let current_client_id = glean .core_metrics @@ -369,7 +368,7 @@ fn client_id_is_set_to_random_value_when_uploading_enabled_at_start() { fn enabling_when_already_enabled_is_a_noop() { let dir = tempfile::tempdir().unwrap(); let tmpname = dir.path().display().to_string(); - let mut glean = Glean::with_options(&tmpname, GLOBAL_APPLICATION_ID, true); + let mut glean = Glean::with_options(&tmpname, GLOBAL_APPLICATION_ID, true, true); assert!(!glean.set_upload_enabled(true)); } @@ -378,7 +377,7 @@ fn enabling_when_already_enabled_is_a_noop() { fn disabling_when_already_disabled_is_a_noop() { let dir = tempfile::tempdir().unwrap(); let tmpname = dir.path().display().to_string(); - let mut glean = Glean::with_options(&tmpname, GLOBAL_APPLICATION_ID, false); + let mut glean = Glean::with_options(&tmpname, GLOBAL_APPLICATION_ID, false, true); assert!(!glean.set_upload_enabled(false)); } @@ -601,14 +600,14 @@ fn test_first_run() { let dir = tempfile::tempdir().unwrap(); let tmpname = dir.path().display().to_string(); { - let glean = Glean::with_options(&tmpname, GLOBAL_APPLICATION_ID, true); + let glean = Glean::with_options(&tmpname, GLOBAL_APPLICATION_ID, true, true); // Check that this is indeed the first run. assert!(glean.is_first_run()); } { // Other runs must be not marked as "first run". - let glean = Glean::with_options(&tmpname, GLOBAL_APPLICATION_ID, true); + let glean = Glean::with_options(&tmpname, GLOBAL_APPLICATION_ID, true, true); assert!(!glean.is_first_run()); } } @@ -618,7 +617,7 @@ fn test_dirty_bit() { let dir = tempfile::tempdir().unwrap(); let tmpname = dir.path().display().to_string(); { - let glean = Glean::with_options(&tmpname, GLOBAL_APPLICATION_ID, true); + let glean = Glean::with_options(&tmpname, GLOBAL_APPLICATION_ID, true, true); // The dirty flag must not be set the first time Glean runs. assert!(!glean.is_dirty_flag_set()); @@ -630,7 +629,7 @@ fn test_dirty_bit() { { // Check that next time Glean runs, it correctly picks up the "dirty flag". // It is expected to be 'true'. - let glean = Glean::with_options(&tmpname, GLOBAL_APPLICATION_ID, true); + let glean = Glean::with_options(&tmpname, GLOBAL_APPLICATION_ID, true, true); assert!(glean.is_dirty_flag_set()); // Set the dirty flag to false. @@ -641,7 +640,7 @@ fn test_dirty_bit() { { // Check that next time Glean runs, it correctly picks up the "dirty flag". // It is expected to be 'false'. - let glean = Glean::with_options(&tmpname, GLOBAL_APPLICATION_ID, true); + let glean = Glean::with_options(&tmpname, GLOBAL_APPLICATION_ID, true, true); assert!(!glean.is_dirty_flag_set()); } } @@ -1065,7 +1064,7 @@ fn test_empty_application_id() { let dir = tempfile::tempdir().unwrap(); let tmpname = dir.path().display().to_string(); - let glean = Glean::with_options(&tmpname, "", true); + let glean = Glean::with_options(&tmpname, "", true, true); // Check that this is indeed the first run. assert!(glean.is_first_run()); } @@ -1080,7 +1079,7 @@ fn records_database_file_size() { let tmpname = dir.path().display().to_string(); // Initialize Glean once to ensure we create the database and did not error. - let glean = Glean::with_options(&tmpname, GLOBAL_APPLICATION_ID, true); + let glean = Glean::with_options(&tmpname, GLOBAL_APPLICATION_ID, true, true); let database_size = &glean.database_metrics.size; let data = database_size.get_value(&glean, "metrics"); @@ -1089,7 +1088,7 @@ fn records_database_file_size() { drop(glean); // Initialize Glean again to record file size. - let glean = Glean::with_options(&tmpname, GLOBAL_APPLICATION_ID, true); + let glean = Glean::with_options(&tmpname, GLOBAL_APPLICATION_ID, true, true); let database_size = &glean.database_metrics.size; let data = database_size.get_value(&glean, "metrics"); @@ -1161,3 +1160,46 @@ fn test_activity_api() { // Check that we set everything we needed for the 'inactive' status. assert!(!glean.is_dirty_flag_set()); } + +#[test] +fn disabled_pings_are_not_submitted() { + let _ = env_logger::builder().is_test(true).try_init(); + + let dir = tempfile::tempdir().unwrap(); + let (mut glean, _t) = new_glean(Some(dir)); + + let ping = PingType::new_internal("custom-disabled", true, false, true, true, vec![], false); + glean.register_ping_type(&ping); + + // We need to store a metric as an empty ping is not stored. + let counter = CounterMetric::new(CommonMetricData { + name: "counter".into(), + category: "local".into(), + send_in_pings: vec!["custom-disabled".into()], + ..Default::default() + }); + counter.add_sync(&glean, 1); + + assert!(!ping.submit_sync(&glean, None)); +} + +#[test] +fn internal_pings_can_be_disabled() { + let _ = env_logger::builder().is_test(true).try_init(); + + let dir = tempfile::tempdir().unwrap(); + let tmpname = dir.path().display().to_string(); + let glean = Glean::with_options(&tmpname, GLOBAL_APPLICATION_ID, true, false); + + // We need to store a metric as an empty ping is not stored. + let counter = CounterMetric::new(CommonMetricData { + name: "counter".into(), + category: "local".into(), + send_in_pings: vec!["baseline".into()], + ..Default::default() + }); + counter.add_sync(&glean, 1); + + let submitted = glean.internal_pings.baseline.submit_sync(&glean, None); + assert!(!submitted); +} diff --git a/third_party/rust/glean-core/src/metrics/event.rs b/third_party/rust/glean-core/src/metrics/event.rs index c7aefd9cd6..74f90f4867 100644 --- a/third_party/rust/glean-core/src/metrics/event.rs +++ b/third_party/rust/glean-core/src/metrics/event.rs @@ -175,9 +175,21 @@ impl EventMetric { .into() .unwrap_or_else(|| &self.meta().inner.send_in_pings[0]); - glean + let events = glean .event_storage() - .test_get_value(&self.meta, queried_ping_name) + .test_get_value(&self.meta, queried_ping_name); + + events.map(|mut evts| { + for ev in &mut evts { + let Some(extra) = &mut ev.extra else { continue }; + extra.remove("glean_timestamp"); + if extra.is_empty() { + ev.extra = None; + } + } + + evts + }) } /// **Test-only API (exported for FFI purposes).** diff --git a/third_party/rust/glean-core/src/metrics/memory_unit.rs b/third_party/rust/glean-core/src/metrics/memory_unit.rs index ce51b975fa..19006a594e 100644 --- a/third_party/rust/glean-core/src/metrics/memory_unit.rs +++ b/third_party/rust/glean-core/src/metrics/memory_unit.rs @@ -2,8 +2,6 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use std::convert::TryFrom; - use serde::{Deserialize, Serialize}; use crate::error::{Error, ErrorKind}; diff --git a/third_party/rust/glean-core/src/metrics/metrics_enabled_config.rs b/third_party/rust/glean-core/src/metrics/metrics_enabled_config.rs index 26d0deff31..b36cbc150a 100644 --- a/third_party/rust/glean-core/src/metrics/metrics_enabled_config.rs +++ b/third_party/rust/glean-core/src/metrics/metrics_enabled_config.rs @@ -2,7 +2,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use std::{collections::HashMap, convert::TryFrom}; +use std::collections::HashMap; use serde::{Deserialize, Serialize}; diff --git a/third_party/rust/glean-core/src/metrics/ping.rs b/third_party/rust/glean-core/src/metrics/ping.rs index e60284b1e2..5defab7a71 100644 --- a/third_party/rust/glean-core/src/metrics/ping.rs +++ b/third_party/rust/glean-core/src/metrics/ping.rs @@ -31,6 +31,11 @@ struct InnerPing { pub include_info_sections: bool, /// The "reason" codes that this ping can send pub reason_codes: Vec, + + /// Whether this ping is enabled. + /// Note: Data for disabled pings is still recorded. + /// It will not be cleared out on submit. + enabled: bool, } impl fmt::Debug for PingType { @@ -67,6 +72,26 @@ impl PingType { precise_timestamps: bool, include_info_sections: bool, reason_codes: Vec, + ) -> Self { + Self::new_internal( + name, + include_client_id, + send_if_empty, + precise_timestamps, + include_info_sections, + reason_codes, + true, + ) + } + + pub(crate) fn new_internal>( + name: A, + include_client_id: bool, + send_if_empty: bool, + precise_timestamps: bool, + include_info_sections: bool, + reason_codes: Vec, + enabled: bool, ) -> Self { let this = Self(Arc::new(InnerPing { name: name.into(), @@ -75,6 +100,7 @@ impl PingType { precise_timestamps, include_info_sections, reason_codes, + enabled, })); // Register this ping. @@ -140,6 +166,11 @@ impl PingType { /// Whether the ping was succesfully assembled and queued. #[doc(hidden)] pub fn submit_sync(&self, glean: &Glean, reason: Option<&str>) -> bool { + if !self.0.enabled { + log::info!("Ping disabled: not submitting '{}' ping.", self.0.name); + return false; + } + if !glean.is_upload_enabled() { log::info!("Glean disabled: not submitting any pings."); return false; diff --git a/third_party/rust/glean-core/src/metrics/string.rs b/third_party/rust/glean-core/src/metrics/string.rs index 4aa30a8d7e..a56ffab648 100644 --- a/third_party/rust/glean-core/src/metrics/string.rs +++ b/third_party/rust/glean-core/src/metrics/string.rs @@ -149,10 +149,8 @@ impl StringMetric { #[cfg(test)] mod test { use super::*; - use crate::test_get_num_recorded_errors; use crate::tests::new_glean; use crate::util::truncate_string_at_boundary; - use crate::ErrorType; use crate::Lifetime; #[test] diff --git a/third_party/rust/glean-core/src/metrics/text.rs b/third_party/rust/glean-core/src/metrics/text.rs index baa8e88d75..35f803c728 100644 --- a/third_party/rust/glean-core/src/metrics/text.rs +++ b/third_party/rust/glean-core/src/metrics/text.rs @@ -153,10 +153,8 @@ impl TextMetric { #[cfg(test)] mod test { use super::*; - use crate::test_get_num_recorded_errors; use crate::tests::new_glean; use crate::util::truncate_string_at_boundary; - use crate::ErrorType; use crate::Lifetime; #[test] diff --git a/third_party/rust/glean-core/src/metrics/time_unit.rs b/third_party/rust/glean-core/src/metrics/time_unit.rs index 6d61a8a242..6c68d5dff0 100644 --- a/third_party/rust/glean-core/src/metrics/time_unit.rs +++ b/third_party/rust/glean-core/src/metrics/time_unit.rs @@ -2,7 +2,6 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use std::convert::TryFrom; use std::time::Duration; use serde::{Deserialize, Serialize}; diff --git a/third_party/rust/glean-core/src/metrics/timespan.rs b/third_party/rust/glean-core/src/metrics/timespan.rs index ee63fb52f8..d72492a590 100644 --- a/third_party/rust/glean-core/src/metrics/timespan.rs +++ b/third_party/rust/glean-core/src/metrics/timespan.rs @@ -2,7 +2,6 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use std::convert::TryInto; use std::sync::{Arc, RwLock}; use std::time::Duration; diff --git a/third_party/rust/glean-core/src/metrics/timing_distribution.rs b/third_party/rust/glean-core/src/metrics/timing_distribution.rs index 3293be9518..776935afea 100644 --- a/third_party/rust/glean-core/src/metrics/timing_distribution.rs +++ b/third_party/rust/glean-core/src/metrics/timing_distribution.rs @@ -96,7 +96,7 @@ impl TimingDistributionMetric { Self { meta: Arc::new(meta.into()), time_unit, - next_id: Arc::new(AtomicUsize::new(0)), + next_id: Arc::new(AtomicUsize::new(1)), start_times: Arc::new(Mutex::new(Default::default())), } } diff --git a/third_party/rust/glean-core/src/metrics/url.rs b/third_party/rust/glean-core/src/metrics/url.rs index 48b3f9e7ae..0fd5712eeb 100644 --- a/third_party/rust/glean-core/src/metrics/url.rs +++ b/third_party/rust/glean-core/src/metrics/url.rs @@ -168,9 +168,7 @@ impl UrlMetric { #[cfg(test)] mod test { use super::*; - use crate::test_get_num_recorded_errors; use crate::tests::new_glean; - use crate::ErrorType; use crate::Lifetime; #[test] diff --git a/third_party/rust/glean-core/src/storage/mod.rs b/third_party/rust/glean-core/src/storage/mod.rs index 67cb9a1552..a4225e21ed 100644 --- a/third_party/rust/glean-core/src/storage/mod.rs +++ b/third_party/rust/glean-core/src/storage/mod.rs @@ -235,7 +235,7 @@ mod test { fn test_experiments_json_serialization() { let t = tempfile::tempdir().unwrap(); let name = t.path().display().to_string(); - let glean = Glean::with_options(&name, "org.mozilla.glean", true); + let glean = Glean::with_options(&name, "org.mozilla.glean", true, true); let extra: HashMap = [("test-key".into(), "test-value".into())] .iter() @@ -264,7 +264,7 @@ mod test { fn test_experiments_json_serialization_empty() { let t = tempfile::tempdir().unwrap(); let name = t.path().display().to_string(); - let glean = Glean::with_options(&name, "org.mozilla.glean", true); + let glean = Glean::with_options(&name, "org.mozilla.glean", true, true); let metric = ExperimentMetric::new(&glean, "some-experiment".to_string()); diff --git a/third_party/rust/glean-core/src/traits/event.rs b/third_party/rust/glean-core/src/traits/event.rs index aa84699b30..ba8c0e5609 100644 --- a/third_party/rust/glean-core/src/traits/event.rs +++ b/third_party/rust/glean-core/src/traits/event.rs @@ -3,7 +3,6 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. use std::collections::HashMap; -use std::convert::TryFrom; use std::hash::Hash; use crate::event_database::RecordedEvent; diff --git a/third_party/rust/glean-core/src/upload/directory.rs b/third_party/rust/glean-core/src/upload/directory.rs index 706550fe6c..91a4d061d1 100644 --- a/third_party/rust/glean-core/src/upload/directory.rs +++ b/third_party/rust/glean-core/src/upload/directory.rs @@ -317,8 +317,6 @@ impl PingDirectoryManager { #[cfg(test)] mod test { - use std::fs::File; - use super::*; use crate::metrics::PingType; use crate::tests::new_glean; diff --git a/third_party/rust/glean-core/src/upload/mod.rs b/third_party/rust/glean-core/src/upload/mod.rs index e51a9d9508..f217137f00 100644 --- a/third_party/rust/glean-core/src/upload/mod.rs +++ b/third_party/rust/glean-core/src/upload/mod.rs @@ -14,7 +14,6 @@ use std::collections::HashMap; use std::collections::VecDeque; -use std::convert::TryInto; use std::path::PathBuf; use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; use std::sync::{Arc, RwLock, RwLockWriteGuard}; @@ -856,9 +855,6 @@ pub fn chunked_log_info(_path: &str, payload: &str) { #[cfg(test)] mod test { - use std::thread; - use std::time::Duration; - use uuid::Uuid; use super::*; diff --git a/third_party/rust/glean-core/src/upload/request.rs b/third_party/rust/glean-core/src/upload/request.rs index b4ac6eba97..6f3b0c0e5c 100644 --- a/third_party/rust/glean-core/src/upload/request.rs +++ b/third_party/rust/glean-core/src/upload/request.rs @@ -8,7 +8,7 @@ use std::collections::HashMap; use chrono::prelude::{DateTime, Utc}; use flate2::{read::GzDecoder, write::GzEncoder, Compression}; -use serde_json::{self, Value as JsonValue}; +use serde_json::Value as JsonValue; use std::io::prelude::*; use crate::error::{ErrorKind, Result}; diff --git a/third_party/rust/glean-core/tests/common/mod.rs b/third_party/rust/glean-core/tests/common/mod.rs index ebc4f14045..cbc6201d02 100644 --- a/third_party/rust/glean-core/tests/common/mod.rs +++ b/third_party/rust/glean-core/tests/common/mod.rs @@ -63,6 +63,7 @@ pub fn new_glean(tempdir: Option) -> (Glean, tempfile::TempDi rate_limit: None, enable_event_timestamps: false, experimentation_id: None, + enable_internal_pings: true, }; let glean = Glean::new(cfg).unwrap(); diff --git a/third_party/rust/glean-core/tests/event.rs b/third_party/rust/glean-core/tests/event.rs index c83e225ca2..48120956d7 100644 --- a/third_party/rust/glean-core/tests/event.rs +++ b/third_party/rust/glean-core/tests/event.rs @@ -481,6 +481,7 @@ fn with_event_timestamps() { rate_limit: None, enable_event_timestamps: true, experimentation_id: None, // Enabling event timestamps + enable_internal_pings: true, }; let glean = Glean::new(cfg).unwrap(); diff --git a/third_party/rust/glean-core/tests/ping_maker.rs b/third_party/rust/glean-core/tests/ping_maker.rs index bc3aac6311..f716dc4692 100644 --- a/third_party/rust/glean-core/tests/ping_maker.rs +++ b/third_party/rust/glean-core/tests/ping_maker.rs @@ -91,6 +91,7 @@ fn test_metrics_must_report_experimentation_id() { rate_limit: None, enable_event_timestamps: true, experimentation_id: Some("test-experimentation-id".to_string()), + enable_internal_pings: true, }) .unwrap(); let ping_maker = PingMaker::new(); @@ -143,6 +144,7 @@ fn experimentation_id_is_removed_if_send_if_empty_is_false() { rate_limit: None, enable_event_timestamps: true, experimentation_id: Some("test-experimentation-id".to_string()), + enable_internal_pings: true, }) .unwrap(); let ping_maker = PingMaker::new(); diff --git a/third_party/rust/glean/.cargo-checksum.json b/third_party/rust/glean/.cargo-checksum.json index f624e73c99..7cb5c7390c 100644 --- a/third_party/rust/glean/.cargo-checksum.json +++ b/third_party/rust/glean/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"Cargo.toml":"29b8551de6fff2f0fa3a821eb933f71a2a326b3ce3d37c25bcef3001f9146dfb","LICENSE":"1f256ecad192880510e84ad60474eab7589218784b9a50bc7ceee34c2b91f1d5","README.md":"5627cc81e6187ab6c2b4dff061af16d559edcab64ba786bac39daa69c703c595","src/common_test.rs":"454df3d99eef045270e813946f921f56c39b16c18a5fadedc32829c3d44129cf","src/configuration.rs":"82b3a7933d913e1e2a4f328a76621db2d2e618d209d9785086d64c5c78c2a2d6","src/core_metrics.rs":"fef8fb4e5fa57c179836c6eb2cf59278fe3b8b036dbe57b0ff02971b4acd822f","src/lib.rs":"aa9c81fc6dc19ca1cb4bede25d554377a5d717fb3b246967edb1be12a395ce61","src/net/http_uploader.rs":"01ad5bd91384411a12c74434cd1c5cd585078cb34faba4615c70bdb669a9bccb","src/net/mod.rs":"f47b96bb878f1a6c771cedbaeaeefb270bc87fb1d1bbbed1b282dddca16216ed","src/private/event.rs":"d7c70c02648584c19c73af89e5180d3c6153c911f2c6830f7d1599b18d6150eb","src/private/mod.rs":"3565eb569d2b96f938f130abe0fc3ee3f55e7e03fd6501e309d3ef6af72ef6ee","src/private/object.rs":"3f70363a196aea46cc163af025a53e48c117c6208babc4bce772bb4c337cced8","src/private/ping.rs":"a6262a3453c77cbf30766c19b535a1bf66a37b2a316e8f87baee03025255c33e","src/system.rs":"6eae5b41c15eba9cad6dbd116abe3519ee3e1fe034e79bdd692b029829a8c384","src/test.rs":"6388b9e8bf96e0fb56ad71b7a5b5630d209ae62f1a65c62e878cbc1757ddd585","tests/common/mod.rs":"08fb9483d9b6ed9fe873b4395245166ae8a15263be750c7a8e298c41d9604745","tests/init_fails.rs":"906bbf0faa613976623e0cf782bd86545b49d76afaab182af7634690b747ebf7","tests/never_init.rs":"19bad996e22f7d6958cc1a650528530aa7d1aeb4a8ab42229a90bbc0315c8ed1","tests/no_time_to_init.rs":"06c81148c27d383cb708c0c80a2e806024c9955337d7adfba8c53aaeade9be67","tests/overflowing_preinit.rs":"7ad4b2274dd9240b53430859a4eb1d2597cf508a5a678333f3d3abbadd2ed4a7","tests/persist_ping_lifetime.rs":"81415dc1d74743f02269f0d0dfa524003147056853f080276972e64a0b761d3c","tests/persist_ping_lifetime_nopanic.rs":"18379d3ffbf4a2c8c684c04ff7a0660b86dfbbb447db2d24dfed6073cb7ddf8f","tests/schema.rs":"9d24028cab4dc60fe3c4d7a0bafbff0815cbc0249fa3e23625d42c3b4fa71734","tests/simple.rs":"1b8b227249ae9d3cc281db07ed779bc75252c7849b1c48b4ac3d765228d65b20","tests/test-shutdown-blocking.sh":"9b16a01c190c7062474dd92182298a3d9a27928c8fa990340fdd798e6cdb7ab2","tests/test-thread-crashing.sh":"ff1bc8e5d7e4ba3a10d0d38bef222db8bfba469e7d30e45b1053d177a4084f09","tests/upload_timing.rs":"3024b7999a0c23f2c3d7e59725b5455522e4e9fdf63e3265b93fea4cec18725f"},"package":"f58388f10d013e2d12bb58e6e76983ede120789956fe827913a3d2560c66d44d"} \ No newline at end of file +{"files":{"Cargo.toml":"af0535de86b60e3e08cadcdb9e61ce4a699c168608d7e9e2ebb92d949e7f31ef","LICENSE":"1f256ecad192880510e84ad60474eab7589218784b9a50bc7ceee34c2b91f1d5","README.md":"5627cc81e6187ab6c2b4dff061af16d559edcab64ba786bac39daa69c703c595","src/common_test.rs":"de47b53dcca37985c0a2b8c02daecbf32309aa54f5a4dd9290719c2c1fd0fa55","src/configuration.rs":"27075b12236021c54d0c99427bcbd417933ca02545275604d3c13f32ca25af13","src/core_metrics.rs":"fef8fb4e5fa57c179836c6eb2cf59278fe3b8b036dbe57b0ff02971b4acd822f","src/lib.rs":"d4010f265de330081467673df05bbd45efbdfeef28823f7dc11a903b11fb8976","src/net/http_uploader.rs":"01ad5bd91384411a12c74434cd1c5cd585078cb34faba4615c70bdb669a9bccb","src/net/mod.rs":"f47b96bb878f1a6c771cedbaeaeefb270bc87fb1d1bbbed1b282dddca16216ed","src/private/event.rs":"d7c70c02648584c19c73af89e5180d3c6153c911f2c6830f7d1599b18d6150eb","src/private/mod.rs":"3565eb569d2b96f938f130abe0fc3ee3f55e7e03fd6501e309d3ef6af72ef6ee","src/private/object.rs":"3f70363a196aea46cc163af025a53e48c117c6208babc4bce772bb4c337cced8","src/private/ping.rs":"a6262a3453c77cbf30766c19b535a1bf66a37b2a316e8f87baee03025255c33e","src/system.rs":"6eae5b41c15eba9cad6dbd116abe3519ee3e1fe034e79bdd692b029829a8c384","src/test.rs":"6388b9e8bf96e0fb56ad71b7a5b5630d209ae62f1a65c62e878cbc1757ddd585","tests/common/mod.rs":"08fb9483d9b6ed9fe873b4395245166ae8a15263be750c7a8e298c41d9604745","tests/init_fails.rs":"906bbf0faa613976623e0cf782bd86545b49d76afaab182af7634690b747ebf7","tests/never_init.rs":"19bad996e22f7d6958cc1a650528530aa7d1aeb4a8ab42229a90bbc0315c8ed1","tests/no_time_to_init.rs":"06c81148c27d383cb708c0c80a2e806024c9955337d7adfba8c53aaeade9be67","tests/overflowing_preinit.rs":"7ad4b2274dd9240b53430859a4eb1d2597cf508a5a678333f3d3abbadd2ed4a7","tests/persist_ping_lifetime.rs":"81415dc1d74743f02269f0d0dfa524003147056853f080276972e64a0b761d3c","tests/persist_ping_lifetime_nopanic.rs":"18379d3ffbf4a2c8c684c04ff7a0660b86dfbbb447db2d24dfed6073cb7ddf8f","tests/schema.rs":"9615eded31a2582c8f04c729d551c0c81a57029ba62a19184221c2e1cd39baf0","tests/simple.rs":"1b8b227249ae9d3cc281db07ed779bc75252c7849b1c48b4ac3d765228d65b20","tests/test-shutdown-blocking.sh":"9b16a01c190c7062474dd92182298a3d9a27928c8fa990340fdd798e6cdb7ab2","tests/test-thread-crashing.sh":"ff1bc8e5d7e4ba3a10d0d38bef222db8bfba469e7d30e45b1053d177a4084f09","tests/upload_timing.rs":"3024b7999a0c23f2c3d7e59725b5455522e4e9fdf63e3265b93fea4cec18725f"},"package":"0ceede8fb9c90ba1b77fb8290d3ae7b62bfcb422ad1d6e46bae1c8af3f22f12d"} \ No newline at end of file diff --git a/third_party/rust/glean/Cargo.toml b/third_party/rust/glean/Cargo.toml index bc25a08940..edcc84d5d6 100644 --- a/third_party/rust/glean/Cargo.toml +++ b/third_party/rust/glean/Cargo.toml @@ -13,7 +13,7 @@ edition = "2021" rust-version = "1.66" name = "glean" -version = "58.1.0" +version = "59.0.0" authors = [ "Jan-Erik Rediger ", "The Glean Team ", @@ -35,7 +35,7 @@ license = "MPL-2.0" repository = "https://github.com/mozilla/glean" [dependencies.glean-core] -version = "58.1.0" +version = "59.0.0" [dependencies.inherent] version = "1" diff --git a/third_party/rust/glean/src/common_test.rs b/third_party/rust/glean/src/common_test.rs index e3c80da5f2..fdb7cfadbf 100644 --- a/third_party/rust/glean/src/common_test.rs +++ b/third_party/rust/glean/src/common_test.rs @@ -42,7 +42,6 @@ pub(crate) fn new_glean( Some(c) => c, None => ConfigurationBuilder::new(true, tmpname, GLOBAL_APPLICATION_ID) .with_server_endpoint("invalid-test-host") - .with_event_timestamps(false) .build(), }; diff --git a/third_party/rust/glean/src/configuration.rs b/third_party/rust/glean/src/configuration.rs index ca0a39c3f1..462bedaa7a 100644 --- a/third_party/rust/glean/src/configuration.rs +++ b/third_party/rust/glean/src/configuration.rs @@ -46,6 +46,8 @@ pub struct Configuration { /// be noted that this has an underlying StringMetric and so should conform to the limitations that /// StringMetric places on length, etc. pub experimentation_id: Option, + /// Whether to enable internal pings. Default: true + pub enable_internal_pings: bool, } /// Configuration builder. @@ -92,6 +94,8 @@ pub struct Builder { /// be noted that this has an underlying StringMetric and so should conform to the limitations that /// StringMetric places on length, etc. pub experimentation_id: Option, + /// Whether to enable internal pings. Default: true + pub enable_internal_pings: bool, } impl Builder { @@ -115,6 +119,7 @@ impl Builder { rate_limit: None, enable_event_timestamps: true, experimentation_id: None, + enable_internal_pings: true, } } @@ -134,6 +139,7 @@ impl Builder { rate_limit: self.rate_limit, enable_event_timestamps: self.enable_event_timestamps, experimentation_id: self.experimentation_id, + enable_internal_pings: self.enable_internal_pings, } } @@ -184,4 +190,10 @@ impl Builder { self.experimentation_id = Some(value); self } + + /// Set whether to enable internal pings. + pub fn with_internal_pings(mut self, value: bool) -> Self { + self.enable_internal_pings = value; + self + } } diff --git a/third_party/rust/glean/src/lib.rs b/third_party/rust/glean/src/lib.rs index 5c5b945a95..81899d42ee 100644 --- a/third_party/rust/glean/src/lib.rs +++ b/third_party/rust/glean/src/lib.rs @@ -122,6 +122,7 @@ fn initialize_internal(cfg: Configuration, client_info: ClientInfoMetrics) -> Op rate_limit: cfg.rate_limit, enable_event_timestamps: cfg.enable_event_timestamps, experimentation_id: cfg.experimentation_id, + enable_internal_pings: cfg.enable_internal_pings, }; glean_core::glean_initialize(core_cfg, client_info.into(), callbacks); diff --git a/third_party/rust/glean/tests/schema.rs b/third_party/rust/glean/tests/schema.rs index 01a2108b3c..59132cd82a 100644 --- a/third_party/rust/glean/tests/schema.rs +++ b/third_party/rust/glean/tests/schema.rs @@ -6,8 +6,7 @@ use std::collections::HashMap; use std::io::Read; use flate2::read::GzDecoder; -use glean_core::TextMetric; -use jsonschema_valid::{self, schemas::Draft}; +use jsonschema_valid::schemas::Draft; use serde_json::Value; use glean::net::{PingUploadRequest, UploadResult}; diff --git a/third_party/rust/goblin/.cargo-checksum.json b/third_party/rust/goblin/.cargo-checksum.json index 4eff08bdd6..619a6b6211 100644 --- a/third_party/rust/goblin/.cargo-checksum.json +++ b/third_party/rust/goblin/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"CHANGELOG.md":"ade9f25d4bd1545f2ff2661d6a1301fe228cf2551a9cb27fcaa17c8119b73c8b","Cargo.toml":"09b271ef4ee3491cb1f6309cef8b60471b960c057c6e57fc90ed579adcc57453","LICENSE":"036bf6b6d6fd6dd1abda2ff6cdb672a63bdf32c468048720072910f2268a965f","README.md":"c09b08f3d5e7e33c4a8fd647708d313ee2ba98b165a1d077fb90f280dcb4da31","src/archive/mod.rs":"ae739638d7267011bedf51712516d3485171d8f2df2ab6746a0d942d86efd6a6","src/elf/compression_header.rs":"2eb5fdda9177c1c897310d86714967de019b39c6e23b1f3a890dd3a659be0acc","src/elf/constants_header.rs":"f2ede290ecacf60b1719e9994aa45612bf0f7baf63806a293d4530a674e5861a","src/elf/constants_relocation.rs":"a010071cd2a25ab71e0c7181eb1d9f417daa2d1ec25a09c74bd12ad944892225","src/elf/dynamic.rs":"c26e75311f2da9e34dc4c0a2120dfcc20df88a41d67c52b9bf703258de018fd8","src/elf/gnu_hash.rs":"7a9fcaf6cb38167d20527364bdf9bc2379c44dede5d7666275a1eb20dc665179","src/elf/header.rs":"3391a1fa9b8e3923f7ce74caff0668d8ddb5b34767bf3da309ff497fd81c34c7","src/elf/mod.rs":"2ee0faa0917deb5e90ca60e9c852434745a4c7f553e609e9603a57b7d55b739f","src/elf/note.rs":"bf5e45e2697f7700d5adbb52f890ea4c63b70b7077ca0e7c751420bb92923529","src/elf/program_header.rs":"4c322eb124c4e2bdeec4915067d2bb11fe9e7fba1811dc351a3f7581df121da0","src/elf/reloc.rs":"8b29162055b2846342b49e5e9e0a1482786fb92b4787bb9eb1c6d04f38b94e87","src/elf/section_header.rs":"f55f4d263f618bd1dec76ff0483f3b2dc3791c8e5c5c2b6ff296a5bc26001666","src/elf/sym.rs":"045c01107f4e100d6827cb819b82a28ea10c0d9bc00a1cdddb04a0865f1162ec","src/elf/symver.rs":"3f899201f64a702653d44288f860003e7acd75e38111d36479af823ed92b1341","src/error.rs":"af620a5692bca070dc727d49cdbb566a533bfb97724ca68932ae7fec7dc05cf6","src/lib.rs":"465eb53b540dfd142d204984ee7280130542d7f83d6c53691299d773f7394faf","src/mach/bind_opcodes.rs":"1dcacfb853d05c2c7e6dbb4509ee705a8ea645db0d334991a2293fef92eee851","src/mach/constants.rs":"c2a2381a0b9c3047d37582465e8965d995dca414d0da21fb7bcc6b8334e49eb6","src/mach/exports.rs":"d22122744673a3ce5f54b2b4b20bfa47d17378e64d3dda2858dd13add74ed3dc","src/mach/fat.rs":"45a3228aaa1ab8b77f322dd4924b7383f1357e226ffc079846d67c0268389ea7","src/mach/header.rs":"006619188f51fa43051dc04aa4b2ecd5f89136cf05cb6a7b23a228228008e6ae","src/mach/imports.rs":"2153269dfff32e23d72f76a82d658be06bd79b7e35d79b7e17115e4eb24b13d5","src/mach/load_command.rs":"0a689e774ae96212666165909c026037f22a3c4e3645250b9bae60c957d50ca4","src/mach/mod.rs":"53ad219fd2265a5689ab38d5031722268eab6bbb649c75756e74295df4b611b7","src/mach/relocation.rs":"11b0b76ed7d997c87e396100515f931fe84473c228bed0e980fbab311530070a","src/mach/segment.rs":"0dc29bf42b25f60c7258bc8b757f6a862e846582dd6d2e70737933ad6334a0e4","src/mach/symbols.rs":"d2505fa8d65ea267abfcb6a9fc4d1acd47d5605aa6775935757e2fa8e92af507","src/pe/authenticode.rs":"c3df9266c4f0a865e0da4b10fa1494eca083953fc4ded0b707b547a7d4ef296a","src/pe/certificate_table.rs":"75ab5dce6bc0c28d3687a5c119c0fa0d00e4796c8959a32d9d208f2369273c50","src/pe/characteristic.rs":"6f810a6e5646b922cf7e3ca6d314677a4e1e1ad5695278c2b1b527a05f4299f3","src/pe/data_directories.rs":"d4e156f0c5b509860ceb3c7d42e1621e6c2143b90fc412806b3cefab1acc577a","src/pe/debug.rs":"3811c616a9b6d6b54e15348bb369b794bb89532e04fe19eca91b745d7c51a553","src/pe/exception.rs":"de2c9c07812ecd315c8400fc8fdcadc6a44d7a8be96e69a3f4ccf14ef8cf8426","src/pe/export.rs":"c98f5ce0b1b18bb87f06d1d41dbf70f443d65ecb1624cb23a1ef6c5f93a892e1","src/pe/header.rs":"f02a4beddc00ddd6624df7defc42991ceb507360b5aa1003cf33332c1c89a743","src/pe/import.rs":"855276e46c01ccd7631104e4d1265592e36c9468aadcacc937a40c29d94aabe3","src/pe/mod.rs":"ec958ee9a717672dec7b56d9d7d33e444c37eb781f299a920a60eb7fa39ef7a1","src/pe/optional_header.rs":"4fd94187fb343756817f23ccc58ec035a1b462b69457c706d9e2f11225d0cb1c","src/pe/options.rs":"b38f4e87f13ae381712621786f89e931452b2b4857a7bb6f140c4c21a63aa652","src/pe/relocation.rs":"c479b80bb1d6910f2168505dda4f2d8925b7edc34bed4e25d069546f88f52bb3","src/pe/section_table.rs":"d7144c7be3242d7aa653d22dca1cf15f7110f79a946a15cbe6ecf531e0cacb19","src/pe/symbol.rs":"9a65226c93c4499e21d094ceb838d58db706951580a1c43dfb36b95dbaff70f0","src/pe/utils.rs":"88e1cd9114c5d4ad58a09c39b312689de20ddd7382654ec660b00424f5c3129c","src/strtab.rs":"6d122084cf5d5244b2bd734b1d6d2c018116cc537ffc0c81d042d5b8815d7782","tests/bins/elf/gnu_hash/README.md":"52581e2ea7067a55bd8aedf4079200fb76448573ae9ffef7d886b9556e980db9"},"package":"f27c1b4369c2cd341b5de549380158b105a04c331be5db9110eef7b6d2742134"} \ No newline at end of file +{"files":{"CHANGELOG.md":"2d45bc2d0db50fd4416e2123f8b98c7288935b3be7985bdd115ecbd236acea41","Cargo.toml":"0d8dade295950e9f63574e7a74390ddec56c039cb44d2507df7e6ff832b49a0d","LICENSE":"036bf6b6d6fd6dd1abda2ff6cdb672a63bdf32c468048720072910f2268a965f","README.md":"302466b411dc5bc705cdf563b928c14755342ab6f2dff371be064446fa0aa0a9","src/archive/mod.rs":"ae739638d7267011bedf51712516d3485171d8f2df2ab6746a0d942d86efd6a6","src/elf/compression_header.rs":"2eb5fdda9177c1c897310d86714967de019b39c6e23b1f3a890dd3a659be0acc","src/elf/constants_header.rs":"f2ede290ecacf60b1719e9994aa45612bf0f7baf63806a293d4530a674e5861a","src/elf/constants_relocation.rs":"a010071cd2a25ab71e0c7181eb1d9f417daa2d1ec25a09c74bd12ad944892225","src/elf/dynamic.rs":"c26e75311f2da9e34dc4c0a2120dfcc20df88a41d67c52b9bf703258de018fd8","src/elf/gnu_hash.rs":"7a9fcaf6cb38167d20527364bdf9bc2379c44dede5d7666275a1eb20dc665179","src/elf/header.rs":"3391a1fa9b8e3923f7ce74caff0668d8ddb5b34767bf3da309ff497fd81c34c7","src/elf/mod.rs":"2ee0faa0917deb5e90ca60e9c852434745a4c7f553e609e9603a57b7d55b739f","src/elf/note.rs":"bf5e45e2697f7700d5adbb52f890ea4c63b70b7077ca0e7c751420bb92923529","src/elf/program_header.rs":"4c322eb124c4e2bdeec4915067d2bb11fe9e7fba1811dc351a3f7581df121da0","src/elf/reloc.rs":"a5d21f9d1ddae8e730e852fcaf1cd2dd194e35fbac8f86fb8fd9033a03bdc66d","src/elf/section_header.rs":"f55f4d263f618bd1dec76ff0483f3b2dc3791c8e5c5c2b6ff296a5bc26001666","src/elf/sym.rs":"045c01107f4e100d6827cb819b82a28ea10c0d9bc00a1cdddb04a0865f1162ec","src/elf/symver.rs":"3f899201f64a702653d44288f860003e7acd75e38111d36479af823ed92b1341","src/error.rs":"a1bb56d82db52ac627e55b163f489f06a78c939a8ccfdec210b4f726d6ed6e9d","src/lib.rs":"f29832bdf7d7f7d9e34f65704afea2710d578df60cc171dd179b5ce889faaf12","src/mach/bind_opcodes.rs":"1dcacfb853d05c2c7e6dbb4509ee705a8ea645db0d334991a2293fef92eee851","src/mach/constants.rs":"c2a2381a0b9c3047d37582465e8965d995dca414d0da21fb7bcc6b8334e49eb6","src/mach/exports.rs":"d22122744673a3ce5f54b2b4b20bfa47d17378e64d3dda2858dd13add74ed3dc","src/mach/fat.rs":"45a3228aaa1ab8b77f322dd4924b7383f1357e226ffc079846d67c0268389ea7","src/mach/header.rs":"006619188f51fa43051dc04aa4b2ecd5f89136cf05cb6a7b23a228228008e6ae","src/mach/imports.rs":"2153269dfff32e23d72f76a82d658be06bd79b7e35d79b7e17115e4eb24b13d5","src/mach/load_command.rs":"42e6f0973092185db233230e71e9312bbac7c2e1090bb6d713804020319dfa33","src/mach/mod.rs":"53ad219fd2265a5689ab38d5031722268eab6bbb649c75756e74295df4b611b7","src/mach/relocation.rs":"11b0b76ed7d997c87e396100515f931fe84473c228bed0e980fbab311530070a","src/mach/segment.rs":"0dc29bf42b25f60c7258bc8b757f6a862e846582dd6d2e70737933ad6334a0e4","src/mach/symbols.rs":"d2505fa8d65ea267abfcb6a9fc4d1acd47d5605aa6775935757e2fa8e92af507","src/pe/authenticode.rs":"ad9c77e42392b49114cf8ce2839111f3231dcfe21cbb8e402ee14e568f5ae657","src/pe/certificate_table.rs":"f6c31ba518d9fc4b6e12d2f24d6c9d58b21b341a1f189cbcf2aae0ae51304ad3","src/pe/characteristic.rs":"6f810a6e5646b922cf7e3ca6d314677a4e1e1ad5695278c2b1b527a05f4299f3","src/pe/data_directories.rs":"d0352ccc03e0ab2935235e91b391cc55828406087f026f90ec11ca5906fd8c8c","src/pe/debug.rs":"3811c616a9b6d6b54e15348bb369b794bb89532e04fe19eca91b745d7c51a553","src/pe/exception.rs":"de2c9c07812ecd315c8400fc8fdcadc6a44d7a8be96e69a3f4ccf14ef8cf8426","src/pe/export.rs":"c98f5ce0b1b18bb87f06d1d41dbf70f443d65ecb1624cb23a1ef6c5f93a892e1","src/pe/header.rs":"879c2ddc8318ab37b4577ac34241fa039d106e0e530dab07edfc9b4e13b08356","src/pe/import.rs":"855276e46c01ccd7631104e4d1265592e36c9468aadcacc937a40c29d94aabe3","src/pe/mod.rs":"ffaeca313ea2fb31c41eb0ede0ef28fede2276b0bb7d81dfc08b4ead6289600d","src/pe/optional_header.rs":"4048151649a7fe3f8f2d7bb67e784bae889eeb1651bf924f9fbe92400b809217","src/pe/options.rs":"457877197f768c331437297d787dc718b1053b813e3a1dd9b968133fb1540d44","src/pe/relocation.rs":"c479b80bb1d6910f2168505dda4f2d8925b7edc34bed4e25d069546f88f52bb3","src/pe/section_table.rs":"e4b1a2f78c2336aaa0355b5ef102dbe29138c4fa1ba29ed3f379aad1fc64bdff","src/pe/symbol.rs":"1a5fb5bec5727752a6506682ed2ab57829ea810f21f951932a0107861ec0e092","src/pe/utils.rs":"e6da9979ba5f2ae7d1274eef8230cdc4dd90c90a79c7bb9438f8b8ff0aef74be","src/strtab.rs":"dcbd0592c7f032980d112a5f752c175fe8dd257a948892e1f060d25ab52328f5","tests/bins/elf/gnu_hash/README.md":"52581e2ea7067a55bd8aedf4079200fb76448573ae9ffef7d886b9556e980db9"},"package":"bb07a4ffed2093b118a525b1d8f5204ae274faed5604537caf7135d0f18d9887"} \ No newline at end of file diff --git a/third_party/rust/goblin/CHANGELOG.md b/third_party/rust/goblin/CHANGELOG.md index 393105f351..71c79c7b49 100644 --- a/third_party/rust/goblin/CHANGELOG.md +++ b/third_party/rust/goblin/CHANGELOG.md @@ -3,9 +3,32 @@ All notable changes to this project will be documented in this file. Before 1.0, this project does not adhere to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). -Goblin is now 0.7, which means we will try our best to ease breaking changes. Tracking issue is here: https://github.com/m4b/goblin/issues/97 +Goblin is now 0.8, which means we will try our best to ease breaking changes. Tracking issue is here: https://github.com/m4b/goblin/issues/97 -## [0.7.0] - unreleased +## [0.8.0] - 2023-12-31 - Happy New Years! +### Breaking +msrv: bumped to 1.63.0 since scroll bumped as well +pe: new field added to parse options: https://github.com/m4b/goblin/pull/377 +pe: attribute certs now non-exhaustive: https://github.com/m4b/goblin/pull/378 +goblin: hint and object enum is now non-exhaustive +pe: write support introduced some breaking changes, e.g., data directories array adds a tuple of usize and data directory, + DosHeader has all the fields filled out, Header struct has a dos_stub field added, + symbols and strings fields is made optional in Coff struct, see: https://github.com/m4b/goblin/pull/361 +### Fixed +elf: fix documentation, thanks @crzysdrs: https://github.com/m4b/goblin/pull/374 +pe: attribute certificates non-exhaustive, thanks @RaitoBezarius: https://github.com/m4b/goblin/pull/378 +pe: fix authenticode parsing, thanks @baloo: https://github.com/m4b/goblin/pull/383 +### Added +strtab: len method added to return number of bytes of the strtab +pe: absolutely epic pe write support PR, thanks @RaitoBezarius and @Baloo: https://github.com/m4b/goblin/pull/361 +pe: add coff object file support, thanks @vadimcn, https://github.com/m4b/goblin/pull/379 +pe: allow toggling parsing of attribute certs, thanks @suttonbradley: https://github.com/m4b/goblin/pull/377 +mach: add new mach-o constants, thanks @keith: https://github.com/m4b/goblin/pull/372 + +## [0.7.1] - 2023-6-11 +### MSRV bump from log + +## [0.7.0] - 2023-6-11 ### Breaking mach: Implement `LC_NOTE`, (breakage=load commands are marked non-exhaustive), thanks @messense: https://github.com/m4b/goblin/pull/342 ### Fixed diff --git a/third_party/rust/goblin/Cargo.toml b/third_party/rust/goblin/Cargo.toml index 7c586a3e0d..f2d6acc389 100644 --- a/third_party/rust/goblin/Cargo.toml +++ b/third_party/rust/goblin/Cargo.toml @@ -11,9 +11,9 @@ [package] edition = "2021" -rust-version = "1.60.0" +rust-version = "1.63.0" name = "goblin" -version = "0.7.1" +version = "0.8.0" authors = [ "m4b ", "seu ", @@ -44,6 +44,7 @@ categories = [ ] license = "MIT" repository = "https://github.com/m4b/goblin" +resolver = "2" [dependencies.log] version = "0.4" @@ -54,9 +55,12 @@ default-features = false version = "0.2.3" [dependencies.scroll] -version = "0.11" +version = "0.12" default_features = false +[dev-dependencies.stderrlog] +version = "0.5.4" + [features] alloc = [ "scroll/derive", diff --git a/third_party/rust/goblin/README.md b/third_party/rust/goblin/README.md index 76c2260457..ee9462d2ff 100644 --- a/third_party/rust/goblin/README.md +++ b/third_party/rust/goblin/README.md @@ -20,13 +20,13 @@ https://docs.rs/goblin/ ### Usage -Goblin requires `rustc` 1.60.0 (Rust 2021 edition). +Goblin requires `rustc` 1.63.0 (Rust 2021 edition). Add to your `Cargo.toml` ```toml [dependencies] -goblin = "0.7" +goblin = "0.8" ``` ### Features @@ -190,6 +190,7 @@ In lexicographic order: [@baloo]: https://github.com/baloo [@burjui]: https://github.com/burjui [@connorkuehl]: https://github.com/connorkuehl +[@crzysdrs]: https://github.com/crzysdrs [@dancrossnyc]: https://github.com/dancrossnyc [@dureuill]: https://github.com/dureuill [@Evian-Zhang]: https://github.com/Evian-Zhang @@ -238,6 +239,7 @@ In lexicographic order: [@sanxiyn]: https://github.com/sanxiyn [@skdltmxn]: https://github.com/skdltmxn [@sollyucko]: https://github.com/sollyucko +[@suttonbradley]: https://github.com/suttonbradley [@Swatinem]: https://github.com/Swatinem [@SweetVishnya]: https://github.com/SweetVishnya [@SquareMan]: https://github.com/SquareMan @@ -248,6 +250,7 @@ In lexicographic order: [@Tiwalun]: https://github.com/Tiwalun [@track-5]: https://github.com/track-5 [@tux3]: https://github.com/tux3 +[@vadimcn]: https://github.com/vadimcn [@wickerwacka]: https://github.com/wickerwaka [@willglynn]: https://github.com/willglynn [@woodruffw]: https://github.com/woodruffw diff --git a/third_party/rust/goblin/src/elf/reloc.rs b/third_party/rust/goblin/src/elf/reloc.rs index eeb3e1a2ac..d8a1df9d6a 100644 --- a/third_party/rust/goblin/src/elf/reloc.rs +++ b/third_party/rust/goblin/src/elf/reloc.rs @@ -51,7 +51,7 @@ //! | `R_X86_64_GOTPC32` | 26 | 32 | GOT + A - P | //! | `R_X86_64_SIZE32` | 32 | 32 | Z + A | //! | `R_X86_64_SIZE64` | 33 | 64 | Z + A | -//! | `R_X86_64_GOTPC32_TLSDESC` 34 | 32 | | +//! | `R_X86_64_GOTPC32_TLSDESC`| 34 | 32 | | //! | `R_X86_64_TLSDESC_CALL` | 35 | NONE | | //! | `R_X86_64_TLSDESC` | 36 | 64 × 2 | | //! | `R_X86_64_IRELATIVE` | 37 | 64 | indirect (B + A) | diff --git a/third_party/rust/goblin/src/error.rs b/third_party/rust/goblin/src/error.rs index e2dd517e15..0ec4e1e0c9 100644 --- a/third_party/rust/goblin/src/error.rs +++ b/third_party/rust/goblin/src/error.rs @@ -3,6 +3,7 @@ use alloc::string::String; use core::fmt; +use core::num::TryFromIntError; use core::result; #[cfg(feature = "std")] use std::{error, io}; @@ -42,6 +43,12 @@ impl From for Error { } } +impl From for Error { + fn from(err: TryFromIntError) -> Error { + Error::Malformed(format!("Integer do not fit: {err}")) + } +} + impl From for Error { fn from(err: scroll::Error) -> Error { Error::Scroll(err) diff --git a/third_party/rust/goblin/src/lib.rs b/third_party/rust/goblin/src/lib.rs index 25ab841322..ec77f93d5c 100644 --- a/third_party/rust/goblin/src/lib.rs +++ b/third_party/rust/goblin/src/lib.rs @@ -42,13 +42,17 @@ //! Object::PE(pe) => { //! println!("pe: {:#?}", &pe); //! }, +//! Object::COFF(coff) => { +//! println!("coff: {:#?}", &coff); +//! }, //! Object::Mach(mach) => { //! println!("mach: {:#?}", &mach); //! }, //! Object::Archive(archive) => { //! println!("archive: {:#?}", &archive); //! }, -//! Object::Unknown(magic) => { println!("unknown magic: {:#x}", magic) } +//! Object::Unknown(magic) => { println!("unknown magic: {:#x}", magic) }, +//! _ => { } //! } //! } //! } @@ -218,12 +222,14 @@ pub struct HintData { } #[derive(Debug)] +#[non_exhaustive] /// A hint at the underlying binary format for 16 bytes of arbitrary data pub enum Hint { Elf(HintData), Mach(HintData), MachFat(usize), PE, + COFF, Archive, Unknown(u64), } @@ -253,10 +259,14 @@ if_everything! { Ok(Hint::Elf(HintData { is_lsb, is_64 })) } else if &bytes[0..archive::SIZEOF_MAGIC] == archive::MAGIC { Ok(Hint::Archive) - } else if (&bytes[0..2]).pread_with::(0, LE)? == pe::header::DOS_MAGIC { - Ok(Hint::PE) } else { - mach::peek_bytes(bytes) + match *&bytes[0..2].pread_with::(0, LE)? { + pe::header::DOS_MAGIC => Ok(Hint::PE), + pe::header::COFF_MACHINE_X86 | + pe::header::COFF_MACHINE_X86_64 | + pe::header::COFF_MACHINE_ARM64 => Ok(Hint::COFF), + _ => mach::peek_bytes(bytes) + } } } @@ -273,12 +283,15 @@ if_everything! { #[derive(Debug)] #[allow(clippy::large_enum_variant)] + #[non_exhaustive] /// A parseable object that goblin understands pub enum Object<'a> { /// An ELF32/ELF64! Elf(elf::Elf<'a>), /// A PE32/PE32+! PE(pe::PE<'a>), + /// A COFF + COFF(pe::Coff<'a>), /// A 32/64-bit Mach-o binary _OR_ it is a multi-architecture binary container! Mach(mach::Mach<'a>), /// A Unix archive @@ -296,7 +309,8 @@ if_everything! { Hint::Mach(_) | Hint::MachFat(_) => Ok(Object::Mach(mach::Mach::parse(bytes)?)), Hint::Archive => Ok(Object::Archive(archive::Archive::parse(bytes)?)), Hint::PE => Ok(Object::PE(pe::PE::parse(bytes)?)), - Hint::Unknown(magic) => Ok(Object::Unknown(magic)) + Hint::COFF => Ok(Object::COFF(pe::Coff::parse(bytes)?)), + Hint::Unknown(magic) => Ok(Object::Unknown(magic)), } } else { Err(error::Error::Malformed(format!("Object is too small."))) diff --git a/third_party/rust/goblin/src/mach/load_command.rs b/third_party/rust/goblin/src/mach/load_command.rs index 34a44e2bae..305c3b823f 100644 --- a/third_party/rust/goblin/src/mach/load_command.rs +++ b/third_party/rust/goblin/src/mach/load_command.rs @@ -1343,9 +1343,12 @@ pub const PLATFORM_IOSSIMULATOR: u32 = 7; pub const PLATFORM_TVOSSIMULATOR: u32 = 8; pub const PLATFORM_WATCHOSSIMULATOR: u32 = 9; pub const PLATFORM_DRIVERKIT: u32 = 10; +pub const PLATFORM_VISIONOS: u32 = 11; +pub const PLATFORM_VISIONOSSIMULATOR: u32 = 12; pub const TOOL_CLANG: u32 = 1; pub const TOOL_SWIFT: u32 = 2; pub const TOOL_LD: u32 = 3; +pub const TOOL_LLD: u32 = 4; pub fn cmd_to_str(cmd: u32) -> &'static str { match cmd { diff --git a/third_party/rust/goblin/src/pe/authenticode.rs b/third_party/rust/goblin/src/pe/authenticode.rs index e16b7997cd..0f738e6888 100644 --- a/third_party/rust/goblin/src/pe/authenticode.rs +++ b/third_party/rust/goblin/src/pe/authenticode.rs @@ -8,9 +8,13 @@ // - data directory entry for certtable // - certtable +use alloc::collections::VecDeque; use core::ops::Range; +use log::debug; -use super::PE; +use super::{section_table::SectionTable, PE}; + +static PADDING: [u8; 7] = [0; 7]; impl PE<'_> { /// [`authenticode_ranges`] returns the various ranges of the binary that are relevant for @@ -19,6 +23,7 @@ impl PE<'_> { ExcludedSectionsIter { pe: self, state: IterState::default(), + sections: VecDeque::default(), } } } @@ -29,19 +34,22 @@ impl PE<'_> { pub(super) struct ExcludedSections { checksum: Range, datadir_entry_certtable: Range, - certtable: Option>, + certificate_table_size: usize, + end_image_header: usize, } impl ExcludedSections { pub(super) fn new( checksum: Range, datadir_entry_certtable: Range, - certtable: Option>, + certificate_table_size: usize, + end_image_header: usize, ) -> Self { Self { checksum, datadir_entry_certtable, - certtable, + certificate_table_size, + end_image_header, } } } @@ -49,14 +57,26 @@ impl ExcludedSections { pub struct ExcludedSectionsIter<'s> { pe: &'s PE<'s>, state: IterState, + sections: VecDeque, } #[derive(Debug, PartialEq)] enum IterState { Initial, - DatadirEntry(usize), - CertTable(usize), - Final(usize), + ChecksumEnd(usize), + CertificateTableEnd(usize), + HeaderEnd { + end_image_header: usize, + sum_of_bytes_hashed: usize, + }, + Sections { + tail: usize, + sum_of_bytes_hashed: usize, + }, + Final { + sum_of_bytes_hashed: usize, + }, + Padding(usize), Done, } @@ -76,24 +96,166 @@ impl<'s> Iterator for ExcludedSectionsIter<'s> { loop { match self.state { IterState::Initial => { - self.state = IterState::DatadirEntry(sections.checksum.end); - return Some(&bytes[..sections.checksum.start]); + // 3. Hash the image header from its base to immediately before the start of the + // checksum address, as specified in Optional Header Windows-Specific Fields. + let out = Some(&bytes[..sections.checksum.start]); + debug!("hashing {:#x} {:#x}", 0, sections.checksum.start); + + // 4. Skip over the checksum, which is a 4-byte field. + debug_assert_eq!(sections.checksum.end - sections.checksum.start, 4); + self.state = IterState::ChecksumEnd(sections.checksum.end); + + return out; + } + IterState::ChecksumEnd(checksum_end) => { + // 5. Hash everything from the end of the checksum field to immediately before the start + // of the Certificate Table entry, as specified in Optional Header Data Directories. + let out = + Some(&bytes[checksum_end..sections.datadir_entry_certtable.start]); + debug!( + "hashing {checksum_end:#x} {:#x}", + sections.datadir_entry_certtable.start + ); + + // 6. Get the Attribute Certificate Table address and size from the Certificate Table entry. + // For details, see section 5.7 of the PE/COFF specification. + // 7. Exclude the Certificate Table entry from the calculation + self.state = + IterState::CertificateTableEnd(sections.datadir_entry_certtable.end); + + return out; + } + IterState::CertificateTableEnd(start) => { + // 7. Exclude the Certificate Table entry from the calculation and hash everything from + // the end of the Certificate Table entry to the end of image header, including + // Section Table (headers). The Certificate Table entry is 8 bytes long, as specified + // in Optional Header Data Directories. + let end_image_header = sections.end_image_header; + let buf = Some(&bytes[start..end_image_header]); + debug!("hashing {start:#x} {:#x}", end_image_header - start); + + // 8. Create a counter called SUM_OF_BYTES_HASHED, which is not part of the signature. + // Set this counter to the SizeOfHeaders field, as specified in + // Optional Header Windows-Specific Field. + let sum_of_bytes_hashed = end_image_header; + + self.state = IterState::HeaderEnd { + end_image_header, + sum_of_bytes_hashed, + }; + + return buf; } - IterState::DatadirEntry(start) => { - self.state = IterState::CertTable(sections.datadir_entry_certtable.end); - return Some(&bytes[start..sections.datadir_entry_certtable.start]); + IterState::HeaderEnd { + end_image_header, + sum_of_bytes_hashed, + } => { + // 9. Build a temporary table of pointers to all of the section headers in the + // image. The NumberOfSections field of COFF File Header indicates how big + // the table should be. Do not include any section headers in the table whose + // SizeOfRawData field is zero. + + // Implementation detail: + // We require allocation here because the section table has a variable size and + // needs to be sorted. + let mut sections: VecDeque = self + .pe + .sections + .iter() + .filter(|section| section.size_of_raw_data != 0) + .cloned() + .collect(); + + // 10. Using the PointerToRawData field (offset 20) in the referenced SectionHeader + // structure as a key, arrange the table's elements in ascending order. In + // other words, sort the section headers in ascending order according to the + // disk-file offset of the sections. + sections + .make_contiguous() + .sort_by_key(|section| section.pointer_to_raw_data); + + self.sections = sections; + + self.state = IterState::Sections { + tail: end_image_header, + sum_of_bytes_hashed, + }; } - IterState::CertTable(start) => { - if let Some(certtable) = sections.certtable.as_ref() { - self.state = IterState::Final(certtable.end); - return Some(&bytes[start..certtable.start]); + IterState::Sections { + mut tail, + mut sum_of_bytes_hashed, + } => { + // 11. Walk through the sorted table, load the corresponding section into memory, + // and hash the entire section. Use the SizeOfRawData field in the SectionHeader + // structure to determine the amount of data to hash. + if let Some(section) = self.sections.pop_front() { + let start = section.pointer_to_raw_data as usize; + let end = start + section.size_of_raw_data as usize; + tail = end; + + // 12. Add the section’s SizeOfRawData value to SUM_OF_BYTES_HASHED. + sum_of_bytes_hashed += section.size_of_raw_data as usize; + + debug!("hashing {start:#x} {:#x}", end - start); + let buf = &bytes[start..end]; + + // 13. Repeat steps 11 and 12 for all of the sections in the sorted table. + self.state = IterState::Sections { + tail, + sum_of_bytes_hashed, + }; + + return Some(buf); } else { - self.state = IterState::Final(start) + self.state = IterState::Final { + sum_of_bytes_hashed, + }; + } + } + IterState::Final { + sum_of_bytes_hashed, + } => { + // 14. Create a value called FILE_SIZE, which is not part of the signature. + // Set this value to the image’s file size, acquired from the underlying + // file system. If FILE_SIZE is greater than SUM_OF_BYTES_HASHED, the + // file contains extra data that must be added to the hash. This data + // begins at the SUM_OF_BYTES_HASHED file offset, and its length is: + // (File Size) - ((Size of AttributeCertificateTable) + SUM_OF_BYTES_HASHED) + // + // Note: The size of Attribute Certificate Table is specified in the second + // ULONG value in the Certificate Table entry (32 bit: offset 132, + // 64 bit: offset 148) in Optional Header Data Directories. + let file_size = bytes.len(); + + // If FILE_SIZE is not a multiple of 8 bytes, the data added to the hash must + // be appended with zero padding of length (8 – (FILE_SIZE % 8)) bytes + let pad_size = (8 - file_size % 8) % 8; + self.state = IterState::Padding(pad_size); + + if file_size > sum_of_bytes_hashed { + let extra_data_start = sum_of_bytes_hashed; + let len = + file_size - sections.certificate_table_size - sum_of_bytes_hashed; + + debug!("hashing {extra_data_start:#x} {len:#x}",); + let buf = &bytes[extra_data_start..extra_data_start + len]; + + return Some(buf); } } - IterState::Final(start) => { + IterState::Padding(pad_size) => { self.state = IterState::Done; - return Some(&bytes[start..]); + + if pad_size != 0 { + debug!("hashing {pad_size:#x}"); + + // NOTE (safety): pad size will be at most 7, and PADDING has a size of 7 + // pad_size is computed ~10 lines above. + debug_assert!(pad_size <= 7); + debug_assert_eq!(PADDING.len(), 7); + + return Some(&PADDING[..pad_size]); + } } IterState::Done => return None, } diff --git a/third_party/rust/goblin/src/pe/certificate_table.rs b/third_party/rust/goblin/src/pe/certificate_table.rs index 353a6c70ce..be45ce48dc 100644 --- a/third_party/rust/goblin/src/pe/certificate_table.rs +++ b/third_party/rust/goblin/src/pe/certificate_table.rs @@ -3,12 +3,15 @@ /// https://learn.microsoft.com/en-us/windows/win32/debug/pe-format#the-attribute-certificate-table-image-only /// https://learn.microsoft.com/en-us/windows/win32/api/wintrust/ns-wintrust-win_certificate use crate::error; -use scroll::Pread; +use scroll::{ctx, Pread, Pwrite}; use alloc::string::ToString; use alloc::vec::Vec; +use super::utils::pad; + #[repr(u16)] +#[non_exhaustive] #[derive(Debug, PartialEq, Copy, Clone)] pub enum AttributeCertificateRevision { /// WIN_CERT_REVISION_1_0 @@ -38,7 +41,7 @@ impl TryFrom for AttributeCertificateRevision { } #[repr(u16)] -#[derive(Debug)] +#[derive(Debug, PartialEq, Copy, Clone)] pub enum AttributeCertificateType { /// WIN_CERT_TYPE_X509 X509 = 0x0001, @@ -127,7 +130,28 @@ impl<'a> AttributeCertificate<'a> { } } +impl<'a> ctx::TryIntoCtx for &AttributeCertificate<'a> { + type Error = error::Error; + + /// Writes an aligned attribute certificate in the buffer. + fn try_into_ctx(self, bytes: &mut [u8], ctx: scroll::Endian) -> Result { + let offset = &mut 0; + bytes.gwrite_with(self.length, offset, ctx)?; + bytes.gwrite_with(self.revision as u16, offset, ctx)?; + bytes.gwrite_with(self.certificate_type as u16, offset, ctx)?; + // Extend by zero the buffer until it is aligned on a quadword (16 bytes). + let maybe_certificate_padding = pad(self.certificate.len(), Some(16usize)); + bytes.gwrite(self.certificate, offset)?; + if let Some(cert_padding) = maybe_certificate_padding { + bytes.gwrite(&cert_padding[..], offset)?; + } + + Ok(*offset) + } +} + pub type CertificateDirectoryTable<'a> = Vec>; + pub(crate) fn enumerate_certificates( bytes: &[u8], table_virtual_address: u32, diff --git a/third_party/rust/goblin/src/pe/data_directories.rs b/third_party/rust/goblin/src/pe/data_directories.rs index 265e4e27f7..e65db5953d 100644 --- a/third_party/rust/goblin/src/pe/data_directories.rs +++ b/third_party/rust/goblin/src/pe/data_directories.rs @@ -1,5 +1,8 @@ use crate::error; -use scroll::{Pread, Pwrite, SizeWith}; +use scroll::{ + ctx::{self}, + Pread, Pwrite, SizeWith, +}; #[repr(C)] #[derive(Debug, PartialEq, Copy, Clone, Default, Pread, Pwrite, SizeWith)] @@ -13,14 +16,86 @@ const NUM_DATA_DIRECTORIES: usize = 16; impl DataDirectory { pub fn parse(bytes: &[u8], offset: &mut usize) -> error::Result { - let dd = bytes.gread_with(offset, scroll::LE)?; - Ok(dd) + Ok(bytes.gread_with(offset, scroll::LE)?) + } +} + +#[derive(Debug, PartialEq, Copy, Clone)] +pub enum DataDirectoryType { + ExportTable, + ImportTable, + ResourceTable, + ExceptionTable, + CertificateTable, + BaseRelocationTable, + DebugTable, + Architecture, + GlobalPtr, + TlsTable, + LoadConfigTable, + BoundImportTable, + ImportAddressTable, + DelayImportDescriptor, + ClrRuntimeHeader, +} + +impl TryFrom for DataDirectoryType { + type Error = error::Error; + fn try_from(value: usize) -> Result { + Ok(match value { + 0 => Self::ExportTable, + 1 => Self::ImportTable, + 2 => Self::ResourceTable, + 3 => Self::ExceptionTable, + 4 => Self::CertificateTable, + 5 => Self::BaseRelocationTable, + 6 => Self::DebugTable, + 7 => Self::Architecture, + 8 => Self::GlobalPtr, + 9 => Self::TlsTable, + 10 => Self::LoadConfigTable, + 11 => Self::BoundImportTable, + 12 => Self::ImportAddressTable, + 13 => Self::DelayImportDescriptor, + 14 => Self::ClrRuntimeHeader, + _ => { + return Err(error::Error::Malformed( + "Wrong data directory index number".into(), + )) + } + }) } } #[derive(Debug, PartialEq, Copy, Clone, Default)] pub struct DataDirectories { - pub data_directories: [Option; NUM_DATA_DIRECTORIES], + pub data_directories: [Option<(usize, DataDirectory)>; NUM_DATA_DIRECTORIES], +} + +impl ctx::TryIntoCtx for DataDirectories { + type Error = error::Error; + + fn try_into_ctx(self, bytes: &mut [u8], ctx: scroll::Endian) -> Result { + let offset = &mut 0; + for opt_dd in self.data_directories { + if let Some((dd_offset, dd)) = opt_dd { + bytes.pwrite_with(dd, dd_offset, ctx)?; + *offset += dd_offset; + } else { + bytes.gwrite(&[0; SIZEOF_DATA_DIRECTORY][..], offset)?; + } + } + Ok(NUM_DATA_DIRECTORIES * SIZEOF_DATA_DIRECTORY) + } +} + +macro_rules! build_dd_getter { + ($dd_name:tt, $index:tt) => { + pub fn $dd_name(&self) -> Option<&DataDirectory> { + let idx = $index; + self.data_directories[idx].as_ref().map(|(_, dd)| dd) + } + }; } impl DataDirectories { @@ -37,70 +112,42 @@ impl DataDirectories { let dd = if dd.virtual_address == 0 && dd.size == 0 { None } else { - Some(dd) + Some((*offset, dd)) }; *dir = dd; } Ok(DataDirectories { data_directories }) } - pub fn get_export_table(&self) -> &Option { - let idx = 0; - &self.data_directories[idx] - } - pub fn get_import_table(&self) -> &Option { - let idx = 1; - &self.data_directories[idx] - } - pub fn get_resource_table(&self) -> &Option { - let idx = 2; - &self.data_directories[idx] - } - pub fn get_exception_table(&self) -> &Option { - let idx = 3; - &self.data_directories[idx] - } - pub fn get_certificate_table(&self) -> &Option { - let idx = 4; - &self.data_directories[idx] - } - pub fn get_base_relocation_table(&self) -> &Option { - let idx = 5; - &self.data_directories[idx] - } - pub fn get_debug_table(&self) -> &Option { - let idx = 6; - &self.data_directories[idx] - } - pub fn get_architecture(&self) -> &Option { - let idx = 7; - &self.data_directories[idx] - } - pub fn get_global_ptr(&self) -> &Option { - let idx = 8; - &self.data_directories[idx] - } - pub fn get_tls_table(&self) -> &Option { - let idx = 9; - &self.data_directories[idx] - } - pub fn get_load_config_table(&self) -> &Option { - let idx = 10; - &self.data_directories[idx] - } - pub fn get_bound_import_table(&self) -> &Option { - let idx = 11; - &self.data_directories[idx] - } - pub fn get_import_address_table(&self) -> &Option { - let idx = 12; - &self.data_directories[idx] - } - pub fn get_delay_import_descriptor(&self) -> &Option { - let idx = 13; - &self.data_directories[idx] - } - pub fn get_clr_runtime_header(&self) -> &Option { - let idx = 14; - &self.data_directories[idx] + + build_dd_getter!(get_export_table, 0); + build_dd_getter!(get_import_table, 1); + build_dd_getter!(get_resource_table, 2); + build_dd_getter!(get_exception_table, 3); + build_dd_getter!(get_certificate_table, 4); + build_dd_getter!(get_base_relocation_table, 5); + build_dd_getter!(get_debug_table, 6); + build_dd_getter!(get_architecture, 7); + build_dd_getter!(get_global_ptr, 8); + build_dd_getter!(get_tls_table, 9); + build_dd_getter!(get_load_config_table, 10); + build_dd_getter!(get_bound_import_table, 11); + build_dd_getter!(get_import_address_table, 12); + build_dd_getter!(get_delay_import_descriptor, 13); + build_dd_getter!(get_clr_runtime_header, 14); + + pub fn dirs(&self) -> impl Iterator { + self.data_directories + .into_iter() + .enumerate() + // (Index, Option
) -> Option<(Index, DD)> -> (DDT, DD) + .filter_map(|(i, o)| + // We should not have invalid indexes. + // Indeed: `data_directories: &[_; N]` where N is the number + // of data directories. + // The `TryFrom` trait for integers to DataDirectoryType + // takes into account the N possible data directories. + // Therefore, the unwrap can never fail as long as Rust guarantees + // on types are honored. + o.map(|(_, v)| (i.try_into().unwrap(), v))) } } diff --git a/third_party/rust/goblin/src/pe/header.rs b/third_party/rust/goblin/src/pe/header.rs index c1ded9dd9f..06e23c0b03 100644 --- a/third_party/rust/goblin/src/pe/header.rs +++ b/third_party/rust/goblin/src/pe/header.rs @@ -3,24 +3,60 @@ use crate::pe::{optional_header, section_table, symbol}; use crate::strtab; use alloc::vec::Vec; use log::debug; -use scroll::{IOread, IOwrite, Pread, Pwrite, SizeWith}; +use scroll::{ctx, IOread, IOwrite, Pread, Pwrite, SizeWith}; /// DOS header present in all PE binaries #[repr(C)] -#[derive(Debug, PartialEq, Copy, Clone, Default)] +#[derive(Debug, PartialEq, Copy, Clone, Default, Pwrite)] pub struct DosHeader { /// Magic number: 5a4d pub signature: u16, - /// Pointer to PE header, always at offset 0x3c + /// e_cblp + pub bytes_on_last_page: u16, + /// e_cp + pub pages_in_file: u16, + /// e_crlc + pub relocations: u16, + /// e_cparhdr + pub size_of_header_in_paragraphs: u16, + /// e_minalloc + pub minimum_extra_paragraphs_needed: u16, + /// e_maxalloc + pub maximum_extra_paragraphs_needed: u16, + /// e_ss + pub initial_relative_ss: u16, + /// e_sp + pub initial_sp: u16, + /// e_csum + pub checksum: u16, + /// e_ip + pub initial_ip: u16, + /// e_cs + pub initial_relative_cs: u16, + /// e_lfarlc + pub file_address_of_relocation_table: u16, + /// e_ovno + pub overlay_number: u16, + /// e_res[4] + pub reserved: [u16; 4], + /// e_oemid + pub oem_id: u16, + /// e_oeminfo + pub oem_info: u16, + /// e_res2[10] + pub reserved2: [u16; 10], + /// e_lfanew: pointer to PE header, always at offset 0x3c pub pe_pointer: u32, } pub const DOS_MAGIC: u16 = 0x5a4d; pub const PE_POINTER_OFFSET: u32 = 0x3c; +pub const DOS_STUB_OFFSET: u32 = PE_POINTER_OFFSET + (core::mem::size_of::() as u32); impl DosHeader { pub fn parse(bytes: &[u8]) -> error::Result { - let signature = bytes.pread_with(0, scroll::LE).map_err(|_| { + let mut offset = 0; + let signature = bytes.gread_with(&mut offset, scroll::LE).map_err(|_| { error::Error::Malformed(format!("cannot parse DOS signature (offset {:#x})", 0)) })?; if signature != DOS_MAGIC { @@ -29,6 +65,33 @@ impl DosHeader { signature ))); } + + let bytes_on_last_page = bytes.gread_with(&mut offset, scroll::LE)?; + let pages_in_file = bytes.gread_with(&mut offset, scroll::LE)?; + let relocations = bytes.gread_with(&mut offset, scroll::LE)?; + let size_of_header_in_paragraphs = bytes.gread_with(&mut offset, scroll::LE)?; + let minimum_extra_paragraphs_needed = bytes.gread_with(&mut offset, scroll::LE)?; + let maximum_extra_paragraphs_needed = bytes.gread_with(&mut offset, scroll::LE)?; + let initial_relative_ss = bytes.gread_with(&mut offset, scroll::LE)?; + let initial_sp = bytes.gread_with(&mut offset, scroll::LE)?; + let checksum = bytes.gread_with(&mut offset, scroll::LE)?; + let initial_ip = bytes.gread_with(&mut offset, scroll::LE)?; + let initial_relative_cs = bytes.gread_with(&mut offset, scroll::LE)?; + let file_address_of_relocation_table = bytes.gread_with(&mut offset, scroll::LE)?; + let overlay_number = bytes.gread_with(&mut offset, scroll::LE)?; + let reserved = [0x0; 4]; + offset += core::mem::size_of_val(&reserved); + let oem_id = bytes.gread_with(&mut offset, scroll::LE)?; + let oem_info = bytes.gread_with(&mut offset, scroll::LE)?; + let reserved2 = [0x0; 10]; + offset += core::mem::size_of_val(&reserved2); + + debug_assert!( + offset == PE_POINTER_OFFSET as usize, + "expected offset ({:#x}) after reading DOS header to be at 0x3C", + offset + ); + let pe_pointer = bytes .pread_with(PE_POINTER_OFFSET as usize, scroll::LE) .map_err(|_| { @@ -37,6 +100,7 @@ impl DosHeader { PE_POINTER_OFFSET )) })?; + let pe_signature: u32 = bytes .pread_with(pe_pointer as usize, scroll::LE) @@ -52,13 +116,48 @@ impl DosHeader { pe_signature ))); } + Ok(DosHeader { signature, + bytes_on_last_page, + pages_in_file, + relocations, + size_of_header_in_paragraphs, + minimum_extra_paragraphs_needed, + maximum_extra_paragraphs_needed, + initial_relative_ss, + initial_sp, + checksum, + initial_ip, + initial_relative_cs, + file_address_of_relocation_table, + overlay_number, + reserved, + oem_id, + oem_info, + reserved2, pe_pointer, }) } } +#[repr(C)] +#[derive(Debug, PartialEq, Copy, Clone, Pread, Pwrite)] +/// The DOS stub program which should be executed in DOS mode +pub struct DosStub(pub [u8; 0x40]); +impl Default for DosStub { + fn default() -> Self { + // "This program cannot be run in DOS mode" error program + Self([ + 0x0E, 0x1F, 0xBA, 0x0E, 0x00, 0xB4, 0x09, 0xCD, 0x21, 0xB8, 0x01, 0x4C, 0xCD, 0x21, + 0x54, 0x68, 0x69, 0x73, 0x20, 0x70, 0x72, 0x6F, 0x67, 0x72, 0x61, 0x6D, 0x20, 0x63, + 0x61, 0x6E, 0x6E, 0x6F, 0x74, 0x20, 0x62, 0x65, 0x20, 0x72, 0x75, 0x6E, 0x20, 0x69, + 0x6E, 0x20, 0x44, 0x4F, 0x53, 0x20, 0x6D, 0x6F, 0x64, 0x65, 0x2E, 0x0D, 0x0D, 0x0A, + 0x24, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + ]) + } +} + /// COFF Header #[repr(C)] #[derive(Debug, PartialEq, Copy, Clone, Default, Pread, Pwrite, IOread, IOwrite, SizeWith)] @@ -163,14 +262,24 @@ impl CoffHeader { } /// Return the COFF symbol table. - pub fn symbols<'a>(&self, bytes: &'a [u8]) -> error::Result> { + pub fn symbols<'a>(&self, bytes: &'a [u8]) -> error::Result>> { let offset = self.pointer_to_symbol_table as usize; let number = self.number_of_symbol_table as usize; - symbol::SymbolTable::parse(bytes, offset, number) + if offset == 0 { + Ok(None) + } else { + symbol::SymbolTable::parse(bytes, offset, number).map(Some) + } } /// Return the COFF string table. - pub fn strings<'a>(&self, bytes: &'a [u8]) -> error::Result> { + pub fn strings<'a>(&self, bytes: &'a [u8]) -> error::Result>> { + // > The file offset of the COFF symbol table, or zero if no COFF symbol table is present. + // > This value should be zero for an image because COFF debugging information is deprecated. + if self.pointer_to_symbol_table == 0 { + return Ok(None); + } + let mut offset = self.pointer_to_symbol_table as usize + symbol::SymbolTable::size(self.number_of_symbol_table as usize); @@ -180,13 +289,15 @@ impl CoffHeader { // The offset needs to be advanced in order to read the strings. offset += length_field_size; - Ok(strtab::Strtab::parse(bytes, offset, length, 0)?) + Ok(Some(strtab::Strtab::parse(bytes, offset, length, 0)?)) } } #[derive(Debug, PartialEq, Copy, Clone, Default)] pub struct Header { pub dos_header: DosHeader, + /// DOS program for legacy loaders + pub dos_stub: DosStub, /// PE Magic: PE\0\0, little endian pub signature: u32, pub coff_header: CoffHeader, @@ -196,6 +307,12 @@ pub struct Header { impl Header { pub fn parse(bytes: &[u8]) -> error::Result { let dos_header = DosHeader::parse(&bytes)?; + let dos_stub = bytes.pread(DOS_STUB_OFFSET as usize).map_err(|_| { + error::Error::Malformed(format!( + "cannot parse DOS stub (offset {:#x})", + DOS_STUB_OFFSET + )) + })?; let mut offset = dos_header.pe_pointer as usize; let signature = bytes.gread_with(&mut offset, scroll::LE).map_err(|_| { error::Error::Malformed(format!("cannot parse PE signature (offset {:#x})", offset)) @@ -208,6 +325,7 @@ impl Header { }; Ok(Header { dos_header, + dos_stub, signature, coff_header, optional_header, @@ -215,6 +333,22 @@ impl Header { } } +impl ctx::TryIntoCtx for Header { + type Error = error::Error; + + fn try_into_ctx(self, bytes: &mut [u8], ctx: scroll::Endian) -> Result { + let offset = &mut 0; + bytes.gwrite_with(self.dos_header, offset, ctx)?; + bytes.gwrite_with(self.dos_stub, offset, ctx)?; + bytes.gwrite_with(self.signature, offset, scroll::LE)?; + bytes.gwrite_with(self.coff_header, offset, ctx)?; + if let Some(opt_header) = self.optional_header { + bytes.gwrite_with(opt_header, offset, ctx)?; + } + Ok(*offset) + } +} + /// Convert machine to str representation pub fn machine_to_str(machine: u16) -> &'static str { match machine { diff --git a/third_party/rust/goblin/src/pe/mod.rs b/third_party/rust/goblin/src/pe/mod.rs index 5a7337630a..2336fddc57 100644 --- a/third_party/rust/goblin/src/pe/mod.rs +++ b/third_party/rust/goblin/src/pe/mod.rs @@ -3,7 +3,12 @@ // TODO: panics with unwrap on None for apisetschema.dll, fhuxgraphics.dll and some others +use core::cmp::max; + +use alloc::borrow::Cow; +use alloc::string::String; use alloc::vec::Vec; +use log::warn; pub mod authenticode; pub mod certificate_table; @@ -23,8 +28,11 @@ pub mod utils; use crate::container; use crate::error; +use crate::pe::utils::pad; use crate::strtab; +use scroll::{ctx, Pwrite}; + use log::debug; #[derive(Debug)] @@ -140,7 +148,7 @@ impl<'a> PE<'a> { entry, image_base, is_64 ); let file_alignment = optional_header.windows_fields.file_alignment; - if let Some(export_table) = *optional_header.data_directories.get_export_table() { + if let Some(&export_table) = optional_header.data_directories.get_export_table() { if let Ok(ed) = export::ExportData::parse_with_opts( bytes, export_table, @@ -162,7 +170,7 @@ impl<'a> PE<'a> { } } debug!("exports: {:#?}", exports); - if let Some(import_table) = *optional_header.data_directories.get_import_table() { + if let Some(&import_table) = optional_header.data_directories.get_import_table() { let id = if is_64 { import::ImportData::parse_with_opts::( bytes, @@ -196,7 +204,7 @@ impl<'a> PE<'a> { import_data = Some(id); } debug!("imports: {:#?}", imports); - if let Some(debug_table) = *optional_header.data_directories.get_debug_table() { + if let Some(&debug_table) = optional_header.data_directories.get_debug_table() { debug_data = Some(debug::DebugData::parse_with_opts( bytes, debug_table, @@ -209,8 +217,8 @@ impl<'a> PE<'a> { if header.coff_header.machine == header::COFF_MACHINE_X86_64 { // currently only x86_64 is supported debug!("exception data: {:#?}", exception_data); - if let Some(exception_table) = - *optional_header.data_directories.get_exception_table() + if let Some(&exception_table) = + optional_header.data_directories.get_exception_table() { exception_data = Some(exception::ExceptionData::parse_with_opts( bytes, @@ -222,26 +230,30 @@ impl<'a> PE<'a> { } } - let certtable = if let Some(certificate_table) = - *optional_header.data_directories.get_certificate_table() - { - certificates = certificate_table::enumerate_certificates( - bytes, - certificate_table.virtual_address, - certificate_table.size, - )?; + // Parse attribute certificates unless opted out of + let certificate_table_size = if opts.parse_attribute_certificates { + if let Some(&certificate_table) = + optional_header.data_directories.get_certificate_table() + { + certificates = certificate_table::enumerate_certificates( + bytes, + certificate_table.virtual_address, + certificate_table.size, + )?; - let start = certificate_table.virtual_address as usize; - let end = start + certificate_table.size as usize; - Some(start..end) + certificate_table.size as usize + } else { + 0 + } } else { - None + 0 }; authenticode_excluded_sections = Some(authenticode::ExcludedSections::new( checksum, datadir_entry_certtable, - certtable, + certificate_table_size, + optional_header.windows_fields.size_of_headers as usize, )); } Ok(PE { @@ -265,6 +277,192 @@ impl<'a> PE<'a> { certificates, }) } + + pub fn write_sections( + &self, + bytes: &mut [u8], + offset: &mut usize, + file_alignment: Option, + ctx: scroll::Endian, + ) -> Result { + // sections table and data + debug_assert!( + self.sections + .iter() + .flat_map(|section_a| { + self.sections + .iter() + .map(move |section_b| (section_a, section_b)) + }) + // given sections = (s_1, …, s_n) + // for all (s_i, s_j), i != j, verify that s_i does not overlap with s_j and vice versa. + .all(|(section_i, section_j)| section_i == section_j + || !section_i.overlaps_with(section_j)), + "Overlapping sections were found, this is not supported." + ); + + for section in &self.sections { + let section_data = section.data(&self.bytes)?.ok_or_else(|| { + error::Error::Malformed(format!( + "Section data `{}` is malformed", + section.name().unwrap_or("unknown name") + )) + })?; + let file_section_offset = + usize::try_from(section.pointer_to_raw_data).map_err(|_| { + error::Error::Malformed(format!( + "Section `{}`'s pointer to raw data does not fit in platform `usize`", + section.name().unwrap_or("unknown name") + )) + })?; + let vsize: usize = section.virtual_size.try_into()?; + let ondisk_size: usize = section.size_of_raw_data.try_into()?; + let section_name = String::from(section.name().unwrap_or("unknown name")); + + let mut file_offset = file_section_offset; + // `file_section_offset` is a on-disk offset which can be anywhere in the file. + // Write section data first to avoid the final consumption. + match section_data { + Cow::Borrowed(borrowed) => bytes.gwrite(borrowed, &mut file_offset)?, + Cow::Owned(owned) => bytes.gwrite(owned.as_slice(), &mut file_offset)?, + }; + + // Section tables follows the header. + bytes.gwrite_with(section, offset, ctx)?; + + // for size size_of_raw_data + // if < virtual_size, pad with 0 + // Pad with zeros if necessary + if file_offset < vsize { + bytes.gwrite(vec![0u8; vsize - file_offset].as_slice(), &mut file_offset)?; + } + + // Align on a boundary as per file alignement field. + if let Some(pad) = pad(file_offset - file_section_offset, file_alignment) { + debug!( + "aligning `{}` {:#x} -> {:#x} bytes'", + section_name, + file_offset - file_section_offset, + file_offset - file_section_offset + pad.len() + ); + bytes.gwrite(pad.as_slice(), &mut file_offset)?; + } + + let written_data_size = file_offset - file_section_offset; + if ondisk_size != written_data_size { + warn!("Original PE is inefficient or bug (on-disk data size in PE: {:#x}), we wrote {:#x} bytes", + ondisk_size, + written_data_size); + } + } + + Ok(*offset) + } + + pub fn write_certificates( + &self, + bytes: &mut [u8], + ctx: scroll::Endian, + ) -> Result { + let opt_header = self + .header + .optional_header + .ok_or(error::Error::Malformed(format!( + "This PE binary has no optional header; it is required to write certificates" + )))?; + let mut max_offset = 0; + + if let Some(certificate_directory) = opt_header.data_directories.get_certificate_table() { + let mut certificate_start = certificate_directory.virtual_address.try_into()?; + for certificate in &self.certificates { + bytes.gwrite_with(certificate, &mut certificate_start, ctx)?; + max_offset = max(certificate_start, max_offset); + } + } + + Ok(max_offset) + } +} + +impl<'a> ctx::TryIntoCtx for PE<'a> { + type Error = error::Error; + + fn try_into_ctx(self, bytes: &mut [u8], ctx: scroll::Endian) -> Result { + let mut offset = 0; + // We need to maintain a `max_offset` because + // we could be writing sections in the wrong order (i.e. not an increasing order for the + // pointer on raw disk) + // and there could be holes between sections. + // If we don't re-layout sections, we cannot fix that ourselves. + // Same can be said about the certificate table, there could be a hole between sections + // and the certificate data. + // To avoid those troubles, we maintain the max over all offsets we see so far. + let mut max_offset = 0; + let file_alignment: Option = match self.header.optional_header { + Some(opt_header) => { + debug_assert!( + opt_header.windows_fields.file_alignment.count_ones() == 1, + "file alignment should be a power of 2" + ); + Some(opt_header.windows_fields.file_alignment.try_into()?) + } + _ => None, + }; + bytes.gwrite_with(self.header, &mut offset, ctx)?; + max_offset = max(offset, max_offset); + self.write_sections(bytes, &mut offset, file_alignment, ctx)?; + // We want the section offset for which we have the highest pointer on disk. + // The next offset is reserved for debug tables (outside of sections) and/or certificate + // tables. + max_offset = max( + self.sections + .iter() + .max_by_key(|section| section.pointer_to_raw_data as usize) + .map(|section| (section.pointer_to_raw_data + section.size_of_raw_data) as usize) + .unwrap_or(offset), + max_offset, + ); + + // COFF Symbol Table + // Auxiliary Symbol Records + // COFF String Table + assert!( + self.header.coff_header.pointer_to_symbol_table == 0, + "Symbol tables in PE are deprecated and not supported to write" + ); + + // The following data directories are + // taken care inside a section: + // - export table (.edata) + // - import table (.idata) + // - bound import table + // - import address table + // - delay import tables + // - resource table (.rsrc) + // - exception table (.pdata) + // - base relocation table (.reloc) + // - debug table (.debug) <- this one is special, it can be outside of a + // section. + // - load config table + // - tls table (.tls) + // - architecture (reserved, 0 for now) + // - global ptr is a "empty" data directory (header-only) + // - clr runtime header (.cormeta is object-only) + // + // Nonetheless, we need to write the attribute certificate table one. + max_offset = max(max_offset, self.write_certificates(bytes, ctx)?); + + // TODO: we would like to support debug table outside of a section. + // i.e. debug tables that are never mapped in memory + // See https://learn.microsoft.com/en-us/windows/win32/debug/pe-format#debug-directory-image-only + // > The debug directory can be in a discardable .debug section (if one exists), or it can be included in any other section in the image file, or not be in a section at all. + // In case it's not in a section at all, we need to find a way + // to rewrite it again. + // and we need to respect the ordering between attribute certificates + // and debug table. + + Ok(max_offset) + } } /// An analyzed COFF object @@ -274,10 +472,12 @@ pub struct Coff<'a> { pub header: header::CoffHeader, /// A list of the sections in this COFF binary pub sections: Vec, - /// The COFF symbol table. - pub symbols: symbol::SymbolTable<'a>, - /// The string table. - pub strings: strtab::Strtab<'a>, + /// The COFF symbol table, they are not guaranteed to exist. + /// For an image, this is expected to be None as COFF debugging information + /// has been deprecated. + pub symbols: Option>, + /// The string table, they don't exist if COFF symbol table does not exist. + pub strings: Option>, } impl<'a> Coff<'a> { @@ -414,7 +614,7 @@ mod tests { #[test] fn string_table_excludes_length() { let coff = Coff::parse(&&COFF_FILE_SINGLE_STRING_IN_STRING_TABLE[..]).unwrap(); - let string_table = coff.strings.to_vec().unwrap(); + let string_table = coff.strings.unwrap().to_vec().unwrap(); assert!(string_table == vec!["ExitProcess"]); } @@ -422,9 +622,10 @@ mod tests { #[test] fn symbol_name_excludes_length() { let coff = Coff::parse(&COFF_FILE_SINGLE_STRING_IN_STRING_TABLE).unwrap(); - let strings = coff.strings; + let strings = coff.strings.unwrap(); let symbols = coff .symbols + .unwrap() .iter() .filter(|(_, name, _)| name.is_none()) .map(|(_, _, sym)| sym.name(&strings).unwrap().to_owned()) diff --git a/third_party/rust/goblin/src/pe/optional_header.rs b/third_party/rust/goblin/src/pe/optional_header.rs index b1d3192764..852f4dced0 100644 --- a/third_party/rust/goblin/src/pe/optional_header.rs +++ b/third_party/rust/goblin/src/pe/optional_header.rs @@ -71,6 +71,22 @@ impl From for StandardFields { } } +impl From for StandardFields32 { + fn from(fields: StandardFields) -> Self { + StandardFields32 { + magic: fields.magic, + major_linker_version: fields.major_linker_version, + minor_linker_version: fields.minor_linker_version, + size_of_code: fields.size_of_code as u32, + size_of_initialized_data: fields.size_of_initialized_data as u32, + size_of_uninitialized_data: fields.size_of_uninitialized_data as u32, + address_of_entry_point: fields.address_of_entry_point as u32, + base_of_code: fields.base_of_code as u32, + base_of_data: fields.base_of_data, + } + } +} + impl From for StandardFields { fn from(fields: StandardFields64) -> Self { StandardFields { @@ -87,6 +103,21 @@ impl From for StandardFields { } } +impl From for StandardFields64 { + fn from(fields: StandardFields) -> Self { + StandardFields64 { + magic: fields.magic, + major_linker_version: fields.major_linker_version, + minor_linker_version: fields.minor_linker_version, + size_of_code: fields.size_of_code as u32, + size_of_initialized_data: fields.size_of_initialized_data as u32, + size_of_uninitialized_data: fields.size_of_uninitialized_data as u32, + address_of_entry_point: fields.address_of_entry_point as u32, + base_of_code: fields.base_of_code as u32, + } + } +} + /// Standard fields magic number for 32-bit binary pub const MAGIC_32: u16 = 0x10b; /// Standard fields magic number for 64-bit binary @@ -208,6 +239,36 @@ impl From for WindowsFields { } } +impl TryFrom for WindowsFields32 { + type Error = crate::error::Error; + + fn try_from(value: WindowsFields64) -> Result { + Ok(WindowsFields32 { + image_base: value.image_base.try_into()?, + section_alignment: value.section_alignment, + file_alignment: value.file_alignment, + major_operating_system_version: value.major_operating_system_version, + minor_operating_system_version: value.minor_operating_system_version, + major_image_version: value.major_image_version, + minor_image_version: value.minor_image_version, + major_subsystem_version: value.major_subsystem_version, + minor_subsystem_version: value.minor_subsystem_version, + win32_version_value: value.win32_version_value, + size_of_image: value.size_of_image, + size_of_headers: value.size_of_headers, + check_sum: value.check_sum, + subsystem: value.subsystem, + dll_characteristics: value.dll_characteristics, + size_of_stack_reserve: value.size_of_stack_reserve.try_into()?, + size_of_stack_commit: value.size_of_stack_commit.try_into()?, + size_of_heap_reserve: value.size_of_heap_reserve.try_into()?, + size_of_heap_commit: value.size_of_heap_commit.try_into()?, + loader_flags: value.loader_flags, + number_of_rva_and_sizes: value.number_of_rva_and_sizes, + }) + } +} + // impl From for WindowsFields { // fn from(windows: WindowsFields32) -> Self { // WindowsFields { @@ -289,6 +350,28 @@ impl<'a> ctx::TryFromCtx<'a, Endian> for OptionalHeader { } } +impl ctx::TryIntoCtx for OptionalHeader { + type Error = error::Error; + + fn try_into_ctx(self, bytes: &mut [u8], ctx: scroll::Endian) -> Result { + let offset = &mut 0; + match self.standard_fields.magic { + MAGIC_32 => { + bytes.gwrite_with::(self.standard_fields.into(), offset, ctx)?; + bytes.gwrite_with(WindowsFields32::try_from(self.windows_fields)?, offset, ctx)?; + bytes.gwrite_with(self.data_directories, offset, ctx)?; + } + MAGIC_64 => { + bytes.gwrite_with::(self.standard_fields.into(), offset, ctx)?; + bytes.gwrite_with(self.windows_fields, offset, ctx)?; + bytes.gwrite_with(self.data_directories, offset, ctx)?; + } + _ => panic!(), + } + Ok(*offset) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/third_party/rust/goblin/src/pe/options.rs b/third_party/rust/goblin/src/pe/options.rs index 5fea632f75..4461a4c7bd 100644 --- a/third_party/rust/goblin/src/pe/options.rs +++ b/third_party/rust/goblin/src/pe/options.rs @@ -3,11 +3,19 @@ pub struct ParseOptions { /// Wether the parser should resolve rvas or not. Default: true pub resolve_rva: bool, + /// Whether or not to parse attribute certificates. + /// Set to false for in-memory representation, as the [loader does not map this info into + /// memory](https://learn.microsoft.com/en-us/windows/win32/debug/pe-format#other-contents-of-the-file). + /// For on-disk representations, leave as true. Default: true + pub parse_attribute_certificates: bool, } impl ParseOptions { /// Returns a parse options structure with default values pub fn default() -> Self { - ParseOptions { resolve_rva: true } + ParseOptions { + resolve_rva: true, + parse_attribute_certificates: true, + } } } diff --git a/third_party/rust/goblin/src/pe/section_table.rs b/third_party/rust/goblin/src/pe/section_table.rs index 4491827f16..f49c2b689c 100644 --- a/third_party/rust/goblin/src/pe/section_table.rs +++ b/third_party/rust/goblin/src/pe/section_table.rs @@ -1,6 +1,8 @@ use crate::error::{self, Error}; use crate::pe::relocation; +use alloc::borrow::Cow; use alloc::string::{String, ToString}; +use alloc::vec::Vec; use scroll::{ctx, Pread, Pwrite}; #[repr(C)] @@ -79,6 +81,32 @@ impl SectionTable { Ok(table) } + pub fn data<'a, 'b: 'a>(&'a self, pe_bytes: &'b [u8]) -> error::Result>> { + let section_start: usize = self.pointer_to_raw_data.try_into().map_err(|_| { + Error::Malformed(format!("Virtual address cannot fit in platform `usize`")) + })?; + + // assert!(self.virtual_size <= self.size_of_raw_data); + // if vsize > size_of_raw_data, the section is zero padded. + let section_end: usize = section_start + + usize::try_from(self.size_of_raw_data).map_err(|_| { + Error::Malformed(format!("Virtual size cannot fit in platform `usize`")) + })?; + + let original_bytes = pe_bytes.get(section_start..section_end).map(Cow::Borrowed); + + if original_bytes.is_some() && self.virtual_size > self.size_of_raw_data { + let mut bytes: Vec = Vec::new(); + bytes.resize(self.size_of_raw_data.try_into()?, 0); + bytes.copy_from_slice(&original_bytes.unwrap()); + bytes.resize(self.virtual_size.try_into()?, 0); + + Ok(Some(Cow::Owned(bytes))) + } else { + Ok(original_bytes) + } + } + pub fn name_offset(&self) -> error::Result> { // Based on https://github.com/llvm-mirror/llvm/blob/af7b1832a03ab6486c42a40d21695b2c03b2d8a3/lib/Object/COFFObjectFile.cpp#L1054 if self.name[0] == b'/' { @@ -163,6 +191,15 @@ impl SectionTable { let number = self.number_of_relocations as usize; relocation::Relocations::parse(bytes, offset, number) } + + /// Tests if `another_section` on-disk ranges will collide. + pub fn overlaps_with(&self, another_section: &SectionTable) -> bool { + let self_end = self.pointer_to_raw_data + self.size_of_raw_data; + let another_end = another_section.pointer_to_raw_data + another_section.size_of_raw_data; + + !((self_end <= another_section.pointer_to_raw_data) + || (another_end <= self.pointer_to_raw_data)) + } } impl ctx::SizeWith for SectionTable { @@ -171,7 +208,7 @@ impl ctx::SizeWith for SectionTable { } } -impl ctx::TryIntoCtx for SectionTable { +impl ctx::TryIntoCtx for &SectionTable { type Error = error::Error; fn try_into_ctx(self, bytes: &mut [u8], ctx: scroll::Endian) -> Result { let offset = &mut 0; @@ -189,7 +226,7 @@ impl ctx::TryIntoCtx for SectionTable { } } -impl ctx::IntoCtx for SectionTable { +impl ctx::IntoCtx for &SectionTable { fn into_ctx(self, bytes: &mut [u8], ctx: scroll::Endian) { bytes.pwrite_with(self, 0, ctx).unwrap(); } diff --git a/third_party/rust/goblin/src/pe/symbol.rs b/third_party/rust/goblin/src/pe/symbol.rs index b2ced808a1..e5aba25bec 100644 --- a/third_party/rust/goblin/src/pe/symbol.rs +++ b/third_party/rust/goblin/src/pe/symbol.rs @@ -412,6 +412,7 @@ pub struct AuxSectionDefinition { } /// A COFF symbol table. +// TODO: #[derive(Pwrite)] produce unparseable tokens pub struct SymbolTable<'a> { symbols: &'a [u8], } @@ -483,6 +484,14 @@ impl<'a> SymbolTable<'a> { } } +impl<'a> ctx::TryIntoCtx for SymbolTable<'a> { + type Error = error::Error; + + fn try_into_ctx(self, bytes: &mut [u8], _ctx: scroll::Endian) -> Result { + bytes.pwrite(self.symbols, 0).map_err(|err| err.into()) + } +} + impl<'a> Debug for SymbolTable<'a> { fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { fmt.debug_struct("SymbolTable") diff --git a/third_party/rust/goblin/src/pe/utils.rs b/third_party/rust/goblin/src/pe/utils.rs index 07a320d57b..289ccc529b 100644 --- a/third_party/rust/goblin/src/pe/utils.rs +++ b/third_party/rust/goblin/src/pe/utils.rs @@ -1,5 +1,6 @@ use crate::error; use alloc::string::ToString; +use alloc::vec::Vec; use scroll::Pread; use super::options; @@ -178,3 +179,18 @@ where let result: T = bytes.pread_with(offset, scroll::LE)?; Ok(result) } + +pub(crate) fn pad(length: usize, alignment: Option) -> Option> { + match alignment { + Some(alignment) => { + let overhang = length % alignment; + if overhang != 0 { + let repeat = alignment - overhang; + Some(vec![0u8; repeat]) + } else { + None + } + } + None => None, + } +} diff --git a/third_party/rust/goblin/src/strtab.rs b/third_party/rust/goblin/src/strtab.rs index dc7b8080f0..487d8d4279 100644 --- a/third_party/rust/goblin/src/strtab.rs +++ b/third_party/rust/goblin/src/strtab.rs @@ -34,6 +34,11 @@ impl<'a> Strtab<'a> { Self::from_slice_unparsed(bytes, 0, bytes.len(), delim) } + /// Returns the length of this `Strtab` in bytes + pub fn len(&self) -> usize { + self.bytes.len() + } + /// Creates a `Strtab` directly without bounds check and without parsing it. /// /// This is potentially unsafe and should only be used if `feature = "alloc"` is disabled. diff --git a/third_party/rust/naga/.cargo-checksum.json b/third_party/rust/naga/.cargo-checksum.json index 8ac93423af..8a1163971f 100644 --- a/third_party/rust/naga/.cargo-checksum.json +++ b/third_party/rust/naga/.cargo-checksum.json @@ -1 +1 @@ -{"files":{".cargo/config.toml":"d7389d2a0c08ec72b79e83a3c76980903e3f9123625c32e69c798721193e2e74","CHANGELOG.md":"6b2c4d8dfd8c537811c33744703b4c03fa8aa15f5fab8f0e2be76f597cb7e273","Cargo.toml":"eab8de21e33a65dbcddbdfd97fca5b98d5cf684f288ac32cf6177a761e44a2e0","README.md":"daa4717a9952b52604bbc3a55af902b252adeacc779991317d8f301f07faa94b","benches/criterion.rs":"f45e38b26e1323e934d32623572ff5395a53fed06f760eb1e07b22ed07858a38","src/arena.rs":"33ed2ec7b36429b133ed2a7de6fb9735827f69ea8b6c2ce97f64746a24a5bf36","src/back/dot/mod.rs":"a40050a73ac00c8fa43dd0b45a84fca6959d28c8c99ab3046b01f33c02f8c8f4","src/back/glsl/features.rs":"3d12147d201aaed746a94741356458a435a1ff7cf30b66baf44ba0b8dfe4b0ca","src/back/glsl/keywords.rs":"1546facbaddf696602102f32e47db7afc875f8ca3fbccc2122e0bcc45e022b53","src/back/glsl/mod.rs":"9e8b34a09401744a2ad4deae4d4863bd0be1d7d5da6ca72a98ca80fe0e3dfde6","src/back/hlsl/conv.rs":"2d7a8e7753b8fb21659e582612eea82e42e353abd23df719de450074a4da731e","src/back/hlsl/help.rs":"06da97ea0d58e2b94823ca1dae67a8611be6d5d047649b1d83755acb4c110808","src/back/hlsl/keywords.rs":"a7164690a4da866e6bfb18ced20e32cc8c42dd7387e0e84addf0c2674f529cf5","src/back/hlsl/mod.rs":"2f5296c45a2147093cae17250321580e7f01c57f907e529d19521eccd0cd4147","src/back/hlsl/storage.rs":"2c2a0071cafe487a398e396dddc85bdb319b1a5d74c097d529078e247a904359","src/back/hlsl/writer.rs":"36f0410edf9c0a8295e4916ca0e7a4e98cd170fcd5ecf6826df6051bef003b9c","src/back/mod.rs":"b941caed50c086f49d25e76228d247ba6c2da6dbeea18d968c02dc68bb97f409","src/back/msl/keywords.rs":"e6a4ef77363f995de1f8079c0b8591497cbf9520c5d3b2d41c7e1f483e8abd24","src/back/msl/mod.rs":"15fdb90b8cd2b98273b22b9569fc322eb473fd135865eef82cc615d27320d779","src/back/msl/sampler.rs":"9b01d68669e12ff7123243284b85e1a9d2c4d49140bd74ca32dedc007cbf15af","src/back/msl/writer.rs":"27e8604a5d11391b91b328f420e93f7cf475364a783b3dc5ba8bb720f17d9d86","src/back/spv/block.rs":"e2326e10cc8ca64398636c1b27166b406611006ffc2388c20fca4a271d609afe","src/back/spv/helpers.rs":"a4e260130f39c7345decec40dadf1e94419c8f6d236ce7a53b5300aa72952a1b","src/back/spv/image.rs":"e4b982ce430e17881d6370191d849dbe6bb8f6d86f4896815eb1736e43b4e302","src/back/spv/index.rs":"26611dd50df5cfd214900e19415f5374dd301d3b7d3bfedbc5ec0f254328287a","src/back/spv/instructions.rs":"d0ced535fdec49323105a7d6ee40a8ed6b4966ac5f0f40b062f0eb11a531b106","src/back/spv/layout.rs":"e263de53cd2f9a03ad94b82b434ce636609bc1ed435a2d1132951663bfaa8ebd","src/back/spv/mod.rs":"31b0229f59b5784b57851fcf6325095add58af6de3afa85d518a4e266c4b99a9","src/back/spv/ray.rs":"a34bf6b26d873f7270caa45841d9ef291aca8d9732ecd086b14d8856038e1e41","src/back/spv/recyclable.rs":"114db0ea12774d6514f995d07295cb9a42631ab75165fc60980c10e9b5ecb832","src/back/spv/selection.rs":"81e404abfa0a977f7c1f76ccb37a78d13ccadbda229048dad53cc67687cc39db","src/back/spv/writer.rs":"e90f76d7de82429db5d375b679de5dd73f205e245c97622924271995db373d1e","src/back/wgsl/mod.rs":"2dd12bbea9ace835850192bb68c5760953da6bac6a636073d1eca19381c0c0b6","src/back/wgsl/writer.rs":"96795df390cffade8ca27baea88ab29592d7e425c4351a9e2591d4f4ecdc73f3","src/block.rs":"c69089e5bbb6de6ba24efb15b21d5d434fcabfbc4d48feae948d2a4da135aae7","src/compact/expressions.rs":"7a4c916282a5b484519ed29ab451c7b595d8dea73c83c5c2cf7efc6fbc648fda","src/compact/functions.rs":"174bd9167ecf6353afb8c36d365ba3f9b483233eb4bacf578e50183c7433aa15","src/compact/handle_set_map.rs":"817c5193352d5fd6a61a5c970daba23224e14a65aea15f8f1c8679c99f834ca2","src/compact/mod.rs":"f1a606e8732f3c5837ab40ba5569eb1687336ef412f7f4b6cc348dd52b8076b3","src/compact/statements.rs":"4df33ee9589300e769e75c674bdc30578e93704ec3eb2aabc7132121745b55c8","src/compact/types.rs":"18343f2ca2c123eea2531cffc1d54a7798797caccecaa1f9b8c4fd5dd6ca1a05","src/front/glsl/ast.rs":"a4615f0c52b0dc9fdb07f816b4534c1ca547c2d176008ca86d66f9e6874f227d","src/front/glsl/builtins.rs":"d35501d5b42b61c261da24436b82eafdf96371b1600d148648d90d041f736ae4","src/front/glsl/context.rs":"066203c24ff5bc6154aa671f4492b5e8dfede8b57ef886f093cc95470d66411b","src/front/glsl/error.rs":"cca4a3aa9de2808952ff68c183755df5fdf6a7cb81f170ba747795176c0342fd","src/front/glsl/functions.rs":"b420be6b54195e9cdabdf76bb854e3e1f3be6542c6c129656fd0b1bd900dcebd","src/front/glsl/lex.rs":"08736ae8beb955da5b0e6e3e0f45995a824995f7096d516a2910417e9c7afa32","src/front/glsl/mod.rs":"c6e81710ae94a52583ba6f2a80a505d6bcd6ea6552008b80b27539af48838df1","src/front/glsl/offset.rs":"9358602ca4f9ef21d5066d674dae757bf88fdf5c289c4360534354d13bd41dc0","src/front/glsl/parser.rs":"fe5291512db412b33b6c09d5b3dcf7c54ff6ec55b47f0a078dcc11695e78471d","src/front/glsl/parser/declarations.rs":"d637cc52e553910a2e97b70b3366c15aefbe737f413adb11c27efd184c1fbf9d","src/front/glsl/parser/expressions.rs":"520cfc9402d5fbd48e52ef1d36562c6b74794c09ec33ec1ebb10aa48d129b66f","src/front/glsl/parser/functions.rs":"75aedcea4133bc4aba06ef49b1697eac96cc28d191e9830689fc4a6c0c4856eb","src/front/glsl/parser/types.rs":"aeb97e1a5fb03205cd5630c29da59d81a376ce9a83a603b62b037e63ad948e88","src/front/glsl/parser_tests.rs":"bfd4dff2580f4369a57edbcace47d23e2666751ffc3ab55f8d7dfe01f1a66311","src/front/glsl/token.rs":"c25c489b152ee2d445ace3c2046473abe64d558b8d27fa08709110e58718b6ac","src/front/glsl/types.rs":"58c9cf3d570dff8cb68f2931faf5b18e875e510741bf035ec10b9ff6df27c5d8","src/front/glsl/variables.rs":"fb2a09e386b6e98ca9fb8fb744afac1e8b19d1b67c6ede5c474e3ba860d3d4cb","src/front/interpolator.rs":"9b6ca498d5fbd9bc1515510a04e303a00b324121d7285da3c955cfe18eb4224c","src/front/mod.rs":"77acd7fb71d004969d1ee69fc728647f03242762988786c4e15fadf8315600af","src/front/spv/convert.rs":"dccc6671e6a4a7f1139aecdf979faa3689609081af5fa2cbbd6a2e8c4128c004","src/front/spv/error.rs":"6438aac57cfcf5d3858dd7652ccda1967a3123c6374f1cab829092b00549f70f","src/front/spv/function.rs":"1acb7bdd34ecfe08c6f4b4d06c2a0ea74aaf9975352e8804e3e4fab90745132f","src/front/spv/image.rs":"5d55cfbf6752732a594114cd09a9a207216e1ee85d8f2c9bc4310217a55ea321","src/front/spv/mod.rs":"22d0de7c43c42279e788144ff806cadfe3a3ea7923d961d11740af22492c4087","src/front/spv/null.rs":"e1446d99d04c76a9c3bbd24dd9b20c4711ce8a918a9b403be6cccbde1175b3b4","src/front/type_gen.rs":"b4f1df23380e06c9fdad4140810ce96ab041dbb1d371a07045b4e0069aa8ba55","src/front/wgsl/error.rs":"e1efd61062a5eb5f7e0413dc05d17abdbe0679c08f2fbdb7478e2b6e8dd13b25","src/front/wgsl/index.rs":"2b9a4929a46bd822d3ed6f9a150e24d437e5bdca8293eb748aebe80ce7e74153","src/front/wgsl/lower/construction.rs":"92342e27f5bdeb598e178799b74aa610788549c19a49fe0ae8914916bfa3c7be","src/front/wgsl/lower/conversion.rs":"961b19bf8ddd4667c6caf854a1889f3d6477757f4125538c3e9ca7d730975dd7","src/front/wgsl/lower/mod.rs":"08eece7a5460e414e2f8398cec96f12a8b9f6a457270426d8e4f045b62290d1f","src/front/wgsl/mod.rs":"02b194a0a29ef7281f71b424564e18ada4a8b1a0d8c26ec40b6be195bd4c4904","src/front/wgsl/parse/ast.rs":"c7eaae40179f0889f2b142d3b31968cbfab6d3cfe02e425912c6da8dadac51df","src/front/wgsl/parse/conv.rs":"9b2a06b5fd577e1881b2212e1d675d3aefe4d1fee99a17b5f7b07c36913e8186","src/front/wgsl/parse/lexer.rs":"17db87d0017f8f9a80fa151b8545f04e1b40c4e5feef6197f4a117efa03488bf","src/front/wgsl/parse/mod.rs":"3b4895a2baf91c719b95f0afb6441ffac2036c2a9ff817e633882fd257afcc38","src/front/wgsl/parse/number.rs":"dafd3d8651cfa1389cb359d76d39bd689e54f8d5025aa23e06c6edd871369efd","src/front/wgsl/tests.rs":"7a0a083a5b66af8e7d4b1a02401b27f077eb72d07181b610693f35b11f107c6c","src/front/wgsl/to_wgsl.rs":"2e2e30d86b07f209b866e530d3a882803bf28b39ce379052561a749f628e8e28","src/keywords/mod.rs":"0138f3931f8af0b0a05174549d0fd2152945b027dc3febefc1bbd676581d2e45","src/keywords/wgsl.rs":"c648ac44241ad55c8c8bad3d8f1bab973d11ddb9c380dcca369b735ed3975309","src/lib.rs":"f2072172957699d4282f247c452d8d8f0a0da08b9d78b279ee010296669d28d8","src/proc/constant_evaluator.rs":"bea5d259dbc4d9f9dacf3717dcb17a0774a22f1b3e5251b7e5b6991330ed3057","src/proc/emitter.rs":"39ac886c651e2ad33c06a676a7e4826a0e93de0af660c01e8e4b1f7406742f88","src/proc/index.rs":"f4250f6944c2b631e8140979024e8deb86fa8d5352d8641ba954a388b2c0940e","src/proc/layouter.rs":"b3d061c87424f36981c902716f37ab7b72f2bb2d0c2d7e900c51149318ea1a0a","src/proc/mod.rs":"4be5dcb137147cd8182a291f90959c46f1681c2d2c7da9e63f702a5f84c8809d","src/proc/namer.rs":"7328fac41e40890c64c7ee2fa985a4395424f18b08d30f30ca2583fdabd2fd35","src/proc/terminator.rs":"13c59bf00f5b26171d971effc421091f5e00dedddd246c2daa44fe65aeda060a","src/proc/typifier.rs":"99de19270d01c12ec49d14323aa1d9b8774f1ee715804af7235deff70739ba3d","src/span.rs":"6560599f20b8bc2de746ee9fd6b05c32bb630af914fce8845d84fdc72f9a636c","src/valid/analyzer.rs":"8472b98f16a4a4a0fa7079197db25696f77ef3e1602a7cddea1930daebd27917","src/valid/compose.rs":"83e4c09c39f853cf085b83b87e48b3db571da619132960d3ec954ebdfb0a74f2","src/valid/expression.rs":"7a8d5f74677c627dee3e15d223e83453ea7f6567dc806fcdfeebd32081012779","src/valid/function.rs":"40754e51906b053becdd8813b189fe709b7693c08babd28b5d3f5c576475b171","src/valid/handles.rs":"0878915e67b16d7c41cf8245d9ab3b3f4a604e7d4e87527ea40e03efcbf1f74a","src/valid/interface.rs":"32ef8e4665106b5c71540833e17ee9cd1dde5a900c9b81f61e0b7b8192c4aaf2","src/valid/mod.rs":"3b2772c88561aeb4dc8f5ce0d8ed5169bcdf5f7db04a62aaf22d04c171cb4f35","src/valid/type.rs":"61357577fa2dffa9c7326504f5c1a5fe7c44afd5d6f439c2354b390c6783fc86"},"package":null} \ No newline at end of file +{"files":{".cargo/config.toml":"d7389d2a0c08ec72b79e83a3c76980903e3f9123625c32e69c798721193e2e74","CHANGELOG.md":"6b2c4d8dfd8c537811c33744703b4c03fa8aa15f5fab8f0e2be76f597cb7e273","Cargo.toml":"dd06bf980497dcb0d35c2c33d845ba80cb7a2f78bd0e824e3d6be6b950b8088b","README.md":"468211bb7457683510ff862f31f27e3ba07514f851a4390570d0e9259cbeaf6f","benches/criterion.rs":"f45e38b26e1323e934d32623572ff5395a53fed06f760eb1e07b22ed07858a38","src/arena.rs":"33ed2ec7b36429b133ed2a7de6fb9735827f69ea8b6c2ce97f64746a24a5bf36","src/back/dot/mod.rs":"a40050a73ac00c8fa43dd0b45a84fca6959d28c8c99ab3046b01f33c02f8c8f4","src/back/glsl/features.rs":"3d12147d201aaed746a94741356458a435a1ff7cf30b66baf44ba0b8dfe4b0ca","src/back/glsl/keywords.rs":"1546facbaddf696602102f32e47db7afc875f8ca3fbccc2122e0bcc45e022b53","src/back/glsl/mod.rs":"9e8b34a09401744a2ad4deae4d4863bd0be1d7d5da6ca72a98ca80fe0e3dfde6","src/back/hlsl/conv.rs":"2d7a8e7753b8fb21659e582612eea82e42e353abd23df719de450074a4da731e","src/back/hlsl/help.rs":"06da97ea0d58e2b94823ca1dae67a8611be6d5d047649b1d83755acb4c110808","src/back/hlsl/keywords.rs":"a7164690a4da866e6bfb18ced20e32cc8c42dd7387e0e84addf0c2674f529cf5","src/back/hlsl/mod.rs":"2f5296c45a2147093cae17250321580e7f01c57f907e529d19521eccd0cd4147","src/back/hlsl/storage.rs":"2c2a0071cafe487a398e396dddc85bdb319b1a5d74c097d529078e247a904359","src/back/hlsl/writer.rs":"36f0410edf9c0a8295e4916ca0e7a4e98cd170fcd5ecf6826df6051bef003b9c","src/back/mod.rs":"6101d92bff1c5b7fe3e503188554dfdcce56d4eb610237e0126bf6d8c4e1a85d","src/back/msl/keywords.rs":"e6a4ef77363f995de1f8079c0b8591497cbf9520c5d3b2d41c7e1f483e8abd24","src/back/msl/mod.rs":"15fdb90b8cd2b98273b22b9569fc322eb473fd135865eef82cc615d27320d779","src/back/msl/sampler.rs":"9b01d68669e12ff7123243284b85e1a9d2c4d49140bd74ca32dedc007cbf15af","src/back/msl/writer.rs":"27e8604a5d11391b91b328f420e93f7cf475364a783b3dc5ba8bb720f17d9d86","src/back/spv/block.rs":"e2326e10cc8ca64398636c1b27166b406611006ffc2388c20fca4a271d609afe","src/back/spv/helpers.rs":"a4e260130f39c7345decec40dadf1e94419c8f6d236ce7a53b5300aa72952a1b","src/back/spv/image.rs":"e4b982ce430e17881d6370191d849dbe6bb8f6d86f4896815eb1736e43b4e302","src/back/spv/index.rs":"54bb90176be12a9a243099343e62b8dd35dd019d237d4d7dcdb9f0da7747df09","src/back/spv/instructions.rs":"d0ced535fdec49323105a7d6ee40a8ed6b4966ac5f0f40b062f0eb11a531b106","src/back/spv/layout.rs":"e263de53cd2f9a03ad94b82b434ce636609bc1ed435a2d1132951663bfaa8ebd","src/back/spv/mod.rs":"bd0589e0f6ce33b294f1f907dd5a1c03b46429b1b0d77986f8bb98ad317b11e3","src/back/spv/ray.rs":"a34bf6b26d873f7270caa45841d9ef291aca8d9732ecd086b14d8856038e1e41","src/back/spv/recyclable.rs":"114db0ea12774d6514f995d07295cb9a42631ab75165fc60980c10e9b5ecb832","src/back/spv/selection.rs":"81e404abfa0a977f7c1f76ccb37a78d13ccadbda229048dad53cc67687cc39db","src/back/spv/writer.rs":"87460df9918c1f104213394c0532e384ae282d132fdb7e09823eeb10c2392e3d","src/back/wgsl/mod.rs":"2dd12bbea9ace835850192bb68c5760953da6bac6a636073d1eca19381c0c0b6","src/back/wgsl/writer.rs":"96795df390cffade8ca27baea88ab29592d7e425c4351a9e2591d4f4ecdc73f3","src/block.rs":"c69089e5bbb6de6ba24efb15b21d5d434fcabfbc4d48feae948d2a4da135aae7","src/compact/expressions.rs":"7a4c916282a5b484519ed29ab451c7b595d8dea73c83c5c2cf7efc6fbc648fda","src/compact/functions.rs":"174bd9167ecf6353afb8c36d365ba3f9b483233eb4bacf578e50183c7433aa15","src/compact/handle_set_map.rs":"817c5193352d5fd6a61a5c970daba23224e14a65aea15f8f1c8679c99f834ca2","src/compact/mod.rs":"f1a606e8732f3c5837ab40ba5569eb1687336ef412f7f4b6cc348dd52b8076b3","src/compact/statements.rs":"4df33ee9589300e769e75c674bdc30578e93704ec3eb2aabc7132121745b55c8","src/compact/types.rs":"18343f2ca2c123eea2531cffc1d54a7798797caccecaa1f9b8c4fd5dd6ca1a05","src/front/glsl/ast.rs":"a4615f0c52b0dc9fdb07f816b4534c1ca547c2d176008ca86d66f9e6874f227d","src/front/glsl/builtins.rs":"d35501d5b42b61c261da24436b82eafdf96371b1600d148648d90d041f736ae4","src/front/glsl/context.rs":"066203c24ff5bc6154aa671f4492b5e8dfede8b57ef886f093cc95470d66411b","src/front/glsl/error.rs":"cca4a3aa9de2808952ff68c183755df5fdf6a7cb81f170ba747795176c0342fd","src/front/glsl/functions.rs":"b420be6b54195e9cdabdf76bb854e3e1f3be6542c6c129656fd0b1bd900dcebd","src/front/glsl/lex.rs":"08736ae8beb955da5b0e6e3e0f45995a824995f7096d516a2910417e9c7afa32","src/front/glsl/mod.rs":"c6e81710ae94a52583ba6f2a80a505d6bcd6ea6552008b80b27539af48838df1","src/front/glsl/offset.rs":"9358602ca4f9ef21d5066d674dae757bf88fdf5c289c4360534354d13bd41dc0","src/front/glsl/parser.rs":"fe5291512db412b33b6c09d5b3dcf7c54ff6ec55b47f0a078dcc11695e78471d","src/front/glsl/parser/declarations.rs":"d637cc52e553910a2e97b70b3366c15aefbe737f413adb11c27efd184c1fbf9d","src/front/glsl/parser/expressions.rs":"520cfc9402d5fbd48e52ef1d36562c6b74794c09ec33ec1ebb10aa48d129b66f","src/front/glsl/parser/functions.rs":"75aedcea4133bc4aba06ef49b1697eac96cc28d191e9830689fc4a6c0c4856eb","src/front/glsl/parser/types.rs":"aeb97e1a5fb03205cd5630c29da59d81a376ce9a83a603b62b037e63ad948e88","src/front/glsl/parser_tests.rs":"bfd4dff2580f4369a57edbcace47d23e2666751ffc3ab55f8d7dfe01f1a66311","src/front/glsl/token.rs":"c25c489b152ee2d445ace3c2046473abe64d558b8d27fa08709110e58718b6ac","src/front/glsl/types.rs":"58c9cf3d570dff8cb68f2931faf5b18e875e510741bf035ec10b9ff6df27c5d8","src/front/glsl/variables.rs":"bfed08368b56dc1f55fe487b240cf7b8e09443e508ea410cfac48df477591eee","src/front/interpolator.rs":"9b6ca498d5fbd9bc1515510a04e303a00b324121d7285da3c955cfe18eb4224c","src/front/mod.rs":"77acd7fb71d004969d1ee69fc728647f03242762988786c4e15fadf8315600af","src/front/spv/convert.rs":"dccc6671e6a4a7f1139aecdf979faa3689609081af5fa2cbbd6a2e8c4128c004","src/front/spv/error.rs":"6438aac57cfcf5d3858dd7652ccda1967a3123c6374f1cab829092b00549f70f","src/front/spv/function.rs":"1acb7bdd34ecfe08c6f4b4d06c2a0ea74aaf9975352e8804e3e4fab90745132f","src/front/spv/image.rs":"5d55cfbf6752732a594114cd09a9a207216e1ee85d8f2c9bc4310217a55ea321","src/front/spv/mod.rs":"22d0de7c43c42279e788144ff806cadfe3a3ea7923d961d11740af22492c4087","src/front/spv/null.rs":"e1446d99d04c76a9c3bbd24dd9b20c4711ce8a918a9b403be6cccbde1175b3b4","src/front/type_gen.rs":"b4f1df23380e06c9fdad4140810ce96ab041dbb1d371a07045b4e0069aa8ba55","src/front/wgsl/error.rs":"e1efd61062a5eb5f7e0413dc05d17abdbe0679c08f2fbdb7478e2b6e8dd13b25","src/front/wgsl/index.rs":"2b9a4929a46bd822d3ed6f9a150e24d437e5bdca8293eb748aebe80ce7e74153","src/front/wgsl/lower/construction.rs":"92342e27f5bdeb598e178799b74aa610788549c19a49fe0ae8914916bfa3c7be","src/front/wgsl/lower/conversion.rs":"961b19bf8ddd4667c6caf854a1889f3d6477757f4125538c3e9ca7d730975dd7","src/front/wgsl/lower/mod.rs":"08eece7a5460e414e2f8398cec96f12a8b9f6a457270426d8e4f045b62290d1f","src/front/wgsl/mod.rs":"02b194a0a29ef7281f71b424564e18ada4a8b1a0d8c26ec40b6be195bd4c4904","src/front/wgsl/parse/ast.rs":"c7eaae40179f0889f2b142d3b31968cbfab6d3cfe02e425912c6da8dadac51df","src/front/wgsl/parse/conv.rs":"9b2a06b5fd577e1881b2212e1d675d3aefe4d1fee99a17b5f7b07c36913e8186","src/front/wgsl/parse/lexer.rs":"17db87d0017f8f9a80fa151b8545f04e1b40c4e5feef6197f4a117efa03488bf","src/front/wgsl/parse/mod.rs":"3b4895a2baf91c719b95f0afb6441ffac2036c2a9ff817e633882fd257afcc38","src/front/wgsl/parse/number.rs":"dafd3d8651cfa1389cb359d76d39bd689e54f8d5025aa23e06c6edd871369efd","src/front/wgsl/tests.rs":"7a0a083a5b66af8e7d4b1a02401b27f077eb72d07181b610693f35b11f107c6c","src/front/wgsl/to_wgsl.rs":"2e2e30d86b07f209b866e530d3a882803bf28b39ce379052561a749f628e8e28","src/keywords/mod.rs":"0138f3931f8af0b0a05174549d0fd2152945b027dc3febefc1bbd676581d2e45","src/keywords/wgsl.rs":"c648ac44241ad55c8c8bad3d8f1bab973d11ddb9c380dcca369b735ed3975309","src/lib.rs":"f2072172957699d4282f247c452d8d8f0a0da08b9d78b279ee010296669d28d8","src/proc/constant_evaluator.rs":"bea5d259dbc4d9f9dacf3717dcb17a0774a22f1b3e5251b7e5b6991330ed3057","src/proc/emitter.rs":"39ac886c651e2ad33c06a676a7e4826a0e93de0af660c01e8e4b1f7406742f88","src/proc/index.rs":"f4250f6944c2b631e8140979024e8deb86fa8d5352d8641ba954a388b2c0940e","src/proc/layouter.rs":"b3d061c87424f36981c902716f37ab7b72f2bb2d0c2d7e900c51149318ea1a0a","src/proc/mod.rs":"4be5dcb137147cd8182a291f90959c46f1681c2d2c7da9e63f702a5f84c8809d","src/proc/namer.rs":"7328fac41e40890c64c7ee2fa985a4395424f18b08d30f30ca2583fdabd2fd35","src/proc/terminator.rs":"13c59bf00f5b26171d971effc421091f5e00dedddd246c2daa44fe65aeda060a","src/proc/typifier.rs":"99de19270d01c12ec49d14323aa1d9b8774f1ee715804af7235deff70739ba3d","src/span.rs":"47d92ea25ce7178ad903ac4f5acd0bc40ebd90de1e470e538a2aa3063d980890","src/valid/analyzer.rs":"65ed1487c5a169688714a7386046b30ec380eaf44e5500f366b9c9a308f59464","src/valid/compose.rs":"83e4c09c39f853cf085b83b87e48b3db571da619132960d3ec954ebdfb0a74f2","src/valid/expression.rs":"7a8d5f74677c627dee3e15d223e83453ea7f6567dc806fcdfeebd32081012779","src/valid/function.rs":"40754e51906b053becdd8813b189fe709b7693c08babd28b5d3f5c576475b171","src/valid/handles.rs":"0878915e67b16d7c41cf8245d9ab3b3f4a604e7d4e87527ea40e03efcbf1f74a","src/valid/interface.rs":"32ef8e4665106b5c71540833e17ee9cd1dde5a900c9b81f61e0b7b8192c4aaf2","src/valid/mod.rs":"3b2772c88561aeb4dc8f5ce0d8ed5169bcdf5f7db04a62aaf22d04c171cb4f35","src/valid/type.rs":"635a4f7b2681d2cc7645a5589640098efb3cdaf3c492d79924c9e80c806c7890"},"package":null} \ No newline at end of file diff --git a/third_party/rust/naga/Cargo.toml b/third_party/rust/naga/Cargo.toml index dc2434e4b0..04375a9960 100644 --- a/third_party/rust/naga/Cargo.toml +++ b/third_party/rust/naga/Cargo.toml @@ -11,7 +11,7 @@ [package] edition = "2021" -rust-version = "1.70" +rust-version = "1.74" name = "naga" version = "0.19.0" authors = ["gfx-rs developers"] @@ -97,7 +97,7 @@ optional = true [dev-dependencies] bincode = "1" diff = "0.1" -env_logger = "0.10" +env_logger = "0.11" ron = "0.8.0" [dev-dependencies.hlsl-snapshots] diff --git a/third_party/rust/naga/README.md b/third_party/rust/naga/README.md index b7f352fc91..0e07d40496 100644 --- a/third_party/rust/naga/README.md +++ b/third_party/rust/naga/README.md @@ -4,7 +4,7 @@ [![Crates.io](https://img.shields.io/crates/v/naga.svg?label=naga)](https://crates.io/crates/naga) [![Docs.rs](https://docs.rs/naga/badge.svg)](https://docs.rs/naga) [![Build Status](https://github.com/gfx-rs/naga/workflows/pipeline/badge.svg)](https://github.com/gfx-rs/naga/actions) -![MSRV](https://img.shields.io/badge/rustc-1.70+-blue.svg) +![MSRV](https://img.shields.io/badge/rustc-1.74+-blue.svg) [![codecov.io](https://codecov.io/gh/gfx-rs/naga/branch/master/graph/badge.svg?token=9VOKYO8BM2)](https://codecov.io/gh/gfx-rs/naga) The shader translation library for the needs of [wgpu](https://github.com/gfx-rs/wgpu). @@ -42,7 +42,7 @@ First, install `naga-cli` from crates.io or directly from GitHub. cargo install naga-cli # development version -cargo install naga-cli --git https://github.com/gfx-rs/naga.git +cargo install naga-cli --git https://github.com/gfx-rs/wgpu.git ``` Then, you can run `naga` command. diff --git a/third_party/rust/naga/src/back/mod.rs b/third_party/rust/naga/src/back/mod.rs index 8100b930e9..c8f091decb 100644 --- a/third_party/rust/naga/src/back/mod.rs +++ b/third_party/rust/naga/src/back/mod.rs @@ -16,14 +16,19 @@ pub mod spv; #[cfg(feature = "wgsl-out")] pub mod wgsl; -const COMPONENTS: &[char] = &['x', 'y', 'z', 'w']; -const INDENT: &str = " "; -const BAKE_PREFIX: &str = "_e"; +/// Names of vector components. +pub const COMPONENTS: &[char] = &['x', 'y', 'z', 'w']; +/// Indent for backends. +pub const INDENT: &str = " "; +/// Prefix used for baking. +pub const BAKE_PREFIX: &str = "_e"; -type NeedBakeExpressions = crate::FastHashSet>; +/// Expressions that need baking. +pub type NeedBakeExpressions = crate::FastHashSet>; +/// Indentation level. #[derive(Clone, Copy)] -struct Level(usize); +pub struct Level(pub usize); impl Level { const fn next(&self) -> Self { @@ -52,7 +57,7 @@ impl std::fmt::Display for Level { /// [`EntryPoint`]: crate::EntryPoint /// [`Module`]: crate::Module /// [`Module::entry_points`]: crate::Module::entry_points -enum FunctionType { +pub enum FunctionType { /// A regular function. Function(crate::Handle), /// An [`EntryPoint`], and its index in [`Module::entry_points`]. @@ -63,7 +68,8 @@ enum FunctionType { } impl FunctionType { - fn is_compute_entry_point(&self, module: &crate::Module) -> bool { + /// Returns true if the function is an entry point for a compute shader. + pub fn is_compute_entry_point(&self, module: &crate::Module) -> bool { match *self { FunctionType::EntryPoint(index) => { module.entry_points[index as usize].stage == crate::ShaderStage::Compute @@ -74,19 +80,20 @@ impl FunctionType { } /// Helper structure that stores data needed when writing the function -struct FunctionCtx<'a> { +pub struct FunctionCtx<'a> { /// The current function being written - ty: FunctionType, + pub ty: FunctionType, /// Analysis about the function - info: &'a crate::valid::FunctionInfo, + pub info: &'a crate::valid::FunctionInfo, /// The expression arena of the current function being written - expressions: &'a crate::Arena, + pub expressions: &'a crate::Arena, /// Map of expressions that have associated variable names - named_expressions: &'a crate::NamedExpressions, + pub named_expressions: &'a crate::NamedExpressions, } impl FunctionCtx<'_> { - fn resolve_type<'a>( + /// Helper method that resolves a type of a given expression. + pub fn resolve_type<'a>( &'a self, handle: crate::Handle, types: &'a crate::UniqueArena, @@ -95,7 +102,10 @@ impl FunctionCtx<'_> { } /// Helper method that generates a [`NameKey`](crate::proc::NameKey) for a local in the current function - const fn name_key(&self, local: crate::Handle) -> crate::proc::NameKey { + pub const fn name_key( + &self, + local: crate::Handle, + ) -> crate::proc::NameKey { match self.ty { FunctionType::Function(handle) => crate::proc::NameKey::FunctionLocal(handle, local), FunctionType::EntryPoint(idx) => crate::proc::NameKey::EntryPointLocal(idx, local), @@ -106,7 +116,7 @@ impl FunctionCtx<'_> { /// /// # Panics /// - If the function arguments are less or equal to `arg` - const fn argument_key(&self, arg: u32) -> crate::proc::NameKey { + pub const fn argument_key(&self, arg: u32) -> crate::proc::NameKey { match self.ty { FunctionType::Function(handle) => crate::proc::NameKey::FunctionArgument(handle, arg), FunctionType::EntryPoint(ep_index) => { @@ -115,8 +125,8 @@ impl FunctionCtx<'_> { } } - // Returns true if the given expression points to a fixed-function pipeline input. - fn is_fixed_function_input( + /// Returns true if the given expression points to a fixed-function pipeline input. + pub fn is_fixed_function_input( &self, mut expression: crate::Handle, module: &crate::Module, @@ -162,7 +172,7 @@ impl crate::Expression { /// See the [module-level documentation][emit] for details. /// /// [emit]: index.html#expression-evaluation-time - const fn bake_ref_count(&self) -> usize { + pub const fn bake_ref_count(&self) -> usize { match *self { // accesses are never cached, only loads are crate::Expression::Access { .. } | crate::Expression::AccessIndex { .. } => usize::MAX, @@ -181,9 +191,7 @@ impl crate::Expression { } /// Helper function that returns the string corresponding to the [`BinaryOperator`](crate::BinaryOperator) -/// # Notes -/// Used by `glsl-out`, `msl-out`, `wgsl-out`, `hlsl-out`. -const fn binary_operation_str(op: crate::BinaryOperator) -> &'static str { +pub const fn binary_operation_str(op: crate::BinaryOperator) -> &'static str { use crate::BinaryOperator as Bo; match op { Bo::Add => "+", @@ -208,8 +216,6 @@ const fn binary_operation_str(op: crate::BinaryOperator) -> &'static str { } /// Helper function that returns the string corresponding to the [`VectorSize`](crate::VectorSize) -/// # Notes -/// Used by `msl-out`, `wgsl-out`, `hlsl-out`. const fn vector_size_str(size: crate::VectorSize) -> &'static str { match size { crate::VectorSize::Bi => "2", @@ -219,7 +225,8 @@ const fn vector_size_str(size: crate::VectorSize) -> &'static str { } impl crate::TypeInner { - const fn is_handle(&self) -> bool { + /// Returns true if this is a handle to a type rather than the type directly. + pub const fn is_handle(&self) -> bool { match *self { crate::TypeInner::Image { .. } | crate::TypeInner::Sampler { .. } => true, _ => false, @@ -266,8 +273,9 @@ bitflags::bitflags! { } } +/// The intersection test to use for ray queries. #[repr(u32)] -enum RayIntersectionType { +pub enum RayIntersectionType { Triangle = 1, BoundingBox = 4, } diff --git a/third_party/rust/naga/src/back/spv/index.rs b/third_party/rust/naga/src/back/spv/index.rs index 92e0f88d9a..0effb568be 100644 --- a/third_party/rust/naga/src/back/spv/index.rs +++ b/third_party/rust/naga/src/back/spv/index.rs @@ -3,8 +3,9 @@ Bounds-checking for SPIR-V output. */ use super::{ - helpers::global_needs_wrapper, selection::Selection, Block, BlockContext, Error, IdGenerator, - Instruction, Word, + helpers::{global_needs_wrapper, map_storage_class}, + selection::Selection, + Block, BlockContext, Error, IdGenerator, Instruction, Word, }; use crate::{arena::Handle, proc::BoundsCheckPolicy}; @@ -42,32 +43,113 @@ impl<'w> BlockContext<'w> { array: Handle, block: &mut Block, ) -> Result { - // Naga IR permits runtime-sized arrays as global variables or as the - // final member of a struct that is a global variable. SPIR-V permits - // only the latter, so this back end wraps bare runtime-sized arrays - // in a made-up struct; see `helpers::global_needs_wrapper` and its uses. - // This code must handle both cases. - let (structure_id, last_member_index) = match self.ir_function.expressions[array] { + // Naga IR permits runtime-sized arrays as global variables, or as the + // final member of a struct that is a global variable, or one of these + // inside a buffer that is itself an element in a buffer bindings array. + // SPIR-V requires that runtime-sized arrays are wrapped in structs. + // See `helpers::global_needs_wrapper` and its uses. + let (opt_array_index_id, global_handle, opt_last_member_index) = match self + .ir_function + .expressions[array] + { crate::Expression::AccessIndex { base, index } => { match self.ir_function.expressions[base] { - crate::Expression::GlobalVariable(handle) => ( - self.writer.global_variables[handle.index()].access_id, - index, - ), - _ => return Err(Error::Validation("array length expression")), + // The global variable is an array of buffer bindings of structs, + // we are accessing one of them with a static index, + // and the last member of it. + crate::Expression::AccessIndex { + base: base_outer, + index: index_outer, + } => match self.ir_function.expressions[base_outer] { + crate::Expression::GlobalVariable(handle) => { + let index_id = self.get_index_constant(index_outer); + (Some(index_id), handle, Some(index)) + } + _ => return Err(Error::Validation("array length expression case-1a")), + }, + // The global variable is an array of buffer bindings of structs, + // we are accessing one of them with a dynamic index, + // and the last member of it. + crate::Expression::Access { + base: base_outer, + index: index_outer, + } => match self.ir_function.expressions[base_outer] { + crate::Expression::GlobalVariable(handle) => { + let index_id = self.cached[index_outer]; + (Some(index_id), handle, Some(index)) + } + _ => return Err(Error::Validation("array length expression case-1b")), + }, + // The global variable is a buffer, and we are accessing the last member. + crate::Expression::GlobalVariable(handle) => { + let global = &self.ir_module.global_variables[handle]; + match self.ir_module.types[global.ty].inner { + // The global variable is an array of buffer bindings of run-time arrays. + crate::TypeInner::BindingArray { .. } => (Some(index), handle, None), + // The global variable is a struct, and we are accessing the last member + _ => (None, handle, Some(index)), + } + } + _ => return Err(Error::Validation("array length expression case-1c")), } } + // The global variable is an array of buffer bindings of arrays. + crate::Expression::Access { base, index } => match self.ir_function.expressions[base] { + crate::Expression::GlobalVariable(handle) => { + let index_id = self.cached[index]; + let global = &self.ir_module.global_variables[handle]; + match self.ir_module.types[global.ty].inner { + crate::TypeInner::BindingArray { .. } => (Some(index_id), handle, None), + _ => return Err(Error::Validation("array length expression case-2a")), + } + } + _ => return Err(Error::Validation("array length expression case-2b")), + }, + // The global variable is a run-time array. crate::Expression::GlobalVariable(handle) => { let global = &self.ir_module.global_variables[handle]; if !global_needs_wrapper(self.ir_module, global) { - return Err(Error::Validation("array length expression")); + return Err(Error::Validation("array length expression case-3")); } - - (self.writer.global_variables[handle.index()].var_id, 0) + (None, handle, None) } - _ => return Err(Error::Validation("array length expression")), + _ => return Err(Error::Validation("array length expression case-4")), }; + let gvar = self.writer.global_variables[global_handle.index()].clone(); + let global = &self.ir_module.global_variables[global_handle]; + let (last_member_index, gvar_id) = match opt_last_member_index { + Some(index) => (index, gvar.access_id), + None => { + if !global_needs_wrapper(self.ir_module, global) { + return Err(Error::Validation( + "pointer to a global that is not a wrapped array", + )); + } + (0, gvar.var_id) + } + }; + let structure_id = match opt_array_index_id { + // We are indexing inside a binding array, generate the access op. + Some(index_id) => { + let element_type_id = match self.ir_module.types[global.ty].inner { + crate::TypeInner::BindingArray { base, size: _ } => { + let class = map_storage_class(global.space); + self.get_pointer_id(base, class)? + } + _ => return Err(Error::Validation("array length expression case-5")), + }; + let structure_id = self.gen_id(); + block.body.push(Instruction::access_chain( + element_type_id, + structure_id, + gvar_id, + &[index_id], + )); + structure_id + } + None => gvar_id, + }; let length_id = self.gen_id(); block.body.push(Instruction::array_length( self.writer.get_uint_type_id(), diff --git a/third_party/rust/naga/src/back/spv/mod.rs b/third_party/rust/naga/src/back/spv/mod.rs index b7d57be0d4..eb29e3cd8b 100644 --- a/third_party/rust/naga/src/back/spv/mod.rs +++ b/third_party/rust/naga/src/back/spv/mod.rs @@ -576,6 +576,15 @@ impl BlockContext<'_> { self.writer .get_constant_scalar(crate::Literal::I32(scope as _)) } + + fn get_pointer_id( + &mut self, + handle: Handle, + class: spirv::StorageClass, + ) -> Result { + self.writer + .get_pointer_id(&self.ir_module.types, handle, class) + } } #[derive(Clone, Copy, Default)] diff --git a/third_party/rust/naga/src/back/spv/writer.rs b/third_party/rust/naga/src/back/spv/writer.rs index de3220bbda..a5065e0623 100644 --- a/third_party/rust/naga/src/back/spv/writer.rs +++ b/third_party/rust/naga/src/back/spv/writer.rs @@ -565,36 +565,38 @@ impl Writer { // Handle globals are pre-emitted and should be loaded automatically. // // Any that are binding arrays we skip as we cannot load the array, we must load the result after indexing. - let is_binding_array = match ir_module.types[var.ty].inner { - crate::TypeInner::BindingArray { .. } => true, - _ => false, - }; - - if var.space == crate::AddressSpace::Handle && !is_binding_array { - let var_type_id = self.get_type_id(LookupType::Handle(var.ty)); - let id = self.id_gen.next(); - prelude - .body - .push(Instruction::load(var_type_id, id, gv.var_id, None)); - gv.access_id = gv.var_id; - gv.handle_id = id; - } else if global_needs_wrapper(ir_module, var) { - let class = map_storage_class(var.space); - let pointer_type_id = self.get_pointer_id(&ir_module.types, var.ty, class)?; - let index_id = self.get_index_constant(0); - - let id = self.id_gen.next(); - prelude.body.push(Instruction::access_chain( - pointer_type_id, - id, - gv.var_id, - &[index_id], - )); - gv.access_id = id; - } else { - // by default, the variable ID is accessed as is - gv.access_id = gv.var_id; - }; + match ir_module.types[var.ty].inner { + crate::TypeInner::BindingArray { .. } => { + gv.access_id = gv.var_id; + } + _ => { + if var.space == crate::AddressSpace::Handle { + let var_type_id = self.get_type_id(LookupType::Handle(var.ty)); + let id = self.id_gen.next(); + prelude + .body + .push(Instruction::load(var_type_id, id, gv.var_id, None)); + gv.access_id = gv.var_id; + gv.handle_id = id; + } else if global_needs_wrapper(ir_module, var) { + let class = map_storage_class(var.space); + let pointer_type_id = + self.get_pointer_id(&ir_module.types, var.ty, class)?; + let index_id = self.get_index_constant(0); + let id = self.id_gen.next(); + prelude.body.push(Instruction::access_chain( + pointer_type_id, + id, + gv.var_id, + &[index_id], + )); + gv.access_id = id; + } else { + // by default, the variable ID is accessed as is + gv.access_id = gv.var_id; + }; + } + } // work around borrow checking in the presence of `self.xxx()` calls self.global_variables[handle.index()] = gv; @@ -1858,9 +1860,15 @@ impl Writer { .iter() .flat_map(|entry| entry.function.arguments.iter()) .any(|arg| has_view_index_check(ir_module, arg.binding.as_ref(), arg.ty)); - let has_ray_query = ir_module.special_types.ray_desc.is_some() + let mut has_ray_query = ir_module.special_types.ray_desc.is_some() | ir_module.special_types.ray_intersection.is_some(); + for (_, &crate::Type { ref inner, .. }) in ir_module.types.iter() { + if let &crate::TypeInner::AccelerationStructure | &crate::TypeInner::RayQuery = inner { + has_ray_query = true + } + } + if self.physical_layout.version < 0x10300 && has_storage_buffers { // enable the storage buffer class on < SPV-1.3 Instruction::extension("SPV_KHR_storage_buffer_storage_class") diff --git a/third_party/rust/naga/src/front/glsl/variables.rs b/third_party/rust/naga/src/front/glsl/variables.rs index 5af2b228f0..9d2e7a0e7b 100644 --- a/third_party/rust/naga/src/front/glsl/variables.rs +++ b/third_party/rust/naga/src/front/glsl/variables.rs @@ -65,7 +65,7 @@ impl Frontend { let idx = self.entry_args.len(); self.entry_args.push(EntryArg { - name: None, + name: Some(name.into()), binding: Binding::BuiltIn(data.builtin), handle, storage: data.storage, diff --git a/third_party/rust/naga/src/span.rs b/third_party/rust/naga/src/span.rs index 53246b25d6..10744647e9 100644 --- a/third_party/rust/naga/src/span.rs +++ b/third_party/rust/naga/src/span.rs @@ -104,16 +104,17 @@ impl std::ops::Index for str { /// A human-readable representation for a span, tailored for text source. /// -/// Corresponds to the positional members of [`GPUCompilationMessage`][gcm] from -/// the WebGPU specification, except that `offset` and `length` are in bytes -/// (UTF-8 code units), instead of UTF-16 code units. +/// Roughly corresponds to the positional members of [`GPUCompilationMessage`][gcm] from +/// the WebGPU specification, except +/// - `offset` and `length` are in bytes (UTF-8 code units), instead of UTF-16 code units. +/// - `line_position` counts entire Unicode code points, instead of UTF-16 code units. /// /// [gcm]: https://www.w3.org/TR/webgpu/#gpucompilationmessage #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub struct SourceLocation { /// 1-based line number. pub line_number: u32, - /// 1-based column of the start of this span + /// 1-based column of the start of this span, counted in Unicode code points. pub line_position: u32, /// 0-based Offset in code units (in bytes) of the start of the span. pub offset: u32, diff --git a/third_party/rust/naga/src/valid/analyzer.rs b/third_party/rust/naga/src/valid/analyzer.rs index df6fc5e9b0..03fbc4089b 100644 --- a/third_party/rust/naga/src/valid/analyzer.rs +++ b/third_party/rust/naga/src/valid/analyzer.rs @@ -145,10 +145,35 @@ pub struct SamplingKey { #[derive(Clone, Debug)] #[cfg_attr(feature = "serialize", derive(serde::Serialize))] #[cfg_attr(feature = "deserialize", derive(serde::Deserialize))] +/// Information about an expression in a function body. pub struct ExpressionInfo { + /// Whether this expression is uniform, and why. + /// + /// If this expression's value is not uniform, this is the handle + /// of the expression from which this one's non-uniformity + /// originates. Otherwise, this is `None`. pub uniformity: Uniformity, + + /// The number of statements and other expressions using this + /// expression's value. pub ref_count: usize, + + /// The global variable into which this expression produces a pointer. + /// + /// This is `None` unless this expression is either a + /// [`GlobalVariable`], or an [`Access`] or [`AccessIndex`] that + /// ultimately refers to some part of a global. + /// + /// [`Load`] expressions applied to pointer-typed arguments could + /// refer to globals, but we leave this as `None` for them. + /// + /// [`GlobalVariable`]: crate::Expression::GlobalVariable + /// [`Access`]: crate::Expression::Access + /// [`AccessIndex`]: crate::Expression::AccessIndex + /// [`Load`]: crate::Expression::Load assignable_global: Option>, + + /// The type of this expression. pub ty: TypeResolution, } @@ -311,14 +336,20 @@ pub enum UniformityDisruptor { } impl FunctionInfo { - /// Adds a value-type reference to an expression. + /// Record a use of `expr` of the sort given by `global_use`. + /// + /// Bump `expr`'s reference count, and return its uniformity. + /// + /// If `expr` is a pointer to a global variable, or some part of + /// a global variable, add `global_use` to that global's set of + /// uses. #[must_use] fn add_ref_impl( &mut self, - handle: Handle, + expr: Handle, global_use: GlobalUse, ) -> NonUniformResult { - let info = &mut self.expressions[handle.index()]; + let info = &mut self.expressions[expr.index()]; info.ref_count += 1; // mark the used global as read if let Some(global) = info.assignable_global { @@ -327,22 +358,38 @@ impl FunctionInfo { info.uniformity.non_uniform_result } - /// Adds a value-type reference to an expression. + /// Record a use of `expr` for its value. + /// + /// This is used for almost all expression references. Anything + /// that writes to the value `expr` points to, or otherwise wants + /// contribute flags other than `GlobalUse::READ`, should use + /// `add_ref_impl` directly. #[must_use] - fn add_ref(&mut self, handle: Handle) -> NonUniformResult { - self.add_ref_impl(handle, GlobalUse::READ) + fn add_ref(&mut self, expr: Handle) -> NonUniformResult { + self.add_ref_impl(expr, GlobalUse::READ) } - /// Adds a potentially assignable reference to an expression. - /// These are destinations for `Store` and `ImageStore` statements, - /// which can transit through `Access` and `AccessIndex`. + /// Record a use of `expr`, and indicate which global variable it + /// refers to, if any. + /// + /// Bump `expr`'s reference count, and return its uniformity. + /// + /// If `expr` is a pointer to a global variable, or some part + /// thereof, store that global in `*assignable_global`. Leave the + /// global's uses unchanged. + /// + /// This is used to determine the [`assignable_global`] for + /// [`Access`] and [`AccessIndex`] expressions that ultimately + /// refer to a global variable. Those expressions don't contribute + /// any usage to the global themselves; that depends on how other + /// expressions use them. #[must_use] fn add_assignable_ref( &mut self, - handle: Handle, + expr: Handle, assignable_global: &mut Option>, ) -> NonUniformResult { - let info = &mut self.expressions[handle.index()]; + let info = &mut self.expressions[expr.index()]; info.ref_count += 1; // propagate the assignable global up the chain, till it either hits // a value-type expression, or the assignment statement. diff --git a/third_party/rust/naga/src/valid/type.rs b/third_party/rust/naga/src/valid/type.rs index d44a295b1a..b8eb618ed4 100644 --- a/third_party/rust/naga/src/valid/type.rs +++ b/third_party/rust/naga/src/valid/type.rs @@ -510,7 +510,6 @@ impl super::Validator { ti.uniform_layout = Ok(Alignment::MIN_UNIFORM); let mut min_offset = 0; - let mut prev_struct_data: Option<(u32, u32)> = None; for (i, member) in members.iter().enumerate() { @@ -662,6 +661,7 @@ impl super::Validator { // Currently Naga only supports binding arrays of structs for non-handle types. match gctx.types[base].inner { crate::TypeInner::Struct { .. } => {} + crate::TypeInner::Array { .. } => {} _ => return Err(TypeError::BindingArrayBaseTypeNotStruct(base)), }; } diff --git a/third_party/rust/neqo-common/.cargo-checksum.json b/third_party/rust/neqo-common/.cargo-checksum.json index e7daca1191..64d5739014 100644 --- a/third_party/rust/neqo-common/.cargo-checksum.json +++ b/third_party/rust/neqo-common/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"Cargo.toml":"b49758e5e8f0a6955d761e689be39530f193f7089de07f2295a7a3aef4df5898","build.rs":"306b2f909a25ae38daf5404a4e128d2a94e8975b70870864c2a71cafec9717c7","src/codec.rs":"fd239f75d374db6ff744211344c82bcd19ecf753e07410e1fe37732bbb81dfe9","src/datagram.rs":"f2ff56faa0e513edbf4331b6ee2c9e6d6111483bda7aff08d16b9f05bce5c320","src/event.rs":"106ca6c4afb107fa49a1bc72f5eb4ae95f4baa1ba19736aa38c8ba973774c160","src/header.rs":"467b947f78bfe354d8bb51e8df0c2be69e75a45e2be688d81f0d268aa77c89ef","src/hrtime.rs":"112dc758e65301b8a7a508b125d3d61063180d432bffaec566a050d4f907ab18","src/incrdecoder.rs":"577c32b9ace51f2daaf940be6d0c391c4f55cd42ef6848c68c1ffc970d8c57b5","src/lib.rs":"a86aae69900933bf83044fa96166ee51216277415eafcdb15c04a907bb2dd10e","src/log.rs":"7246053bffd704b264d42fc82f986b9d62079472a76a9fc3749c25cfc7698532","src/qlog.rs":"9b081f32bf158fd340300693acc97fe0554b617ae664eba86e4d3572e2b1e16e","src/timer.rs":"350a730cc5a159dfdac5d78ec8e8a34c5172a476d827a566703edec24c791842","src/tos.rs":"440616cb0aee9082abe00623b33e68dbe80eda47aec889ac5f4145b1566bf692","src/udp.rs":"2b92132e078791e35b66f68d99d79ff5df55efd03e788474f7781a00403a5533","tests/log.rs":"a11e21fb570258ca93bb40e3923817d381e1e605accbc3aed1df5a0a9918b41d"},"package":null} \ No newline at end of file +{"files":{"Cargo.toml":"28a963b1a9067fef18e945a938a6ae5ea14a3a29a937f3be2ec4c0e3ae33854f","benches/timer.rs":"52d35abe1e06b92e913f43d95295b4eded0f19809677a7a63857fe92dad2c6fa","build.rs":"306b2f909a25ae38daf5404a4e128d2a94e8975b70870864c2a71cafec9717c7","src/codec.rs":"fd239f75d374db6ff744211344c82bcd19ecf753e07410e1fe37732bbb81dfe9","src/datagram.rs":"691ad94a3618d6bf5202a7911419b5e75e318d09c8cc57a9a542a864dcc764ec","src/event.rs":"106ca6c4afb107fa49a1bc72f5eb4ae95f4baa1ba19736aa38c8ba973774c160","src/header.rs":"467b947f78bfe354d8bb51e8df0c2be69e75a45e2be688d81f0d268aa77c89ef","src/hrtime.rs":"112dc758e65301b8a7a508b125d3d61063180d432bffaec566a050d4f907ab18","src/incrdecoder.rs":"577c32b9ace51f2daaf940be6d0c391c4f55cd42ef6848c68c1ffc970d8c57b5","src/lib.rs":"c917282134f43d0ddfbd67bbceea9f615a7db8a23608f809b4746808c08a9b3f","src/log.rs":"6ed99e15707c4256ae793011ed2f4b33aa81fed70205aaf5f8d3cd11ad451cf0","src/qlog.rs":"9b081f32bf158fd340300693acc97fe0554b617ae664eba86e4d3572e2b1e16e","src/timer.rs":"f6da86baf3b5d91c1230d5296ef886fb7233cdefa8c8e2b4197fcf82425a54fa","src/tos.rs":"baec87b4f8a6253b88cd257730bd1e3147c046ef993288b08235d54a24f88fbe","tests/log.rs":"a11e21fb570258ca93bb40e3923817d381e1e605accbc3aed1df5a0a9918b41d"},"package":null} \ No newline at end of file diff --git a/third_party/rust/neqo-common/Cargo.toml b/third_party/rust/neqo-common/Cargo.toml index dc5bed385f..90b254c888 100644 --- a/third_party/rust/neqo-common/Cargo.toml +++ b/third_party/rust/neqo-common/Cargo.toml @@ -13,7 +13,7 @@ edition = "2021" rust-version = "1.74.0" name = "neqo-common" -version = "0.7.2" +version = "0.7.5" authors = ["The Neqo Authors "] build = "build.rs" homepage = "https://github.com/mozilla/neqo/" @@ -23,6 +23,10 @@ repository = "https://github.com/mozilla/neqo/" [lib] bench = false +[[bench]] +name = "timer" +harness = false + [dependencies.enum-map] version = "2.7" default-features = false @@ -39,26 +43,14 @@ default-features = false version = "0.12" default-features = false -[dependencies.quinn-udp] -git = "https://github.com/quinn-rs/quinn/" -rev = "a947962131aba8a6521253d03cc948b20098a2d6" -optional = true - [dependencies.time] version = "0.3" features = ["formatting"] default-features = false -[dependencies.tokio] -version = "1" -features = [ - "net", - "time", - "macros", - "rt", - "rt-multi-thread", -] -optional = true +[dev-dependencies.criterion] +version = "0.5" +features = ["html_reports"] default-features = false [dev-dependencies.test-fixture] @@ -66,10 +58,6 @@ path = "../test-fixture" [features] ci = [] -udp = [ - "dep:quinn-udp", - "dep:tokio", -] [target."cfg(windows)".dependencies.winapi] version = "0.3" diff --git a/third_party/rust/neqo-common/benches/timer.rs b/third_party/rust/neqo-common/benches/timer.rs new file mode 100644 index 0000000000..5ac8019db4 --- /dev/null +++ b/third_party/rust/neqo-common/benches/timer.rs @@ -0,0 +1,39 @@ +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +use std::time::{Duration, Instant}; + +use criterion::{criterion_group, criterion_main, Criterion}; +use neqo_common::timer::Timer; +use test_fixture::now; + +fn benchmark_timer(c: &mut Criterion) { + c.bench_function("drain a timer quickly", |b| { + b.iter_batched_ref( + make_timer, + |(_now, timer)| { + while let Some(t) = timer.next_time() { + assert!(timer.take_next(t).is_some()); + } + }, + criterion::BatchSize::SmallInput, + ); + }); +} + +fn make_timer() -> (Instant, Timer<()>) { + const TIMES: &[u64] = &[1, 2, 3, 5, 8, 13, 21, 34]; + + let now = now(); + let mut timer = Timer::new(now, Duration::from_millis(777), 100); + for &t in TIMES { + timer.add(now + Duration::from_secs(t), ()); + } + (now, timer) +} + +criterion_group!(benches, benchmark_timer); +criterion_main!(benches); diff --git a/third_party/rust/neqo-common/src/datagram.rs b/third_party/rust/neqo-common/src/datagram.rs index 04ba1a45a1..cc2cb7d113 100644 --- a/third_party/rust/neqo-common/src/datagram.rs +++ b/third_party/rust/neqo-common/src/datagram.rs @@ -54,10 +54,8 @@ impl Datagram { self.ttl } - #[cfg(feature = "udp")] - #[must_use] - pub(crate) fn into_data(self) -> Vec { - self.d + pub fn set_tos(&mut self, tos: IpTos) { + self.tos = tos; } } @@ -83,6 +81,12 @@ impl std::fmt::Debug for Datagram { } } +impl From for Vec { + fn from(datagram: Datagram) -> Self { + datagram.d + } +} + #[cfg(test)] use test_fixture::datagram; @@ -90,8 +94,7 @@ use test_fixture::datagram; fn fmt_datagram() { let d = datagram([0; 1].to_vec()); assert_eq!( - format!("{d:?}"), + &format!("{d:?}"), "Datagram IpTos(Cs0, NotEct) TTL Some(128) [fe80::1]:443->[fe80::1]:443: [1]: 00" - .to_string() ); } diff --git a/third_party/rust/neqo-common/src/lib.rs b/third_party/rust/neqo-common/src/lib.rs index fe88097983..e988c6071d 100644 --- a/third_party/rust/neqo-common/src/lib.rs +++ b/third_party/rust/neqo-common/src/lib.rs @@ -16,8 +16,6 @@ pub mod log; pub mod qlog; pub mod timer; pub mod tos; -#[cfg(feature = "udp")] -pub mod udp; use std::fmt::Write; diff --git a/third_party/rust/neqo-common/src/log.rs b/third_party/rust/neqo-common/src/log.rs index c5b89be8a6..04028a26bd 100644 --- a/third_party/rust/neqo-common/src/log.rs +++ b/third_party/rust/neqo-common/src/log.rs @@ -50,7 +50,7 @@ fn since_start() -> Duration { START_TIME.get_or_init(Instant::now).elapsed() } -pub fn init() { +pub fn init(level_filter: Option) { static INIT_ONCE: Once = Once::new(); if ::log::STATIC_MAX_LEVEL == ::log::LevelFilter::Off { @@ -59,6 +59,9 @@ pub fn init() { INIT_ONCE.call_once(|| { let mut builder = Builder::from_env("RUST_LOG"); + if let Some(filter) = level_filter { + builder.filter_level(filter); + } builder.format(|buf, record| { let elapsed = since_start(); writeln!( @@ -71,9 +74,9 @@ pub fn init() { ) }); if let Err(e) = builder.try_init() { - do_log!(::log::Level::Info, "Logging initialization error {:?}", e); + do_log!(::log::Level::Warn, "Logging initialization error {:?}", e); } else { - do_log!(::log::Level::Info, "Logging initialized"); + do_log!(::log::Level::Debug, "Logging initialized"); } }); } @@ -81,32 +84,32 @@ pub fn init() { #[macro_export] macro_rules! log_invoke { ($lvl:expr, $ctx:expr, $($arg:tt)*) => ( { - ::neqo_common::log::init(); + ::neqo_common::log::init(None); ::neqo_common::do_log!($lvl, "[{}] {}", $ctx, format!($($arg)*)); } ) } #[macro_export] macro_rules! qerror { ([$ctx:expr], $($arg:tt)*) => (::neqo_common::log_invoke!(::log::Level::Error, $ctx, $($arg)*);); - ($($arg:tt)*) => ( { ::neqo_common::log::init(); ::neqo_common::do_log!(::log::Level::Error, $($arg)*); } ); + ($($arg:tt)*) => ( { ::neqo_common::log::init(None); ::neqo_common::do_log!(::log::Level::Error, $($arg)*); } ); } #[macro_export] macro_rules! qwarn { ([$ctx:expr], $($arg:tt)*) => (::neqo_common::log_invoke!(::log::Level::Warn, $ctx, $($arg)*);); - ($($arg:tt)*) => ( { ::neqo_common::log::init(); ::neqo_common::do_log!(::log::Level::Warn, $($arg)*); } ); + ($($arg:tt)*) => ( { ::neqo_common::log::init(None); ::neqo_common::do_log!(::log::Level::Warn, $($arg)*); } ); } #[macro_export] macro_rules! qinfo { ([$ctx:expr], $($arg:tt)*) => (::neqo_common::log_invoke!(::log::Level::Info, $ctx, $($arg)*);); - ($($arg:tt)*) => ( { ::neqo_common::log::init(); ::neqo_common::do_log!(::log::Level::Info, $($arg)*); } ); + ($($arg:tt)*) => ( { ::neqo_common::log::init(None); ::neqo_common::do_log!(::log::Level::Info, $($arg)*); } ); } #[macro_export] macro_rules! qdebug { ([$ctx:expr], $($arg:tt)*) => (::neqo_common::log_invoke!(::log::Level::Debug, $ctx, $($arg)*);); - ($($arg:tt)*) => ( { ::neqo_common::log::init(); ::neqo_common::do_log!(::log::Level::Debug, $($arg)*); } ); + ($($arg:tt)*) => ( { ::neqo_common::log::init(None); ::neqo_common::do_log!(::log::Level::Debug, $($arg)*); } ); } #[macro_export] macro_rules! qtrace { ([$ctx:expr], $($arg:tt)*) => (::neqo_common::log_invoke!(::log::Level::Trace, $ctx, $($arg)*);); - ($($arg:tt)*) => ( { ::neqo_common::log::init(); ::neqo_common::do_log!(::log::Level::Trace, $($arg)*); } ); + ($($arg:tt)*) => ( { ::neqo_common::log::init(None); ::neqo_common::do_log!(::log::Level::Trace, $($arg)*); } ); } diff --git a/third_party/rust/neqo-common/src/timer.rs b/third_party/rust/neqo-common/src/timer.rs index a413252e08..3feddb2226 100644 --- a/third_party/rust/neqo-common/src/timer.rs +++ b/third_party/rust/neqo-common/src/timer.rs @@ -5,6 +5,7 @@ // except according to those terms. use std::{ + collections::VecDeque, mem, time::{Duration, Instant}, }; @@ -27,7 +28,7 @@ impl TimerItem { /// points). Time is relative, the wheel has an origin time and it is unable to represent times that /// are more than `granularity * capacity` past that time. pub struct Timer { - items: Vec>>, + items: Vec>>, now: Instant, granularity: Duration, cursor: usize, @@ -55,9 +56,14 @@ impl Timer { /// Return a reference to the time of the next entry. #[must_use] pub fn next_time(&self) -> Option { - for i in 0..self.items.len() { - let idx = self.bucket(i); - if let Some(t) = self.items[idx].first() { + let idx = self.bucket(0); + for i in idx..self.items.len() { + if let Some(t) = self.items[i].front() { + return Some(t.time); + } + } + for i in 0..idx { + if let Some(t) = self.items[i].front() { return Some(t.time); } } @@ -145,6 +151,9 @@ impl Timer { /// Given knowledge of the time an item was added, remove it. /// This requires use of a predicate that identifies matching items. + /// + /// # Panics + /// Impossible, I think. pub fn remove(&mut self, time: Instant, mut selector: F) -> Option where F: FnMut(&T) -> bool, @@ -167,7 +176,7 @@ impl Timer { break; } if selector(&self.items[bucket][i].item) { - return Some(self.items[bucket].remove(i).item); + return Some(self.items[bucket].remove(i).unwrap().item); } } // ... then forwards. @@ -176,7 +185,7 @@ impl Timer { break; } if selector(&self.items[bucket][i].item) { - return Some(self.items[bucket].remove(i).item); + return Some(self.items[bucket].remove(i).unwrap().item); } } None @@ -185,10 +194,25 @@ impl Timer { /// Take the next item, unless there are no items with /// a timeout in the past relative to `until`. pub fn take_next(&mut self, until: Instant) -> Option { - for i in 0..self.items.len() { - let idx = self.bucket(i); - if !self.items[idx].is_empty() && self.items[idx][0].time <= until { - return Some(self.items[idx].remove(0).item); + fn maybe_take(v: &mut VecDeque>, until: Instant) -> Option { + if !v.is_empty() && v[0].time <= until { + Some(v.pop_front().unwrap().item) + } else { + None + } + } + + let idx = self.bucket(0); + for i in idx..self.items.len() { + let res = maybe_take(&mut self.items[i], until); + if res.is_some() { + return res; + } + } + for i in 0..idx { + let res = maybe_take(&mut self.items[i], until); + if res.is_some() { + return res; } } None @@ -201,7 +225,7 @@ impl Timer { if until >= self.now + self.span() { // Drain everything, so a clean sweep. let mut empty_items = Vec::with_capacity(self.items.len()); - empty_items.resize_with(self.items.len(), Vec::default); + empty_items.resize_with(self.items.len(), VecDeque::default); let mut items = mem::replace(&mut self.items, empty_items); self.now = until; self.cursor = 0; diff --git a/third_party/rust/neqo-common/src/tos.rs b/third_party/rust/neqo-common/src/tos.rs index 3610f72750..533c5447e2 100644 --- a/third_party/rust/neqo-common/src/tos.rs +++ b/third_party/rust/neqo-common/src/tos.rs @@ -36,7 +36,7 @@ impl From for u8 { impl From for IpTosEcn { fn from(v: u8) -> Self { - match v & 0b11 { + match v & 0b0000_0011 { 0b00 => IpTosEcn::NotEct, 0b01 => IpTosEcn::Ect1, 0b10 => IpTosEcn::Ect0, @@ -47,8 +47,8 @@ impl From for IpTosEcn { } impl From for IpTosEcn { - fn from(value: IpTos) -> Self { - IpTosEcn::from(value.0 & 0x3) + fn from(v: IpTos) -> Self { + IpTosEcn::from(u8::from(v)) } } @@ -166,14 +166,13 @@ impl From for IpTosDscp { } impl From for IpTosDscp { - fn from(value: IpTos) -> Self { - IpTosDscp::from(value.0 & 0xfc) + fn from(v: IpTos) -> Self { + IpTosDscp::from(u8::from(v)) } } /// The type-of-service field in an IP packet. -#[allow(clippy::module_name_repetitions)] -#[derive(Copy, Clone, PartialEq, Eq)] +#[derive(Copy, Clone, PartialEq, Eq, Default)] pub struct IpTos(u8); impl From for IpTos { @@ -215,15 +214,19 @@ impl From for IpTos { impl Debug for IpTos { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_tuple("IpTos") - .field(&IpTosDscp::from(self.0 & 0xfc)) - .field(&IpTosEcn::from(self.0 & 0x3)) + .field(&IpTosDscp::from(*self)) + .field(&IpTosEcn::from(*self)) .finish() } } -impl Default for IpTos { - fn default() -> Self { - (IpTosDscp::default(), IpTosEcn::default()).into() +impl IpTos { + pub fn set_ecn(&mut self, ecn: IpTosEcn) { + self.0 = u8::from(IpTosDscp::from(*self)) | u8::from(ecn); + } + + pub fn set_dscp(&mut self, dscp: IpTosDscp) { + self.0 = u8::from(IpTosEcn::from(*self)) | u8::from(dscp); } } @@ -322,4 +325,25 @@ mod tests { assert_eq!(tos, u8::from(iptos)); assert_eq!(IpTos::from(tos), iptos); } + + #[test] + fn iptos_to_iptosdscp() { + let tos = IpTos::from((IpTosDscp::Af41, IpTosEcn::NotEct)); + let dscp = IpTosDscp::from(tos); + assert_eq!(dscp, IpTosDscp::Af41); + } + + #[test] + fn tos_modify_ecn() { + let mut iptos: IpTos = (IpTosDscp::Af41, IpTosEcn::NotEct).into(); + iptos.set_ecn(IpTosEcn::Ce); + assert_eq!(u8::from(iptos), 0b1000_1011); + } + + #[test] + fn tos_modify_dscp() { + let mut iptos: IpTos = (IpTosDscp::Af41, IpTosEcn::Ect1).into(); + iptos.set_dscp(IpTosDscp::Le); + assert_eq!(u8::from(iptos), 0b0000_0101); + } } diff --git a/third_party/rust/neqo-common/src/udp.rs b/third_party/rust/neqo-common/src/udp.rs deleted file mode 100644 index c27b0632ff..0000000000 --- a/third_party/rust/neqo-common/src/udp.rs +++ /dev/null @@ -1,222 +0,0 @@ -// Licensed under the Apache License, Version 2.0 or the MIT license -// , at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - -#![allow(clippy::missing_errors_doc)] // Functions simply delegate to tokio and quinn-udp. -#![allow(clippy::missing_panics_doc)] // Functions simply delegate to tokio and quinn-udp. - -use std::{ - io::{self, IoSliceMut}, - net::{SocketAddr, ToSocketAddrs}, - slice, -}; - -use quinn_udp::{EcnCodepoint, RecvMeta, Transmit, UdpSocketState}; -use tokio::io::Interest; - -use crate::{Datagram, IpTos}; - -/// Socket receive buffer size. -/// -/// Allows reading multiple datagrams in a single [`Socket::recv`] call. -const RECV_BUF_SIZE: usize = u16::MAX as usize; - -pub struct Socket { - socket: tokio::net::UdpSocket, - state: UdpSocketState, - recv_buf: Vec, -} - -impl Socket { - /// Calls [`std::net::UdpSocket::bind`] and instantiates [`quinn_udp::UdpSocketState`]. - pub fn bind(addr: A) -> Result { - let socket = std::net::UdpSocket::bind(addr)?; - - Ok(Self { - state: quinn_udp::UdpSocketState::new((&socket).into())?, - socket: tokio::net::UdpSocket::from_std(socket)?, - recv_buf: vec![0; RECV_BUF_SIZE], - }) - } - - /// See [`tokio::net::UdpSocket::local_addr`]. - pub fn local_addr(&self) -> io::Result { - self.socket.local_addr() - } - - /// See [`tokio::net::UdpSocket::writable`]. - pub async fn writable(&self) -> Result<(), io::Error> { - self.socket.writable().await - } - - /// See [`tokio::net::UdpSocket::readable`]. - pub async fn readable(&self) -> Result<(), io::Error> { - self.socket.readable().await - } - - /// Send the UDP datagram on the specified socket. - pub fn send(&self, d: Datagram) -> io::Result { - let transmit = Transmit { - destination: d.destination(), - ecn: EcnCodepoint::from_bits(Into::::into(d.tos())), - contents: d.into_data().into(), - segment_size: None, - src_ip: None, - }; - - let n = self.socket.try_io(Interest::WRITABLE, || { - self.state - .send((&self.socket).into(), slice::from_ref(&transmit)) - })?; - - assert_eq!(n, 1, "only passed one slice"); - - Ok(n) - } - - /// Receive a UDP datagram on the specified socket. - pub fn recv(&mut self, local_address: &SocketAddr) -> Result, io::Error> { - let mut meta = RecvMeta::default(); - - match self.socket.try_io(Interest::READABLE, || { - self.state.recv( - (&self.socket).into(), - &mut [IoSliceMut::new(&mut self.recv_buf)], - slice::from_mut(&mut meta), - ) - }) { - Ok(n) => { - assert_eq!(n, 1, "only passed one slice"); - } - Err(ref err) - if err.kind() == io::ErrorKind::WouldBlock - || err.kind() == io::ErrorKind::Interrupted => - { - return Ok(vec![]) - } - Err(err) => { - return Err(err); - } - }; - - if meta.len == 0 { - eprintln!("zero length datagram received?"); - return Ok(vec![]); - } - if meta.len == self.recv_buf.len() { - eprintln!( - "Might have received more than {} bytes", - self.recv_buf.len() - ); - } - - Ok(self.recv_buf[0..meta.len] - .chunks(meta.stride.min(self.recv_buf.len())) - .map(|d| { - Datagram::new( - meta.addr, - *local_address, - meta.ecn.map(|n| IpTos::from(n as u8)).unwrap_or_default(), - None, // TODO: get the real TTL https://github.com/quinn-rs/quinn/issues/1749 - d, - ) - }) - .collect()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::{IpTosDscp, IpTosEcn}; - - #[tokio::test] - async fn datagram_tos() -> Result<(), io::Error> { - let sender = Socket::bind("127.0.0.1:0")?; - let receiver_addr: SocketAddr = "127.0.0.1:0".parse().unwrap(); - let mut receiver = Socket::bind(receiver_addr)?; - - let datagram = Datagram::new( - sender.local_addr()?, - receiver.local_addr()?, - IpTos::from((IpTosDscp::Le, IpTosEcn::Ect1)), - None, - "Hello, world!".as_bytes().to_vec(), - ); - - sender.writable().await?; - sender.send(datagram.clone())?; - - receiver.readable().await?; - let received_datagram = receiver - .recv(&receiver_addr) - .expect("receive to succeed") - .into_iter() - .next() - .expect("receive to yield datagram"); - - // Assert that the ECN is correct. - assert_eq!( - IpTosEcn::from(datagram.tos()), - IpTosEcn::from(received_datagram.tos()) - ); - - Ok(()) - } - - /// Expect [`Socket::recv`] to handle multiple [`Datagram`]s on GRO read. - #[tokio::test] - #[cfg_attr(not(any(target_os = "linux", target_os = "windows")), ignore)] - async fn many_datagrams_through_gro() -> Result<(), io::Error> { - const SEGMENT_SIZE: usize = 128; - - let sender = Socket::bind("127.0.0.1:0")?; - let receiver_addr: SocketAddr = "127.0.0.1:0".parse().unwrap(); - let mut receiver = Socket::bind(receiver_addr)?; - - // `neqo_common::udp::Socket::send` does not yet - // (https://github.com/mozilla/neqo/issues/1693) support GSO. Use - // `quinn_udp` directly. - let max_gso_segments = sender.state.max_gso_segments(); - let msg = vec![0xAB; SEGMENT_SIZE * max_gso_segments]; - let transmit = Transmit { - destination: receiver.local_addr()?, - ecn: EcnCodepoint::from_bits(Into::::into(IpTos::from(( - IpTosDscp::Le, - IpTosEcn::Ect1, - )))), - contents: msg.clone().into(), - segment_size: Some(SEGMENT_SIZE), - src_ip: None, - }; - sender.writable().await?; - let n = sender.socket.try_io(Interest::WRITABLE, || { - sender - .state - .send((&sender.socket).into(), slice::from_ref(&transmit)) - })?; - assert_eq!(n, 1, "only passed one slice"); - - // Allow for one GSO sendmmsg to result in multiple GRO recvmmsg. - let mut num_received = 0; - while num_received < max_gso_segments { - receiver.readable().await?; - receiver - .recv(&receiver_addr) - .expect("receive to succeed") - .into_iter() - .for_each(|d| { - assert_eq!( - SEGMENT_SIZE, - d.len(), - "Expect received datagrams to have same length as sent datagrams." - ); - num_received += 1; - }); - } - - Ok(()) - } -} diff --git a/third_party/rust/neqo-crypto/.cargo-checksum.json b/third_party/rust/neqo-crypto/.cargo-checksum.json index 5622e7f4ad..c3e603bd5a 100644 --- a/third_party/rust/neqo-crypto/.cargo-checksum.json +++ b/third_party/rust/neqo-crypto/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"Cargo.toml":"6f1917fbd4cbf53cb4883c30e8fcb9c20f8ebe15e19576c7d37cb6ba0ab9e42b","bindings/bindings.toml":"0660c1661318b8a5094834c2f1bb12266287ef467307f66947eff7762528f70a","bindings/mozpkix.hpp":"77072c8bb0f6eb6bfe8cbadc111dcd92e0c79936d13f2e501aae1e5d289a6675","bindings/nspr_err.h":"2d5205d017b536c2d838bcf9bc4ec79f96dd50e7bb9b73892328781f1ee6629d","bindings/nspr_error.h":"e41c03c77b8c22046f8618832c9569fbcc7b26d8b9bbc35eea7168f35e346889","bindings/nspr_io.h":"085b289849ef0e77f88512a27b4d9bdc28252bd4d39c6a17303204e46ef45f72","bindings/nspr_time.h":"2e637fd338a5cf0fd3fb0070a47f474a34c2a7f4447f31b6875f5a9928d0a261","bindings/nss_ciphers.h":"95ec6344a607558b3c5ba8510f463b6295f3a2fb3f538a01410531045a5f62d1","bindings/nss_init.h":"ef49045063782fb612aff459172cc6a89340f15005808608ade5320ca9974310","bindings/nss_p11.h":"0b81e64fe6db49b2ecff94edd850be111ef99ec11220e88ceb1c67be90143a78","bindings/nss_secerr.h":"713e8368bdae5159af7893cfa517dabfe5103cede051dee9c9557c850a2defc6","bindings/nss_ssl.h":"af222fb957b989e392e762fa2125c82608a0053aff4fb97e556691646c88c335","bindings/nss_sslerr.h":"24b97f092183d8486f774cdaef5030d0249221c78343570d83a4ee5b594210ae","bindings/nss_sslopt.h":"b7807eb7abdad14db6ad7bc51048a46b065a0ea65a4508c95a12ce90e59d1eea","build.rs":"21d9a0140b2afd708583f58f2af0a4ba93ab07ec088680b4cbf0e184aeb8785b","src/aead.rs":"8f50e4557b7829edb67f57c80c777c6ae23c868e2b2eeaaae0736af04dc0d298","src/aead_fuzzing.rs":"c3e590572314e0bb3fafa13dac3c831358b8a7b5570fe9cfe592752fce8cbdee","src/agent.rs":"e995e9cc5108470594bae1b0d4e4bc6b7a8ac2b66488f71ea99e2836c0edbd7e","src/agentio.rs":"c4cb1b3cd92ef53eb0b4fb0b34a597068d82d78ba470dae5821670a0f06c9cda","src/auth.rs":"ced1a18f691894984244088020ea25dc1ee678603317f0c7dfc8b8842fa750b4","src/cert.rs":"8942cb3ce25a61f92b6ffc30fb286052ed6f56eeda3be12fd46ea76ceba6c1cf","src/constants.rs":"f22bf16bd8cb539862cb1e47138dbba79e93fe738f4b907e465891326f98883c","src/ech.rs":"9d322fcc01c0886f1dfe9bb6273cb9f88a746452ac9a802761b1816a05930c1f","src/err.rs":"fca0222167883231a5e0a569a593f44214501819adf5aadf814be27891c87c24","src/exp.rs":"cec59d61fc95914f9703d2fb6490a8507af993c9db710dde894f2f8fd38123c7","src/ext.rs":"cbf7d9f5ecabf4b8c9efd6c334637ab1596ec5266d38ab8d2d6ceae305283deb","src/hkdf.rs":"ef32f20e30a9bd7f094199536d19c87c4231b7fbbe4a9c54c70e84ca9c6575be","src/hp.rs":"644f1bed67f1c6189a67c8d02ab3358aaa7f63af4b913dd7395becbc01a84291","src/lib.rs":"23732c7799be038c0e0835b54e7c40cf6c6536113e0adb6ae3b41b216a6e5220","src/p11.rs":"e8c366def0df470101f3d120dcc4391f74f921fe59e2f3db2a56832e2852b855","src/prio.rs":"e5e169296c0ac69919c59fb6c1f8bd6bf079452eaa13d75da0edd41d435d3f6f","src/replay.rs":"96b7af8eff9e14313e79303092018b12e8834f780c96b8e247c497fdc680c696","src/result.rs":"0587cbb6aace71a7f9765ef7c01dcd9f73a49dcc6331e1d8fe4de2aef6ca65b6","src/secrets.rs":"4ffaa66f25df47dadf042063bff5953effa7bf2f4920cafe827757d6a659cb58","src/selfencrypt.rs":"ac65b13f5bade9d03ab4709364f9ec937fa4ca009965c77ca73b481534a0a470","src/ssl.rs":"c83baa5518b81dd06f2e4072ea3c2d666ccdeb8b1ff6e3746eea9f1af47023a6","src/time.rs":"3b2829a98a1648eb052db19bb470808b6b015a1eca27ab7be64b5d196c0271c0","tests/aead.rs":"3ac4fe4ab79922b5d0191a9717058fc8d0710380ce9b25448095f870f511844f","tests/agent.rs":"824735f88e487a3748200844e9481e81a72163ad74d82faa9aa16594d9b9bb25","tests/ext.rs":"1b047d23d9b224ad06eb65d8f3a7b351e263774e404c79bbcbe8f43790e29c18","tests/handshake.rs":"e892a2839b31414be16e96cdf3b1a65978716094700c1a4989229f7edbf578a0","tests/hkdf.rs":"1d2098dc8398395864baf13e4886cfd1da6d36118727c3b264f457ee3da6b048","tests/hp.rs":"b24fec53771c169be788772532d2617a5349196cf87d6444dc74214f7c73e92c","tests/init.rs":"44fe7626b75ab8c57adfee361bb70a83d5958797e1eb6c4531bb74988ba3a990","tests/selfencrypt.rs":"25813b0c6f32fc8383bb7685745feb750eb3fdc0a6a172a50d961c68d39f2a46"},"package":null} \ No newline at end of file +{"files":{"Cargo.toml":"e0ddb5aa433c742c87b94760fc995afb8091b8fa1360bf1ce66ed59d4e34a44d","bindings/bindings.toml":"29ec7a8ef3d5f1e4a632003e2d36c270e1caf12fd3fcf108a22d1893b90a41a6","bindings/nspr_err.h":"2d5205d017b536c2d838bcf9bc4ec79f96dd50e7bb9b73892328781f1ee6629d","bindings/nspr_error.h":"e41c03c77b8c22046f8618832c9569fbcc7b26d8b9bbc35eea7168f35e346889","bindings/nspr_io.h":"085b289849ef0e77f88512a27b4d9bdc28252bd4d39c6a17303204e46ef45f72","bindings/nspr_time.h":"2e637fd338a5cf0fd3fb0070a47f474a34c2a7f4447f31b6875f5a9928d0a261","bindings/nss_ciphers.h":"95ec6344a607558b3c5ba8510f463b6295f3a2fb3f538a01410531045a5f62d1","bindings/nss_init.h":"ef49045063782fb612aff459172cc6a89340f15005808608ade5320ca9974310","bindings/nss_p11.h":"0b81e64fe6db49b2ecff94edd850be111ef99ec11220e88ceb1c67be90143a78","bindings/nss_secerr.h":"713e8368bdae5159af7893cfa517dabfe5103cede051dee9c9557c850a2defc6","bindings/nss_ssl.h":"af222fb957b989e392e762fa2125c82608a0053aff4fb97e556691646c88c335","bindings/nss_sslerr.h":"24b97f092183d8486f774cdaef5030d0249221c78343570d83a4ee5b594210ae","bindings/nss_sslopt.h":"b7807eb7abdad14db6ad7bc51048a46b065a0ea65a4508c95a12ce90e59d1eea","build.rs":"cbf6a7d912314784c8c124cf7319c910a786d0e263f466843edd3f43826f036c","min_version.txt":"7e98f86c69cddb4f65cf96a6de1f4297e3ce224a4c4628609e29042b6c4dcfb9","src/aead.rs":"fc42bc20b84d2e5ccfd56271ae2d2db082e55586ea2926470c102da177f22296","src/aead_null.rs":"664f80bbb56d0abd3794b99cc927fd5f678ddb4ce95456001413ec18a6c6a6a9","src/agent.rs":"b12004faee4a136c10e8168848d397443b5927e9497edb62c72e6db3eb1c10a0","src/agentio.rs":"c4cb1b3cd92ef53eb0b4fb0b34a597068d82d78ba470dae5821670a0f06c9cda","src/auth.rs":"ced1a18f691894984244088020ea25dc1ee678603317f0c7dfc8b8842fa750b4","src/cert.rs":"8942cb3ce25a61f92b6ffc30fb286052ed6f56eeda3be12fd46ea76ceba6c1cf","src/constants.rs":"f22bf16bd8cb539862cb1e47138dbba79e93fe738f4b907e465891326f98883c","src/ech.rs":"9d322fcc01c0886f1dfe9bb6273cb9f88a746452ac9a802761b1816a05930c1f","src/err.rs":"ae979f334604aba89640c4491262641910033f0bd790d58671f649f5039b291c","src/exp.rs":"cec59d61fc95914f9703d2fb6490a8507af993c9db710dde894f2f8fd38123c7","src/ext.rs":"cbf7d9f5ecabf4b8c9efd6c334637ab1596ec5266d38ab8d2d6ceae305283deb","src/hkdf.rs":"ef32f20e30a9bd7f094199536d19c87c4231b7fbbe4a9c54c70e84ca9c6575be","src/hp.rs":"644f1bed67f1c6189a67c8d02ab3358aaa7f63af4b913dd7395becbc01a84291","src/lib.rs":"6b2d0eb2c55f6351d673d3a3e5fc5adac8d1030c67dae9af4c79552de0f57455","src/min_version.rs":"89b7ef6f9d2301db4f689f4d963b58375d577f705b92003a804048441e00cfd1","src/p11.rs":"e8c366def0df470101f3d120dcc4391f74f921fe59e2f3db2a56832e2852b855","src/prio.rs":"e5e169296c0ac69919c59fb6c1f8bd6bf079452eaa13d75da0edd41d435d3f6f","src/replay.rs":"96b7af8eff9e14313e79303092018b12e8834f780c96b8e247c497fdc680c696","src/result.rs":"0587cbb6aace71a7f9765ef7c01dcd9f73a49dcc6331e1d8fe4de2aef6ca65b6","src/secrets.rs":"4ffaa66f25df47dadf042063bff5953effa7bf2f4920cafe827757d6a659cb58","src/selfencrypt.rs":"b7cc1c896c7661c37461fc3a8bcbfdf2589433b907fa5f968ae4f6907704b441","src/ssl.rs":"c83baa5518b81dd06f2e4072ea3c2d666ccdeb8b1ff6e3746eea9f1af47023a6","src/time.rs":"c71a01ff8aa2c0e97fb16ad620df4ed6b7cc1819ff93f46634e2f1c9551627ec","tests/aead.rs":"e36ae77802df1ea6d17cfd1bd2178a3706089577d6fd1554ca86e748b8b235b9","tests/agent.rs":"824735f88e487a3748200844e9481e81a72163ad74d82faa9aa16594d9b9bb25","tests/ext.rs":"1b047d23d9b224ad06eb65d8f3a7b351e263774e404c79bbcbe8f43790e29c18","tests/handshake.rs":"e892a2839b31414be16e96cdf3b1a65978716094700c1a4989229f7edbf578a0","tests/hkdf.rs":"1d2098dc8398395864baf13e4886cfd1da6d36118727c3b264f457ee3da6b048","tests/hp.rs":"b24fec53771c169be788772532d2617a5349196cf87d6444dc74214f7c73e92c","tests/init.rs":"616313cb38eac44b8c71a1d23a52a7d7b4c7c07d4c20dc9ea6600c3317f92613","tests/selfencrypt.rs":"8d10840b41629bf449a6b3a551377315e8a05ca26c6b041548748196652c5909"},"package":null} \ No newline at end of file diff --git a/third_party/rust/neqo-crypto/Cargo.toml b/third_party/rust/neqo-crypto/Cargo.toml index 499921e531..9507a066df 100644 --- a/third_party/rust/neqo-crypto/Cargo.toml +++ b/third_party/rust/neqo-crypto/Cargo.toml @@ -13,7 +13,7 @@ edition = "2021" rust-version = "1.74.0" name = "neqo-crypto" -version = "0.7.2" +version = "0.7.5" authors = ["The Neqo Authors "] build = "build.rs" homepage = "https://github.com/mozilla/neqo/" @@ -43,6 +43,10 @@ version = "0.1" optional = true default-features = false +[build-dependencies.semver] +version = "1.0" +default-features = false + [build-dependencies.serde] version = "1.0" default-features = false @@ -56,7 +60,7 @@ version = "0.5" default-features = false [features] -fuzzing = [] +disable-encryption = [] gecko = ["mozbuild"] [lints.clippy.pedantic] diff --git a/third_party/rust/neqo-crypto/bindings/bindings.toml b/third_party/rust/neqo-crypto/bindings/bindings.toml index 3e5c1fdf7d..72c6d524d5 100644 --- a/third_party/rust/neqo-crypto/bindings/bindings.toml +++ b/third_party/rust/neqo-crypto/bindings/bindings.toml @@ -265,8 +265,3 @@ enums = [ [nspr_time] types = ["PRTime"] functions = ["PR_Now"] - -[mozpkix] -cplusplus = true -types = ["mozilla::pkix::ErrorCode"] -enums = ["mozilla::pkix::ErrorCode"] diff --git a/third_party/rust/neqo-crypto/bindings/mozpkix.hpp b/third_party/rust/neqo-crypto/bindings/mozpkix.hpp deleted file mode 100644 index d0a6cb5861..0000000000 --- a/third_party/rust/neqo-crypto/bindings/mozpkix.hpp +++ /dev/null @@ -1 +0,0 @@ -#include "mozpkix/pkixnss.h" \ No newline at end of file diff --git a/third_party/rust/neqo-crypto/build.rs b/third_party/rust/neqo-crypto/build.rs index c4c2a73e75..2dd4543797 100644 --- a/third_party/rust/neqo-crypto/build.rs +++ b/third_party/rust/neqo-crypto/build.rs @@ -12,8 +12,13 @@ use std::{ }; use bindgen::Builder; +use semver::{Version, VersionReq}; use serde_derive::Deserialize; +#[path = "src/min_version.rs"] +mod min_version; +use min_version::MINIMUM_NSS_VERSION; + const BINDINGS_DIR: &str = "bindings"; const BINDINGS_CONFIG: &str = "bindings.toml"; @@ -90,46 +95,6 @@ fn setup_clang() { } } -fn nss_dir() -> PathBuf { - let dir = if let Ok(dir) = env::var("NSS_DIR") { - let path = PathBuf::from(dir.trim()); - assert!( - !path.is_relative(), - "The NSS_DIR environment variable is expected to be an absolute path." - ); - path - } else { - let out_dir = env::var("OUT_DIR").unwrap(); - let dir = Path::new(&out_dir).join("nss"); - if !dir.exists() { - Command::new("hg") - .args([ - "clone", - "https://hg.mozilla.org/projects/nss", - dir.to_str().unwrap(), - ]) - .status() - .expect("can't clone nss"); - } - let nspr_dir = Path::new(&out_dir).join("nspr"); - if !nspr_dir.exists() { - Command::new("hg") - .args([ - "clone", - "https://hg.mozilla.org/projects/nspr", - nspr_dir.to_str().unwrap(), - ]) - .status() - .expect("can't clone nspr"); - } - dir - }; - assert!(dir.is_dir(), "NSS_DIR {dir:?} doesn't exist"); - // Note that this returns a relative path because UNC - // paths on windows cause certain tools to explode. - dir -} - fn get_bash() -> PathBuf { // If BASH is set, use that. if let Ok(bash) = env::var("BASH") { @@ -295,11 +260,63 @@ fn build_bindings(base: &str, bindings: &Bindings, flags: &[String], gecko: bool .expect("couldn't write bindings"); } -fn setup_standalone() -> Vec { +fn pkg_config() -> Vec { + let modversion = Command::new("pkg-config") + .args(["--modversion", "nss"]) + .output() + .expect("pkg-config reports NSS as absent") + .stdout; + let modversion = String::from_utf8(modversion).expect("non-UTF8 from pkg-config"); + let modversion = modversion.trim(); + // The NSS version number does not follow semver numbering, because it omits the patch version + // when that's 0. Deal with that. + let modversion_for_cmp = if modversion.chars().filter(|c| *c == '.').count() == 1 { + modversion.to_owned() + ".0" + } else { + modversion.to_owned() + }; + let modversion_for_cmp = + Version::parse(&modversion_for_cmp).expect("NSS version not in semver format"); + let version_req = VersionReq::parse(&format!(">={}", MINIMUM_NSS_VERSION.trim())).unwrap(); + assert!( + version_req.matches(&modversion_for_cmp), + "neqo has NSS version requirement {version_req}, found {modversion}" + ); + + let cfg = Command::new("pkg-config") + .args(["--cflags", "--libs", "nss"]) + .output() + .expect("NSS flags not returned by pkg-config") + .stdout; + let cfg_str = String::from_utf8(cfg).expect("non-UTF8 from pkg-config"); + + let mut flags: Vec = Vec::new(); + for f in cfg_str.split(' ') { + if let Some(include) = f.strip_prefix("-I") { + flags.push(String::from(f)); + println!("cargo:include={include}"); + } else if let Some(path) = f.strip_prefix("-L") { + println!("cargo:rustc-link-search=native={path}"); + } else if let Some(lib) = f.strip_prefix("-l") { + println!("cargo:rustc-link-lib=dylib={lib}"); + } else { + println!("Warning: Unknown flag from pkg-config: {f}"); + } + } + + flags +} + +fn setup_standalone(nss: &str) -> Vec { setup_clang(); println!("cargo:rerun-if-env-changed=NSS_DIR"); - let nss = nss_dir(); + let nss = PathBuf::from(nss); + assert!( + !nss.is_relative(), + "The NSS_DIR environment variable is expected to be an absolute path." + ); + build_nss(nss.clone()); // $NSS_DIR/../dist/ @@ -406,8 +423,10 @@ fn setup_for_gecko() -> Vec { fn main() { let flags = if cfg!(feature = "gecko") { setup_for_gecko() + } else if let Ok(nss_dir) = env::var("NSS_DIR") { + setup_standalone(nss_dir.trim()) } else { - setup_standalone() + pkg_config() }; let config_file = PathBuf::from(BINDINGS_DIR).join(BINDINGS_CONFIG); diff --git a/third_party/rust/neqo-crypto/min_version.txt b/third_party/rust/neqo-crypto/min_version.txt new file mode 100644 index 0000000000..422c9c7093 --- /dev/null +++ b/third_party/rust/neqo-crypto/min_version.txt @@ -0,0 +1 @@ +3.98 diff --git a/third_party/rust/neqo-crypto/src/aead.rs b/third_party/rust/neqo-crypto/src/aead.rs index bf7d7fe9d7..21027d55b2 100644 --- a/third_party/rust/neqo-crypto/src/aead.rs +++ b/third_party/rust/neqo-crypto/src/aead.rs @@ -63,13 +63,7 @@ impl RealAead { /// # Errors /// /// Returns `Error` when the supporting NSS functions fail. - pub fn new( - _fuzzing: bool, - version: Version, - cipher: Cipher, - secret: &SymKey, - prefix: &str, - ) -> Res { + pub fn new(version: Version, cipher: Cipher, secret: &SymKey, prefix: &str) -> Res { let s: *mut PK11SymKey = **secret; unsafe { Self::from_raw(version, cipher, s, prefix) } } diff --git a/third_party/rust/neqo-crypto/src/aead_fuzzing.rs b/third_party/rust/neqo-crypto/src/aead_fuzzing.rs deleted file mode 100644 index 4e5a6de07f..0000000000 --- a/third_party/rust/neqo-crypto/src/aead_fuzzing.rs +++ /dev/null @@ -1,103 +0,0 @@ -// Licensed under the Apache License, Version 2.0 or the MIT license -// , at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - -use std::fmt; - -use crate::{ - constants::{Cipher, Version}, - err::{sec::SEC_ERROR_BAD_DATA, Error, Res}, - p11::SymKey, - RealAead, -}; - -pub const FIXED_TAG_FUZZING: &[u8] = &[0x0a; 16]; - -pub struct FuzzingAead { - real: Option, -} - -impl FuzzingAead { - pub fn new( - fuzzing: bool, - version: Version, - cipher: Cipher, - secret: &SymKey, - prefix: &str, - ) -> Res { - let real = if fuzzing { - None - } else { - Some(RealAead::new(false, version, cipher, secret, prefix)?) - }; - Ok(Self { real }) - } - - #[must_use] - pub fn expansion(&self) -> usize { - if let Some(aead) = &self.real { - aead.expansion() - } else { - FIXED_TAG_FUZZING.len() - } - } - - pub fn encrypt<'a>( - &self, - count: u64, - aad: &[u8], - input: &[u8], - output: &'a mut [u8], - ) -> Res<&'a [u8]> { - if let Some(aead) = &self.real { - return aead.encrypt(count, aad, input, output); - } - - let l = input.len(); - output[..l].copy_from_slice(input); - output[l..l + 16].copy_from_slice(FIXED_TAG_FUZZING); - Ok(&output[..l + 16]) - } - - pub fn decrypt<'a>( - &self, - count: u64, - aad: &[u8], - input: &[u8], - output: &'a mut [u8], - ) -> Res<&'a [u8]> { - if let Some(aead) = &self.real { - return aead.decrypt(count, aad, input, output); - } - - if input.len() < FIXED_TAG_FUZZING.len() { - return Err(Error::from(SEC_ERROR_BAD_DATA)); - } - - let len_encrypted = input.len() - FIXED_TAG_FUZZING.len(); - // Check that: - // 1) expansion is all zeros and - // 2) if the encrypted data is also supplied that at least some values are no zero - // (otherwise padding will be interpreted as a valid packet) - if &input[len_encrypted..] == FIXED_TAG_FUZZING - && (len_encrypted == 0 || input[..len_encrypted].iter().any(|x| *x != 0x0)) - { - output[..len_encrypted].copy_from_slice(&input[..len_encrypted]); - Ok(&output[..len_encrypted]) - } else { - Err(Error::from(SEC_ERROR_BAD_DATA)) - } - } -} - -impl fmt::Debug for FuzzingAead { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - if let Some(a) = &self.real { - a.fmt(f) - } else { - write!(f, "[FUZZING AEAD]") - } - } -} diff --git a/third_party/rust/neqo-crypto/src/aead_null.rs b/third_party/rust/neqo-crypto/src/aead_null.rs new file mode 100644 index 0000000000..2d5656de73 --- /dev/null +++ b/third_party/rust/neqo-crypto/src/aead_null.rs @@ -0,0 +1,78 @@ +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +#![cfg(feature = "disable-encryption")] + +use std::fmt; + +use crate::{ + constants::{Cipher, Version}, + err::{sec::SEC_ERROR_BAD_DATA, Error, Res}, + p11::SymKey, +}; + +pub const AEAD_NULL_TAG: &[u8] = &[0x0a; 16]; + +pub struct AeadNull {} + +impl AeadNull { + #[allow(clippy::missing_errors_doc)] + pub fn new(_version: Version, _cipher: Cipher, _secret: &SymKey, _prefix: &str) -> Res { + Ok(Self {}) + } + + #[must_use] + pub fn expansion(&self) -> usize { + AEAD_NULL_TAG.len() + } + + #[allow(clippy::missing_errors_doc)] + pub fn encrypt<'a>( + &self, + _count: u64, + _aad: &[u8], + input: &[u8], + output: &'a mut [u8], + ) -> Res<&'a [u8]> { + let l = input.len(); + output[..l].copy_from_slice(input); + output[l..l + 16].copy_from_slice(AEAD_NULL_TAG); + Ok(&output[..l + 16]) + } + + #[allow(clippy::missing_errors_doc)] + pub fn decrypt<'a>( + &self, + _count: u64, + _aad: &[u8], + input: &[u8], + output: &'a mut [u8], + ) -> Res<&'a [u8]> { + if input.len() < AEAD_NULL_TAG.len() { + return Err(Error::from(SEC_ERROR_BAD_DATA)); + } + + let len_encrypted = input.len() - AEAD_NULL_TAG.len(); + // Check that: + // 1) expansion is all zeros and + // 2) if the encrypted data is also supplied that at least some values are no zero + // (otherwise padding will be interpreted as a valid packet) + if &input[len_encrypted..] == AEAD_NULL_TAG + && (len_encrypted == 0 || input[..len_encrypted].iter().any(|x| *x != 0x0)) + { + output[..len_encrypted].copy_from_slice(&input[..len_encrypted]); + Ok(&output[..len_encrypted]) + } else { + Err(Error::from(SEC_ERROR_BAD_DATA)) + } + } +} + +impl fmt::Debug for AeadNull { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "[NULL AEAD]") + } +} diff --git a/third_party/rust/neqo-crypto/src/agent.rs b/third_party/rust/neqo-crypto/src/agent.rs index 82a6dacd48..3d5a8b9f35 100644 --- a/third_party/rust/neqo-crypto/src/agent.rs +++ b/third_party/rust/neqo-crypto/src/agent.rs @@ -16,7 +16,7 @@ use std::{ time::Instant, }; -use neqo_common::{hex_snip_middle, hex_with_len, qdebug, qinfo, qtrace, qwarn}; +use neqo_common::{hex_snip_middle, hex_with_len, qdebug, qtrace, qwarn}; pub use crate::{ agentio::{as_c_void, Record, RecordList}, @@ -406,10 +406,7 @@ impl SecretAgent { self.set_option(ssl::Opt::Locking, false)?; self.set_option(ssl::Opt::Tickets, false)?; self.set_option(ssl::Opt::OcspStapling, true)?; - if let Err(e) = self.set_option(ssl::Opt::Grease, grease) { - // Until NSS supports greasing, it's OK to fail here. - qinfo!([self], "Failed to enable greasing {:?}", e); - } + self.set_option(ssl::Opt::Grease, grease)?; Ok(()) } @@ -670,7 +667,7 @@ impl SecretAgent { let info = self.capture_error(SecretAgentInfo::new(self.fd))?; HandshakeState::Complete(info) }; - qinfo!([self], "state -> {:?}", self.state); + qdebug!([self], "state -> {:?}", self.state); Ok(()) } @@ -898,7 +895,7 @@ impl Client { let len = usize::try_from(len).unwrap(); let mut v = Vec::with_capacity(len); v.extend_from_slice(null_safe_slice(token, len)); - qinfo!( + qdebug!( [format!("{fd:p}")], "Got resumption token {}", hex_snip_middle(&v) diff --git a/third_party/rust/neqo-crypto/src/err.rs b/third_party/rust/neqo-crypto/src/err.rs index 187303d2a9..8d4f239a0b 100644 --- a/third_party/rust/neqo-crypto/src/err.rs +++ b/third_party/rust/neqo-crypto/src/err.rs @@ -16,13 +16,39 @@ mod codes { #![allow(non_snake_case)] include!(concat!(env!("OUT_DIR"), "/nss_secerr.rs")); include!(concat!(env!("OUT_DIR"), "/nss_sslerr.rs")); - include!(concat!(env!("OUT_DIR"), "/mozpkix.rs")); } -pub use codes::{mozilla_pkix_ErrorCode as mozpkix, SECErrorCodes as sec, SSLErrorCodes as ssl}; +pub use codes::{SECErrorCodes as sec, SSLErrorCodes as ssl}; pub mod nspr { include!(concat!(env!("OUT_DIR"), "/nspr_err.rs")); } +pub mod mozpkix { + // These are manually extracted from the many bindings generated + // by bindgen when provided with the simple header: + // #include "mozpkix/pkixnss.h" + + #[allow(non_camel_case_types)] + pub type mozilla_pkix_ErrorCode = ::std::os::raw::c_int; + pub const MOZILLA_PKIX_ERROR_KEY_PINNING_FAILURE: mozilla_pkix_ErrorCode = -16384; + pub const MOZILLA_PKIX_ERROR_CA_CERT_USED_AS_END_ENTITY: mozilla_pkix_ErrorCode = -16383; + pub const MOZILLA_PKIX_ERROR_INADEQUATE_KEY_SIZE: mozilla_pkix_ErrorCode = -16382; + pub const MOZILLA_PKIX_ERROR_V1_CERT_USED_AS_CA: mozilla_pkix_ErrorCode = -16381; + pub const MOZILLA_PKIX_ERROR_NO_RFC822NAME_MATCH: mozilla_pkix_ErrorCode = -16380; + pub const MOZILLA_PKIX_ERROR_NOT_YET_VALID_CERTIFICATE: mozilla_pkix_ErrorCode = -16379; + pub const MOZILLA_PKIX_ERROR_NOT_YET_VALID_ISSUER_CERTIFICATE: mozilla_pkix_ErrorCode = -16378; + pub const MOZILLA_PKIX_ERROR_SIGNATURE_ALGORITHM_MISMATCH: mozilla_pkix_ErrorCode = -16377; + pub const MOZILLA_PKIX_ERROR_OCSP_RESPONSE_FOR_CERT_MISSING: mozilla_pkix_ErrorCode = -16376; + pub const MOZILLA_PKIX_ERROR_VALIDITY_TOO_LONG: mozilla_pkix_ErrorCode = -16375; + pub const MOZILLA_PKIX_ERROR_REQUIRED_TLS_FEATURE_MISSING: mozilla_pkix_ErrorCode = -16374; + pub const MOZILLA_PKIX_ERROR_INVALID_INTEGER_ENCODING: mozilla_pkix_ErrorCode = -16373; + pub const MOZILLA_PKIX_ERROR_EMPTY_ISSUER_NAME: mozilla_pkix_ErrorCode = -16372; + pub const MOZILLA_PKIX_ERROR_ADDITIONAL_POLICY_CONSTRAINT_FAILED: mozilla_pkix_ErrorCode = + -16371; + pub const MOZILLA_PKIX_ERROR_SELF_SIGNED_CERT: mozilla_pkix_ErrorCode = -16370; + pub const MOZILLA_PKIX_ERROR_MITM_DETECTED: mozilla_pkix_ErrorCode = -16369; + pub const END_OF_LIST: mozilla_pkix_ErrorCode = -16368; +} + pub type Res = Result; #[derive(Clone, Debug, PartialEq, PartialOrd, Ord, Eq)] diff --git a/third_party/rust/neqo-crypto/src/lib.rs b/third_party/rust/neqo-crypto/src/lib.rs index 2ec1b4a3ea..2db985e8ee 100644 --- a/third_party/rust/neqo-crypto/src/lib.rs +++ b/third_party/rust/neqo-crypto/src/lib.rs @@ -8,8 +8,8 @@ #![allow(clippy::unseparated_literal_suffix, clippy::used_underscore_binding)] // For bindgen code. mod aead; -#[cfg(feature = "fuzzing")] -mod aead_fuzzing; +#[cfg(feature = "disable-encryption")] +pub mod aead_null; pub mod agent; mod agentio; mod auth; @@ -33,12 +33,12 @@ mod time; use std::{ffi::CString, path::PathBuf, ptr::null, sync::OnceLock}; -#[cfg(not(feature = "fuzzing"))] +#[cfg(not(feature = "disable-encryption"))] pub use self::aead::RealAead as Aead; -#[cfg(feature = "fuzzing")] +#[cfg(feature = "disable-encryption")] pub use self::aead::RealAead; -#[cfg(feature = "fuzzing")] -pub use self::aead_fuzzing::FuzzingAead as Aead; +#[cfg(feature = "disable-encryption")] +pub use self::aead_null::AeadNull as Aead; pub use self::{ agent::{ Agent, AllowZeroRtt, Client, HandshakeState, Record, RecordList, ResumptionToken, @@ -59,7 +59,8 @@ pub use self::{ ssl::Opt, }; -const MINIMUM_NSS_VERSION: &str = "3.97"; +mod min_version; +use min_version::MINIMUM_NSS_VERSION; #[allow(non_upper_case_globals, clippy::redundant_static_lifetimes)] #[allow(clippy::upper_case_acronyms)] @@ -89,7 +90,7 @@ impl Drop for NssLoaded { } } -static INITIALIZED: OnceLock = OnceLock::new(); +static INITIALIZED: OnceLock> = OnceLock::new(); fn already_initialized() -> bool { unsafe { nss::NSS_IsInitialized() != 0 } @@ -107,24 +108,24 @@ fn version_check() { /// Initialize NSS. This only executes the initialization routines once, so if there is any chance /// that /// -/// # Panics +/// # Errors /// /// When NSS initialization fails. -pub fn init() { +pub fn init() -> Res<()> { // Set time zero. time::init(); - _ = INITIALIZED.get_or_init(|| { + let res = INITIALIZED.get_or_init(|| { version_check(); if already_initialized() { - return NssLoaded::External; + return Ok(NssLoaded::External); } - secstatus_to_res(unsafe { nss::NSS_NoDB_Init(null()) }).expect("NSS_NoDB_Init failed"); - secstatus_to_res(unsafe { nss::NSS_SetDomesticPolicy() }) - .expect("NSS_SetDomesticPolicy failed"); + secstatus_to_res(unsafe { nss::NSS_NoDB_Init(null()) })?; + secstatus_to_res(unsafe { nss::NSS_SetDomesticPolicy() })?; - NssLoaded::NoDb + Ok(NssLoaded::NoDb) }); + res.as_ref().map(|_| ()).map_err(Clone::clone) } /// This enables SSLTRACE by calling a simple, harmless function to trigger its @@ -132,31 +133,32 @@ pub fn init() { /// global options are accessed. Reading an option is the least impact approach. /// This allows us to use SSLTRACE in all of our unit tests and programs. #[cfg(debug_assertions)] -fn enable_ssl_trace() { +fn enable_ssl_trace() -> Res<()> { let opt = ssl::Opt::Locking.as_int(); let mut v: ::std::os::raw::c_int = 0; secstatus_to_res(unsafe { ssl::SSL_OptionGetDefault(opt, &mut v) }) - .expect("SSL_OptionGetDefault failed"); } /// Initialize with a database. /// -/// # Panics +/// # Errors /// /// If NSS cannot be initialized. -pub fn init_db>(dir: P) { +pub fn init_db>(dir: P) -> Res<()> { time::init(); - _ = INITIALIZED.get_or_init(|| { + let res = INITIALIZED.get_or_init(|| { version_check(); if already_initialized() { - return NssLoaded::External; + return Ok(NssLoaded::External); } let path = dir.into(); - assert!(path.is_dir()); - let pathstr = path.to_str().expect("path converts to string").to_string(); - let dircstr = CString::new(pathstr).unwrap(); - let empty = CString::new("").unwrap(); + if !path.is_dir() { + return Err(Error::InternalError); + } + let pathstr = path.to_str().ok_or(Error::InternalError)?; + let dircstr = CString::new(pathstr)?; + let empty = CString::new("")?; secstatus_to_res(unsafe { nss::NSS_Initialize( dircstr.as_ptr(), @@ -165,21 +167,19 @@ pub fn init_db>(dir: P) { nss::SECMOD_DB.as_ptr().cast(), nss::NSS_INIT_READONLY, ) - }) - .expect("NSS_Initialize failed"); + })?; - secstatus_to_res(unsafe { nss::NSS_SetDomesticPolicy() }) - .expect("NSS_SetDomesticPolicy failed"); + secstatus_to_res(unsafe { nss::NSS_SetDomesticPolicy() })?; secstatus_to_res(unsafe { ssl::SSL_ConfigServerSessionIDCache(1024, 0, 0, dircstr.as_ptr()) - }) - .expect("SSL_ConfigServerSessionIDCache failed"); + })?; #[cfg(debug_assertions)] - enable_ssl_trace(); + enable_ssl_trace()?; - NssLoaded::Db + Ok(NssLoaded::Db) }); + res.as_ref().map(|_| ()).map_err(Clone::clone) } /// # Panics diff --git a/third_party/rust/neqo-crypto/src/min_version.rs b/third_party/rust/neqo-crypto/src/min_version.rs new file mode 100644 index 0000000000..4386371b1b --- /dev/null +++ b/third_party/rust/neqo-crypto/src/min_version.rs @@ -0,0 +1,9 @@ +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +/// The minimum version of NSS that is required by this version of neqo. +/// Note that the string may contain whitespace at the beginning and/or end. +pub(crate) const MINIMUM_NSS_VERSION: &str = include_str!("../min_version.txt"); diff --git a/third_party/rust/neqo-crypto/src/selfencrypt.rs b/third_party/rust/neqo-crypto/src/selfencrypt.rs index 1130c35250..d0a85830b0 100644 --- a/third_party/rust/neqo-crypto/src/selfencrypt.rs +++ b/third_party/rust/neqo-crypto/src/selfencrypt.rs @@ -47,7 +47,7 @@ impl SelfEncrypt { debug_assert_eq!(salt.len(), Self::SALT_LENGTH); let salt = hkdf::import_key(self.version, salt)?; let secret = hkdf::extract(self.version, self.cipher, Some(&salt), k)?; - Aead::new(false, self.version, self.cipher, &secret, "neqo self") + Aead::new(self.version, self.cipher, &secret, "neqo self") } /// Rotate keys. This causes any previous key that is being held to be replaced by the current diff --git a/third_party/rust/neqo-crypto/src/time.rs b/third_party/rust/neqo-crypto/src/time.rs index 0e59c4f5e2..359436a854 100644 --- a/third_party/rust/neqo-crypto/src/time.rs +++ b/third_party/rust/neqo-crypto/src/time.rs @@ -258,11 +258,11 @@ mod test { #[test] // We allow replace_consts here because - // std::u64::max_value() isn't available + // std::u64::MAX isn't available // in all of our targets fn overflow_interval() { init(); - let interval = Interval::from(Duration::from_micros(u64::max_value())); + let interval = Interval::from(Duration::from_micros(u64::MAX)); let res: Res = interval.try_into(); assert!(res.is_err()); } diff --git a/third_party/rust/neqo-crypto/tests/aead.rs b/third_party/rust/neqo-crypto/tests/aead.rs index 5cf0034aec..f8416ed9a7 100644 --- a/third_party/rust/neqo-crypto/tests/aead.rs +++ b/third_party/rust/neqo-crypto/tests/aead.rs @@ -5,7 +5,7 @@ // except according to those terms. #![warn(clippy::pedantic)] -#![cfg(not(feature = "fuzzing"))] +#![cfg(not(feature = "disable-encryption"))] use neqo_crypto::{ constants::{Cipher, TLS_AES_128_GCM_SHA256, TLS_VERSION_1_3}, @@ -40,7 +40,6 @@ fn make_aead(cipher: Cipher) -> Aead { ) .expect("make a secret"); Aead::new( - false, TLS_VERSION_1_3, cipher, &secret, diff --git a/third_party/rust/neqo-crypto/tests/init.rs b/third_party/rust/neqo-crypto/tests/init.rs index 13218cc340..ee7d808e29 100644 --- a/third_party/rust/neqo-crypto/tests/init.rs +++ b/third_party/rust/neqo-crypto/tests/init.rs @@ -15,13 +15,7 @@ use neqo_crypto::{assert_initialized, init_db}; // Pull in the NSS internals so that we can ask NSS if it thinks that // it is properly initialized. -#[allow( - dead_code, - non_upper_case_globals, - clippy::redundant_static_lifetimes, - clippy::unseparated_literal_suffix, - clippy::upper_case_acronyms -)] +#[allow(dead_code, non_upper_case_globals)] mod nss { include!(concat!(env!("OUT_DIR"), "/nss_init.rs")); } @@ -29,19 +23,54 @@ mod nss { #[cfg(nss_nodb)] #[test] fn init_nodb() { - init(); + neqo_crypto::init().unwrap(); assert_initialized(); unsafe { - assert!(nss::NSS_IsInitialized() != 0); + assert_ne!(nss::NSS_IsInitialized(), 0); } } +#[cfg(nss_nodb)] +#[test] +fn init_twice_nodb() { + unsafe { + nss::NSS_NoDB_Init(std::ptr::null()); + assert_ne!(nss::NSS_IsInitialized(), 0); + } + // Now do it again + init_nodb(); +} + #[cfg(not(nss_nodb))] #[test] fn init_withdb() { - init_db(::test_fixture::NSS_DB_PATH); + init_db(::test_fixture::NSS_DB_PATH).unwrap(); assert_initialized(); unsafe { - assert!(nss::NSS_IsInitialized() != 0); + assert_ne!(nss::NSS_IsInitialized(), 0); + } +} + +#[cfg(not(nss_nodb))] +#[test] +fn init_twice_withdb() { + use std::{ffi::CString, path::PathBuf}; + + let empty = CString::new("").unwrap(); + let path: PathBuf = ::test_fixture::NSS_DB_PATH.into(); + assert!(path.is_dir()); + let pathstr = path.to_str().unwrap(); + let dircstr = CString::new(pathstr).unwrap(); + unsafe { + nss::NSS_Initialize( + dircstr.as_ptr(), + empty.as_ptr(), + empty.as_ptr(), + nss::SECMOD_DB.as_ptr().cast(), + nss::NSS_INIT_READONLY, + ); + assert_ne!(nss::NSS_IsInitialized(), 0); } + // Now do it again + init_withdb(); } diff --git a/third_party/rust/neqo-crypto/tests/selfencrypt.rs b/third_party/rust/neqo-crypto/tests/selfencrypt.rs index 4c574a3ae9..9fc2162fe2 100644 --- a/third_party/rust/neqo-crypto/tests/selfencrypt.rs +++ b/third_party/rust/neqo-crypto/tests/selfencrypt.rs @@ -4,7 +4,7 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. -#![cfg(not(feature = "fuzzing"))] +#![cfg(not(feature = "disable-encryption"))] use neqo_crypto::{ constants::{TLS_AES_128_GCM_SHA256, TLS_VERSION_1_3}, @@ -15,7 +15,7 @@ use neqo_crypto::{ #[test] fn se_create() { - init(); + init().unwrap(); SelfEncrypt::new(TLS_VERSION_1_3, TLS_AES_128_GCM_SHA256).expect("constructor works"); } @@ -23,7 +23,7 @@ const PLAINTEXT: &[u8] = b"PLAINTEXT"; const AAD: &[u8] = b"AAD"; fn sealed() -> (SelfEncrypt, Vec) { - init(); + init().unwrap(); let se = SelfEncrypt::new(TLS_VERSION_1_3, TLS_AES_128_GCM_SHA256).unwrap(); let sealed = se.seal(AAD, PLAINTEXT).expect("sealing works"); (se, sealed) diff --git a/third_party/rust/neqo-http3/.cargo-checksum.json b/third_party/rust/neqo-http3/.cargo-checksum.json index 0459fea7cc..dff33cc964 100644 --- a/third_party/rust/neqo-http3/.cargo-checksum.json +++ b/third_party/rust/neqo-http3/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"Cargo.toml":"458f04261cda071d61402c52cf64062ad7cfc24f3f312bfaa5d52cae47409010","src/buffered_send_stream.rs":"f45bdf9ad2a04b3828c74ff5440681d3c9d1af39b55470e4f729842dc2412295","src/client_events.rs":"77fedca72ce54956eaba3fb7103085d196a631b764662584ea2629224c5c234e","src/conn_params.rs":"224a8ea6ef632930a7788a1cabf47ce69ad41bd4bc8dcf3053fbd998fdb38e82","src/connection.rs":"9384cdfd8481a30a0cd13f56f590188ccfa47b4472f35f7a4978537bab19adc1","src/connection_client.rs":"8db29409f3a265f7dff7c7a7eaf2ac607d6923e4b3238e82eab6dc22854e4303","src/connection_server.rs":"ca33b50650bd1ca2a952851b72712d55ec2e48b48f1f06e4184c808b8e1e009a","src/control_stream_local.rs":"ae52e3286f1686ca1265e7de841392addd42616db02799bb967a59feb6039cb5","src/control_stream_remote.rs":"59eb4041e366d92f9f294e8446755caa5e91fd943bba7b79b726698ba13be248","src/features/extended_connect/mod.rs":"3b02f6b18627f3855465a81b1d9b285e6f13839e75a8a6db648ed9082908d7f0","src/features/extended_connect/tests/mod.rs":"fd6aee37243713e80fc526552f21f0222338cec9890409b6575a2a637b17ec1f","src/features/extended_connect/tests/webtransport/datagrams.rs":"4c85a90afb753ce588e3fdeb773669bc49c013aebc28912340359eb01b74fd70","src/features/extended_connect/tests/webtransport/mod.rs":"a30ea715f5271a826a739278b18e145964dedbce7026eed45f1b7d0355c407d5","src/features/extended_connect/tests/webtransport/negotiation.rs":"98254ef8446581ec520026b04ef9549645602181b61602c9936f6660141edf0b","src/features/extended_connect/tests/webtransport/sessions.rs":"de3d836f666c2bec31e70b33bdc2669572cabbe17df2225db7282613a224a364","src/features/extended_connect/tests/webtransport/streams.rs":"8b3c34cac1b2171252a4bb53d420ac2098549a20309c327bf56e2e9ba9e33538","src/features/extended_connect/webtransport_session.rs":"239d92c06fbc5f6226078bb411a803f57b555dea0077349d49d7f57671cf2eab","src/features/extended_connect/webtransport_streams.rs":"5d7507aaf6a819d266fbea9b7a415c8324329df0f6936d9045b73e17a5b844ee","src/features/mod.rs":"925aae4427ad82e4d019354802b223d53db5e5585d4a940f5417a24a9503d7ee","src/frames/hframe.rs":"56c36ac597504f28c73cf2370acd82104f8c7a7b9ffc0f6d222378abc524482d","src/frames/mod.rs":"7d0a46ca147336d14781edb8dbee8b03c2e4bcd6646f5473a9d93d31fe73fecb","src/frames/reader.rs":"e07ee9de74bc499c10afcda592fefd9a7eef3381c045aa14f6596d67313546ca","src/frames/tests/hframe.rs":"01ec74eb3eb25d95042aa0263f9267f89535e6b7b8c1161fab4ba9ee5352d4a7","src/frames/tests/mod.rs":"0610609b316767a6a022837d32ee0452e37ea296fde37e51bec87e7c77e923a3","src/frames/tests/reader.rs":"2bfadc7afbc41bff9f5f930b31550259a8a92484d35f6c5d8dd8fd9acfb88f5b","src/frames/tests/wtframe.rs":"589ebe1e62ce4da63b37b7d22cde7ba572ddbf29336fdcdbbcd0a745f79dacd8","src/frames/wtframe.rs":"1d9d0256ace2ba7262343ed035df795f21a4d45065792d3fd45b3391b6916b2f","src/headers_checks.rs":"be0f0109298dcc3a40350b7c0950076ddfe20617d195b305e3ffc8582557ab18","src/lib.rs":"4f908a021222bcc79b9d569bc3759a493379a20b47dfa228fddf51600bf6e446","src/priority.rs":"f3b77c208962e44a4e2d13138c6998b703d40e7bcf8f73ea84d8ef5b556e0aee","src/push_controller.rs":"13bccf2834ae19109504cf695a5948c3b2d03fd101bc032a92bb77a033423854","src/qlog.rs":"2debd75c7ea103c95ff79e44412f1408c3e496e324976100c55d5a833912b6c3","src/qpack_decoder_receiver.rs":"c927dfc3e58c71d282210ba79280f6f03e789733bc3bedc247e68bab516b9e9e","src/qpack_encoder_receiver.rs":"d0ac03cc111b6e1c555a8654d3234116f2b135b5b040edac23cefe2d640beba9","src/recv_message.rs":"eb711dbc6b3371373c26b75333ac5858edf0d30184b0e05d67ab02c656eb6619","src/request_target.rs":"6041a69a0a74969ec08bc164509c055e9bad99f53bbeb16c0aa17d108dd68b8c","src/send_message.rs":"7785af11b77cee398faf3f7a2875b41e251ed7a1b272c23f81a48334596ab836","src/server.rs":"b9e6060da36cfb467478f5b78b17e22a123214ad2d64c919ce688ea2bc0e24bb","src/server_connection_events.rs":"12d353ca6301467f6d475dde3b789951a5716c89ddd7dbf1383efef8082361f3","src/server_events.rs":"463dd2cb6f97a800bac32c93c4aa2a6289f71e33a89f3b33152460cb941fc378","src/settings.rs":"476b154b5eea4c8d69a4a790fee3e527cef4d375df1cfb5eed04ec56406fe15a","src/stream_type_reader.rs":"7a7226b7911d69f7e00ec4987c2a32a5e8a33463203398cbee1e6645d2691478","tests/httpconn.rs":"bb6927801a8c75e4f05eb6cdb1e7f2d57be69b74e68ddad2a1614f2aeed04369","tests/priority.rs":"364754507873298612ad12e8d1d106d26d993712142d0be4cbf056da5338854c","tests/send_message.rs":"b5435045b16429d9e626ea94a8f10e2937e1a5a878af0035763a4f5ec09bf53c","tests/webtransport.rs":"25794305017ff58e57dc3c3b9b078e5bfc1814ea82a521b7b7156228e613c092"},"package":null} \ No newline at end of file +{"files":{"Cargo.toml":"f3d4e7ace03bbc6ee36fb30540e0a96aebdf5222090cd95abdff14b840cfa68e","src/buffered_send_stream.rs":"f45bdf9ad2a04b3828c74ff5440681d3c9d1af39b55470e4f729842dc2412295","src/client_events.rs":"77fedca72ce54956eaba3fb7103085d196a631b764662584ea2629224c5c234e","src/conn_params.rs":"224a8ea6ef632930a7788a1cabf47ce69ad41bd4bc8dcf3053fbd998fdb38e82","src/connection.rs":"e9c869ec4c650162927543cfe608721fad037554c5101962b323496bf69e29a5","src/connection_client.rs":"2026e7b2c11f4d5a71b35eedaf14175d7d7cf96ce2bb64c4a97d6062cbb646c7","src/connection_server.rs":"de5a6cb42b8c4dc08fb1f626f681b45cd22435892b11e6053b61a5401490db94","src/control_stream_local.rs":"ae52e3286f1686ca1265e7de841392addd42616db02799bb967a59feb6039cb5","src/control_stream_remote.rs":"59eb4041e366d92f9f294e8446755caa5e91fd943bba7b79b726698ba13be248","src/features/extended_connect/mod.rs":"3b02f6b18627f3855465a81b1d9b285e6f13839e75a8a6db648ed9082908d7f0","src/features/extended_connect/tests/mod.rs":"fd6aee37243713e80fc526552f21f0222338cec9890409b6575a2a637b17ec1f","src/features/extended_connect/tests/webtransport/datagrams.rs":"4c85a90afb753ce588e3fdeb773669bc49c013aebc28912340359eb01b74fd70","src/features/extended_connect/tests/webtransport/mod.rs":"a30ea715f5271a826a739278b18e145964dedbce7026eed45f1b7d0355c407d5","src/features/extended_connect/tests/webtransport/negotiation.rs":"98254ef8446581ec520026b04ef9549645602181b61602c9936f6660141edf0b","src/features/extended_connect/tests/webtransport/sessions.rs":"de3d836f666c2bec31e70b33bdc2669572cabbe17df2225db7282613a224a364","src/features/extended_connect/tests/webtransport/streams.rs":"8b3c34cac1b2171252a4bb53d420ac2098549a20309c327bf56e2e9ba9e33538","src/features/extended_connect/webtransport_session.rs":"239d92c06fbc5f6226078bb411a803f57b555dea0077349d49d7f57671cf2eab","src/features/extended_connect/webtransport_streams.rs":"5d7507aaf6a819d266fbea9b7a415c8324329df0f6936d9045b73e17a5b844ee","src/features/mod.rs":"925aae4427ad82e4d019354802b223d53db5e5585d4a940f5417a24a9503d7ee","src/frames/hframe.rs":"56c36ac597504f28c73cf2370acd82104f8c7a7b9ffc0f6d222378abc524482d","src/frames/mod.rs":"7d0a46ca147336d14781edb8dbee8b03c2e4bcd6646f5473a9d93d31fe73fecb","src/frames/reader.rs":"e07ee9de74bc499c10afcda592fefd9a7eef3381c045aa14f6596d67313546ca","src/frames/tests/hframe.rs":"01ec74eb3eb25d95042aa0263f9267f89535e6b7b8c1161fab4ba9ee5352d4a7","src/frames/tests/mod.rs":"0610609b316767a6a022837d32ee0452e37ea296fde37e51bec87e7c77e923a3","src/frames/tests/reader.rs":"2bfadc7afbc41bff9f5f930b31550259a8a92484d35f6c5d8dd8fd9acfb88f5b","src/frames/tests/wtframe.rs":"589ebe1e62ce4da63b37b7d22cde7ba572ddbf29336fdcdbbcd0a745f79dacd8","src/frames/wtframe.rs":"1d9d0256ace2ba7262343ed035df795f21a4d45065792d3fd45b3391b6916b2f","src/headers_checks.rs":"be0f0109298dcc3a40350b7c0950076ddfe20617d195b305e3ffc8582557ab18","src/lib.rs":"4f908a021222bcc79b9d569bc3759a493379a20b47dfa228fddf51600bf6e446","src/priority.rs":"f3b77c208962e44a4e2d13138c6998b703d40e7bcf8f73ea84d8ef5b556e0aee","src/push_controller.rs":"13bccf2834ae19109504cf695a5948c3b2d03fd101bc032a92bb77a033423854","src/qlog.rs":"2debd75c7ea103c95ff79e44412f1408c3e496e324976100c55d5a833912b6c3","src/qpack_decoder_receiver.rs":"c927dfc3e58c71d282210ba79280f6f03e789733bc3bedc247e68bab516b9e9e","src/qpack_encoder_receiver.rs":"d0ac03cc111b6e1c555a8654d3234116f2b135b5b040edac23cefe2d640beba9","src/recv_message.rs":"7ac8d4057ba53874e4edfc62cd25ad5d3f0b10aaac5bf6e156103c3bc44e18cc","src/request_target.rs":"6041a69a0a74969ec08bc164509c055e9bad99f53bbeb16c0aa17d108dd68b8c","src/send_message.rs":"374e168f60063b8102a2aff52c719ae2e1e5078527cf50d095b3e7217f6ec7d2","src/server.rs":"b9e6060da36cfb467478f5b78b17e22a123214ad2d64c919ce688ea2bc0e24bb","src/server_connection_events.rs":"12d353ca6301467f6d475dde3b789951a5716c89ddd7dbf1383efef8082361f3","src/server_events.rs":"1cda8d6c413fad0fa67fcfd7cb78e795bf7ef7f0e09b5720992646a82d51ce16","src/settings.rs":"476b154b5eea4c8d69a4a790fee3e527cef4d375df1cfb5eed04ec56406fe15a","src/stream_type_reader.rs":"7a7226b7911d69f7e00ec4987c2a32a5e8a33463203398cbee1e6645d2691478","tests/httpconn.rs":"bb6927801a8c75e4f05eb6cdb1e7f2d57be69b74e68ddad2a1614f2aeed04369","tests/priority.rs":"364754507873298612ad12e8d1d106d26d993712142d0be4cbf056da5338854c","tests/send_message.rs":"b5435045b16429d9e626ea94a8f10e2937e1a5a878af0035763a4f5ec09bf53c","tests/webtransport.rs":"25794305017ff58e57dc3c3b9b078e5bfc1814ea82a521b7b7156228e613c092"},"package":null} \ No newline at end of file diff --git a/third_party/rust/neqo-http3/Cargo.toml b/third_party/rust/neqo-http3/Cargo.toml index 12b9a125d0..c844ba43ef 100644 --- a/third_party/rust/neqo-http3/Cargo.toml +++ b/third_party/rust/neqo-http3/Cargo.toml @@ -13,7 +13,7 @@ edition = "2021" rust-version = "1.74.0" name = "neqo-http3" -version = "0.7.2" +version = "0.7.5" authors = ["The Neqo Authors "] homepage = "https://github.com/mozilla/neqo/" license = "MIT OR Apache-2.0" @@ -62,9 +62,9 @@ default-features = false path = "../test-fixture" [features] -fuzzing = [ - "neqo-transport/fuzzing", - "neqo-crypto/fuzzing", +disable-encryption = [ + "neqo-transport/disable-encryption", + "neqo-crypto/disable-encryption", ] [lints.clippy.pedantic] diff --git a/third_party/rust/neqo-http3/src/connection.rs b/third_party/rust/neqo-http3/src/connection.rs index 287ea2c2af..dd45797baa 100644 --- a/third_party/rust/neqo-http3/src/connection.rs +++ b/third_party/rust/neqo-http3/src/connection.rs @@ -354,7 +354,7 @@ impl Http3Connection { /// This function creates and initializes, i.e. send stream type, the control and qpack /// streams. fn initialize_http3_connection(&mut self, conn: &mut Connection) -> Res<()> { - qinfo!([self], "Initialize the http3 connection."); + qdebug!([self], "Initialize the http3 connection."); self.control_stream_local.create(conn)?; self.send_settings(); @@ -386,8 +386,8 @@ impl Http3Connection { Ok(()) } - /// Inform a `HttpConnection` that a stream has data to send and that `send` should be called - /// for the stream. + /// Inform an [`Http3Connection`] that a stream has data to send and that + /// [`SendStream::send`] should be called for the stream. pub fn stream_has_pending_data(&mut self, stream_id: StreamId) { self.streams_with_pending_data.insert(stream_id); } @@ -704,7 +704,7 @@ impl Http3Connection { ); } NewStreamType::Decoder => { - qinfo!([self], "A new remote qpack encoder stream {}", stream_id); + qdebug!([self], "A new remote qpack encoder stream {}", stream_id); self.check_stream_exists(Http3StreamType::Decoder)?; self.recv_streams.insert( stream_id, @@ -715,7 +715,7 @@ impl Http3Connection { ); } NewStreamType::Encoder => { - qinfo!([self], "A new remote qpack decoder stream {}", stream_id); + qdebug!([self], "A new remote qpack decoder stream {}", stream_id); self.check_stream_exists(Http3StreamType::Encoder)?; self.recv_streams.insert( stream_id, @@ -766,7 +766,7 @@ impl Http3Connection { /// This is called when an application closes the connection. pub fn close(&mut self, error: AppError) { - qinfo!([self], "Close connection error {:?}.", error); + qdebug!([self], "Close connection error {:?}.", error); self.state = Http3State::Closing(ConnectionError::Application(error)); if (!self.send_streams.is_empty() || !self.recv_streams.is_empty()) && (error == 0) { qwarn!("close(0) called when streams still active"); @@ -952,7 +952,7 @@ impl Http3Connection { stream_id: StreamId, buf: &mut [u8], ) -> Res<(usize, bool)> { - qinfo!([self], "read_data from stream {}.", stream_id); + qdebug!([self], "read_data from stream {}.", stream_id); let res = self .recv_streams .get_mut(&stream_id) @@ -1091,7 +1091,7 @@ impl Http3Connection { /// This is called when an application wants to close the sending side of a stream. pub fn stream_close_send(&mut self, conn: &mut Connection, stream_id: StreamId) -> Res<()> { - qinfo!([self], "Close the sending side for stream {}.", stream_id); + qdebug!([self], "Close the sending side for stream {}.", stream_id); debug_assert!(self.state.active()); let send_stream = self .send_streams @@ -1402,7 +1402,7 @@ impl Http3Connection { /// `PriorityUpdateRequestPush` which handling is specific to the client and server, we must /// give them to the specific client/server handler. fn handle_control_frame(&mut self, f: HFrame) -> Res> { - qinfo!([self], "Handle a control frame {:?}", f); + qdebug!([self], "Handle a control frame {:?}", f); if !matches!(f, HFrame::Settings { .. }) && !matches!( self.settings_state, @@ -1433,7 +1433,7 @@ impl Http3Connection { } fn handle_settings(&mut self, new_settings: HSettings) -> Res<()> { - qinfo!([self], "Handle SETTINGS frame."); + qdebug!([self], "Handle SETTINGS frame."); match &self.settings_state { Http3RemoteSettingsState::NotReceived => { self.set_qpack_settings(&new_settings)?; diff --git a/third_party/rust/neqo-http3/src/connection_client.rs b/third_party/rust/neqo-http3/src/connection_client.rs index 52572a760d..4c8772d14a 100644 --- a/third_party/rust/neqo-http3/src/connection_client.rs +++ b/third_party/rust/neqo-http3/src/connection_client.rs @@ -7,7 +7,7 @@ use std::{ cell::RefCell, fmt::{Debug, Display}, - mem, + iter, mem, net::SocketAddr, rc::Rc, time::Instant, @@ -590,7 +590,7 @@ impl Http3Client { /// /// An error will be return if stream does not exist. pub fn stream_close_send(&mut self, stream_id: StreamId) -> Res<()> { - qinfo!([self], "Close sending side stream={}.", stream_id); + qdebug!([self], "Close sending side stream={}.", stream_id); self.base_handler .stream_close_send(&mut self.conn, stream_id) } @@ -652,7 +652,7 @@ impl Http3Client { stream_id: StreamId, buf: &mut [u8], ) -> Res<(usize, bool)> { - qinfo!([self], "read_data from stream {}.", stream_id); + qdebug!([self], "read_data from stream {}.", stream_id); let res = self.base_handler.read_data(&mut self.conn, stream_id, buf); if let Err(e) = &res { if e.connection_error() { @@ -874,19 +874,16 @@ impl Http3Client { /// /// [1]: ../neqo_transport/enum.ConnectionEvent.html pub fn process_input(&mut self, dgram: &Datagram, now: Instant) { - qtrace!([self], "Process input."); - self.conn.process_input(dgram, now); - self.process_http3(now); + self.process_multiple_input(iter::once(dgram), now); } pub fn process_multiple_input<'a, I>(&mut self, dgrams: I, now: Instant) where I: IntoIterator, - I::IntoIter: ExactSizeIterator, { - let dgrams = dgrams.into_iter(); - qtrace!([self], "Process multiple datagrams, len={}", dgrams.len()); - if dgrams.len() == 0 { + let mut dgrams = dgrams.into_iter().peekable(); + qtrace!([self], "Process multiple datagrams"); + if dgrams.peek().is_none() { return; } self.conn.process_multiple_input(dgrams, now); diff --git a/third_party/rust/neqo-http3/src/connection_server.rs b/third_party/rust/neqo-http3/src/connection_server.rs index 097209a226..cc887a26fc 100644 --- a/third_party/rust/neqo-http3/src/connection_server.rs +++ b/third_party/rust/neqo-http3/src/connection_server.rs @@ -64,13 +64,17 @@ impl Http3ServerHandler { data: &[u8], conn: &mut Connection, ) -> Res { - self.base_handler.stream_has_pending_data(stream_id); - self.needs_processing = true; - self.base_handler + let n = self + .base_handler .send_streams .get_mut(&stream_id) .ok_or(Error::InvalidStreamId)? - .send_data(conn, data) + .send_data(conn, data)?; + if n > 0 { + self.base_handler.stream_has_pending_data(stream_id); + } + self.needs_processing = true; + Ok(n) } /// Supply response heeaders for a request. @@ -98,9 +102,8 @@ impl Http3ServerHandler { /// /// An error will be returned if stream does not exist. pub fn stream_close_send(&mut self, stream_id: StreamId, conn: &mut Connection) -> Res<()> { - qinfo!([self], "Close sending side stream={}.", stream_id); + qdebug!([self], "Close sending side stream={}.", stream_id); self.base_handler.stream_close_send(conn, stream_id)?; - self.base_handler.stream_has_pending_data(stream_id); self.needs_processing = true; Ok(()) } @@ -408,7 +411,7 @@ impl Http3ServerHandler { stream_id: StreamId, buf: &mut [u8], ) -> Res<(usize, bool)> { - qinfo!([self], "read_data from stream {}.", stream_id); + qdebug!([self], "read_data from stream {}.", stream_id); let res = self.base_handler.read_data(conn, stream_id, buf); if let Err(e) = &res { if e.connection_error() { diff --git a/third_party/rust/neqo-http3/src/recv_message.rs b/third_party/rust/neqo-http3/src/recv_message.rs index be58b7e47c..55970849ef 100644 --- a/third_party/rust/neqo-http3/src/recv_message.rs +++ b/third_party/rust/neqo-http3/src/recv_message.rs @@ -271,7 +271,7 @@ impl RecvMessage { } (None, false) => break Ok(()), (Some(frame), fin) => { - qinfo!( + qdebug!( [self], "A new frame has been received: {:?}; state={:?} fin={}", frame, diff --git a/third_party/rust/neqo-http3/src/send_message.rs b/third_party/rust/neqo-http3/src/send_message.rs index c50e3e056a..15965c44f6 100644 --- a/third_party/rust/neqo-http3/src/send_message.rs +++ b/third_party/rust/neqo-http3/src/send_message.rs @@ -6,7 +6,7 @@ use std::{cell::RefCell, cmp::min, fmt::Debug, rc::Rc}; -use neqo_common::{qdebug, qinfo, qtrace, Encoder, Header, MessageType}; +use neqo_common::{qdebug, qtrace, Encoder, Header, MessageType}; use neqo_qpack::encoder::QPackEncoder; use neqo_transport::{Connection, StreamId}; @@ -119,7 +119,7 @@ impl SendMessage { encoder: Rc>, conn_events: Box, ) -> Self { - qinfo!("Create a request stream_id={}", stream_id); + qdebug!("Create a request stream_id={}", stream_id); Self { state: MessageState::WaitingForHeaders, message_type, @@ -193,7 +193,7 @@ impl SendStream for SendMessage { min(buf.len(), available - 9) }; - qinfo!( + qdebug!( [self], "send_request_body: available={} to_send={}.", available, diff --git a/third_party/rust/neqo-http3/src/server_events.rs b/third_party/rust/neqo-http3/src/server_events.rs index a85ece0bfb..214a48c757 100644 --- a/third_party/rust/neqo-http3/src/server_events.rs +++ b/third_party/rust/neqo-http3/src/server_events.rs @@ -13,7 +13,7 @@ use std::{ rc::Rc, }; -use neqo_common::{qdebug, qinfo, Encoder, Header}; +use neqo_common::{qdebug, Encoder, Header}; use neqo_transport::{ server::ActiveConnectionRef, AppError, Connection, DatagramTracking, StreamId, StreamType, }; @@ -189,7 +189,7 @@ impl Http3OrWebTransportStream { /// /// It may return `InvalidStreamId` if a stream does not exist anymore. pub fn send_data(&mut self, data: &[u8]) -> Res { - qinfo!([self], "Set new response."); + qdebug!([self], "Set new response."); self.stream_handler.send_data(data) } @@ -199,7 +199,7 @@ impl Http3OrWebTransportStream { /// /// It may return `InvalidStreamId` if a stream does not exist anymore. pub fn stream_close_send(&mut self) -> Res<()> { - qinfo!([self], "Set new response."); + qdebug!([self], "Set new response."); self.stream_handler.stream_close_send() } } @@ -270,7 +270,7 @@ impl WebTransportRequest { /// /// It may return `InvalidStreamId` if a stream does not exist anymore. pub fn response(&mut self, accept: &WebTransportSessionAcceptAction) -> Res<()> { - qinfo!([self], "Set a response for a WebTransport session."); + qdebug!([self], "Set a response for a WebTransport session."); self.stream_handler .handler .borrow_mut() diff --git a/third_party/rust/neqo-qpack/.cargo-checksum.json b/third_party/rust/neqo-qpack/.cargo-checksum.json index aae0a1e594..d717487018 100644 --- a/third_party/rust/neqo-qpack/.cargo-checksum.json +++ b/third_party/rust/neqo-qpack/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"Cargo.toml":"c2152600379c3961ba79e661e164630a63531744f79e082fce39cdf1cbe75ddd","src/decoder.rs":"0675444129e074e9d5d56f0d45d2eaed614c85e22cfe9f2d28cdee912c15b420","src/decoder_instructions.rs":"d991d70e51f079bc5b30d3982fd0176edfa9bb7ba14c17a20ec3eea878c56206","src/encoder.rs":"84649cbee81e050f55d7ea691ac871e072741abd8bbf96303eb2e98aa8ee0aea","src/encoder_instructions.rs":"86e3abbd9cf94332041326ac6cf806ed64623e3fd38dbc0385b1f63c37e73fd9","src/header_block.rs":"3925476df69b90d950594faadc5cb24c374d46de8c75a374a235f0d27323a7d8","src/huffman.rs":"71ec740426eee0abb6205104e504f5b97f525a76c4a5f5827b78034d28ce1876","src/huffman_decode_helper.rs":"9ce470e318b3664f58aa109bed483ab15bfd9e0b17d261ea2b609668a42a9d80","src/huffman_table.rs":"06fea766a6276ac56c7ee0326faed800a742c15fda1f33bf2513e6cc6a5e6d27","src/lib.rs":"fd673630b5ed64197851c9a9758685096d3c0aa04f4994290733a38057004ee6","src/prefix.rs":"fb4a9acbcf6fd3178f4474404cd3d3b131abca934f69fe14a9d744bc7e636dc5","src/qlog.rs":"e320007ea8309546b26f9c0019ab8722da80dbd38fa976233fd8ae19a0af637c","src/qpack_send_buf.rs":"755af90fe077b1bcca34a1a2a1bdce5ce601ea490b2ca3f1313e0107d13e67e2","src/reader.rs":"1581261741a0922b147a6975cc8b1a3503846f6dbfdb771d254760c298996982","src/static_table.rs":"fda9d5c6f38f94b0bf92d3afdf8432dce6e27e189736596e16727090c77b78ec","src/stats.rs":"624dfa3b40858c304097bb0ce5b1be1bb4d7916b1abfc222f1aa705907009730","src/table.rs":"6e16debdceadc453546f247f8316883af9eeeedd12f2070219d8484a0a131d46"},"package":null} \ No newline at end of file +{"files":{"Cargo.toml":"b963363c6eb1b0b3318d5901d372599db89dff87131e0979b2eb09f42c84d72a","src/decoder.rs":"0675444129e074e9d5d56f0d45d2eaed614c85e22cfe9f2d28cdee912c15b420","src/decoder_instructions.rs":"d991d70e51f079bc5b30d3982fd0176edfa9bb7ba14c17a20ec3eea878c56206","src/encoder.rs":"84649cbee81e050f55d7ea691ac871e072741abd8bbf96303eb2e98aa8ee0aea","src/encoder_instructions.rs":"86e3abbd9cf94332041326ac6cf806ed64623e3fd38dbc0385b1f63c37e73fd9","src/header_block.rs":"3925476df69b90d950594faadc5cb24c374d46de8c75a374a235f0d27323a7d8","src/huffman.rs":"71ec740426eee0abb6205104e504f5b97f525a76c4a5f5827b78034d28ce1876","src/huffman_decode_helper.rs":"9ce470e318b3664f58aa109bed483ab15bfd9e0b17d261ea2b609668a42a9d80","src/huffman_table.rs":"06fea766a6276ac56c7ee0326faed800a742c15fda1f33bf2513e6cc6a5e6d27","src/lib.rs":"fd673630b5ed64197851c9a9758685096d3c0aa04f4994290733a38057004ee6","src/prefix.rs":"fb4a9acbcf6fd3178f4474404cd3d3b131abca934f69fe14a9d744bc7e636dc5","src/qlog.rs":"e320007ea8309546b26f9c0019ab8722da80dbd38fa976233fd8ae19a0af637c","src/qpack_send_buf.rs":"755af90fe077b1bcca34a1a2a1bdce5ce601ea490b2ca3f1313e0107d13e67e2","src/reader.rs":"1581261741a0922b147a6975cc8b1a3503846f6dbfdb771d254760c298996982","src/static_table.rs":"fda9d5c6f38f94b0bf92d3afdf8432dce6e27e189736596e16727090c77b78ec","src/stats.rs":"624dfa3b40858c304097bb0ce5b1be1bb4d7916b1abfc222f1aa705907009730","src/table.rs":"ddf055a228bed575d640d9a06e19e1e9fd98a48e393a7d326f8254438fb94889"},"package":null} \ No newline at end of file diff --git a/third_party/rust/neqo-qpack/Cargo.toml b/third_party/rust/neqo-qpack/Cargo.toml index 1becac8190..822cddd9c0 100644 --- a/third_party/rust/neqo-qpack/Cargo.toml +++ b/third_party/rust/neqo-qpack/Cargo.toml @@ -13,7 +13,7 @@ edition = "2021" rust-version = "1.74.0" name = "neqo-qpack" -version = "0.7.2" +version = "0.7.5" authors = ["The Neqo Authors "] homepage = "https://github.com/mozilla/neqo/" license = "MIT OR Apache-2.0" diff --git a/third_party/rust/neqo-qpack/src/table.rs b/third_party/rust/neqo-qpack/src/table.rs index 517e98db09..d5275ec98f 100644 --- a/third_party/rust/neqo-qpack/src/table.rs +++ b/third_party/rust/neqo-qpack/src/table.rs @@ -94,7 +94,7 @@ impl HeaderTable { capacity: 0, used: 0, base: 0, - acked_inserts_cnt: if encoder { 0 } else { u64::max_value() }, + acked_inserts_cnt: if encoder { 0 } else { u64::MAX }, } } diff --git a/third_party/rust/neqo-transport/.cargo-checksum.json b/third_party/rust/neqo-transport/.cargo-checksum.json index 669c0120f0..3fc724515c 100644 --- a/third_party/rust/neqo-transport/.cargo-checksum.json +++ b/third_party/rust/neqo-transport/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"Cargo.toml":"8eac0a271cef9232d100eb45093a3f7978a93e0576c64c99c161092ff445825d","benches/range_tracker.rs":"4821443d3cccc697b8976b7c50d787a7aa8cb86ab8633a7582be3f85135168db","benches/rx_stream_orderer.rs":"a8db922390d8506c483a3a1f40ac9bf12806ebdb4f501716904776dd58e995be","benches/transfer.rs":"11343c1ac9131585c42236749d32d9e272a33b6acd58831fa3415be4d4f1cf86","src/ackrate.rs":"4bb882e1069a0707dc85338b75327e2910c93ee5f36575767a0d58c4c41c9d4f","src/addr_valid.rs":"03c0b2ff85254179c5d425b12acfdcc6b1ea5735aeb0f604b9b3603451b3ef0a","src/cc/classic_cc.rs":"4528bb4e9059524942ee7ef931de5de90c78ee13f76489185a964ad45c12c0b3","src/cc/cubic.rs":"24c6913cc6346e5361007221c26e8096ece51583431fc3ab9c99e4ce4b0a9f5d","src/cc/mod.rs":"e0837937c9991b37edad15cd870ea9e0623489babfccc340074dd8322e1ef401","src/cc/new_reno.rs":"25d0921005688e0f0666efd0a4931b4f8cd44363587d98e5b6404818c5d05dd4","src/cc/tests/cubic.rs":"109fc8be5efba8959e777288c32ae8f2db581fc08719f318ad676e187f655478","src/cc/tests/mod.rs":"44f8df551e742ae1037cd1cdb85b2c1334c2e5ab3c23ed63d856dbc6b8743afc","src/cc/tests/new_reno.rs":"5414e26b6c928c5f82c5eeb42f04772b05be1ec2c8ee21c2b698ce8cb32829a1","src/cid.rs":"9686a3070c593cfca846d7549863728e31211b304b9fa876220f79bff5e24173","src/connection/dump.rs":"c539caffdf5b4dfaf0173bb20d1974f5242b5432a0a32fc0b8ab56ee682cb1eb","src/connection/idle.rs":"b3bc2ad1290e54278d8703092d135eda973eb12316d1f6dffedaffdf25e2a47e","src/connection/mod.rs":"dcfba9574b707318292f460dc40f54f3cdf8fd883f5f0d604f1d0d466f99f481","src/connection/params.rs":"9731bc5faa584874c48538ed19839c7a310277df39144c580cdf3001153f5a56","src/connection/saved.rs":"97eb19792be3c4d721057021a43ea50a52f89a3cfa583d3d3dcf5d9144b332f5","src/connection/state.rs":"c1820864cc63073e1f44b875be1fcde9835df644e0fa8c2e05652421ad78b7b2","src/connection/test_internal.rs":"f3ebfe97b25c9c716d41406066295e5aff4e96a3051ef4e2b5fb258282bbc14c","src/connection/tests/ackrate.rs":"4a2b835575850ae4a14209d3e51883ecb1e69afb44ef91b5e13a5e6cb7174fab","src/connection/tests/cc.rs":"d0d6ac038572ad3dcd9e6734596eaeedc6d3222d24e31b023aaab3540d195e46","src/connection/tests/close.rs":"20bf9b044ba52517693c2bd10820ff04a8c07de01d59c8c47b4e9478aa730211","src/connection/tests/datagram.rs":"f4c85099b6a8739fb99eadd8711b02066ad80fc8341a2e5e0dae2520378af9fe","src/connection/tests/fuzzing.rs":"79d9ac83fe2d952a3a13140d603569c332d29dbba2e0d2b8ee5f6e42e8f4708a","src/connection/tests/handshake.rs":"eda7308fdd46570ee3b5569ad34e63761ccde89eb5ca854c877e3a53e7de5ec8","src/connection/tests/idle.rs":"f3bcb12cd79cb8eabc969ce3fb0fab4eea26d6383b81a323c0e18ca9c42cfb59","src/connection/tests/keys.rs":"55558c057beb4221245beb262076de3991dca3f2661411db61c09d21194873df","src/connection/tests/migration.rs":"624985d925236be197eee52208dbdebe35c0df5bd9d30982d6f183dfda4cbab5","src/connection/tests/mod.rs":"8b6709a5c89becf2daed407515f894ba3337e87b2d45b21acffa02e67f37eeec","src/connection/tests/priority.rs":"dd3504f52d3fce7a96441624bc1c82c733e6bb556b9b79d24d0f4fb4efaf5a9e","src/connection/tests/recovery.rs":"7f28767f3cca2ff60e3dcfa803e12ef043486a222f54681a8faf2ea2fee564a1","src/connection/tests/resumption.rs":"1a0de0993cd325224fc79a3c094d22636d5b122ab1123d16265d4fafb23574bd","src/connection/tests/stream.rs":"e5590c2b52d33fbe1b4e995edf1c016dda460ecfa2a9f021005e4abe8ea13580","src/connection/tests/vn.rs":"550eb6b4d39d5960aafc70037c25a1a0f5db1232ce0ec6080b2c29a731a9574e","src/connection/tests/zerortt.rs":"67f77721e33d9fa2691c5ea3ef4a90935987541d81f0f42fbcfca31e690b352a","src/crypto.rs":"c5780ab85ca84e830024c31346a416f1f470694372d732ee5e5b7c5df3adc202","src/events.rs":"6e115f309c5c46f30f6223e1347bea477ada457f8bb2189ecccc6d65483318d6","src/fc.rs":"ec9de1028286870c0adf88a92e1355acf13dede8b1e91179230df3263e3827a9","src/frame.rs":"eb35c4add314f0013ad7837157fa9daeb76a2286fc7f8c922993624f54a09569","src/lib.rs":"f8d83b370cab19b3d172d0689f8d76115f5fd26c742e394fca62e253809cedc4","src/pace.rs":"86a674ac4d086148ea297214910458c3705918bd627b996ba8bbb12f2c4bf99e","src/packet/mod.rs":"9fac8f4046ada084dbbcc6601391a2bf8bbc23a09d6fe7df3c135a36840dbee3","src/packet/retry.rs":"1f10bb2c39ae4335e65b8d5d97f2b6df62e04877740af27c7b965a65e7f7ca66","src/path.rs":"3eb7e5e3bc496bfefc729c1e15fba0f9f83572151a850bf13b9c931297789279","src/qlog.rs":"b94aa36d5bac2799d8635cf6b25b9bb9383944d5607ea85aff55715f70af5f7b","src/quic_datagrams.rs":"3d33ecb9e6e80c77b812e8260fc807352300fb2305a29b797259ae34c52b67c5","src/recovery.rs":"1dadc6717dd133007943e762231a50680087392466904c2f2e6fface084e2ba9","src/recv_stream.rs":"f21ae0bb786901bb7d726a94cb3352607b0057128beaa331808137f2f57a330b","src/rtt.rs":"4635dc0c401b78a1fd9d34da6f9bf7f6e7f5be3a57ed9716e0efc8f0f94f1e47","src/send_stream.rs":"f717f64b75e368cf5fa4ca43078aa7c1b5aff48b4f6266713e6fa7dc458328aa","src/sender.rs":"5f760988bdd6fbbd5956877a97abe7c17370dd531f68b751a9e4e4459583f87b","src/server.rs":"048aaac84e15d49fd25850294759107fe1855bbbc0481c16f8bd888d8f2a8f6d","src/stats.rs":"b2a4c03d5b24edeecd00d809019c56f1a22a4e35967309ae6e6053331aafcf30","src/stream_id.rs":"fd07cbb81709a54bdb0659f676ef851cd145c004b817044ede5b21e54fdb60e4","src/streams.rs":"062b1b61edd1a76a86914f2cc1ca007c03edd9136c0c3409d960ddb805805fc6","src/tparams.rs":"10d62ac64865e0606c3c14141f9674631c610b3f042e274e110bdcef5d388491","src/tracking.rs":"f9a9aa01abc79fdd7a2cfb2c3ae342b9ab709e6a2a11076ec5c475fc89c1f598","src/version.rs":"182484ed9ecc2e17cab73cc61914a86a2d206936cab313825ae76fd37eeade77","tests/common/mod.rs":"0aa6674ae4efd2f151a65737ed5eab9e700bd1b3da5b4c165cb24de2b01598ce","tests/conn_vectors.rs":"290550072bd0c37652b79ac119729064dd486452c3a740353a6669bcdb2b82cf","tests/connection.rs":"b3c2ce0c62c4b79f80e42289eadd51933931b0ae44c0adc20ce5141edd454e00","tests/network.rs":"9e30b8610124250262fceef27d09fdecf2d6e9c3a96b1e676ff4189b9e06d5ba","tests/retry.rs":"da5c6a6f9ec1a8f556073b2d2e11fbcd2f58463818b0f08f8d23158016fea0d5","tests/server.rs":"cb83de909d858950bfd75a789fc23c3c44fcdf1d965b63800b2c7b498507987f"},"package":null} \ No newline at end of file +{"files":{"Cargo.toml":"28025605522925d25700870cd043f2a169acad5193b2da09c7a296bd9f73d389","benches/range_tracker.rs":"590dd1f81c92e89ce28af1efdda583d85240438bd9c4c68767286d22a299ad4b","benches/rx_stream_orderer.rs":"53a008357703251a18100521a12d8fa9443c5601ddc3cbd1b3c2899074da4c4f","benches/transfer.rs":"94eb0ec1a0a7d0a4863ddc1c6d006521e52c1f2e7f03c69428b18f7eb827d33f","src/ackrate.rs":"4bb882e1069a0707dc85338b75327e2910c93ee5f36575767a0d58c4c41c9d4f","src/addr_valid.rs":"03c0b2ff85254179c5d425b12acfdcc6b1ea5735aeb0f604b9b3603451b3ef0a","src/cc/classic_cc.rs":"aaaf3670bfaacf10c2ab77b547b4aeec2618ec6b2bb2d921000a8b795f38ea87","src/cc/cubic.rs":"24c6913cc6346e5361007221c26e8096ece51583431fc3ab9c99e4ce4b0a9f5d","src/cc/mod.rs":"e0837937c9991b37edad15cd870ea9e0623489babfccc340074dd8322e1ef401","src/cc/new_reno.rs":"25d0921005688e0f0666efd0a4931b4f8cd44363587d98e5b6404818c5d05dd4","src/cc/tests/cubic.rs":"109fc8be5efba8959e777288c32ae8f2db581fc08719f318ad676e187f655478","src/cc/tests/mod.rs":"44f8df551e742ae1037cd1cdb85b2c1334c2e5ab3c23ed63d856dbc6b8743afc","src/cc/tests/new_reno.rs":"5414e26b6c928c5f82c5eeb42f04772b05be1ec2c8ee21c2b698ce8cb32829a1","src/cid.rs":"9686a3070c593cfca846d7549863728e31211b304b9fa876220f79bff5e24173","src/connection/dump.rs":"bd4fb55785fe42f5c94f7bcc14ccf4ae377d28b691fb55dbf1139ae9412b0ea9","src/connection/idle.rs":"b3bc2ad1290e54278d8703092d135eda973eb12316d1f6dffedaffdf25e2a47e","src/connection/mod.rs":"907ded3ba14ec8ef675e1ea55c5698d6ffe023de5e81a006746d9759eb243640","src/connection/params.rs":"38e0b47c8cc5fbe602e3174d7a70df410829bc240b42f21cebd10818e606ef7c","src/connection/saved.rs":"97eb19792be3c4d721057021a43ea50a52f89a3cfa583d3d3dcf5d9144b332f5","src/connection/state.rs":"c0c4b1c15624a8762eabc8d5fa76f169f3f93945a9ee86f30fbd7714f1ac1d37","src/connection/test_internal.rs":"f3ebfe97b25c9c716d41406066295e5aff4e96a3051ef4e2b5fb258282bbc14c","src/connection/tests/ackrate.rs":"4a2b835575850ae4a14209d3e51883ecb1e69afb44ef91b5e13a5e6cb7174fab","src/connection/tests/cc.rs":"d0d6ac038572ad3dcd9e6734596eaeedc6d3222d24e31b023aaab3540d195e46","src/connection/tests/close.rs":"20bf9b044ba52517693c2bd10820ff04a8c07de01d59c8c47b4e9478aa730211","src/connection/tests/datagram.rs":"f4c85099b6a8739fb99eadd8711b02066ad80fc8341a2e5e0dae2520378af9fe","src/connection/tests/handshake.rs":"c759737ee98c7b33b2327eb7d521f45c63aed15dc8f272b7bbcc510ee8e48877","src/connection/tests/idle.rs":"f3bcb12cd79cb8eabc969ce3fb0fab4eea26d6383b81a323c0e18ca9c42cfb59","src/connection/tests/keys.rs":"55558c057beb4221245beb262076de3991dca3f2661411db61c09d21194873df","src/connection/tests/migration.rs":"624985d925236be197eee52208dbdebe35c0df5bd9d30982d6f183dfda4cbab5","src/connection/tests/mod.rs":"280077d4e69faabd2fc1fe03f754096b8b83a8e2b2438fd05b3d7cd924154489","src/connection/tests/null.rs":"38f76a4ea15e6b11634d4374cb0f2a68bd250e5d35831edfce0fa48deeaa420d","src/connection/tests/priority.rs":"dd3504f52d3fce7a96441624bc1c82c733e6bb556b9b79d24d0f4fb4efaf5a9e","src/connection/tests/recovery.rs":"7f28767f3cca2ff60e3dcfa803e12ef043486a222f54681a8faf2ea2fee564a1","src/connection/tests/resumption.rs":"1a0de0993cd325224fc79a3c094d22636d5b122ab1123d16265d4fafb23574bd","src/connection/tests/stream.rs":"8e4af07d8033a951fc57f2afda570a08843b36931eca53ba3781a5992978afb2","src/connection/tests/vn.rs":"550eb6b4d39d5960aafc70037c25a1a0f5db1232ce0ec6080b2c29a731a9574e","src/connection/tests/zerortt.rs":"67f77721e33d9fa2691c5ea3ef4a90935987541d81f0f42fbcfca31e690b352a","src/crypto.rs":"416b73c06fcc2812cc252936bcb039fc13cf0b715e7e22a54314a3f72aee743c","src/events.rs":"6e115f309c5c46f30f6223e1347bea477ada457f8bb2189ecccc6d65483318d6","src/fc.rs":"ec9de1028286870c0adf88a92e1355acf13dede8b1e91179230df3263e3827a9","src/frame.rs":"5c8e5bc21e1052367f7db31523ee422efa4278ccdfc8cd581bb44a50ee205f16","src/lib.rs":"95810fd3ec1b7da9e42f4786e1360a6e40444d69c427065856e751fd1cf411bb","src/pace.rs":"86a674ac4d086148ea297214910458c3705918bd627b996ba8bbb12f2c4bf99e","src/packet/mod.rs":"e21e594c28c568c5d21bfa1dff2903ff3fe6f9dcb98c478eeca120c535d763ef","src/packet/retry.rs":"d5f999485f21b388a7383cd011fc6e96109c1a9fb5aef79b19017df6844271ff","src/path.rs":"610e6ce83da91b785d0690995591fa4da7b5a1add3d0022eea0be5050612cee9","src/qlog.rs":"f3d3661835a29e6023014f7a0996494fc8dc1f2d062154b94346a0c21bbf6fd1","src/quic_datagrams.rs":"3d33ecb9e6e80c77b812e8260fc807352300fb2305a29b797259ae34c52b67c5","src/recovery.rs":"1dadc6717dd133007943e762231a50680087392466904c2f2e6fface084e2ba9","src/recv_stream.rs":"f21ae0bb786901bb7d726a94cb3352607b0057128beaa331808137f2f57a330b","src/rtt.rs":"4635dc0c401b78a1fd9d34da6f9bf7f6e7f5be3a57ed9716e0efc8f0f94f1e47","src/send_stream.rs":"f717f64b75e368cf5fa4ca43078aa7c1b5aff48b4f6266713e6fa7dc458328aa","src/sender.rs":"5f760988bdd6fbbd5956877a97abe7c17370dd531f68b751a9e4e4459583f87b","src/server.rs":"048aaac84e15d49fd25850294759107fe1855bbbc0481c16f8bd888d8f2a8f6d","src/stats.rs":"257ab1242ea2e6bfac0900e6c4bdad794bc67b666930323d24e022e46b9be82b","src/stream_id.rs":"fd07cbb81709a54bdb0659f676ef851cd145c004b817044ede5b21e54fdb60e4","src/streams.rs":"062b1b61edd1a76a86914f2cc1ca007c03edd9136c0c3409d960ddb805805fc6","src/tparams.rs":"10d62ac64865e0606c3c14141f9674631c610b3f042e274e110bdcef5d388491","src/tracking.rs":"f9a9aa01abc79fdd7a2cfb2c3ae342b9ab709e6a2a11076ec5c475fc89c1f598","src/version.rs":"182484ed9ecc2e17cab73cc61914a86a2d206936cab313825ae76fd37eeade77","tests/common/mod.rs":"a6584d268da0157190f8f61842a655ffe81ee68702b3e6569ae300a169080eab","tests/conn_vectors.rs":"997702f4d8b8fa3b987b33077a0eb325e968b25b61fb4703532f8d97e1d4c98c","tests/connection.rs":"d1bc28294d70a5a484eb869162115e399862742caa791749fbd6b923b702b7cc","tests/network.rs":"9e30b8610124250262fceef27d09fdecf2d6e9c3a96b1e676ff4189b9e06d5ba","tests/retry.rs":"3225b64c0c0ca918df12d94df21f6023091e72606701c1dc8c060ce3c1e09c52","tests/server.rs":"cb83de909d858950bfd75a789fc23c3c44fcdf1d965b63800b2c7b498507987f"},"package":null} \ No newline at end of file diff --git a/third_party/rust/neqo-transport/Cargo.toml b/third_party/rust/neqo-transport/Cargo.toml index a309987434..9abdf9b984 100644 --- a/third_party/rust/neqo-transport/Cargo.toml +++ b/third_party/rust/neqo-transport/Cargo.toml @@ -13,7 +13,7 @@ edition = "2021" rust-version = "1.74.0" name = "neqo-transport" -version = "0.7.2" +version = "0.7.5" authors = ["The Neqo Authors "] homepage = "https://github.com/mozilla/neqo/" license = "MIT OR Apache-2.0" @@ -73,7 +73,7 @@ path = "../test-fixture" [features] bench = [] -fuzzing = ["neqo-crypto/fuzzing"] +disable-encryption = ["neqo-crypto/disable-encryption"] [lints.clippy.pedantic] level = "warn" diff --git a/third_party/rust/neqo-transport/benches/range_tracker.rs b/third_party/rust/neqo-transport/benches/range_tracker.rs index c2f78f4874..ee611cf4ea 100644 --- a/third_party/rust/neqo-transport/benches/range_tracker.rs +++ b/third_party/rust/neqo-transport/benches/range_tracker.rs @@ -11,30 +11,32 @@ const CHUNK: u64 = 1000; const END: u64 = 100_000; fn build_coalesce(len: u64) -> RangeTracker { let mut used = RangeTracker::default(); - used.mark_acked(0, CHUNK as usize); - used.mark_sent(CHUNK, END as usize); + let chunk = usize::try_from(CHUNK).expect("should fit"); + used.mark_acked(0, chunk); + used.mark_sent(CHUNK, usize::try_from(END).expect("should fit")); // leave a gap or it will coalesce here for i in 2..=len { // These do not get immediately coalesced when marking since they're not at the end or start - used.mark_acked(i * CHUNK, CHUNK as usize); + used.mark_acked(i * CHUNK, chunk); } used } fn coalesce(c: &mut Criterion, count: u64) { + let chunk = usize::try_from(CHUNK).expect("should fit"); c.bench_function( &format!("coalesce_acked_from_zero {count}+1 entries"), |b| { b.iter_batched_ref( || build_coalesce(count), |used| { - used.mark_acked(CHUNK, CHUNK as usize); + used.mark_acked(CHUNK, chunk); let tail = (count + 1) * CHUNK; - used.mark_sent(tail, CHUNK as usize); - used.mark_acked(tail, CHUNK as usize); + used.mark_sent(tail, chunk); + used.mark_acked(tail, chunk); }, criterion::BatchSize::SmallInput, - ) + ); }, ); } diff --git a/third_party/rust/neqo-transport/benches/rx_stream_orderer.rs b/third_party/rust/neqo-transport/benches/rx_stream_orderer.rs index 0a1e763e97..d58e11ee86 100644 --- a/third_party/rust/neqo-transport/benches/rx_stream_orderer.rs +++ b/third_party/rust/neqo-transport/benches/rx_stream_orderer.rs @@ -11,14 +11,14 @@ fn rx_stream_orderer() { let mut rx = RxStreamOrderer::new(); let data: &[u8] = &[0; 1337]; - for i in 0..100000 { + for i in 0..100_000 { rx.inbound_frame(i * 1337, data); } } fn criterion_benchmark(c: &mut Criterion) { c.bench_function("RxStreamOrderer::inbound_frame()", |b| { - b.iter(rx_stream_orderer) + b.iter(rx_stream_orderer); }); } diff --git a/third_party/rust/neqo-transport/benches/transfer.rs b/third_party/rust/neqo-transport/benches/transfer.rs index 444f738f9c..32959f6cb5 100644 --- a/third_party/rust/neqo-transport/benches/transfer.rs +++ b/third_party/rust/neqo-transport/benches/transfer.rs @@ -6,7 +6,7 @@ use std::time::Duration; -use criterion::{criterion_group, criterion_main, BatchSize::SmallInput, Criterion}; +use criterion::{criterion_group, criterion_main, BatchSize::SmallInput, Criterion, Throughput}; use test_fixture::{ boxed, sim::{ @@ -20,8 +20,11 @@ const ZERO: Duration = Duration::from_millis(0); const JITTER: Duration = Duration::from_millis(10); const TRANSFER_AMOUNT: usize = 1 << 22; // 4Mbyte -fn benchmark_transfer(c: &mut Criterion, label: &str, seed: Option>) { - c.bench_function(label, |b| { +fn benchmark_transfer(c: &mut Criterion, label: &str, seed: &Option>) { + let mut group = c.benchmark_group("transfer"); + group.throughput(Throughput::Bytes(u64::try_from(TRANSFER_AMOUNT).unwrap())); + group.noise_threshold(0.03); + group.bench_function(label, |b| { b.iter_batched( || { let nodes = boxed![ @@ -42,15 +45,16 @@ fn benchmark_transfer(c: &mut Criterion, label: &str, seed: Option CongestionControl for ClassicCongestionControl { let mut is_app_limited = true; let mut new_acked = 0; for pkt in acked_pkts { - qinfo!( + qdebug!( "packet_acked this={:p}, pn={}, ps={}, ignored={}, lost={}, rtt_est={:?}", self, pkt.pn, @@ -179,8 +179,9 @@ impl CongestionControl for ClassicCongestionControl { if pkt.pn < self.first_app_limited { is_app_limited = false; } - assert!(self.bytes_in_flight >= pkt.size); - self.bytes_in_flight -= pkt.size; + // BIF is set to 0 on a path change, but in case that was because of a simple rebinding + // event, we may still get ACKs for packets sent before the rebinding. + self.bytes_in_flight = self.bytes_in_flight.saturating_sub(pkt.size); if !self.after_recovery_start(pkt) { // Do not increase congestion window for packets sent before @@ -198,7 +199,7 @@ impl CongestionControl for ClassicCongestionControl { if is_app_limited { self.cc_algorithm.on_app_limited(); - qinfo!("on_packets_acked this={:p}, limited=1, bytes_in_flight={}, cwnd={}, state={:?}, new_acked={}", self, self.bytes_in_flight, self.congestion_window, self.state, new_acked); + qdebug!("on_packets_acked this={:p}, limited=1, bytes_in_flight={}, cwnd={}, state={:?}, new_acked={}", self, self.bytes_in_flight, self.congestion_window, self.state, new_acked); return; } @@ -208,7 +209,7 @@ impl CongestionControl for ClassicCongestionControl { let increase = min(self.ssthresh - self.congestion_window, self.acked_bytes); self.congestion_window += increase; self.acked_bytes -= increase; - qinfo!([self], "slow start += {}", increase); + qdebug!([self], "slow start += {}", increase); if self.congestion_window == self.ssthresh { // This doesn't look like it is necessary, but it can happen // after persistent congestion. @@ -249,7 +250,7 @@ impl CongestionControl for ClassicCongestionControl { QlogMetric::BytesInFlight(self.bytes_in_flight), ], ); - qinfo!([self], "on_packets_acked this={:p}, limited=0, bytes_in_flight={}, cwnd={}, state={:?}, new_acked={}", self, self.bytes_in_flight, self.congestion_window, self.state, new_acked); + qdebug!([self], "on_packets_acked this={:p}, limited=0, bytes_in_flight={}, cwnd={}, state={:?}, new_acked={}", self, self.bytes_in_flight, self.congestion_window, self.state, new_acked); } /// Update congestion controller state based on lost packets. @@ -265,14 +266,15 @@ impl CongestionControl for ClassicCongestionControl { } for pkt in lost_packets.iter().filter(|pkt| pkt.cc_in_flight()) { - qinfo!( + qdebug!( "packet_lost this={:p}, pn={}, ps={}", self, pkt.pn, pkt.size ); - assert!(self.bytes_in_flight >= pkt.size); - self.bytes_in_flight -= pkt.size; + // BIF is set to 0 on a path change, but in case that was because of a simple rebinding + // event, we may still declare packets lost that were sent before the rebinding. + self.bytes_in_flight = self.bytes_in_flight.saturating_sub(pkt.size); } qlog::metrics_updated( &mut self.qlog, @@ -286,7 +288,7 @@ impl CongestionControl for ClassicCongestionControl { pto, lost_packets, ); - qinfo!( + qdebug!( "on_packets_lost this={:p}, bytes_in_flight={}, cwnd={}, state={:?}", self, self.bytes_in_flight, @@ -335,7 +337,7 @@ impl CongestionControl for ClassicCongestionControl { } self.bytes_in_flight += pkt.size; - qinfo!( + qdebug!( "packet_sent this={:p}, pn={}, ps={}", self, pkt.pn, @@ -498,7 +500,7 @@ impl ClassicCongestionControl { self.congestion_window = max(cwnd, CWND_MIN); self.acked_bytes = acked_bytes; self.ssthresh = self.congestion_window; - qinfo!( + qdebug!( [self], "Cong event -> recovery; cwnd {}, ssthresh {}", self.congestion_window, @@ -516,7 +518,6 @@ impl ClassicCongestionControl { true } - #[allow(clippy::unused_self)] fn app_limited(&self) -> bool { if self.bytes_in_flight >= self.congestion_window { false diff --git a/third_party/rust/neqo-transport/src/connection/dump.rs b/third_party/rust/neqo-transport/src/connection/dump.rs index 8a4f34dbb8..12d337c570 100644 --- a/third_party/rust/neqo-transport/src/connection/dump.rs +++ b/third_party/rust/neqo-transport/src/connection/dump.rs @@ -9,7 +9,7 @@ use std::fmt::Write; -use neqo_common::{qdebug, Decoder}; +use neqo_common::{qdebug, Decoder, IpTos}; use crate::{ connection::Connection, @@ -26,6 +26,7 @@ pub fn dump_packet( pt: PacketType, pn: PacketNumber, payload: &[u8], + tos: IpTos, ) { if log::STATIC_MAX_LEVEL == log::LevelFilter::Off || !log::log_enabled!(log::Level::Debug) { return; @@ -38,9 +39,18 @@ pub fn dump_packet( s.push_str(" [broken]..."); break; }; - if let Some(x) = f.dump() { + let x = f.dump(); + if !x.is_empty() { write!(&mut s, "\n {} {}", dir, &x).unwrap(); } } - qdebug!([conn], "pn={} type={:?} {}{}", pn, pt, path.borrow(), s); + qdebug!( + [conn], + "pn={} type={:?} {} {:?}{}", + pn, + pt, + path.borrow(), + tos, + s + ); } diff --git a/third_party/rust/neqo-transport/src/connection/mod.rs b/third_party/rust/neqo-transport/src/connection/mod.rs index c81a3727c6..8522507a69 100644 --- a/third_party/rust/neqo-transport/src/connection/mod.rs +++ b/third_party/rust/neqo-transport/src/connection/mod.rs @@ -10,7 +10,7 @@ use std::{ cell::RefCell, cmp::{max, min}, fmt::{self, Debug}, - mem, + iter, mem, net::{IpAddr, SocketAddr}, ops::RangeInclusive, rc::{Rc, Weak}, @@ -19,7 +19,7 @@ use std::{ use neqo_common::{ event::Provider as EventProvider, hex, hex_snip_middle, hrtime, qdebug, qerror, qinfo, - qlog::NeqoQlog, qtrace, qwarn, Datagram, Decoder, Encoder, Role, + qlog::NeqoQlog, qtrace, qwarn, Datagram, Decoder, Encoder, IpTos, Role, }; use neqo_crypto::{ agent::CertificateInfo, Agent, AntiReplay, AuthenticationStatus, Cipher, Client, Group, @@ -383,7 +383,6 @@ impl Connection { agent, protocols.iter().map(P::as_ref).map(String::from).collect(), Rc::clone(&tphandler), - conn_params.is_fuzzing(), )?; let stats = StatsCell::default(); @@ -461,7 +460,7 @@ impl Connection { } /// # Errors - /// When the operation fails. + /// When the operation fails. pub fn client_enable_ech(&mut self, ech_config_list: impl AsRef<[u8]>) -> Res<()> { self.crypto.client_enable_ech(ech_config_list) } @@ -778,7 +777,7 @@ impl Connection { }); enc.encode(extra); let records = s.send_ticket(now, enc.as_ref())?; - qinfo!([self], "send session ticket {}", hex(&enc)); + qdebug!([self], "send session ticket {}", hex(&enc)); self.crypto.buffer_records(records)?; } else { unreachable!(); @@ -824,7 +823,7 @@ impl Connection { /// the connection to fail. However, if no packets have been /// exchanged, it's not OK. pub fn authenticated(&mut self, status: AuthenticationStatus, now: Instant) { - qinfo!([self], "Authenticated {:?}", status); + qdebug!([self], "Authenticated {:?}", status); self.crypto.tls.authenticated(status); let res = self.handshake(now, self.version, PacketNumberSpace::Handshake, None); self.absorb_error(now, res); @@ -979,19 +978,16 @@ impl Connection { /// Process new input datagrams on the connection. pub fn process_input(&mut self, d: &Datagram, now: Instant) { - self.input(d, now, now); - self.process_saved(now); - self.streams.cleanup_closed_streams(); + self.process_multiple_input(iter::once(d), now); } /// Process new input datagrams on the connection. pub fn process_multiple_input<'a, I>(&mut self, dgrams: I, now: Instant) where I: IntoIterator, - I::IntoIter: ExactSizeIterator, { - let dgrams = dgrams.into_iter(); - if dgrams.len() == 0 { + let mut dgrams = dgrams.into_iter().peekable(); + if dgrams.peek().is_none() { return; } @@ -1154,7 +1150,7 @@ impl Connection { fn discard_keys(&mut self, space: PacketNumberSpace, now: Instant) { if self.crypto.discard(space) { - qinfo!([self], "Drop packet number space {}", space); + qdebug!([self], "Drop packet number space {}", space); let primary = self.paths.primary(); self.loss_recovery.discard(&primary, space, now); self.acks.drop_space(space); @@ -1492,6 +1488,7 @@ impl Connection { payload.packet_type(), payload.pn(), &payload[..], + d.tos(), ); qlog::packet_received(&mut self.qlog, &packet, &payload); @@ -1552,6 +1549,10 @@ impl Connection { packet: &DecryptedPacket, now: Instant, ) -> Res { + (!packet.is_empty()) + .then_some(()) + .ok_or(Error::ProtocolViolation)?; + // TODO(ekr@rtfm.com): Have the server blow away the initial // crypto state if this fails? Otherwise, we will get a panic // on the assert for doesn't exist. @@ -1560,24 +1561,8 @@ impl Connection { let mut ack_eliciting = false; let mut probing = true; let mut d = Decoder::from(&packet[..]); - let mut consecutive_padding = 0; while d.remaining() > 0 { - let mut f = Frame::decode(&mut d)?; - - // Skip padding - while f == Frame::Padding && d.remaining() > 0 { - consecutive_padding += 1; - f = Frame::decode(&mut d)?; - } - if consecutive_padding > 0 { - qdebug!( - [self], - "PADDING frame repeated {} times", - consecutive_padding - ); - consecutive_padding = 0; - } - + let f = Frame::decode(&mut d)?; ack_eliciting |= f.ack_eliciting(); probing &= f.path_probing(); let t = f.get_type(); @@ -1841,7 +1826,7 @@ impl Connection { | State::Connected | State::Confirmed => { if let Some(path) = self.paths.select_path() { - let res = self.output_path(&path, now); + let res = self.output_path(&path, now, &None); self.capture_error(Some(path), now, 0, res) } else { Ok(SendOption::default()) @@ -1850,7 +1835,16 @@ impl Connection { State::Closing { .. } | State::Draining { .. } | State::Closed(_) => { if let Some(details) = self.state_signaling.close_frame() { let path = Rc::clone(details.path()); - let res = self.output_close(&details); + // In some error cases, we will not be able to make a new, permanent path. + // For example, if we run out of connection IDs and the error results from + // a packet on a new path, we avoid sending (and the privacy risk) rather + // than reuse a connection ID. + let res = if path.borrow().is_temporary() { + assert!(!cfg!(test), "attempting to close with a temporary path"); + Err(Error::InternalError) + } else { + self.output_path(&path, now, &Some(details)) + }; self.capture_error(Some(path), now, 0, res) } else { Ok(SendOption::default()) @@ -1927,62 +1921,6 @@ impl Connection { } } - fn output_close(&mut self, close: &ClosingFrame) -> Res { - let mut encoder = Encoder::with_capacity(256); - let grease_quic_bit = self.can_grease_quic_bit(); - let version = self.version(); - for space in PacketNumberSpace::iter() { - let Some((cspace, tx)) = self.crypto.states.select_tx_mut(self.version, *space) else { - continue; - }; - - let path = close.path().borrow(); - // In some error cases, we will not be able to make a new, permanent path. - // For example, if we run out of connection IDs and the error results from - // a packet on a new path, we avoid sending (and the privacy risk) rather - // than reuse a connection ID. - if path.is_temporary() { - assert!(!cfg!(test), "attempting to close with a temporary path"); - return Err(Error::InternalError); - } - let (_, mut builder) = Self::build_packet_header( - &path, - cspace, - encoder, - tx, - &AddressValidationInfo::None, - version, - grease_quic_bit, - ); - _ = Self::add_packet_number( - &mut builder, - tx, - self.loss_recovery.largest_acknowledged_pn(*space), - ); - // The builder will set the limit to 0 if there isn't enough space for the header. - if builder.is_full() { - encoder = builder.abort(); - break; - } - builder.set_limit(min(path.amplification_limit(), path.mtu()) - tx.expansion()); - debug_assert!(builder.limit() <= 2048); - - // ConnectionError::Application is only allowed at 1RTT. - let sanitized = if *space == PacketNumberSpace::ApplicationData { - None - } else { - close.sanitize() - }; - sanitized - .as_ref() - .unwrap_or(close) - .write_frame(&mut builder); - encoder = builder.build(tx)?; - } - - Ok(SendOption::Yes(close.path().borrow().datagram(encoder))) - } - /// Write the frames that are exchanged in the application data space. /// The order of calls here determines the relative priority of frames. fn write_appdata_frames( @@ -2203,7 +2141,12 @@ impl Connection { /// Build a datagram, possibly from multiple packets (for different PN /// spaces) and each containing 1+ frames. #[allow(clippy::too_many_lines)] // Yeah, that's just the way it is. - fn output_path(&mut self, path: &PathRef, now: Instant) -> Res { + fn output_path( + &mut self, + path: &PathRef, + now: Instant, + closing_frame: &Option, + ) -> Res { let mut initial_sent = None; let mut needs_padding = false; let grease_quic_bit = self.can_grease_quic_bit(); @@ -2256,8 +2199,23 @@ impl Connection { // Add frames to the packet. let payload_start = builder.len(); - let (tokens, ack_eliciting, padded) = - self.write_frames(path, *space, &profile, &mut builder, now); + let (mut tokens, mut ack_eliciting, mut padded) = (Vec::new(), false, false); + if let Some(ref close) = closing_frame { + // ConnectionError::Application is only allowed at 1RTT. + let sanitized = if *space == PacketNumberSpace::ApplicationData { + None + } else { + close.sanitize() + }; + sanitized + .as_ref() + .unwrap_or(close) + .write_frame(&mut builder); + self.stats.borrow_mut().frame_tx.connection_close += 1; + } else { + (tokens, ack_eliciting, padded) = + self.write_frames(path, *space, &profile, &mut builder, now); + } if builder.packet_empty() { // Nothing to include in this packet. encoder = builder.abort(); @@ -2271,6 +2229,7 @@ impl Connection { pt, pn, &builder.as_ref()[payload_start..], + IpTos::default(), // TODO: set from path ); qlog::packet_sent( &mut self.qlog, @@ -2323,7 +2282,7 @@ impl Connection { } if encoder.is_empty() { - qinfo!("TX blocked, profile={:?} ", profile); + qdebug!("TX blocked, profile={:?} ", profile); Ok(SendOption::No(profile.paced())) } else { // Perform additional padding for Initial packets as necessary. @@ -2337,6 +2296,8 @@ impl Connection { mtu ); initial.size += mtu - packets.len(); + // These zeros aren't padding frames, they are an invalid all-zero coalesced + // packet, which is why we don't increase `frame_tx.padding` count here. packets.resize(mtu, 0); } self.loss_recovery.on_packet_sent(path, initial); @@ -2367,7 +2328,7 @@ impl Connection { } fn client_start(&mut self, now: Instant) -> Res<()> { - qinfo!([self], "client_start"); + qdebug!([self], "client_start"); debug_assert_eq!(self.role, Role::Client); qlog::client_connection_started(&mut self.qlog, &self.paths.primary()); qlog::client_version_information_initiated(&mut self.qlog, self.conn_params.get_versions()); @@ -2599,7 +2560,7 @@ impl Connection { fn confirm_version(&mut self, v: Version) { if self.version != v { - qinfo!([self], "Compatible upgrade {:?} ==> {:?}", self.version, v); + qdebug!([self], "Compatible upgrade {:?} ==> {:?}", self.version, v); } self.crypto.confirm_version(v); self.version = v; @@ -2694,9 +2655,8 @@ impl Connection { .input_frame(&frame, &mut self.stats.borrow_mut().frame_rx); } match frame { - Frame::Padding => { - // Note: This counts contiguous padding as a single frame. - self.stats.borrow_mut().frame_rx.padding += 1; + Frame::Padding(length) => { + self.stats.borrow_mut().frame_rx.padding += usize::from(length); } Frame::Ping => { // If we get a PING and there are outstanding CRYPTO frames, @@ -2899,7 +2859,7 @@ impl Connection { R: IntoIterator> + Debug, R::IntoIter: ExactSizeIterator, { - qinfo!([self], "Rx ACK space={}, ranges={:?}", space, ack_ranges); + qdebug!([self], "Rx ACK space={}, ranges={:?}", space, ack_ranges); let (acked_packets, lost_packets) = self.loss_recovery.on_ack_received( &self.paths.primary(), @@ -2953,7 +2913,7 @@ impl Connection { } fn set_connected(&mut self, now: Instant) -> Res<()> { - qinfo!([self], "TLS connection complete"); + qdebug!([self], "TLS connection complete"); if self.crypto.tls.info().map(SecretAgentInfo::alpn).is_none() { qwarn!([self], "No ALPN. Closing connection."); // 120 = no_application_protocol @@ -2996,7 +2956,7 @@ impl Connection { fn set_state(&mut self, state: State) { if state > self.state { - qinfo!([self], "State change from {:?} -> {:?}", self.state, state); + qdebug!([self], "State change from {:?} -> {:?}", self.state, state); self.state = state.clone(); if self.state.closed() { self.streams.clear_streams(); diff --git a/third_party/rust/neqo-transport/src/connection/params.rs b/third_party/rust/neqo-transport/src/connection/params.rs index 72d1efa3ee..d8aa617024 100644 --- a/third_party/rust/neqo-transport/src/connection/params.rs +++ b/third_party/rust/neqo-transport/src/connection/params.rs @@ -77,7 +77,6 @@ pub struct ConnectionParameters { outgoing_datagram_queue: usize, incoming_datagram_queue: usize, fast_pto: u8, - fuzzing: bool, grease: bool, pacing: bool, } @@ -100,7 +99,6 @@ impl Default for ConnectionParameters { outgoing_datagram_queue: MAX_QUEUED_DATAGRAMS_DEFAULT, incoming_datagram_queue: MAX_QUEUED_DATAGRAMS_DEFAULT, fast_pto: FAST_PTO_SCALE, - fuzzing: false, grease: true, pacing: true, } @@ -324,17 +322,6 @@ impl ConnectionParameters { self } - #[must_use] - pub fn is_fuzzing(&self) -> bool { - self.fuzzing - } - - #[must_use] - pub fn fuzzing(mut self, enable: bool) -> Self { - self.fuzzing = enable; - self - } - #[must_use] pub fn is_greasing(&self) -> bool { self.grease diff --git a/third_party/rust/neqo-transport/src/connection/state.rs b/third_party/rust/neqo-transport/src/connection/state.rs index 9789151d3f..cc2f6e30d2 100644 --- a/third_party/rust/neqo-transport/src/connection/state.rs +++ b/third_party/rust/neqo-transport/src/connection/state.rs @@ -209,7 +209,10 @@ pub enum StateSignaling { impl StateSignaling { pub fn handshake_done(&mut self) { if !matches!(self, Self::Idle) { - debug_assert!(false, "StateSignaling must be in Idle state."); + debug_assert!( + false, + "StateSignaling must be in Idle state but is in {self:?} state.", + ); return; } *self = Self::HandshakeDone; diff --git a/third_party/rust/neqo-transport/src/connection/tests/fuzzing.rs b/third_party/rust/neqo-transport/src/connection/tests/fuzzing.rs deleted file mode 100644 index 9924c06fa4..0000000000 --- a/third_party/rust/neqo-transport/src/connection/tests/fuzzing.rs +++ /dev/null @@ -1,42 +0,0 @@ -// Licensed under the Apache License, Version 2.0 or the MIT license -// , at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - -#![cfg(feature = "fuzzing")] - -use neqo_crypto::FIXED_TAG_FUZZING; -use test_fixture::now; - -use super::{connect_force_idle, default_client, default_server}; -use crate::StreamType; - -#[test] -fn no_encryption() { - const DATA_CLIENT: &[u8] = &[2; 40]; - const DATA_SERVER: &[u8] = &[3; 50]; - let mut client = default_client(); - let mut server = default_server(); - connect_force_idle(&mut client, &mut server); - - let stream_id = client.stream_create(StreamType::BiDi).unwrap(); - - client.stream_send(stream_id, DATA_CLIENT).unwrap(); - let client_pkt = client.process_output(now()).dgram().unwrap(); - assert!(client_pkt[..client_pkt.len() - FIXED_TAG_FUZZING.len()].ends_with(DATA_CLIENT)); - - server.process_input(&client_pkt, now()); - let mut buf = vec![0; 100]; - let (len, _) = server.stream_recv(stream_id, &mut buf).unwrap(); - assert_eq!(len, DATA_CLIENT.len()); - assert_eq!(&buf[..len], DATA_CLIENT); - server.stream_send(stream_id, DATA_SERVER).unwrap(); - let server_pkt = server.process_output(now()).dgram().unwrap(); - assert!(server_pkt[..server_pkt.len() - FIXED_TAG_FUZZING.len()].ends_with(DATA_SERVER)); - - client.process_input(&server_pkt, now()); - let (len, _) = client.stream_recv(stream_id, &mut buf).unwrap(); - assert_eq!(len, DATA_SERVER.len()); - assert_eq!(&buf[..len], DATA_SERVER); -} diff --git a/third_party/rust/neqo-transport/src/connection/tests/handshake.rs b/third_party/rust/neqo-transport/src/connection/tests/handshake.rs index af0352ce90..f2103523ec 100644 --- a/third_party/rust/neqo-transport/src/connection/tests/handshake.rs +++ b/third_party/rust/neqo-transport/src/connection/tests/handshake.rs @@ -16,9 +16,10 @@ use neqo_common::{event::Provider, qdebug, Datagram}; use neqo_crypto::{ constants::TLS_CHACHA20_POLY1305_SHA256, generate_ech_keys, AuthenticationStatus, }; +#[cfg(not(feature = "disable-encryption"))] +use test_fixture::datagram; use test_fixture::{ - assertions, assertions::assert_coalesced_0rtt, datagram, fixture_init, now, split_datagram, - DEFAULT_ADDR, + assertions, assertions::assert_coalesced_0rtt, fixture_init, now, split_datagram, DEFAULT_ADDR, }; use super::{ @@ -458,7 +459,7 @@ fn coalesce_05rtt() { assert_eq!(client.stats().dropped_rx, 0); // No Initial padding. assert_eq!(client.stats().packets_rx, 4); assert_eq!(client.stats().saved_datagrams, 1); - assert_eq!(client.stats().frame_rx.padding, 1); // Padding uses frames. + assert!(client.stats().frame_rx.padding > 0); // Padding uses frames. // Allow the handshake to complete. now += RTT / 2; @@ -605,7 +606,7 @@ fn reorder_1rtt() { } } -#[cfg(not(feature = "fuzzing"))] +#[cfg(not(feature = "disable-encryption"))] #[test] fn corrupted_initial() { let mut client = default_client(); @@ -808,7 +809,7 @@ fn anti_amplification() { assert_eq!(*server.state(), State::Confirmed); } -#[cfg(not(feature = "fuzzing"))] +#[cfg(not(feature = "disable-encryption"))] #[test] fn garbage_initial() { let mut client = default_client(); diff --git a/third_party/rust/neqo-transport/src/connection/tests/mod.rs b/third_party/rust/neqo-transport/src/connection/tests/mod.rs index b6ce08f8d1..c8c87a0df0 100644 --- a/third_party/rust/neqo-transport/src/connection/tests/mod.rs +++ b/third_party/rust/neqo-transport/src/connection/tests/mod.rs @@ -37,11 +37,11 @@ mod ackrate; mod cc; mod close; mod datagram; -mod fuzzing; mod handshake; mod idle; mod keys; mod migration; +mod null; mod priority; mod recovery; mod resumption; @@ -170,12 +170,17 @@ impl crate::connection::test_internal::FrameWriter for PingWriter { } } +trait DatagramModifier: FnMut(Datagram) -> Option {} + +impl DatagramModifier for T where T: FnMut(Datagram) -> Option {} + /// Drive the handshake between the client and server. -fn handshake( +fn handshake_with_modifier( client: &mut Connection, server: &mut Connection, now: Instant, rtt: Duration, + mut modifier: impl DatagramModifier, ) -> Instant { let mut a = client; let mut b = server; @@ -212,7 +217,11 @@ fn handshake( did_ping[a.role()] = true; } assert!(had_input || output.is_some()); - input = output; + if let Some(d) = output { + input = modifier(d); + } else { + input = output; + } qtrace!("handshake: t += {:?}", rtt / 2); now += rtt / 2; mem::swap(&mut a, &mut b); @@ -223,6 +232,15 @@ fn handshake( now } +fn handshake( + client: &mut Connection, + server: &mut Connection, + now: Instant, + rtt: Duration, +) -> Instant { + handshake_with_modifier(client, server, now, rtt, Some) +} + fn connect_fail( client: &mut Connection, server: &mut Connection, @@ -234,11 +252,12 @@ fn connect_fail( assert_error(server, &ConnectionError::Transport(server_error)); } -fn connect_with_rtt( +fn connect_with_rtt_and_modifier( client: &mut Connection, server: &mut Connection, now: Instant, rtt: Duration, + modifier: impl DatagramModifier, ) -> Instant { fn check_rtt(stats: &Stats, rtt: Duration) { assert_eq!(stats.rtt, rtt); @@ -246,7 +265,7 @@ fn connect_with_rtt( let n = stats.frame_rx.ack + usize::from(stats.rtt_init_guess); assert_eq!(stats.rttvar, rttvar_after_n_updates(n, rtt)); } - let now = handshake(client, server, now, rtt); + let now = handshake_with_modifier(client, server, now, rtt, modifier); assert_eq!(*client.state(), State::Confirmed); assert_eq!(*server.state(), State::Confirmed); @@ -255,6 +274,15 @@ fn connect_with_rtt( now } +fn connect_with_rtt( + client: &mut Connection, + server: &mut Connection, + now: Instant, + rtt: Duration, +) -> Instant { + connect_with_rtt_and_modifier(client, server, now, rtt, Some) +} + fn connect(client: &mut Connection, server: &mut Connection) { connect_with_rtt(client, server, now(), Duration::new(0, 0)); } @@ -301,8 +329,13 @@ fn assert_idle(client: &mut Connection, server: &mut Connection, rtt: Duration, } /// Connect with an RTT and then force both peers to be idle. -fn connect_rtt_idle(client: &mut Connection, server: &mut Connection, rtt: Duration) -> Instant { - let now = connect_with_rtt(client, server, now(), rtt); +fn connect_rtt_idle_with_modifier( + client: &mut Connection, + server: &mut Connection, + rtt: Duration, + modifier: impl DatagramModifier, +) -> Instant { + let now = connect_with_rtt_and_modifier(client, server, now(), rtt, modifier); assert_idle(client, server, rtt, now); // Drain events from both as well. _ = client.events().count(); @@ -311,8 +344,20 @@ fn connect_rtt_idle(client: &mut Connection, server: &mut Connection, rtt: Durat now } +fn connect_rtt_idle(client: &mut Connection, server: &mut Connection, rtt: Duration) -> Instant { + connect_rtt_idle_with_modifier(client, server, rtt, Some) +} + +fn connect_force_idle_with_modifier( + client: &mut Connection, + server: &mut Connection, + modifier: impl DatagramModifier, +) { + connect_rtt_idle_with_modifier(client, server, Duration::new(0, 0), modifier); +} + fn connect_force_idle(client: &mut Connection, server: &mut Connection) { - connect_rtt_idle(client, server, Duration::new(0, 0)); + connect_force_idle_with_modifier(client, server, Some); } fn fill_stream(c: &mut Connection, stream: StreamId) { @@ -524,12 +569,14 @@ fn assert_full_cwnd(packets: &[Datagram], cwnd: usize) { } /// Send something on a stream from `sender` to `receiver`, maybe allowing for pacing. +/// Takes a modifier function that can be used to modify the datagram before it is sent. /// Return the resulting datagram and the new time. #[must_use] -fn send_something_paced( +fn send_something_paced_with_modifier( sender: &mut Connection, mut now: Instant, allow_pacing: bool, + mut modifier: impl DatagramModifier, ) -> (Datagram, Instant) { let stream_id = sender.stream_create(StreamType::UniDi).unwrap(); assert!(sender.stream_send(stream_id, DEFAULT_STREAM_DATA).is_ok()); @@ -544,16 +591,32 @@ fn send_something_paced( .dgram() .expect("send_something: should have something to send") } - Output::Datagram(d) => d, + Output::Datagram(d) => modifier(d).unwrap(), Output::None => panic!("send_something: got Output::None"), }; (dgram, now) } +fn send_something_paced( + sender: &mut Connection, + now: Instant, + allow_pacing: bool, +) -> (Datagram, Instant) { + send_something_paced_with_modifier(sender, now, allow_pacing, Some) +} + +fn send_something_with_modifier( + sender: &mut Connection, + now: Instant, + modifier: impl DatagramModifier, +) -> Datagram { + send_something_paced_with_modifier(sender, now, false, modifier).0 +} + /// Send something on a stream from `sender` to `receiver`. /// Return the resulting datagram. fn send_something(sender: &mut Connection, now: Instant) -> Datagram { - send_something_paced(sender, now, false).0 + send_something_with_modifier(sender, now, Some) } /// Send something on a stream from `sender` to `receiver`. diff --git a/third_party/rust/neqo-transport/src/connection/tests/null.rs b/third_party/rust/neqo-transport/src/connection/tests/null.rs new file mode 100644 index 0000000000..e4d60445c6 --- /dev/null +++ b/third_party/rust/neqo-transport/src/connection/tests/null.rs @@ -0,0 +1,42 @@ +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +#![cfg(feature = "disable-encryption")] + +use neqo_crypto::aead_null::AEAD_NULL_TAG; +use test_fixture::now; + +use super::{connect_force_idle, default_client, default_server}; +use crate::StreamType; + +#[test] +fn no_encryption() { + const DATA_CLIENT: &[u8] = &[2; 40]; + const DATA_SERVER: &[u8] = &[3; 50]; + let mut client = default_client(); + let mut server = default_server(); + connect_force_idle(&mut client, &mut server); + + let stream_id = client.stream_create(StreamType::BiDi).unwrap(); + + client.stream_send(stream_id, DATA_CLIENT).unwrap(); + let client_pkt = client.process_output(now()).dgram().unwrap(); + assert!(client_pkt[..client_pkt.len() - AEAD_NULL_TAG.len()].ends_with(DATA_CLIENT)); + + server.process_input(&client_pkt, now()); + let mut buf = vec![0; 100]; + let (len, _) = server.stream_recv(stream_id, &mut buf).unwrap(); + assert_eq!(len, DATA_CLIENT.len()); + assert_eq!(&buf[..len], DATA_CLIENT); + server.stream_send(stream_id, DATA_SERVER).unwrap(); + let server_pkt = server.process_output(now()).dgram().unwrap(); + assert!(server_pkt[..server_pkt.len() - AEAD_NULL_TAG.len()].ends_with(DATA_SERVER)); + + client.process_input(&server_pkt, now()); + let (len, _) = client.stream_recv(stream_id, &mut buf).unwrap(); + assert_eq!(len, DATA_SERVER.len()); + assert_eq!(&buf[..len], DATA_SERVER); +} diff --git a/third_party/rust/neqo-transport/src/connection/tests/stream.rs b/third_party/rust/neqo-transport/src/connection/tests/stream.rs index f469866d50..66d3bf32f3 100644 --- a/third_party/rust/neqo-transport/src/connection/tests/stream.rs +++ b/third_party/rust/neqo-transport/src/connection/tests/stream.rs @@ -116,12 +116,6 @@ fn transfer() { assert!(fin3); } -#[derive(PartialEq, Eq, PartialOrd, Ord)] -struct IdEntry { - sendorder: StreamOrder, - stream_id: StreamId, -} - // tests stream sendorder priorization fn sendorder_test(order_of_sendorder: &[Option]) { let mut client = default_client(); diff --git a/third_party/rust/neqo-transport/src/crypto.rs b/third_party/rust/neqo-transport/src/crypto.rs index 9840eaa1e1..60d056f2d2 100644 --- a/third_party/rust/neqo-transport/src/crypto.rs +++ b/third_party/rust/neqo-transport/src/crypto.rs @@ -69,7 +69,6 @@ impl Crypto { mut agent: Agent, protocols: Vec, tphandler: TpHandler, - fuzzing: bool, ) -> Res { agent.set_version_range(TLS_VERSION_1_3, TLS_VERSION_1_3)?; agent.set_ciphers(&[ @@ -102,7 +101,6 @@ impl Crypto { tls: agent, streams: CryptoStreams::default(), states: CryptoStates { - fuzzing, ..CryptoStates::default() }, }) @@ -317,7 +315,7 @@ impl Crypto { } pub fn acked(&mut self, token: &CryptoRecoveryToken) { - qinfo!( + qdebug!( "Acked crypto frame space={} offset={} length={}", token.space, token.offset, @@ -367,7 +365,7 @@ impl Crypto { }); enc.encode_vvec(new_token.unwrap_or(&[])); enc.encode(t.as_ref()); - qinfo!("resumption token {}", hex_snip_middle(enc.as_ref())); + qdebug!("resumption token {}", hex_snip_middle(enc.as_ref())); Some(ResumptionToken::new(enc.into(), t.expiration_time())) } else { None @@ -420,7 +418,6 @@ pub struct CryptoDxState { /// The total number of operations that are remaining before the keys /// become exhausted and can't be used any more. invocations: PacketNumber, - fuzzing: bool, } impl CryptoDxState { @@ -431,9 +428,8 @@ impl CryptoDxState { epoch: Epoch, secret: &SymKey, cipher: Cipher, - fuzzing: bool, ) -> Self { - qinfo!( + qdebug!( "Making {:?} {} CryptoDxState, v={:?} cipher={}", direction, epoch, @@ -445,19 +441,11 @@ impl CryptoDxState { version, direction, epoch: usize::from(epoch), - aead: Aead::new( - fuzzing, - TLS_VERSION_1_3, - cipher, - secret, - version.label_prefix(), - ) - .unwrap(), + aead: Aead::new(TLS_VERSION_1_3, cipher, secret, version.label_prefix()).unwrap(), hpkey: HpKey::extract(TLS_VERSION_1_3, cipher, secret, &hplabel).unwrap(), used_pn: 0..0, min_pn: 0, invocations: Self::limit(direction, cipher), - fuzzing, } } @@ -466,7 +454,6 @@ impl CryptoDxState { direction: CryptoDxDirection, label: &str, dcid: &[u8], - fuzzing: bool, ) -> Self { qtrace!("new_initial {:?} {}", version, ConnectionIdRef::from(dcid)); let salt = version.initial_salt(); @@ -482,14 +469,7 @@ impl CryptoDxState { let secret = hkdf::expand_label(TLS_VERSION_1_3, cipher, &initial_secret, &[], label).unwrap(); - Self::new( - version, - direction, - TLS_EPOCH_INITIAL, - &secret, - cipher, - fuzzing, - ) + Self::new(version, direction, TLS_EPOCH_INITIAL, &secret, cipher) } /// Determine the confidentiality and integrity limits for the cipher. @@ -549,7 +529,6 @@ impl CryptoDxState { direction: self.direction, epoch: self.epoch + 1, aead: Aead::new( - self.fuzzing, TLS_VERSION_1_3, cipher, next_secret, @@ -560,7 +539,6 @@ impl CryptoDxState { used_pn: pn..pn, min_pn: pn, invocations, - fuzzing: self.fuzzing, } } @@ -696,7 +674,7 @@ impl CryptoDxState { Ok(res.to_vec()) } - #[cfg(all(test, not(feature = "fuzzing")))] + #[cfg(all(test, not(feature = "disable-encryption")))] pub(crate) fn test_default() -> Self { // This matches the value in packet.rs const CLIENT_CID: &[u8] = &[0x83, 0x94, 0xc8, 0xf0, 0x3e, 0x51, 0x57, 0x08]; @@ -705,7 +683,6 @@ impl CryptoDxState { CryptoDxDirection::Write, "server in", CLIENT_CID, - false, ) } @@ -759,7 +736,6 @@ pub(crate) struct CryptoDxAppData { cipher: Cipher, // Not the secret used to create `self.dx`, but the one needed for the next iteration. next_secret: SymKey, - fuzzing: bool, } impl CryptoDxAppData { @@ -768,20 +744,11 @@ impl CryptoDxAppData { dir: CryptoDxDirection, secret: &SymKey, cipher: Cipher, - fuzzing: bool, ) -> Res { Ok(Self { - dx: CryptoDxState::new( - version, - dir, - TLS_EPOCH_APPLICATION_DATA, - secret, - cipher, - fuzzing, - ), + dx: CryptoDxState::new(version, dir, TLS_EPOCH_APPLICATION_DATA, secret, cipher), cipher, next_secret: Self::update_secret(cipher, secret)?, - fuzzing, }) } @@ -791,7 +758,7 @@ impl CryptoDxAppData { } pub fn next(&self) -> Res { - if self.dx.epoch == usize::max_value() { + if self.dx.epoch == usize::MAX { // Guard against too many key updates. return Err(Error::KeysExhausted); } @@ -800,7 +767,6 @@ impl CryptoDxAppData { dx: self.dx.next(&self.next_secret, self.cipher), cipher: self.cipher, next_secret, - fuzzing: self.fuzzing, }) } @@ -834,7 +800,6 @@ pub struct CryptoStates { // If this is set, then we have noticed a genuine update. // Once this time passes, we should switch in new keys. read_update_time: Option, - fuzzing: bool, } impl CryptoStates { @@ -980,7 +945,7 @@ impl CryptoStates { }; for v in versions { - qinfo!( + qdebug!( [self], "Creating initial cipher state v={:?}, role={:?} dcid={}", v, @@ -989,20 +954,8 @@ impl CryptoStates { ); let mut initial = CryptoState { - tx: CryptoDxState::new_initial( - *v, - CryptoDxDirection::Write, - write, - dcid, - self.fuzzing, - ), - rx: CryptoDxState::new_initial( - *v, - CryptoDxDirection::Read, - read, - dcid, - self.fuzzing, - ), + tx: CryptoDxState::new_initial(*v, CryptoDxDirection::Write, write, dcid), + rx: CryptoDxState::new_initial(*v, CryptoDxDirection::Read, read, dcid), }; if let Some(prev) = self.initials.get(v) { qinfo!( @@ -1056,7 +1009,6 @@ impl CryptoStates { TLS_EPOCH_ZERO_RTT, secret, cipher, - self.fuzzing, )); } @@ -1097,7 +1049,6 @@ impl CryptoStates { TLS_EPOCH_HANDSHAKE, write_secret, cipher, - self.fuzzing, ), rx: CryptoDxState::new( version, @@ -1105,7 +1056,6 @@ impl CryptoStates { TLS_EPOCH_HANDSHAKE, read_secret, cipher, - self.fuzzing, ), }); } @@ -1113,13 +1063,7 @@ impl CryptoStates { pub fn set_application_write_key(&mut self, version: Version, secret: &SymKey) -> Res<()> { debug_assert!(self.app_write.is_none()); debug_assert_ne!(self.cipher, 0); - let mut app = CryptoDxAppData::new( - version, - CryptoDxDirection::Write, - secret, - self.cipher, - self.fuzzing, - )?; + let mut app = CryptoDxAppData::new(version, CryptoDxDirection::Write, secret, self.cipher)?; if let Some(z) = &self.zero_rtt { if z.direction == CryptoDxDirection::Write { app.dx.continuation(z)?; @@ -1138,13 +1082,7 @@ impl CryptoStates { ) -> Res<()> { debug_assert!(self.app_write.is_some(), "should have write keys installed"); debug_assert!(self.app_read.is_none()); - let mut app = CryptoDxAppData::new( - version, - CryptoDxDirection::Read, - secret, - self.cipher, - self.fuzzing, - )?; + let mut app = CryptoDxAppData::new(version, CryptoDxDirection::Read, secret, self.cipher)?; if let Some(z) = &self.zero_rtt { if z.direction == CryptoDxDirection::Read { app.dx.continuation(z)?; @@ -1286,7 +1224,7 @@ impl CryptoStates { } /// Make some state for removing protection in tests. - #[cfg(not(feature = "fuzzing"))] + #[cfg(not(feature = "disable-encryption"))] #[cfg(test)] pub(crate) fn test_default() -> Self { let read = |epoch| { @@ -1299,7 +1237,6 @@ impl CryptoStates { dx: read(epoch), cipher: TLS_AES_128_GCM_SHA256, next_secret: hkdf::import_key(TLS_VERSION_1_3, &[0xaa; 32]).unwrap(), - fuzzing: false, }; let mut initials = HashMap::new(); initials.insert( @@ -1319,11 +1256,10 @@ impl CryptoStates { app_read: Some(app_read(3)), app_read_next: Some(app_read(4)), read_update_time: None, - fuzzing: false, } } - #[cfg(all(not(feature = "fuzzing"), test))] + #[cfg(all(not(feature = "disable-encryption"), test))] pub(crate) fn test_chacha() -> Self { const SECRET: &[u8] = &[ 0x9a, 0xc3, 0x12, 0xa7, 0xf8, 0x77, 0x46, 0x8e, 0xbe, 0x69, 0x42, 0x27, 0x48, 0xad, @@ -1337,7 +1273,6 @@ impl CryptoStates { direction: CryptoDxDirection::Read, epoch, aead: Aead::new( - false, TLS_VERSION_1_3, TLS_CHACHA20_POLY1305_SHA256, &secret, @@ -1354,11 +1289,9 @@ impl CryptoStates { used_pn: 0..645_971_972, min_pn: 0, invocations: 10, - fuzzing: false, }, cipher: TLS_CHACHA20_POLY1305_SHA256, next_secret: secret.clone(), - fuzzing: false, }; Self { initials: HashMap::new(), @@ -1369,7 +1302,6 @@ impl CryptoStates { app_read: Some(app_read(3)), app_read_next: Some(app_read(4)), read_update_time: None, - fuzzing: false, } } } diff --git a/third_party/rust/neqo-transport/src/frame.rs b/third_party/rust/neqo-transport/src/frame.rs index b3bb024a2c..d84eb61ce8 100644 --- a/third_party/rust/neqo-transport/src/frame.rs +++ b/third_party/rust/neqo-transport/src/frame.rs @@ -20,7 +20,7 @@ use crate::{ #[allow(clippy::module_name_repetitions)] pub type FrameType = u64; -const FRAME_TYPE_PADDING: FrameType = 0x0; +pub const FRAME_TYPE_PADDING: FrameType = 0x0; pub const FRAME_TYPE_PING: FrameType = 0x1; pub const FRAME_TYPE_ACK: FrameType = 0x2; const FRAME_TYPE_ACK_ECN: FrameType = 0x3; @@ -95,6 +95,12 @@ impl From for CloseError { } } +impl From for Error { + fn from(_err: std::array::TryFromSliceError) -> Self { + Self::FrameEncodingError + } +} + #[derive(PartialEq, Eq, Debug, Default, Clone)] pub struct AckRange { pub(crate) gap: u64, @@ -103,7 +109,7 @@ pub struct AckRange { #[derive(PartialEq, Eq, Debug, Clone)] pub enum Frame<'a> { - Padding, + Padding(u16), Ping, Ack { largest_acknowledged: u64, @@ -213,9 +219,10 @@ impl<'a> Frame<'a> { } } + #[must_use] pub fn get_type(&self) -> FrameType { match self { - Self::Padding => FRAME_TYPE_PADDING, + Self::Padding { .. } => FRAME_TYPE_PADDING, Self::Ping => FRAME_TYPE_PING, Self::Ack { .. } => FRAME_TYPE_ACK, // We don't do ACK ECN. Self::ResetStream { .. } => FRAME_TYPE_RESET_STREAM, @@ -254,6 +261,7 @@ impl<'a> Frame<'a> { } } + #[must_use] pub fn is_stream(&self) -> bool { matches!( self, @@ -269,6 +277,7 @@ impl<'a> Frame<'a> { ) } + #[must_use] pub fn stream_type(fin: bool, nonzero_offset: bool, fill: bool) -> u64 { let mut t = FRAME_TYPE_STREAM; if fin { @@ -285,19 +294,21 @@ impl<'a> Frame<'a> { /// If the frame causes a recipient to generate an ACK within its /// advertised maximum acknowledgement delay. + #[must_use] pub fn ack_eliciting(&self) -> bool { !matches!( self, - Self::Ack { .. } | Self::Padding | Self::ConnectionClose { .. } + Self::Ack { .. } | Self::Padding { .. } | Self::ConnectionClose { .. } ) } /// If the frame can be sent in a path probe /// without initiating migration to that path. + #[must_use] pub fn path_probing(&self) -> bool { matches!( self, - Self::Padding + Self::Padding { .. } | Self::NewConnectionId { .. } | Self::PathChallenge { .. } | Self::PathResponse { .. } @@ -307,6 +318,10 @@ impl<'a> Frame<'a> { /// Converts `AckRanges` as encoded in a ACK frame (see -transport /// 19.3.1) into ranges of acked packets (end, start), inclusive of /// start and end values. + /// + /// # Errors + /// + /// Returns an error if the ranges are invalid. pub fn decode_ack_frame( largest_acked: u64, first_ack_range: u64, @@ -347,36 +362,36 @@ impl<'a> Frame<'a> { Ok(acked_ranges) } - pub fn dump(&self) -> Option { + #[must_use] + pub fn dump(&self) -> String { match self { - Self::Crypto { offset, data } => Some(format!( - "Crypto {{ offset: {}, len: {} }}", - offset, - data.len() - )), + Self::Crypto { offset, data } => { + format!("Crypto {{ offset: {}, len: {} }}", offset, data.len()) + } Self::Stream { stream_id, offset, fill, data, fin, - } => Some(format!( + } => format!( "Stream {{ stream_id: {}, offset: {}, len: {}{}, fin: {} }}", stream_id.as_u64(), offset, if *fill { ">>" } else { "" }, data.len(), fin, - )), - Self::Padding => None, - Self::Datagram { data, .. } => Some(format!("Datagram {{ len: {} }}", data.len())), - _ => Some(format!("{self:?}")), + ), + Self::Padding(length) => format!("Padding {{ len: {length} }}"), + Self::Datagram { data, .. } => format!("Datagram {{ len: {} }}", data.len()), + _ => format!("{self:?}"), } } + #[must_use] pub fn is_allowed(&self, pt: PacketType) -> bool { match self { - Self::Padding | Self::Ping => true, + Self::Padding { .. } | Self::Ping => true, Self::Crypto { .. } | Self::Ack { .. } | Self::ConnectionClose { @@ -388,6 +403,9 @@ impl<'a> Frame<'a> { } } + /// # Errors + /// + /// Returns an error if the frame cannot be decoded. #[allow(clippy::too_many_lines)] // Yeah, but it's a nice match statement. pub fn decode(dec: &mut Decoder<'a>) -> Res { /// Maximum ACK Range Count in ACK Frame @@ -409,13 +427,23 @@ impl<'a> Frame<'a> { } // TODO(ekr@rtfm.com): check for minimal encoding - let t = d(dec.decode_varint())?; + let t = dv(dec)?; match t { - FRAME_TYPE_PADDING => Ok(Self::Padding), + FRAME_TYPE_PADDING => { + let mut length: u16 = 1; + while let Some(b) = dec.peek_byte() { + if u64::from(b) != FRAME_TYPE_PADDING { + break; + } + length += 1; + dec.skip(1); + } + Ok(Self::Padding(length)) + } FRAME_TYPE_PING => Ok(Self::Ping), FRAME_TYPE_RESET_STREAM => Ok(Self::ResetStream { stream_id: StreamId::from(dv(dec)?), - application_error_code: d(dec.decode_varint())?, + application_error_code: dv(dec)?, final_size: match dec.decode_varint() { Some(v) => v, _ => return Err(Error::NoMoreData), @@ -457,12 +485,12 @@ impl<'a> Frame<'a> { } FRAME_TYPE_STOP_SENDING => Ok(Self::StopSending { stream_id: StreamId::from(dv(dec)?), - application_error_code: d(dec.decode_varint())?, + application_error_code: dv(dec)?, }), FRAME_TYPE_CRYPTO => { let offset = dv(dec)?; let data = d(dec.decode_vvec())?; - if offset + u64::try_from(data.len()).unwrap() > ((1 << 62) - 1) { + if offset + u64::try_from(data.len())? > ((1 << 62) - 1) { return Err(Error::FrameEncodingError); } Ok(Self::Crypto { offset, data }) @@ -489,7 +517,7 @@ impl<'a> Frame<'a> { qtrace!("STREAM frame, with length"); d(dec.decode_vvec())? }; - if o + u64::try_from(data.len()).unwrap() > ((1 << 62) - 1) { + if o + u64::try_from(data.len())? > ((1 << 62) - 1) { return Err(Error::FrameEncodingError); } Ok(Self::Stream { @@ -538,7 +566,7 @@ impl<'a> Frame<'a> { return Err(Error::DecodingFrame); } let srt = d(dec.decode(16))?; - let stateless_reset_token = <&[_; 16]>::try_from(srt).unwrap(); + let stateless_reset_token = <&[_; 16]>::try_from(srt)?; Ok(Self::NewConnectionId { sequence_number, @@ -563,7 +591,7 @@ impl<'a> Frame<'a> { Ok(Self::PathResponse { data: datav }) } FRAME_TYPE_CONNECTION_CLOSE_TRANSPORT | FRAME_TYPE_CONNECTION_CLOSE_APPLICATION => { - let error_code = CloseError::from_type_bit(t, d(dec.decode_varint())?); + let error_code = CloseError::from_type_bit(t, dv(dec)?); let frame_type = if t == FRAME_TYPE_CONNECTION_CLOSE_TRANSPORT { dv(dec)? } else { @@ -631,8 +659,10 @@ mod tests { #[test] fn padding() { - let f = Frame::Padding; + let f = Frame::Padding(1); just_dec(&f, "00"); + let f = Frame::Padding(2); + just_dec(&f, "0000"); } #[test] @@ -888,8 +918,8 @@ mod tests { #[test] fn test_compare() { - let f1 = Frame::Padding; - let f2 = Frame::Padding; + let f1 = Frame::Padding(1); + let f2 = Frame::Padding(1); let f3 = Frame::Crypto { offset: 0, data: &[1, 2, 3], diff --git a/third_party/rust/neqo-transport/src/lib.rs b/third_party/rust/neqo-transport/src/lib.rs index be482c466f..5488472b58 100644 --- a/third_party/rust/neqo-transport/src/lib.rs +++ b/third_party/rust/neqo-transport/src/lib.rs @@ -6,7 +6,7 @@ #![allow(clippy::module_name_repetitions)] // This lint doesn't work here. -use neqo_common::qinfo; +use neqo_common::qwarn; use neqo_crypto::Error as CryptoError; mod ackrate; @@ -70,8 +70,8 @@ const ERROR_AEAD_LIMIT_REACHED: TransportError = 15; #[derive(Clone, Debug, PartialEq, PartialOrd, Ord, Eq)] pub enum Error { NoError, - // Each time tihe error is return a different parameter is supply. - // This will be use to distinguish each occurance of this error. + // Each time this error is returned a different parameter is supplied. + // This will be used to distinguish each occurance of this error. InternalError, ConnectionRefused, FlowControlError, @@ -165,7 +165,7 @@ impl Error { impl From for Error { fn from(err: CryptoError) -> Self { - qinfo!("Crypto operation failed {:?}", err); + qwarn!("Crypto operation failed {:?}", err); match err { CryptoError::EchRetry(config) => Self::EchRetry(config), _ => Self::CryptoError(err), diff --git a/third_party/rust/neqo-transport/src/packet/mod.rs b/third_party/rust/neqo-transport/src/packet/mod.rs index 8458f69779..ce611a9664 100644 --- a/third_party/rust/neqo-transport/src/packet/mod.rs +++ b/third_party/rust/neqo-transport/src/packet/mod.rs @@ -18,6 +18,7 @@ use neqo_crypto::random; use crate::{ cid::{ConnectionId, ConnectionIdDecoder, ConnectionIdRef, MAX_CONNECTION_ID_LEN}, crypto::{CryptoDxState, CryptoSpace, CryptoStates}, + frame::FRAME_TYPE_PADDING, version::{Version, WireVersion}, Error, Res, }; @@ -157,7 +158,7 @@ impl PacketBuilder { } Self { encoder, - pn: u64::max_value(), + pn: u64::MAX, header: header_start..header_start, offsets: PacketBuilderOffsets { first_byte_mask: PACKET_HP_MASK_SHORT, @@ -200,7 +201,7 @@ impl PacketBuilder { Self { encoder, - pn: u64::max_value(), + pn: u64::MAX, header: header_start..header_start, offsets: PacketBuilderOffsets { first_byte_mask: PACKET_HP_MASK_LONG, @@ -255,9 +256,14 @@ impl PacketBuilder { /// Maybe pad with "PADDING" frames. /// Only does so if padding was needed and this is a short packet. /// Returns true if padding was added. + /// + /// # Panics + /// + /// Cannot happen. pub fn pad(&mut self) -> bool { if self.padding && !self.is_long() { - self.encoder.pad_to(self.limit, 0); + self.encoder + .pad_to(self.limit, FRAME_TYPE_PADDING.try_into().unwrap()); true } else { false @@ -288,6 +294,10 @@ impl PacketBuilder { /// The length is filled in after calling `build`. /// Does nothing if there isn't 4 bytes available other than render this builder /// unusable; if `remaining()` returns 0 at any point, call `abort()`. + /// + /// # Panics + /// + /// This will panic if the packet number length is too large. pub fn pn(&mut self, pn: PacketNumber, pn_len: usize) { if self.remaining() < 4 { self.limit = 0; @@ -352,6 +362,10 @@ impl PacketBuilder { } /// Build the packet and return the encoder. + /// + /// # Errors + /// + /// This will return an error if the packet is too large. pub fn build(mut self, crypto: &mut CryptoDxState) -> Res { if self.len() > self.limit { qwarn!("Packet contents are more than the limit"); @@ -376,7 +390,9 @@ impl PacketBuilder { // Calculate the mask. let offset = SAMPLE_OFFSET - self.offsets.pn.len(); - assert!(offset + SAMPLE_SIZE <= ciphertext.len()); + if offset + SAMPLE_SIZE > ciphertext.len() { + return Err(Error::InternalError); + } let sample = &ciphertext[offset..offset + SAMPLE_SIZE]; let mask = crypto.compute_mask(sample)?; @@ -410,6 +426,10 @@ impl PacketBuilder { /// As this is a simple packet, this is just an associated function. /// As Retry is odd (it has to be constructed with leading bytes), /// this returns a [`Vec`] rather than building on an encoder. + /// + /// # Errors + /// + /// This will return an error if AEAD encrypt fails. #[allow(clippy::similar_names)] // scid and dcid are fine here. pub fn retry( version: Version, @@ -443,6 +463,7 @@ impl PacketBuilder { /// Make a Version Negotiation packet. #[allow(clippy::similar_names)] // scid and dcid are fine here. + #[must_use] pub fn version_negotiation( dcid: &[u8], scid: &[u8], @@ -534,7 +555,10 @@ impl<'a> PublicPacket<'a> { if packet_type == PacketType::Retry { let header_len = decoder.offset(); let expansion = retry::expansion(version); - let token = Self::opt(decoder.decode(decoder.remaining() - expansion))?; + let token = decoder + .remaining() + .checked_sub(expansion) + .map_or(Err(Error::InvalidPacket), |v| Self::opt(decoder.decode(v)))?; if token.is_empty() { return Err(Error::InvalidPacket); } @@ -554,6 +578,10 @@ impl<'a> PublicPacket<'a> { /// Decode the common parts of a packet. This provides minimal parsing and validation. /// Returns a tuple of a `PublicPacket` and a slice with any remainder from the datagram. + /// + /// # Errors + /// + /// This will return an error if the packet could not be decoded. #[allow(clippy::similar_names)] // For dcid and scid, which are fine. pub fn decode(data: &'a [u8], dcid_decoder: &dyn ConnectionIdDecoder) -> Res<(Self, &'a [u8])> { let mut decoder = Decoder::new(data); @@ -585,7 +613,7 @@ impl<'a> PublicPacket<'a> { } // Generic long header. - let version = WireVersion::try_from(Self::opt(decoder.decode_uint(4))?).unwrap(); + let version = WireVersion::try_from(Self::opt(decoder.decode_uint(4))?)?; let dcid = ConnectionIdRef::from(Self::opt(decoder.decode_vec(1))?); let scid = ConnectionIdRef::from(Self::opt(decoder.decode_vec(1))?); @@ -645,11 +673,14 @@ impl<'a> PublicPacket<'a> { } /// Validate the given packet as though it were a retry. + #[must_use] pub fn is_valid_retry(&self, odcid: &ConnectionId) -> bool { if self.packet_type != PacketType::Retry { return false; } - let version = self.version().unwrap(); + let Some(version) = self.version() else { + return false; + }; let expansion = retry::expansion(version); if self.data.len() <= expansion { return false; @@ -665,6 +696,7 @@ impl<'a> PublicPacket<'a> { .unwrap_or(false) } + #[must_use] pub fn is_valid_initial(&self) -> bool { // Packet has to be an initial, with a DCID of 8 bytes, or a token. // Note: the Server class validates the token and checks the length. @@ -672,32 +704,42 @@ impl<'a> PublicPacket<'a> { && (self.dcid().len() >= 8 || !self.token.is_empty()) } + #[must_use] pub fn packet_type(&self) -> PacketType { self.packet_type } + #[must_use] pub fn dcid(&self) -> ConnectionIdRef<'a> { self.dcid } + /// # Panics + /// + /// This will panic if called for a short header packet. + #[must_use] pub fn scid(&self) -> ConnectionIdRef<'a> { self.scid .expect("should only be called for long header packets") } + #[must_use] pub fn token(&self) -> &'a [u8] { self.token } + #[must_use] pub fn version(&self) -> Option { self.version.and_then(|v| Version::try_from(v).ok()) } + #[must_use] pub fn wire_version(&self) -> WireVersion { debug_assert!(self.version.is_some()); self.version.unwrap_or(0) } + #[must_use] pub fn len(&self) -> usize { self.data.len() } @@ -725,14 +767,10 @@ impl<'a> PublicPacket<'a> { assert_ne!(self.packet_type, PacketType::Retry); assert_ne!(self.packet_type, PacketType::VersionNegotiation); - qtrace!( - "unmask hdr={}", - hex(&self.data[..self.header_len + SAMPLE_OFFSET]) - ); - let sample_offset = self.header_len + SAMPLE_OFFSET; let mask = if let Some(sample) = self.data.get(sample_offset..(sample_offset + SAMPLE_SIZE)) { + qtrace!("unmask hdr={}", hex(&self.data[..sample_offset])); crypto.compute_mask(sample) } else { Err(Error::NoMoreData) @@ -776,6 +814,9 @@ impl<'a> PublicPacket<'a> { )) } + /// # Errors + /// + /// This will return an error if the packet cannot be decrypted. pub fn decrypt(&self, crypto: &mut CryptoStates, release_at: Instant) -> Res { let cspace: CryptoSpace = self.packet_type.into(); // When we don't have a version, the crypto code doesn't need a version @@ -790,7 +831,9 @@ impl<'a> PublicPacket<'a> { // too small (which is public information). let (key_phase, pn, header, body) = self.decrypt_header(rx)?; qtrace!([rx], "decoded header: {:?}", header); - let rx = crypto.rx(version, cspace, key_phase).unwrap(); + let Some(rx) = crypto.rx(version, cspace, key_phase) else { + return Err(Error::DecryptError); + }; let version = rx.version(); // Version fixup; see above. let d = rx.decrypt(pn, &header, body)?; // If this is the first packet ever successfully decrypted @@ -813,8 +856,14 @@ impl<'a> PublicPacket<'a> { } } + /// # Errors + /// + /// This will return an error if the packet is not a version negotiation packet + /// or if the versions cannot be decoded. pub fn supported_versions(&self) -> Res> { - assert_eq!(self.packet_type, PacketType::VersionNegotiation); + if self.packet_type != PacketType::VersionNegotiation { + return Err(Error::InvalidPacket); + } let mut decoder = Decoder::new(&self.data[self.header_len..]); let mut res = Vec::new(); while decoder.remaining() > 0 { @@ -845,14 +894,17 @@ pub struct DecryptedPacket { } impl DecryptedPacket { + #[must_use] pub fn version(&self) -> Version { self.version } + #[must_use] pub fn packet_type(&self) -> PacketType { self.pt } + #[must_use] pub fn pn(&self) -> PacketNumber { self.pn } @@ -866,7 +918,7 @@ impl Deref for DecryptedPacket { } } -#[cfg(all(test, not(feature = "fuzzing")))] +#[cfg(all(test, not(feature = "disable-encryption")))] mod tests { use neqo_common::Encoder; use test_fixture::{fixture_init, now}; @@ -1469,4 +1521,21 @@ mod tests { assert_eq!(decrypted.pn(), 654_360_564); assert_eq!(&decrypted[..], &[0x01]); } + + #[test] + fn decode_empty() { + neqo_crypto::init().unwrap(); + let res = PublicPacket::decode(&[], &EmptyConnectionIdGenerator::default()); + assert!(res.is_err()); + } + + #[test] + fn decode_too_short() { + neqo_crypto::init().unwrap(); + let res = PublicPacket::decode( + &[179, 255, 0, 0, 32, 0, 0], + &EmptyConnectionIdGenerator::default(), + ); + assert!(res.is_err()); + } } diff --git a/third_party/rust/neqo-transport/src/packet/retry.rs b/third_party/rust/neqo-transport/src/packet/retry.rs index 72036d3b49..71193b9100 100644 --- a/third_party/rust/neqo-transport/src/packet/retry.rs +++ b/third_party/rust/neqo-transport/src/packet/retry.rs @@ -18,7 +18,6 @@ fn make_aead(version: Version) -> Aead { let secret = hkdf::import_key(TLS_VERSION_1_3, version.retry_secret()).unwrap(); Aead::new( - false, TLS_VERSION_1_3, TLS_AES_128_GCM_SHA256, &secret, diff --git a/third_party/rust/neqo-transport/src/path.rs b/third_party/rust/neqo-transport/src/path.rs index 4e8d9958ab..50e458ff36 100644 --- a/third_party/rust/neqo-transport/src/path.rs +++ b/third_party/rust/neqo-transport/src/path.rs @@ -216,7 +216,7 @@ impl Paths { /// to a migration from a peer, in which case the old path needs to be probed. #[must_use] fn select_primary(&mut self, path: &PathRef) -> Option { - qinfo!([path.borrow()], "set as primary path"); + qdebug!([path.borrow()], "set as primary path"); let old_path = self.primary.replace(Rc::clone(path)).map(|old| { old.borrow_mut().set_primary(false); old diff --git a/third_party/rust/neqo-transport/src/qlog.rs b/third_party/rust/neqo-transport/src/qlog.rs index 2572966104..a8ad986d2a 100644 --- a/third_party/rust/neqo-transport/src/qlog.rs +++ b/third_party/rust/neqo-transport/src/qlog.rs @@ -195,7 +195,7 @@ pub fn packet_sent( ) { qlog.add_event_with_stream(|stream| { let mut d = Decoder::from(body); - let header = PacketHeader::with_type(to_qlog_pkt_type(pt), Some(pn), None, None, None); + let header = PacketHeader::with_type(pt.into(), Some(pn), None, None, None); let raw = RawInfo { length: Some(plen as u64), payload_length: None, @@ -205,7 +205,7 @@ pub fn packet_sent( let mut frames = SmallVec::new(); while d.remaining() > 0 { if let Ok(f) = Frame::decode(&mut d) { - frames.push(frame_to_qlogframe(&f)); + frames.push(QuicFrame::from(&f)); } else { qinfo!("qlog: invalid frame"); break; @@ -231,13 +231,8 @@ pub fn packet_sent( pub fn packet_dropped(qlog: &mut NeqoQlog, public_packet: &PublicPacket) { qlog.add_event_data(|| { - let header = PacketHeader::with_type( - to_qlog_pkt_type(public_packet.packet_type()), - None, - None, - None, - None, - ); + let header = + PacketHeader::with_type(public_packet.packet_type().into(), None, None, None, None); let raw = RawInfo { length: Some(public_packet.len() as u64), payload_length: None, @@ -259,8 +254,7 @@ pub fn packet_dropped(qlog: &mut NeqoQlog, public_packet: &PublicPacket) { pub fn packets_lost(qlog: &mut NeqoQlog, pkts: &[SentPacket]) { qlog.add_event_with_stream(|stream| { for pkt in pkts { - let header = - PacketHeader::with_type(to_qlog_pkt_type(pkt.pt), Some(pkt.pn), None, None, None); + let header = PacketHeader::with_type(pkt.pt.into(), Some(pkt.pn), None, None, None); let ev_data = EventData::PacketLost(PacketLost { header: Some(header), @@ -283,7 +277,7 @@ pub fn packet_received( let mut d = Decoder::from(&payload[..]); let header = PacketHeader::with_type( - to_qlog_pkt_type(public_packet.packet_type()), + public_packet.packet_type().into(), Some(payload.pn()), None, None, @@ -299,7 +293,7 @@ pub fn packet_received( while d.remaining() > 0 { if let Ok(f) = Frame::decode(&mut d) { - frames.push(frame_to_qlogframe(&f)); + frames.push(QuicFrame::from(&f)); } else { qinfo!("qlog: invalid frame"); break; @@ -393,173 +387,180 @@ pub fn metrics_updated(qlog: &mut NeqoQlog, updated_metrics: &[QlogMetric]) { #[allow(clippy::too_many_lines)] // Yeah, but it's a nice match. #[allow(clippy::cast_possible_truncation, clippy::cast_precision_loss)] // No choice here. -fn frame_to_qlogframe(frame: &Frame) -> QuicFrame { - match frame { - Frame::Padding => QuicFrame::Padding, - Frame::Ping => QuicFrame::Ping, - Frame::Ack { - largest_acknowledged, - ack_delay, - first_ack_range, - ack_ranges, - } => { - let ranges = - Frame::decode_ack_frame(*largest_acknowledged, *first_ack_range, ack_ranges).ok(); - - let acked_ranges = ranges.map(|all| { - AckedRanges::Double( - all.into_iter() - .map(RangeInclusive::into_inner) - .collect::>(), - ) - }); - - QuicFrame::Ack { - ack_delay: Some(*ack_delay as f32 / 1000.0), - acked_ranges, - ect1: None, - ect0: None, - ce: None, +impl From<&Frame<'_>> for QuicFrame { + fn from(frame: &Frame) -> Self { + match frame { + // TODO: Add payload length to `QuicFrame::Padding` once + // https://github.com/cloudflare/quiche/pull/1745 is available via the qlog crate. + Frame::Padding { .. } => QuicFrame::Padding, + Frame::Ping => QuicFrame::Ping, + Frame::Ack { + largest_acknowledged, + ack_delay, + first_ack_range, + ack_ranges, + } => { + let ranges = + Frame::decode_ack_frame(*largest_acknowledged, *first_ack_range, ack_ranges) + .ok(); + + let acked_ranges = ranges.map(|all| { + AckedRanges::Double( + all.into_iter() + .map(RangeInclusive::into_inner) + .collect::>(), + ) + }); + + QuicFrame::Ack { + ack_delay: Some(*ack_delay as f32 / 1000.0), + acked_ranges, + ect1: None, + ect0: None, + ce: None, + } } - } - Frame::ResetStream { - stream_id, - application_error_code, - final_size, - } => QuicFrame::ResetStream { - stream_id: stream_id.as_u64(), - error_code: *application_error_code, - final_size: *final_size, - }, - Frame::StopSending { - stream_id, - application_error_code, - } => QuicFrame::StopSending { - stream_id: stream_id.as_u64(), - error_code: *application_error_code, - }, - Frame::Crypto { offset, data } => QuicFrame::Crypto { - offset: *offset, - length: data.len() as u64, - }, - Frame::NewToken { token } => QuicFrame::NewToken { - token: qlog::Token { - ty: Some(qlog::TokenType::Retry), - details: None, - raw: Some(RawInfo { - data: Some(hex(token)), - length: Some(token.len() as u64), - payload_length: None, - }), + Frame::ResetStream { + stream_id, + application_error_code, + final_size, + } => QuicFrame::ResetStream { + stream_id: stream_id.as_u64(), + error_code: *application_error_code, + final_size: *final_size, + }, + Frame::StopSending { + stream_id, + application_error_code, + } => QuicFrame::StopSending { + stream_id: stream_id.as_u64(), + error_code: *application_error_code, + }, + Frame::Crypto { offset, data } => QuicFrame::Crypto { + offset: *offset, + length: data.len() as u64, + }, + Frame::NewToken { token } => QuicFrame::NewToken { + token: qlog::Token { + ty: Some(qlog::TokenType::Retry), + details: None, + raw: Some(RawInfo { + data: Some(hex(token)), + length: Some(token.len() as u64), + payload_length: None, + }), + }, }, - }, - Frame::Stream { - fin, - stream_id, - offset, - data, - .. - } => QuicFrame::Stream { - stream_id: stream_id.as_u64(), - offset: *offset, - length: data.len() as u64, - fin: Some(*fin), - raw: None, - }, - Frame::MaxData { maximum_data } => QuicFrame::MaxData { - maximum: *maximum_data, - }, - Frame::MaxStreamData { - stream_id, - maximum_stream_data, - } => QuicFrame::MaxStreamData { - stream_id: stream_id.as_u64(), - maximum: *maximum_stream_data, - }, - Frame::MaxStreams { - stream_type, - maximum_streams, - } => QuicFrame::MaxStreams { - stream_type: match stream_type { - NeqoStreamType::BiDi => StreamType::Bidirectional, - NeqoStreamType::UniDi => StreamType::Unidirectional, + Frame::Stream { + fin, + stream_id, + offset, + data, + .. + } => QuicFrame::Stream { + stream_id: stream_id.as_u64(), + offset: *offset, + length: data.len() as u64, + fin: Some(*fin), + raw: None, }, - maximum: *maximum_streams, - }, - Frame::DataBlocked { data_limit } => QuicFrame::DataBlocked { limit: *data_limit }, - Frame::StreamDataBlocked { - stream_id, - stream_data_limit, - } => QuicFrame::StreamDataBlocked { - stream_id: stream_id.as_u64(), - limit: *stream_data_limit, - }, - Frame::StreamsBlocked { - stream_type, - stream_limit, - } => QuicFrame::StreamsBlocked { - stream_type: match stream_type { - NeqoStreamType::BiDi => StreamType::Bidirectional, - NeqoStreamType::UniDi => StreamType::Unidirectional, + Frame::MaxData { maximum_data } => QuicFrame::MaxData { + maximum: *maximum_data, }, - limit: *stream_limit, - }, - Frame::NewConnectionId { - sequence_number, - retire_prior, - connection_id, - stateless_reset_token, - } => QuicFrame::NewConnectionId { - sequence_number: *sequence_number as u32, - retire_prior_to: *retire_prior as u32, - connection_id_length: Some(connection_id.len() as u8), - connection_id: hex(connection_id), - stateless_reset_token: Some(hex(stateless_reset_token)), - }, - Frame::RetireConnectionId { sequence_number } => QuicFrame::RetireConnectionId { - sequence_number: *sequence_number as u32, - }, - Frame::PathChallenge { data } => QuicFrame::PathChallenge { - data: Some(hex(data)), - }, - Frame::PathResponse { data } => QuicFrame::PathResponse { - data: Some(hex(data)), - }, - Frame::ConnectionClose { - error_code, - frame_type, - reason_phrase, - } => QuicFrame::ConnectionClose { - error_space: match error_code { - CloseError::Transport(_) => Some(ErrorSpace::TransportError), - CloseError::Application(_) => Some(ErrorSpace::ApplicationError), + Frame::MaxStreamData { + stream_id, + maximum_stream_data, + } => QuicFrame::MaxStreamData { + stream_id: stream_id.as_u64(), + maximum: *maximum_stream_data, }, - error_code: Some(error_code.code()), - error_code_value: Some(0), - reason: Some(String::from_utf8_lossy(reason_phrase).to_string()), - trigger_frame_type: Some(*frame_type), - }, - Frame::HandshakeDone => QuicFrame::HandshakeDone, - Frame::AckFrequency { .. } => QuicFrame::Unknown { - frame_type_value: None, - raw_frame_type: frame.get_type(), - raw: None, - }, - Frame::Datagram { data, .. } => QuicFrame::Datagram { - length: data.len() as u64, - raw: None, - }, + Frame::MaxStreams { + stream_type, + maximum_streams, + } => QuicFrame::MaxStreams { + stream_type: match stream_type { + NeqoStreamType::BiDi => StreamType::Bidirectional, + NeqoStreamType::UniDi => StreamType::Unidirectional, + }, + maximum: *maximum_streams, + }, + Frame::DataBlocked { data_limit } => QuicFrame::DataBlocked { limit: *data_limit }, + Frame::StreamDataBlocked { + stream_id, + stream_data_limit, + } => QuicFrame::StreamDataBlocked { + stream_id: stream_id.as_u64(), + limit: *stream_data_limit, + }, + Frame::StreamsBlocked { + stream_type, + stream_limit, + } => QuicFrame::StreamsBlocked { + stream_type: match stream_type { + NeqoStreamType::BiDi => StreamType::Bidirectional, + NeqoStreamType::UniDi => StreamType::Unidirectional, + }, + limit: *stream_limit, + }, + Frame::NewConnectionId { + sequence_number, + retire_prior, + connection_id, + stateless_reset_token, + } => QuicFrame::NewConnectionId { + sequence_number: *sequence_number as u32, + retire_prior_to: *retire_prior as u32, + connection_id_length: Some(connection_id.len() as u8), + connection_id: hex(connection_id), + stateless_reset_token: Some(hex(stateless_reset_token)), + }, + Frame::RetireConnectionId { sequence_number } => QuicFrame::RetireConnectionId { + sequence_number: *sequence_number as u32, + }, + Frame::PathChallenge { data } => QuicFrame::PathChallenge { + data: Some(hex(data)), + }, + Frame::PathResponse { data } => QuicFrame::PathResponse { + data: Some(hex(data)), + }, + Frame::ConnectionClose { + error_code, + frame_type, + reason_phrase, + } => QuicFrame::ConnectionClose { + error_space: match error_code { + CloseError::Transport(_) => Some(ErrorSpace::TransportError), + CloseError::Application(_) => Some(ErrorSpace::ApplicationError), + }, + error_code: Some(error_code.code()), + error_code_value: Some(0), + reason: Some(String::from_utf8_lossy(reason_phrase).to_string()), + trigger_frame_type: Some(*frame_type), + }, + Frame::HandshakeDone => QuicFrame::HandshakeDone, + Frame::AckFrequency { .. } => QuicFrame::Unknown { + frame_type_value: None, + raw_frame_type: frame.get_type(), + raw: None, + }, + Frame::Datagram { data, .. } => QuicFrame::Datagram { + length: data.len() as u64, + raw: None, + }, + } } } -fn to_qlog_pkt_type(ptype: PacketType) -> qlog::events::quic::PacketType { - match ptype { - PacketType::Initial => qlog::events::quic::PacketType::Initial, - PacketType::Handshake => qlog::events::quic::PacketType::Handshake, - PacketType::ZeroRtt => qlog::events::quic::PacketType::ZeroRtt, - PacketType::Short => qlog::events::quic::PacketType::OneRtt, - PacketType::Retry => qlog::events::quic::PacketType::Retry, - PacketType::VersionNegotiation => qlog::events::quic::PacketType::VersionNegotiation, - PacketType::OtherVersion => qlog::events::quic::PacketType::Unknown, +impl From for qlog::events::quic::PacketType { + fn from(value: PacketType) -> Self { + match value { + PacketType::Initial => qlog::events::quic::PacketType::Initial, + PacketType::Handshake => qlog::events::quic::PacketType::Handshake, + PacketType::ZeroRtt => qlog::events::quic::PacketType::ZeroRtt, + PacketType::Short => qlog::events::quic::PacketType::OneRtt, + PacketType::Retry => qlog::events::quic::PacketType::Retry, + PacketType::VersionNegotiation => qlog::events::quic::PacketType::VersionNegotiation, + PacketType::OtherVersion => qlog::events::quic::PacketType::Unknown, + } } } diff --git a/third_party/rust/neqo-transport/src/stats.rs b/third_party/rust/neqo-transport/src/stats.rs index 9eff503dcf..0a61097010 100644 --- a/third_party/rust/neqo-transport/src/stats.rs +++ b/third_party/rust/neqo-transport/src/stats.rs @@ -14,7 +14,7 @@ use std::{ time::Duration, }; -use neqo_common::qinfo; +use neqo_common::qwarn; use crate::packet::PacketNumber; @@ -168,7 +168,7 @@ impl Stats { pub fn pkt_dropped(&mut self, reason: impl AsRef) { self.dropped_rx += 1; - qinfo!( + qwarn!( [self.info], "Dropped received packet: {}; Total: {}", reason.as_ref(), @@ -206,7 +206,7 @@ impl Debug for Stats { " tx: {} lost {} lateack {} ptoack {}", self.packets_tx, self.lost, self.late_ack, self.pto_ack )?; - writeln!(f, " resumed: {} ", self.resumed)?; + writeln!(f, " resumed: {}", self.resumed)?; writeln!(f, " frames rx:")?; self.frame_rx.fmt(f)?; writeln!(f, " frames tx:")?; diff --git a/third_party/rust/neqo-transport/tests/common/mod.rs b/third_party/rust/neqo-transport/tests/common/mod.rs index faff216eb9..e36e66f753 100644 --- a/third_party/rust/neqo-transport/tests/common/mod.rs +++ b/third_party/rust/neqo-transport/tests/common/mod.rs @@ -146,14 +146,7 @@ pub fn initial_aead_and_hp(dcid: &[u8], role: Role) -> (Aead, HpKey) { ) .unwrap(); ( - Aead::new( - false, - TLS_VERSION_1_3, - TLS_AES_128_GCM_SHA256, - &secret, - "quic ", - ) - .unwrap(), + Aead::new(TLS_VERSION_1_3, TLS_AES_128_GCM_SHA256, &secret, "quic ").unwrap(), HpKey::extract(TLS_VERSION_1_3, TLS_AES_128_GCM_SHA256, &secret, "quic hp").unwrap(), ) } diff --git a/third_party/rust/neqo-transport/tests/conn_vectors.rs b/third_party/rust/neqo-transport/tests/conn_vectors.rs index f478883075..86fe9d36fc 100644 --- a/third_party/rust/neqo-transport/tests/conn_vectors.rs +++ b/third_party/rust/neqo-transport/tests/conn_vectors.rs @@ -6,7 +6,7 @@ // Tests with the test vectors from the spec. -#![cfg(not(feature = "fuzzing"))] +#![cfg(not(feature = "disable-encryption"))] use std::{cell::RefCell, rc::Rc}; diff --git a/third_party/rust/neqo-transport/tests/connection.rs b/third_party/rust/neqo-transport/tests/connection.rs index 0b91fcf306..b8877b946d 100644 --- a/third_party/rust/neqo-transport/tests/connection.rs +++ b/third_party/rust/neqo-transport/tests/connection.rs @@ -127,6 +127,76 @@ fn reorder_server_initial() { assert_eq!(*client.state(), State::Confirmed); } +fn set_payload(server_packet: &Option, client_dcid: &[u8], payload: &[u8]) -> Datagram { + let (server_initial, _server_hs) = split_datagram(server_packet.as_ref().unwrap()); + let (protected_header, _, _, orig_payload) = + decode_initial_header(&server_initial, Role::Server); + + // Now decrypt the packet. + let (aead, hp) = initial_aead_and_hp(client_dcid, Role::Server); + let (mut header, pn) = remove_header_protection(&hp, protected_header, orig_payload); + assert_eq!(pn, 0); + // Re-encode the packet number as four bytes, so we have enough material for the header + // protection sample if payload is empty. + let pn_pos = header.len() - 2; + header[pn_pos] = u8::try_from(4 + aead.expansion()).unwrap(); + header.resize(header.len() + 3, 0); + header[0] |= 0b0000_0011; // Set the packet number length to 4. + + // And build a packet containing the given payload. + let mut packet = header.clone(); + packet.resize(header.len() + payload.len() + aead.expansion(), 0); + aead.encrypt(pn, &header, payload, &mut packet[header.len()..]) + .unwrap(); + apply_header_protection(&hp, &mut packet, protected_header.len()..header.len()); + Datagram::new( + server_initial.source(), + server_initial.destination(), + server_initial.tos(), + server_initial.ttl(), + packet, + ) +} + +/// Test that the stack treats a packet without any frames as a protocol violation. +#[test] +fn packet_without_frames() { + let mut client = new_client( + ConnectionParameters::default().versions(Version::Version1, vec![Version::Version1]), + ); + let mut server = default_server(); + + let client_initial = client.process_output(now()); + let (_, client_dcid, _, _) = + decode_initial_header(client_initial.as_dgram_ref().unwrap(), Role::Client); + + let server_packet = server.process(client_initial.as_dgram_ref(), now()).dgram(); + let modified = set_payload(&server_packet, client_dcid, &[]); + client.process_input(&modified, now()); + assert_eq!( + client.state(), + &State::Closed(ConnectionError::Transport(Error::ProtocolViolation)) + ); +} + +/// Test that the stack permits a packet containing only padding. +#[test] +fn packet_with_only_padding() { + let mut client = new_client( + ConnectionParameters::default().versions(Version::Version1, vec![Version::Version1]), + ); + let mut server = default_server(); + + let client_initial = client.process_output(now()); + let (_, client_dcid, _, _) = + decode_initial_header(client_initial.as_dgram_ref().unwrap(), Role::Client); + + let server_packet = server.process(client_initial.as_dgram_ref(), now()).dgram(); + let modified = set_payload(&server_packet, client_dcid, &[0]); + client.process_input(&modified, now()); + assert_eq!(client.state(), &State::WaitInitial); +} + /// Overflow the crypto buffer. #[allow(clippy::similar_names)] // For ..._scid and ..._dcid, which are fine. #[test] diff --git a/third_party/rust/neqo-transport/tests/retry.rs b/third_party/rust/neqo-transport/tests/retry.rs index e583fcae0f..36eff71e7b 100644 --- a/third_party/rust/neqo-transport/tests/retry.rs +++ b/third_party/rust/neqo-transport/tests/retry.rs @@ -4,7 +4,7 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. -#![cfg(not(feature = "fuzzing"))] +#![cfg(not(feature = "disable-encryption"))] mod common; diff --git a/third_party/rust/oneshot-uniffi/.cargo-checksum.json b/third_party/rust/oneshot-uniffi/.cargo-checksum.json index 6b252c6773..985beded51 100644 --- a/third_party/rust/oneshot-uniffi/.cargo-checksum.json +++ b/third_party/rust/oneshot-uniffi/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"CHANGELOG.md":"e1165d97c283b915d87e22f209494be39933723a0b544e725f69cfa5cef3876c","Cargo.lock":"7625529900ca1e3626b90e74ef268b31e06da85e8334a885711fdfd80821ddda","Cargo.toml":"81dde8ad3180c7b97325a6a67bfbefb145590606de9008d881c4e04808865f0a","README.md":"811ea1c958d5a65583d0223b7ab09bb282e7a51ed60f9a2cb90ef6d555325a68","benches/benches.rs":"67dcc916d0b7e28e396c28dac0499726366e1cb10e9991948d1c881a5abf5faa","check_mem_leaks.sh":"c1ab6ef27997c7f971352ab1c86a184004843c499bc24925da953aefcf1c624c","examples/recv_before_send.rs":"9a3cabcc2878990b61787d0048061b382555a8cd1a08b1ddec63a6e8a4a31e56","examples/recv_before_send_then_drop_sender.rs":"14706c6b4308a690662ceaa47f1699588bd833b3ec020eb9f42f220f3ffc7ae7","examples/recv_ref_before_send.rs":"43699f4720c46b5f138c260b866eb708ddf616e2b442ffa74a97373f4f48d4d0","examples/recv_ref_before_send_then_drop_sender.rs":"a190ed220cb4288d4965485365c9afaed30535cbfad5f8cb7389071b82d67cac","examples/recv_timeout_before_send.rs":"2262aa6531afce7816d43182ad9cbec2c04f3dc129064e11e89452278ce8b163","examples/recv_timeout_before_send_then_drop_sender.rs":"4cc8eade4c211f52f5b9be0f72a5906689b894490f4cb5255525e44106e7a4a8","examples/recv_with_dropped_sender.rs":"7906685053ce1c53ff6c26ce11d3221d4bf5ca3429d1d4d2c28de9237cb151c6","examples/send_before_recv.rs":"5555bd61ad52273b663007794128d8f012fc54272bd3225259b5546221bcd591","examples/send_then_drop_receiver.rs":"c3612de207309098404b057468687a2d2311d07f354b7e046398e35e93c4cdcf","examples/send_with_dropped_receiver.rs":"f5a7762b231a24a0db4397c5139437cba155d09b9dbb59872d662c7923080706","src/errors.rs":"df6a1db663fdb1c54d6941d737f6591bfe0dc6f01bd627ba0a94d67ed50b27a9","src/lib.rs":"86893f56e8e762b41ee079b42f4248608e9efb68bd76aa9550fce61e7466bbb0","src/loombox.rs":"fc85d1c2d3fda432be60f0c4d1d528e5998ec2b738a5b395a242285051b94d65","tests/assert_mem.rs":"b1e5190af01af22e55c7c1cd1ff2711807591f788e4eb8b6c6d89123e146105e","tests/async.rs":"6fd2826e589b94677d4eeed1080deda8bcc429aa05a20d843d1442a3a48ea757","tests/future.rs":"0e71f0293cd5a8c44210e8882aca20cfbf1e3771ecd4e4f6b59b924c0d01dd97","tests/helpers/mod.rs":"19161ed33e0ba8862746f04678d0606dee90205896083f85d8c1dcd4d211ccb0","tests/helpers/waker.rs":"77494d49f62d0d320df3830643c306e06e6e20751d210cf6fa58b238bd96c3f9","tests/loom.rs":"ea350fa424a95581e1871bc0037badecc5a090f28fd10532917abbaf561218ab","tests/sync.rs":"1186fa6cdb5a180944fa7d793ccb8be412c4a4e88bb504daa70bc097ee081b06"},"package":"9ae4988774e7a7e6a0783d119bdc683ea8c1d01a24d4fff9b4bdc280e07bd99e"} \ No newline at end of file +{"files":{"CHANGELOG.md":"4ad03d95d5532e8f2551e3e53877e6347c04c32f479c4edf517244ecd5921ac7","Cargo.lock":"5d85bcfda2ee559d243099fb26f3724ae239777d891e780a924804e30f6733ad","Cargo.toml":"07a73ff74274df3a7439fccc8acfe306fae0f51ad79a80edbc54b51a730314c0","README.md":"811ea1c958d5a65583d0223b7ab09bb282e7a51ed60f9a2cb90ef6d555325a68","benches/benches.rs":"67dcc916d0b7e28e396c28dac0499726366e1cb10e9991948d1c881a5abf5faa","check_mem_leaks.sh":"c1ab6ef27997c7f971352ab1c86a184004843c499bc24925da953aefcf1c624c","examples/recv_before_send.rs":"9a3cabcc2878990b61787d0048061b382555a8cd1a08b1ddec63a6e8a4a31e56","examples/recv_before_send_then_drop_sender.rs":"14706c6b4308a690662ceaa47f1699588bd833b3ec020eb9f42f220f3ffc7ae7","examples/recv_ref_before_send.rs":"43699f4720c46b5f138c260b866eb708ddf616e2b442ffa74a97373f4f48d4d0","examples/recv_ref_before_send_then_drop_sender.rs":"a190ed220cb4288d4965485365c9afaed30535cbfad5f8cb7389071b82d67cac","examples/recv_timeout_before_send.rs":"2262aa6531afce7816d43182ad9cbec2c04f3dc129064e11e89452278ce8b163","examples/recv_timeout_before_send_then_drop_sender.rs":"4cc8eade4c211f52f5b9be0f72a5906689b894490f4cb5255525e44106e7a4a8","examples/recv_with_dropped_sender.rs":"7906685053ce1c53ff6c26ce11d3221d4bf5ca3429d1d4d2c28de9237cb151c6","examples/send_before_recv.rs":"5555bd61ad52273b663007794128d8f012fc54272bd3225259b5546221bcd591","examples/send_then_drop_receiver.rs":"c3612de207309098404b057468687a2d2311d07f354b7e046398e35e93c4cdcf","examples/send_with_dropped_receiver.rs":"f5a7762b231a24a0db4397c5139437cba155d09b9dbb59872d662c7923080706","src/errors.rs":"a5aa56bc497dccdbdbe15b9070360f50835c762f11be4ee96e0d25b150168ac9","src/lib.rs":"4bef3602ff4f5d2b42ce963d722a48c9ff07275e75ef6bed7b523e8f45e459fe","src/loombox.rs":"fc85d1c2d3fda432be60f0c4d1d528e5998ec2b738a5b395a242285051b94d65","tests/assert_mem.rs":"b1e5190af01af22e55c7c1cd1ff2711807591f788e4eb8b6c6d89123e146105e","tests/async.rs":"6fd2826e589b94677d4eeed1080deda8bcc429aa05a20d843d1442a3a48ea757","tests/future.rs":"0e71f0293cd5a8c44210e8882aca20cfbf1e3771ecd4e4f6b59b924c0d01dd97","tests/helpers/mod.rs":"19161ed33e0ba8862746f04678d0606dee90205896083f85d8c1dcd4d211ccb0","tests/helpers/waker.rs":"77494d49f62d0d320df3830643c306e06e6e20751d210cf6fa58b238bd96c3f9","tests/loom.rs":"ea350fa424a95581e1871bc0037badecc5a090f28fd10532917abbaf561218ab","tests/raw.rs":"5564615fea811b0061d8ad801356e60e0018ec4e3fb99cc739287ed5b96cb7cf","tests/sync.rs":"1186fa6cdb5a180944fa7d793ccb8be412c4a4e88bb504daa70bc097ee081b06"},"package":"6c548d5c78976f6955d72d0ced18c48ca07030f7a1d4024529fedd7c1c01b29c"} \ No newline at end of file diff --git a/third_party/rust/oneshot-uniffi/CHANGELOG.md b/third_party/rust/oneshot-uniffi/CHANGELOG.md index e12f6cc979..c4c928386b 100644 --- a/third_party/rust/oneshot-uniffi/CHANGELOG.md +++ b/third_party/rust/oneshot-uniffi/CHANGELOG.md @@ -16,6 +16,13 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] + +## [0.1.6] - 2023-09-14 +### Added +* Add `into_raw` and `from_raw` methods on both `Sender` and `Receiver`. Allows passing `oneshot` + channels over FFI without an extra layer of heap allocation. + + ## [0.1.5] - 2022-09-01 ### Fixed - Handle the UNPARKING state correctly in all recv methods. `try_recv` will now not panic diff --git a/third_party/rust/oneshot-uniffi/Cargo.lock b/third_party/rust/oneshot-uniffi/Cargo.lock index 1b75d43675..b71644917f 100644 --- a/third_party/rust/oneshot-uniffi/Cargo.lock +++ b/third_party/rust/oneshot-uniffi/Cargo.lock @@ -601,7 +601,7 @@ checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" [[package]] name = "oneshot-uniffi" -version = "0.1.5" +version = "0.1.6" dependencies = [ "async-std", "criterion", diff --git a/third_party/rust/oneshot-uniffi/Cargo.toml b/third_party/rust/oneshot-uniffi/Cargo.toml index 078a477fb2..cba7266915 100644 --- a/third_party/rust/oneshot-uniffi/Cargo.toml +++ b/third_party/rust/oneshot-uniffi/Cargo.toml @@ -13,7 +13,7 @@ edition = "2021" rust-version = "1.60.0" name = "oneshot-uniffi" -version = "0.1.5" +version = "0.1.6" authors = ["Linus Färnstrand "] description = """ Patched version of oneshot specifically for the UniFFI project. diff --git a/third_party/rust/oneshot-uniffi/src/errors.rs b/third_party/rust/oneshot-uniffi/src/errors.rs index afc48acd03..1fd0de1eed 100644 --- a/third_party/rust/oneshot-uniffi/src/errors.rs +++ b/third_party/rust/oneshot-uniffi/src/errors.rs @@ -4,7 +4,8 @@ use core::mem; use core::ptr::NonNull; /// An error returned when trying to send on a closed channel. Returned from -/// [`Sender::send`] if the corresponding [`Receiver`] has already been dropped. +/// [`Sender::send`](crate::Sender::send) if the corresponding [`Receiver`](crate::Receiver) +/// has already been dropped. /// /// The message that could not be sent can be retreived again with [`SendError::into_inner`]. pub struct SendError { @@ -79,10 +80,10 @@ impl fmt::Debug for SendError { #[cfg(feature = "std")] impl std::error::Error for SendError {} -/// An error returned from the indefinitely blocking recv functions on a [`Receiver`]. +/// An error returned from the blocking [`Receiver::recv`](crate::Receiver::recv) method. /// -/// The recv operation can only fail if the corresponding [`Sender`] was dropped before sending -/// any message. Or if a message has already been sent and received on the channel. +/// The receive operation can only fail if the corresponding [`Sender`](crate::Sender) was dropped +/// before sending any message, or if a message has already been sent and received on the channel. #[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)] pub struct RecvError; @@ -95,7 +96,8 @@ impl fmt::Display for RecvError { #[cfg(feature = "std")] impl std::error::Error for RecvError {} -/// An error returned when trying a non blocking receive on a [`Receiver`]. +/// An error returned when failing to receive a message in the non-blocking +/// [`Receiver::try_recv`](crate::Receiver::try_recv). #[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)] pub enum TryRecvError { /// The channel is still open, but there was no message present in it. @@ -119,7 +121,8 @@ impl fmt::Display for TryRecvError { #[cfg(feature = "std")] impl std::error::Error for TryRecvError {} -/// An error returned when trying a time limited blocking receive on a [`Receiver`]. +/// An error returned when failing to receive a message in +/// [`Receiver::recv_timeout`](crate::Receiver::recv_timeout). #[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)] pub enum RecvTimeoutError { /// No message arrived on the channel before the timeout was reached. The channel is still open. diff --git a/third_party/rust/oneshot-uniffi/src/lib.rs b/third_party/rust/oneshot-uniffi/src/lib.rs index 94bb35d12a..8da012b8c0 100644 --- a/third_party/rust/oneshot-uniffi/src/lib.rs +++ b/third_party/rust/oneshot-uniffi/src/lib.rs @@ -314,6 +314,31 @@ impl Sender { _ => unreachable!(), } } + + /// Consumes the Sender, returning a raw pointer to the channel on the heap. + /// + /// This is intended to simplify using oneshot channels with some FFI code. The only safe thing + /// to do with the returned pointer is to later reconstruct the Sender with [Sender::from_raw]. + /// Memory will leak if the Sender is never reconstructed. + pub fn into_raw(self) -> *mut () { + let raw = self.channel_ptr.as_ptr() as *mut (); + mem::forget(self); + raw + } + + /// Consumes a raw pointer from [Sender::into_raw], recreating the Sender. + /// + /// # Safety + /// + /// This pointer must have come from [`Sender::into_raw`] with the same message type, `T`. + /// At most one Sender must exist for a channel at any point in time. + /// Constructing multiple Senders from the same raw pointer leads to undefined behavior. + pub unsafe fn from_raw(raw: *mut ()) -> Self { + Self { + channel_ptr: NonNull::new_unchecked(raw as *mut Channel), + _invariant: PhantomData, + } + } } impl Drop for Sender { @@ -816,6 +841,30 @@ impl Receiver { _ => unreachable!(), } } + + /// Consumes the Receiver, returning a raw pointer to the channel on the heap. + /// + /// This is intended to simplify using oneshot channels with some FFI code. The only safe thing + /// to do with the returned pointer is to later reconstruct the Receiver with + /// [Receiver::from_raw]. Memory will leak if the Receiver is never reconstructed. + pub fn into_raw(self) -> *mut () { + let raw = self.channel_ptr.as_ptr() as *mut (); + mem::forget(self); + raw + } + + /// Consumes a raw pointer from [Receiver::into_raw], recreating the Receiver. + /// + /// # Safety + /// + /// This pointer must have come from [`Receiver::into_raw`] with the same message type, `T`. + /// At most one Receiver must exist for a channel at any point in time. + /// Constructing multiple Receivers from the same raw pointer leads to undefined behavior. + pub unsafe fn from_raw(raw: *mut ()) -> Self { + Self { + channel_ptr: NonNull::new_unchecked(raw as *mut Channel), + } + } } #[cfg(feature = "async")] @@ -1178,7 +1227,7 @@ fn receiver_waker_size() { (false, false) => 0, (false, true) => 16, (true, false) => 8, - (true, true) => 24, + (true, true) => 16, }; assert_eq!(mem::size_of::(), expected); } diff --git a/third_party/rust/oneshot-uniffi/tests/raw.rs b/third_party/rust/oneshot-uniffi/tests/raw.rs new file mode 100644 index 0000000000..e38dc45c48 --- /dev/null +++ b/third_party/rust/oneshot-uniffi/tests/raw.rs @@ -0,0 +1,46 @@ +#![cfg(not(loom))] + +use oneshot::{channel, Receiver, Sender}; + +#[test] +fn test_raw_sender() { + let (sender, receiver) = channel::(); + let raw = sender.into_raw(); + let recreated = unsafe { Sender::::from_raw(raw) }; + recreated + .send(100) + .unwrap_or_else(|e| panic!("error sending after into_raw/from_raw roundtrip: {e}")); + assert_eq!(receiver.try_recv(), Ok(100)) +} + +#[test] +fn test_raw_receiver() { + let (sender, receiver) = channel::(); + let raw = receiver.into_raw(); + sender.send(100).unwrap(); + let recreated = unsafe { Receiver::::from_raw(raw) }; + assert_eq!( + recreated + .try_recv() + .unwrap_or_else(|e| panic!("error receiving after into_raw/from_raw roundtrip: {e}")), + 100 + ) +} + +#[test] +fn test_raw_sender_and_receiver() { + let (sender, receiver) = channel::(); + let raw_receiver = receiver.into_raw(); + let raw_sender = sender.into_raw(); + + let recreated_sender = unsafe { Sender::::from_raw(raw_sender) }; + recreated_sender.send(100).unwrap(); + + let recreated_receiver = unsafe { Receiver::::from_raw(raw_receiver) }; + assert_eq!( + recreated_receiver + .try_recv() + .unwrap_or_else(|e| panic!("error receiving after into_raw/from_raw roundtrip: {e}")), + 100 + ) +} diff --git a/third_party/rust/relevancy/.cargo-checksum.json b/third_party/rust/relevancy/.cargo-checksum.json new file mode 100644 index 0000000000..c8d8187c8d --- /dev/null +++ b/third_party/rust/relevancy/.cargo-checksum.json @@ -0,0 +1 @@ +{"files":{"Cargo.toml":"76d64a839128f51662d1c10728ceddbb6a9ebdfce803915874cd654117d1b14e","build.rs":"a562bfe527d21c4e8a1a44b892defa83cdff141ec5dd51ed6f3862330e50ddd7","src/bin/generate-test-data.rs":"7f1c9dc445418c7627f89d1f2aa8e550d0f85b3d1f05edb7c378ab9441714f1f","src/db.rs":"0b45180f3031759213a0421231b6f109ed4f5c88aca556df159ce2717416cfec","src/error.rs":"6831fc329044174a8451b8b008c0b96c47404c591eb42e880562e65da0adfd0f","src/interest.rs":"ce6298ef8f69fcb57c8e5797467cbe1c0212a0d94daf828b12845740ac14a166","src/lib.rs":"7a0f0ad0a43f371035d9c0b73d143cf1b387d4b8cfad0d0db79314b5b91fd43c","src/populate_interests.rs":"b8905b52f9fc80719c175253b758413f606b27660e660635094421eec8b24c8f","src/relevancy.udl":"a3fae5097f9e8b39bb6c74ed6789906748c46f22d377e3dcb73b08731908f5bc","src/schema.rs":"f782c712f10c4f1af2f9e1424d6b52f59a2bacfcc452a8feb763f36478f5dd5d","src/url_hash.rs":"5619a249d471e7b642d889bad09e93212559c8b947010d49492c1423da2b310e","test-data":"392fc950363c9953ea6ab144b81d84021c4af1e1177cc0adac4eda5688c8bc33"},"package":null} \ No newline at end of file diff --git a/third_party/rust/relevancy/Cargo.toml b/third_party/rust/relevancy/Cargo.toml new file mode 100644 index 0000000000..eddd8fd25c --- /dev/null +++ b/third_party/rust/relevancy/Cargo.toml @@ -0,0 +1,47 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2021" +name = "relevancy" +version = "0.1.0" +exclude = [ + "/android", + "/ios", +] +license = "MPL-2.0" + +[lib] + +[[bin]] +name = "generate-test-data" + +[dependencies] +log = "0.4" +md-5 = "0.10" +parking_lot = ">=0.11,<=0.12" +thiserror = "1.0" +uniffi = "0.27.1" +url = "2.5" + +[dependencies.error-support] +path = "../support/error" + +[dependencies.rusqlite] +version = "0.30.0" +features = ["bundled"] + +[dependencies.sql-support] +path = "../support/sql" + +[build-dependencies.uniffi] +version = "0.27.1" +features = ["build"] diff --git a/third_party/rust/relevancy/build.rs b/third_party/rust/relevancy/build.rs new file mode 100644 index 0000000000..5439f41d68 --- /dev/null +++ b/third_party/rust/relevancy/build.rs @@ -0,0 +1,8 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +fn main() { + uniffi::generate_scaffolding("./src/relevancy.udl").unwrap(); +} diff --git a/third_party/rust/relevancy/src/bin/generate-test-data.rs b/third_party/rust/relevancy/src/bin/generate-test-data.rs new file mode 100644 index 0000000000..04c5827275 --- /dev/null +++ b/third_party/rust/relevancy/src/bin/generate-test-data.rs @@ -0,0 +1,43 @@ +use relevancy::{ + url_hash::{hash_url, UrlHash}, + Interest, +}; +use std::{collections::HashMap, fs::File, io::Write}; + +// Generate a set of test data and output it to the `test-data` file. +// +// This is meant to be a placeholder until we can get this data stored in remote settings. + +const TEST_INTEREST_DATA: &[(&str, Interest)] = &[ + ("https://espn.com/", Interest::Sports), + ("https://dogs.com/", Interest::Animals), + ("https://cars.com/", Interest::Autos), + ("https://www.vouge.com/", Interest::Fashion), + ("https://slashdot.org/", Interest::Tech), + ("https://www.nascar.com/", Interest::Autos), + ("https://www.nascar.com/", Interest::Sports), +]; + +fn main() { + let mut interest_map: HashMap> = + HashMap::from_iter(Interest::all().into_iter().map(|i| (i, vec![]))); + for (url, interest) in TEST_INTEREST_DATA { + if let Some(hash) = hash_url(url) { + interest_map.get_mut(interest).unwrap().push(hash) + } + } + + let mut f = File::create("test-data").expect("Error opening file"); + // Loop over all possible interests + for interest in Interest::all() { + // Get the list of URL hashes for that interest + let hashes = interest_map.get(&interest).unwrap(); + // Write the count + f.write_all(&(hashes.len() as u32).to_le_bytes()) + .expect("Error writing file"); + // Write the hashes + for hash in hashes { + f.write_all(hash).expect("Error writing file"); + } + } +} diff --git a/third_party/rust/relevancy/src/db.rs b/third_party/rust/relevancy/src/db.rs new file mode 100644 index 0000000000..08684c45af --- /dev/null +++ b/third_party/rust/relevancy/src/db.rs @@ -0,0 +1,118 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +use crate::{ + schema::RelevancyConnectionInitializer, + url_hash::{hash_url, UrlHash}, + Interest, InterestVector, Result, +}; +use parking_lot::Mutex; +use rusqlite::{Connection, OpenFlags}; +use sql_support::{open_database::open_database_with_flags, ConnExt}; +use std::path::Path; + +/// A thread-safe wrapper around an SQLite connection to the Relevancy database +pub struct RelevancyDb { + pub conn: Mutex, +} + +impl RelevancyDb { + pub fn open(path: impl AsRef) -> Result { + let conn = open_database_with_flags( + path, + OpenFlags::SQLITE_OPEN_URI + | OpenFlags::SQLITE_OPEN_NO_MUTEX + | OpenFlags::SQLITE_OPEN_CREATE + | OpenFlags::SQLITE_OPEN_READ_WRITE, + &RelevancyConnectionInitializer, + )?; + Ok(Self { + conn: Mutex::new(conn), + }) + } + + #[cfg(test)] + pub fn open_for_test() -> Self { + use std::sync::atomic::{AtomicU32, Ordering}; + static COUNTER: AtomicU32 = AtomicU32::new(0); + let count = COUNTER.fetch_add(1, Ordering::Relaxed); + Self::open(format!("file:test{count}.sqlite?mode=memory&cache=shared")).unwrap() + } + + /// Accesses the Suggest database in a transaction for reading. + pub fn read(&self, op: impl FnOnce(&RelevancyDao) -> Result) -> Result { + let mut conn = self.conn.lock(); + let tx = conn.transaction()?; + let dao = RelevancyDao::new(&tx); + op(&dao) + } + + /// Accesses the Suggest database in a transaction for reading and writing. + pub fn read_write(&self, op: impl FnOnce(&mut RelevancyDao) -> Result) -> Result { + let mut conn = self.conn.lock(); + let tx = conn.transaction()?; + let mut dao = RelevancyDao::new(&tx); + let result = op(&mut dao)?; + tx.commit()?; + Ok(result) + } +} + +/// A data access object (DAO) that wraps a connection to the Relevancy database +/// +/// Methods that only read from the database take an immutable reference to +/// `self` (`&self`), and methods that write to the database take a mutable +/// reference (`&mut self`). +pub struct RelevancyDao<'a> { + pub conn: &'a Connection, +} + +impl<'a> RelevancyDao<'a> { + fn new(conn: &'a Connection) -> Self { + Self { conn } + } + + /// Associate a URL with an interest + pub fn add_url_interest(&mut self, url_hash: UrlHash, interest: Interest) -> Result<()> { + let sql = " + INSERT OR REPLACE INTO url_interest(url_hash, interest_code) + VALUES (?, ?) + "; + self.conn.execute(sql, (url_hash, interest as u32))?; + Ok(()) + } + + /// Get an interest vector for a URL + pub fn get_url_interest_vector(&self, url: &str) -> Result { + let hash = match hash_url(url) { + Some(u) => u, + None => return Ok(InterestVector::default()), + }; + let mut stmt = self.conn.prepare_cached( + " + SELECT interest_code + FROM url_interest + WHERE url_hash=? + ", + )?; + let interests = stmt.query_and_then((hash,), |row| -> Result { + Ok(row.get::<_, u32>(0)?.into()) + })?; + + let mut interest_vec = InterestVector::default(); + for interest in interests { + interest_vec[interest?] += 1 + } + Ok(interest_vec) + } + + /// Do we need to load the interest data? + pub fn need_to_load_url_interests(&self) -> Result { + // TODO: we probably will need a better check than this. + Ok(self + .conn + .query_one("SELECT NOT EXISTS (SELECT 1 FROM url_interest)")?) + } +} diff --git a/third_party/rust/relevancy/src/error.rs b/third_party/rust/relevancy/src/error.rs new file mode 100644 index 0000000000..93ca7aabaa --- /dev/null +++ b/third_party/rust/relevancy/src/error.rs @@ -0,0 +1,44 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public +* License, v. 2.0. If a copy of the MPL was not distributed with this +* file, You can obtain one at http://mozilla.org/MPL/2.0/. +*/ + +use error_support::{ErrorHandling, GetErrorHandling}; + +/// Errors we return via the public interface. +#[derive(Debug, thiserror::Error)] +pub enum RelevancyApiError { + #[error("Unexpected Error: {reason}")] + Unexpected { reason: String }, +} + +/// Errors we use internally +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("Error opening database: {0}")] + OpenDatabase(#[from] sql_support::open_database::Error), + + #[error("Sql error: {0}")] + SqlError(#[from] rusqlite::Error), + + #[error("Error fetching interest data")] + FetchInterestDataError, +} + +/// Result enum for the public API +pub type ApiResult = std::result::Result; + +/// Result enum for internal functions +pub type Result = std::result::Result; + +// Define how our internal errors are handled and converted to external errors +// See `support/error/README.md` for how this works, especially the warning about PII. +impl GetErrorHandling for Error { + type ExternalError = RelevancyApiError; + + fn get_error_handling(&self) -> ErrorHandling { + ErrorHandling::convert(RelevancyApiError::Unexpected { + reason: self.to_string(), + }) + } +} diff --git a/third_party/rust/relevancy/src/interest.rs b/third_party/rust/relevancy/src/interest.rs new file mode 100644 index 0000000000..0573c743fc --- /dev/null +++ b/third_party/rust/relevancy/src/interest.rs @@ -0,0 +1,167 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/// List of possible interests for a domain. Domains can have be associated with one or multiple +/// interests. `Inconclusive` is used for domains in the user's top sites that we can't classify +/// because there's no corresponding entry in the interest database. +#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)] +#[repr(u32)] +pub enum Interest { + Animals, + Arts, + Autos, + Business, + Career, + Education, + Fashion, + Finance, + Food, + Government, + Health, + Hobbies, + Home, + News, + RealEstate, + Society, + Sports, + Tech, + Travel, + Inconclusive, +} + +impl From for u32 { + fn from(interest: Interest) -> Self { + interest as u32 + } +} + +impl From for usize { + fn from(interest: Interest) -> Self { + interest as usize + } +} + +impl From for Interest { + fn from(code: u32) -> Self { + if code as usize > Self::COUNT { + panic!("Invalid interest code: {code}") + } + // Safety: This is safe since Interest has a u32 representation and we've done a bounds + // check + unsafe { std::mem::transmute(code) } + } +} + +impl Interest { + const COUNT: usize = 20; + + pub fn all() -> [Interest; Self::COUNT] { + [ + Self::Animals, + Self::Arts, + Self::Autos, + Self::Business, + Self::Career, + Self::Education, + Self::Fashion, + Self::Finance, + Self::Food, + Self::Government, + Self::Health, + Self::Hobbies, + Self::Home, + Self::News, + Self::RealEstate, + Self::Society, + Self::Sports, + Self::Tech, + Self::Travel, + Self::Inconclusive, + ] + } +} + +/// Vector storing a count value for each interest +/// +/// Here "vector" refers to the mathematical object, not a Rust `Vec`. It always has a fixed +/// number of elements. +#[derive(Debug, Default, PartialEq, Eq)] +pub struct InterestVector { + pub animals: u32, + pub arts: u32, + pub autos: u32, + pub business: u32, + pub career: u32, + pub education: u32, + pub fashion: u32, + pub finance: u32, + pub food: u32, + pub government: u32, + pub health: u32, + pub hobbies: u32, + pub home: u32, + pub news: u32, + pub real_estate: u32, + pub society: u32, + pub sports: u32, + pub tech: u32, + pub travel: u32, + pub inconclusive: u32, +} + +impl std::ops::Index for InterestVector { + type Output = u32; + + fn index(&self, index: Interest) -> &u32 { + match index { + Interest::Animals => &self.animals, + Interest::Arts => &self.arts, + Interest::Autos => &self.autos, + Interest::Business => &self.business, + Interest::Career => &self.career, + Interest::Education => &self.education, + Interest::Fashion => &self.fashion, + Interest::Finance => &self.finance, + Interest::Food => &self.food, + Interest::Government => &self.government, + Interest::Health => &self.health, + Interest::Hobbies => &self.hobbies, + Interest::Home => &self.home, + Interest::News => &self.news, + Interest::RealEstate => &self.real_estate, + Interest::Society => &self.society, + Interest::Sports => &self.sports, + Interest::Tech => &self.tech, + Interest::Travel => &self.travel, + Interest::Inconclusive => &self.inconclusive, + } + } +} + +impl std::ops::IndexMut for InterestVector { + fn index_mut(&mut self, index: Interest) -> &mut u32 { + match index { + Interest::Animals => &mut self.animals, + Interest::Arts => &mut self.arts, + Interest::Autos => &mut self.autos, + Interest::Business => &mut self.business, + Interest::Career => &mut self.career, + Interest::Education => &mut self.education, + Interest::Fashion => &mut self.fashion, + Interest::Finance => &mut self.finance, + Interest::Food => &mut self.food, + Interest::Government => &mut self.government, + Interest::Health => &mut self.health, + Interest::Hobbies => &mut self.hobbies, + Interest::Home => &mut self.home, + Interest::News => &mut self.news, + Interest::RealEstate => &mut self.real_estate, + Interest::Society => &mut self.society, + Interest::Sports => &mut self.sports, + Interest::Tech => &mut self.tech, + Interest::Travel => &mut self.travel, + Interest::Inconclusive => &mut self.inconclusive, + } + } +} diff --git a/third_party/rust/relevancy/src/lib.rs b/third_party/rust/relevancy/src/lib.rs new file mode 100644 index 0000000000..157a26277e --- /dev/null +++ b/third_party/rust/relevancy/src/lib.rs @@ -0,0 +1,81 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +//! Proposed API for the relevancy component (validation phase) +//! +//! The goal here is to allow us to validate that we can reliably detect user interests from +//! history data, without spending too much time building the API out. There's some hand-waving +//! towards how we would use this data to rank search results, but we don't need to come to a final +//! decision on that yet. + +mod db; +mod error; +mod interest; +mod populate_interests; +mod schema; +pub mod url_hash; + +pub use db::RelevancyDb; +pub use error::{ApiResult, Error, RelevancyApiError, Result}; +pub use interest::{Interest, InterestVector}; + +use error_support::handle_error; + +pub struct RelevancyStore { + db: RelevancyDb, +} + +/// Top-level API for the Relevancy component +impl RelevancyStore { + #[handle_error(Error)] + pub fn new(db_path: String) -> ApiResult { + Ok(Self { + db: RelevancyDb::open(db_path)?, + }) + } + + /// Ingest top URLs to build the user's interest vector. + /// + /// Consumer should pass a list of the user's top URLs by frecency to this method. It will + /// then: + /// + /// - Download the URL interest data from remote settings. Eventually this should be cached / + /// stored in the database, but for now it would be fine to download fresh data each time. + /// - Match the user's top URls against the interest data to build up their interest vector. + /// - Store the user's interest vector in the database. + /// + /// This method may execute for a long time and should only be called from a worker thread. + #[handle_error(Error)] + pub fn ingest(&self, _top_urls_by_frecency: Vec) -> ApiResult<()> { + populate_interests::ensure_interest_data_populated(&self.db)?; + todo!() + } + + /// Calculate metrics for the validation phase + /// + /// This runs after [Self::ingest]. It takes the interest vector that ingest created and + /// calculates a set of metrics that we can report to glean. + #[handle_error(Error)] + pub fn calculate_metrics(&self) -> ApiResult { + todo!() + } + + /// Get the user's interest vector directly. + /// + /// This runs after [Self::ingest]. It returns the interest vector directly so that the + /// consumer can show it in an `about:` page. + #[handle_error(Error)] + pub fn user_interest_vector(&self) -> ApiResult { + todo!() + } +} + +/// Interest metric data. See `relevancy.udl` for details. +pub struct InterestMetrics { + pub top_single_interest_similarity: u32, + pub top_2interest_similarity: u32, + pub top_3interest_similarity: u32, +} + +uniffi::include_scaffolding!("relevancy"); diff --git a/third_party/rust/relevancy/src/populate_interests.rs b/third_party/rust/relevancy/src/populate_interests.rs new file mode 100644 index 0000000000..e33b677dd6 --- /dev/null +++ b/third_party/rust/relevancy/src/populate_interests.rs @@ -0,0 +1,157 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use crate::{url_hash::UrlHash, Error, Interest, RelevancyDb, Result}; +use std::io::{Cursor, Read}; + +pub fn ensure_interest_data_populated(db: &RelevancyDb) -> Result<()> { + if !db.read(|dao| dao.need_to_load_url_interests())? { + return Ok(()); + } + let interest_data = match fetch_interest_data() { + Ok(data) => data, + Err(e) => { + log::warn!("error fetching interest data: {e}"); + return Err(Error::FetchInterestDataError); + } + }; + db.read_write(move |dao| { + for (url_hash, interest) in interest_data { + dao.add_url_interest(url_hash, interest)?; + } + Ok(()) + }) +} + +/// Fetch the interest data +fn fetch_interest_data() -> std::io::Result> { + // TODO: this hack should be replaced with something that fetches from remote settings + let bytes = include_bytes!("../test-data"); + let mut reader = Cursor::new(&bytes); + let mut data = vec![]; + + // Loop over all possible interests + for interest in Interest::all() { + // read the count + let mut buf = [0u8; 4]; + reader.read_exact(&mut buf)?; + let count = u32::from_le_bytes(buf); + for _ in 0..count { + let mut url_hash: UrlHash = [0u8; 16]; + reader.read_exact(&mut url_hash)?; + data.push((url_hash, interest)); + } + } + Ok(data) +} + +#[cfg(test)] +mod test { + use super::*; + use crate::InterestVector; + + #[test] + fn test_interest_vectors() { + let db = RelevancyDb::open_for_test(); + ensure_interest_data_populated(&db).unwrap(); + db.read(|dao| { + // Test that the interest data matches the values we started from in + // `bin/generate-test-data.rs` + assert_eq!( + dao.get_url_interest_vector("https://espn.com/").unwrap(), + InterestVector { + sports: 1, + ..InterestVector::default() + } + ); + assert_eq!( + dao.get_url_interest_vector("https://dogs.com/").unwrap(), + InterestVector { + animals: 1, + ..InterestVector::default() + } + ); + assert_eq!( + dao.get_url_interest_vector("https://cars.com/").unwrap(), + InterestVector { + autos: 1, + ..InterestVector::default() + } + ); + assert_eq!( + dao.get_url_interest_vector("https://www.vouge.com/") + .unwrap(), + InterestVector { + fashion: 1, + ..InterestVector::default() + } + ); + assert_eq!( + dao.get_url_interest_vector("https://slashdot.org/") + .unwrap(), + InterestVector { + tech: 1, + ..InterestVector::default() + } + ); + assert_eq!( + dao.get_url_interest_vector("https://www.nascar.com/") + .unwrap(), + InterestVector { + autos: 1, + sports: 1, + ..InterestVector::default() + } + ); + assert_eq!( + dao.get_url_interest_vector("https://unknown.url/").unwrap(), + InterestVector::default() + ); + Ok(()) + }) + .unwrap(); + } + + #[test] + fn test_variations_on_the_url() { + let db = RelevancyDb::open_for_test(); + ensure_interest_data_populated(&db).unwrap(); + db.read(|dao| { + // Different paths/queries should work + assert_eq!( + dao.get_url_interest_vector("https://espn.com/foo/bar/?baz") + .unwrap(), + InterestVector { + sports: 1, + ..InterestVector::default() + } + ); + // Different schemes should too + assert_eq!( + dao.get_url_interest_vector("http://espn.com/").unwrap(), + InterestVector { + sports: 1, + ..InterestVector::default() + } + ); + // But changes to the domain shouldn't + assert_eq!( + dao.get_url_interest_vector("http://www.espn.com/").unwrap(), + InterestVector::default() + ); + // However, extra components past the 3rd one in the domain are ignored + assert_eq!( + dao.get_url_interest_vector("https://foo.www.nascar.com/") + .unwrap(), + InterestVector { + autos: 1, + sports: 1, + ..InterestVector::default() + } + ); + Ok(()) + }) + .unwrap(); + } +} diff --git a/third_party/rust/relevancy/src/relevancy.udl b/third_party/rust/relevancy/src/relevancy.udl new file mode 100644 index 0000000000..e07243ec28 --- /dev/null +++ b/third_party/rust/relevancy/src/relevancy.udl @@ -0,0 +1,106 @@ +namespace relevancy { }; + +[Error] +interface RelevancyApiError { + Unexpected(string reason); +}; + +// Top-level class for the Relevancy component +interface RelevancyStore { + // Construct a new RelevancyStore + [Throws=RelevancyApiError] + constructor(string dbpath); + + // Ingest the top URLs by frequency to build up the user's interest vector + [Throws=RelevancyApiError] + void ingest(sequence top_urls); + + // Calculate metrics for the user's interest vector in order to measure how strongly we're + // identifying interests. See the `InterestMetrics` struct for details. + [Throws=RelevancyApiError] + InterestMetrics calculate_metrics(); + + // Get the interest vector for the user. + // + // This is intended to be show to the user in an `about:` page so that users can judge if it + // feels correct. + [Throws=RelevancyApiError] + InterestVector user_interest_vector(); +}; + +enum Interest { + "Animals", + "Arts", + "Autos", + "Business", + "Career", + "Education", + "Fashion", + "Finance", + "Food", + "Government", + "Health", + "Hobbies", + "Home", + "News", + "RealEstate", + "Society", + "Sports", + "Tech", + "Travel", + "Inconclusive", +}; + +// Interest metrics that we want to send to Glean as part of the validation process. These contain +// the cosine similarity when comparing the user's interest against various interest vectors that +// consumers may use. +// +// Cosine similary was chosen because it seems easy to calculate. This was then matched against +// some semi-plausible real-world interest vectors that consumers might use. This is all up for +// debate and we may decide to switch to some other metrics. +// +// Similarity values are transformed to integers by multiplying the floating point value by 1000 and +// rounding. This is to make them compatible with Glean's distribution metrics. +dictionary InterestMetrics { + // Similarity between the user's interest vector and an interest vector where the element for + // the user's top interest is copied, but all other interests are set to zero. This measures + // the highest possible similarity with consumers that used interest vectors with a single + // interest set. + u32 top_single_interest_similarity; + + // The same as before, but the top 2 interests are copied. This measures the highest possible + // similarity with consumers that used interest vectors with a two interests (note: this means + // they would need to choose the user's top two interests and have the exact same proportion + // between them as the user). + u32 top_2interest_similarity; + + // The same as before, but the top 3 interests are copied. + u32 top_3interest_similarity; +}; + +// Vector storing a count value for each interest +// +// Here "vector" refers to the mathematical object, not a Rust `Vec`. It always has a fixed +// number of elements. +dictionary InterestVector { + u32 animals; + u32 arts; + u32 autos; + u32 business; + u32 career; + u32 education; + u32 fashion; + u32 finance; + u32 food; + u32 government; + u32 health; + u32 hobbies; + u32 home; + u32 news; + u32 real_estate; + u32 society; + u32 sports; + u32 tech; + u32 travel; + u32 inconclusive; +}; diff --git a/third_party/rust/relevancy/src/schema.rs b/third_party/rust/relevancy/src/schema.rs new file mode 100644 index 0000000000..bcb2f260d9 --- /dev/null +++ b/third_party/rust/relevancy/src/schema.rs @@ -0,0 +1,53 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +use rusqlite::{Connection, Transaction}; +use sql_support::open_database::{self, ConnectionInitializer}; + +/// The current database schema version. +/// +/// For any changes to the schema [`SQL`], please make sure to: +/// +/// 1. Bump this version. +/// 2. Add a migration from the old version to the new version in +/// [`RelevancyConnectionInitializer::upgrade_from`]. +pub const VERSION: u32 = 13; + +/// The current database schema. +pub const SQL: &str = " + CREATE TABLE url_interest( + url_hash BLOB NOT NULL, + interest_code INTEGER NOT NULL, + PRIMARY KEY (url_hash, interest_code) + ) WITHOUT ROWID; +"; + +/// Initializes an SQLite connection to the Relevancy database, performing +/// migrations as needed. +pub struct RelevancyConnectionInitializer; + +impl ConnectionInitializer for RelevancyConnectionInitializer { + const NAME: &'static str = "relevancy db"; + const END_VERSION: u32 = VERSION; + + fn prepare(&self, conn: &Connection, _db_empty: bool) -> open_database::Result<()> { + let initial_pragmas = " + -- Use in-memory storage for TEMP tables. + PRAGMA temp_store = 2; + PRAGMA journal_mode = WAL; + PRAGMA foreign_keys = ON; + "; + conn.execute_batch(initial_pragmas)?; + Ok(()) + } + + fn init(&self, db: &Transaction<'_>) -> open_database::Result<()> { + Ok(db.execute_batch(SQL)?) + } + + fn upgrade_from(&self, _db: &Transaction<'_>, version: u32) -> open_database::Result<()> { + Err(open_database::Error::IncompatibleVersion(version)) + } +} diff --git a/third_party/rust/relevancy/src/url_hash.rs b/third_party/rust/relevancy/src/url_hash.rs new file mode 100644 index 0000000000..d31a45d06b --- /dev/null +++ b/third_party/rust/relevancy/src/url_hash.rs @@ -0,0 +1,63 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use md5::{Digest, Md5}; +use url::{Host, Url}; + +pub type UrlHash = [u8; 16]; + +/// Given a URL, extract the part of it that we want to use to identify it. +/// +/// We currently use the final 3 components of the URL domain. +/// +/// TODO: decide if this should be 3 or 3 components. +pub fn url_hash_source(url: &str) -> Option { + let url = Url::parse(url).ok()?; + let domain = match url.host() { + Some(Host::Domain(d)) => d, + _ => return None, + }; + // This will store indexes of `.` chars as we search backwards. + let mut pos = domain.len(); + for _ in 0..3 { + match domain[0..pos].rfind('.') { + Some(p) => pos = p, + // The domain has less than 3 dots, return it all + None => return Some(domain.to_owned()), + } + } + Some(domain[pos + 1..].to_owned()) +} + +pub fn hash_url(url: &str) -> Option { + url_hash_source(url).map(|hash_source| { + let mut hasher = Md5::new(); + hasher.update(hash_source); + let result = hasher.finalize(); + result.into() + }) +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_url_hash_source() { + let table = [ + ("http://example.com/some-path", Some("example.com")), + ("http://foo.example.com/some-path", Some("foo.example.com")), + ( + "http://foo.bar.baz.example.com/some-path", + Some("baz.example.com"), + ), + ("http://foo.com.uk/some-path", Some("foo.com.uk")), + ("http://amazon.com/some-path", Some("amazon.com")), + ("http://192.168.0.1/some-path", None), + ]; + for (url, expected) in table { + assert_eq!(url_hash_source(url).as_deref(), expected) + } + } +} diff --git a/third_party/rust/relevancy/test-data b/third_party/rust/relevancy/test-data new file mode 100644 index 0000000000..c645914143 Binary files /dev/null and b/third_party/rust/relevancy/test-data differ diff --git a/third_party/rust/remote_settings/.cargo-checksum.json b/third_party/rust/remote_settings/.cargo-checksum.json index 8a2b63f314..619a9d13ca 100644 --- a/third_party/rust/remote_settings/.cargo-checksum.json +++ b/third_party/rust/remote_settings/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"Cargo.toml":"4fa89b0606fe8ec8ac8c479b8b9adf33d0c936b09fa5af108ded74139ace37fb","build.rs":"4326f03729cf8f1673e4228e6dc111de1ea4d8bcc06351f7ae563efb2613f866","src/client.rs":"fb3f2cd47460e5ae07a5e8d61b358d588d14075bd9dd6b6e818e1af74abd5dba","src/config.rs":"7bb678addfae3b4ed5f2892d32263e5b33cc05e5a12a250f664150e78211f94a","src/error.rs":"192ca42af7c6b882f3129378c23b45dab8a0d2b179e23a8813a335ffd56b21dc","src/lib.rs":"416e99894e152f6cea7418ad2fabfd94bc3d907efd9f33fbd2a83fb99452b2df","src/remote_settings.udl":"2e71491ad3894d17e5bde0663d9490bfea6294d99cdbe9d67a36137faeedc593","uniffi.toml":"f8ec8dc593e0d501c2e9e40368ec93ec33b1edd8608e29495e0a54b63144e880"},"package":null} \ No newline at end of file +{"files":{"Cargo.toml":"1029f571c66d33c4dfc5e9fc55287a780329ce183f5d2b672de79737155c4227","build.rs":"4326f03729cf8f1673e4228e6dc111de1ea4d8bcc06351f7ae563efb2613f866","src/client.rs":"7510ae0d5bcb9fbaa2c43c4773aa0fd518edc78fe0f396c0e1d6dd442446f429","src/config.rs":"7bb678addfae3b4ed5f2892d32263e5b33cc05e5a12a250f664150e78211f94a","src/error.rs":"192ca42af7c6b882f3129378c23b45dab8a0d2b179e23a8813a335ffd56b21dc","src/lib.rs":"416e99894e152f6cea7418ad2fabfd94bc3d907efd9f33fbd2a83fb99452b2df","src/remote_settings.udl":"2e71491ad3894d17e5bde0663d9490bfea6294d99cdbe9d67a36137faeedc593","uniffi.toml":"f8ec8dc593e0d501c2e9e40368ec93ec33b1edd8608e29495e0a54b63144e880"},"package":null} \ No newline at end of file diff --git a/third_party/rust/remote_settings/Cargo.toml b/third_party/rust/remote_settings/Cargo.toml index b04e6ed6c6..71d72079b7 100644 --- a/third_party/rust/remote_settings/Cargo.toml +++ b/third_party/rust/remote_settings/Cargo.toml @@ -28,7 +28,7 @@ license = "MPL-2.0" parking_lot = "0.12" serde_json = "1" thiserror = "1.0" -uniffi = "0.25.2" +uniffi = "0.27.1" url = "2.1" [dependencies.serde] @@ -46,5 +46,5 @@ mockito = "0.31" path = "../support/viaduct-reqwest" [build-dependencies.uniffi] -version = "0.25.2" +version = "0.27.1" features = ["build"] diff --git a/third_party/rust/remote_settings/src/client.rs b/third_party/rust/remote_settings/src/client.rs index 0d99de9cc1..2cf26b7319 100644 --- a/third_party/rust/remote_settings/src/client.rs +++ b/third_party/rust/remote_settings/src/client.rs @@ -64,7 +64,7 @@ impl Client { /// collection defined by the [ClientConfig] used to generate this [Client]. pub fn get_records_since(&self, timestamp: u64) -> Result { self.get_records_with_options( - GetItemsOptions::new().gt("last_modified", timestamp.to_string()), + GetItemsOptions::new().filter_gt("last_modified", timestamp.to_string()), ) } @@ -307,7 +307,7 @@ struct AttachmentsCapability { } /// Options for requests to endpoints that return multiple items. -#[derive(Clone, Debug, Default)] +#[derive(Clone, Debug, Default, Hash, PartialEq, Eq, PartialOrd, Ord)] pub struct GetItemsOptions { filters: Vec, sort: Vec, @@ -328,14 +328,14 @@ impl GetItemsOptions { /// `author.name`. `value` can be a bare number or string (like /// `2` or `Ben`), or a stringified JSON value (`"2.0"`, `[1, 2]`, /// `{"checked": true}`). - pub fn eq(&mut self, field: impl Into, value: impl Into) -> &mut Self { + pub fn filter_eq(&mut self, field: impl Into, value: impl Into) -> &mut Self { self.filters.push(Filter::Eq(field.into(), value.into())); self } /// Sets an option to only return items whose `field` is not equal to the /// given `value`. - pub fn not(&mut self, field: impl Into, value: impl Into) -> &mut Self { + pub fn filter_not(&mut self, field: impl Into, value: impl Into) -> &mut Self { self.filters.push(Filter::Not(field.into(), value.into())); self } @@ -343,7 +343,11 @@ impl GetItemsOptions { /// Sets an option to only return items whose `field` is an array that /// contains the given `value`. If `value` is a stringified JSON array, the /// field must contain all its elements. - pub fn contains(&mut self, field: impl Into, value: impl Into) -> &mut Self { + pub fn filter_contains( + &mut self, + field: impl Into, + value: impl Into, + ) -> &mut Self { self.filters .push(Filter::Contains(field.into(), value.into())); self @@ -351,47 +355,47 @@ impl GetItemsOptions { /// Sets an option to only return items whose `field` is strictly less /// than the given `value`. - pub fn lt(&mut self, field: impl Into, value: impl Into) -> &mut Self { + pub fn filter_lt(&mut self, field: impl Into, value: impl Into) -> &mut Self { self.filters.push(Filter::Lt(field.into(), value.into())); self } /// Sets an option to only return items whose `field` is strictly greater /// than the given `value`. - pub fn gt(&mut self, field: impl Into, value: impl Into) -> &mut Self { + pub fn filter_gt(&mut self, field: impl Into, value: impl Into) -> &mut Self { self.filters.push(Filter::Gt(field.into(), value.into())); self } /// Sets an option to only return items whose `field` is less than or equal /// to the given `value`. - pub fn max(&mut self, field: impl Into, value: impl Into) -> &mut Self { + pub fn filter_max(&mut self, field: impl Into, value: impl Into) -> &mut Self { self.filters.push(Filter::Max(field.into(), value.into())); self } /// Sets an option to only return items whose `field` is greater than or /// equal to the given `value`. - pub fn min(&mut self, field: impl Into, value: impl Into) -> &mut Self { + pub fn filter_min(&mut self, field: impl Into, value: impl Into) -> &mut Self { self.filters.push(Filter::Min(field.into(), value.into())); self } /// Sets an option to only return items whose `field` is a string that /// contains the substring `value`. `value` can contain `*` wildcards. - pub fn like(&mut self, field: impl Into, value: impl Into) -> &mut Self { + pub fn filter_like(&mut self, field: impl Into, value: impl Into) -> &mut Self { self.filters.push(Filter::Like(field.into(), value.into())); self } /// Sets an option to only return items that have the given `field`. - pub fn has(&mut self, field: impl Into) -> &mut Self { + pub fn filter_has(&mut self, field: impl Into) -> &mut Self { self.filters.push(Filter::Has(field.into())); self } /// Sets an option to only return items that do not have the given `field`. - pub fn has_not(&mut self, field: impl Into) -> &mut Self { + pub fn filter_has_not(&mut self, field: impl Into) -> &mut Self { self.filters.push(Filter::HasNot(field.into())); self } @@ -454,7 +458,7 @@ impl GetItemsOptions { } /// The order in which to return items. -#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, PartialOrd, Ord)] pub enum SortOrder { /// Smaller values first. Ascending, @@ -462,7 +466,7 @@ pub enum SortOrder { Descending, } -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)] enum Filter { Eq(String, String), Not(String, String), @@ -495,7 +499,7 @@ impl Filter { } } -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)] struct Sort(String, SortOrder); impl Sort { @@ -692,16 +696,16 @@ mod test { .field("a") .field("c") .field("b") - .eq("a", "b") - .lt("c.d", "5") - .gt("e", "15") - .max("f", "20") - .min("g", "10") - .not("h", "i") - .like("j", "*k*") - .has("l") - .has_not("m") - .contains("n", "o") + .filter_eq("a", "b") + .filter_lt("c.d", "5") + .filter_gt("e", "15") + .filter_max("f", "20") + .filter_min("g", "10") + .filter_not("h", "i") + .filter_like("j", "*k*") + .filter_has("l") + .filter_has_not("m") + .filter_contains("n", "o") .sort("b", SortOrder::Descending) .sort("a", SortOrder::Ascending) .limit(3); diff --git a/third_party/rust/rure/src/lib.rs b/third_party/rust/rure/src/lib.rs index aa03c60aff..1c1eef9251 100644 --- a/third_party/rust/rure/src/lib.rs +++ b/third_party/rust/rure/src/lib.rs @@ -1,3 +1,7 @@ +// This library is patched, this considered first-party. Ignore warnings +// as if it were third-party. +#![allow(warnings)] + #[macro_use] mod macros; mod error; diff --git a/third_party/rust/scroll/.cargo-checksum.json b/third_party/rust/scroll/.cargo-checksum.json index 406f97faa2..4cdf2841c7 100644 --- a/third_party/rust/scroll/.cargo-checksum.json +++ b/third_party/rust/scroll/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"CHANGELOG.md":"de2bbf4669561405d402322f4cc2604218d4986b73b75b41708b9505aebcb02c","Cargo.lock":"d6a215b7466d37e08551c56949e77be4ee488f989bdef3e507713c729bbda0e6","Cargo.toml":"c240c5768d23ea9611ef57308f08b8ee4372ede6c04f0783dc9fd1710e664c19","LICENSE":"6e24b7455f0b9afefdf4f3efd59a56ce76a3020c2dc4371937e281fc5e587fd7","README.md":"e4fe9aabcd87d85a5ec93241eeefc0d69aa0d98fbd67da2fe1849e4cbddac3ce","benches/bench.rs":"12ae02c383c91f1b0e11e9201eb8a9d44dadfb2b5987e7e71b0ef7c6589af1ca","examples/data_ctx.rs":"79684fc44d499d0b13a173184793837fbaba70d2f74f075e796eb37a1803ce3d","src/ctx.rs":"8f58672c5f3bc09b8f09c76f1d423431cbff786af75f5b39a0cef23b820d48c6","src/endian.rs":"5b717eb5ed0dc2b536779316b020df4e6489c05b13b4fd9b5f5e683aca1b2c28","src/error.rs":"a6a0ec9a6237d23febd608637c0e3926d147511e7983195366bc5a11f12d9093","src/greater.rs":"29d9736f9d35a0f92ca054c7a36878ade0a77b4e8ee27441c34cd81c6bdb68e6","src/leb128.rs":"e343f4e104ca6d8660a3dded30934b83bad4c04d8888ce2cbebfa562f5ac115d","src/lesser.rs":"d3028781977e60d67003512e45666935deab9a03c76a3ba9316a5dbdddf432eb","src/lib.rs":"49d02fa761bb2a771d1857ffd150aa4b6f55b4f03aee1a7a23d8181c76a55fd6","src/pread.rs":"64afdcf2c2785f1f23d065ec5e565d78569086dfd9ece0a3d2553b05aee5df9b","src/pwrite.rs":"05e3129ec666790a61f5b5f894ad863103e213eb798243cfe5f2cbb54d042ba1","tests/api.rs":"1bef345e020a6a4e590350ea4f6069c5836941656379e252bfbdaee6edbbc0de"},"package":"04c565b551bafbef4157586fa379538366e4385d42082f255bfd96e4fe8519da"} \ No newline at end of file +{"files":{"Cargo.toml":"c9242ab52e0b3ba02d9557c3ef2070bc173dfbef05870c3c4831937233de38c0","LICENSE":"6e24b7455f0b9afefdf4f3efd59a56ce76a3020c2dc4371937e281fc5e587fd7","README.md":"7a7f6695853fbc174e3b016d72a8ef0113e313c897269779c7c368f102ed0c23","src/ctx.rs":"9bd92f1038962a8034450b64818cc7b5eaebacde2a229eec5b9cda3ec99c5ae4","src/endian.rs":"e3e0fcb99d0f71f739b6f0ea466a5d3479ed9c90f29269adb1aa2d725ac12af4","src/error.rs":"d91d332a87bde35738cc5915279fc0fde65301fe86ef98ec36126e1de9fd0474","src/greater.rs":"29d9736f9d35a0f92ca054c7a36878ade0a77b4e8ee27441c34cd81c6bdb68e6","src/leb128.rs":"eb71761d708f78c785e6dbe8d385fd90317d08369d1c3ac57d142ca7c0e09e9e","src/lesser.rs":"16fa2c3a737c126b7ac40117c960bc025fb418abc99559c244e8a5ae4348c730","src/lib.rs":"e9a1b9b0ee06ba39de6925f4bc23cb847c8ec3831ca37280c3660dc6d1b28826","src/pread.rs":"80eb931ad7340bba7e1a03a7cbef62c93537bdf4703e467210957d07b82f6489","src/pwrite.rs":"5384d97a57a245e057bca70bd3a386c2942c89f6f7555bcad498b348ee555543"},"package":"6ab8598aa408498679922eff7fa985c25d58a90771bd6be794434c5277eab1a6"} \ No newline at end of file diff --git a/third_party/rust/scroll/CHANGELOG.md b/third_party/rust/scroll/CHANGELOG.md deleted file mode 100644 index bae87ee590..0000000000 --- a/third_party/rust/scroll/CHANGELOG.md +++ /dev/null @@ -1,17 +0,0 @@ -# Changelog -All notable changes to this project will be documented in this file. - -Before 1.0, this project does not adhere to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). - -## [0.10.0] - unreleased -### Added - - scroll is now 2018 compliant, thanks @lzutao: https://github.com/m4b/scroll/pull/49 - - scroll_derive now lives in scroll repo itself -### Removed - - BREAKING: removed units/size generics in SizeWith, thanks @willglynn: https://github.com/m4b/scroll/pull/45 - -## [0.9.1] - 2018-9-22 -### Added - - pread primitive references: https://github.com/m4b/scroll/pull/35 - - u128/i128 support: https://github.com/m4b/scroll/pull/32 - - CStr support: https://github.com/m4b/scroll/pull/30 diff --git a/third_party/rust/scroll/Cargo.lock b/third_party/rust/scroll/Cargo.lock deleted file mode 100644 index baf29fe049..0000000000 --- a/third_party/rust/scroll/Cargo.lock +++ /dev/null @@ -1,205 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 3 - -[[package]] -name = "autocfg" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" - -[[package]] -name = "byteorder" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae44d1a3d5a19df61dd0c8beb138458ac2a53a7ac09eba97d55592540004306b" - -[[package]] -name = "cfg-if" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" - -[[package]] -name = "const_fn" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28b9d6de7f49e22cf97ad17fc4036ece69300032f45f78f30b4a4482cdc3f4a6" - -[[package]] -name = "crossbeam-channel" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dca26ee1f8d361640700bde38b2c37d8c22b3ce2d360e1fc1c74ea4b0aa7d775" -dependencies = [ - "cfg-if", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-deque" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94af6efb46fef72616855b036a624cf27ba656ffc9be1b9a3c931cfc7749a9a9" -dependencies = [ - "cfg-if", - "crossbeam-epoch", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1aaa739f95311c2c7887a76863f500026092fb1dce0161dab577e559ef3569d" -dependencies = [ - "cfg-if", - "const_fn", - "crossbeam-utils", - "lazy_static", - "memoffset", - "scopeguard", -] - -[[package]] -name = "crossbeam-utils" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02d96d1e189ef58269ebe5b97953da3274d83a93af647c2ddd6f9dab28cedb8d" -dependencies = [ - "autocfg", - "cfg-if", - "lazy_static", -] - -[[package]] -name = "either" -version = "1.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" - -[[package]] -name = "hermit-abi" -version = "0.1.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "322f4de77956e22ed0e5032c359a0f1273f1f7f0d79bfa3b8ffbc730d7fbcc5c" -dependencies = [ - "libc", -] - -[[package]] -name = "lazy_static" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" - -[[package]] -name = "libc" -version = "0.2.82" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89203f3fba0a3795506acaad8ebce3c80c0af93f994d5a1d7a0b1eeb23271929" - -[[package]] -name = "memoffset" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "157b4208e3059a8f9e78d559edc658e13df41410cb3ae03979c83130067fdd87" -dependencies = [ - "autocfg", -] - -[[package]] -name = "num_cpus" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" -dependencies = [ - "hermit-abi", - "libc", -] - -[[package]] -name = "proc-macro2" -version = "1.0.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71" -dependencies = [ - "unicode-xid", -] - -[[package]] -name = "quote" -version = "1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "991431c3519a3f36861882da93630ce66b52918dcf1b8e2fd66b397fc96f28df" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "rayon" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b0d8e0819fadc20c74ea8373106ead0600e3a67ef1fe8da56e39b9ae7275674" -dependencies = [ - "autocfg", - "crossbeam-deque", - "either", - "rayon-core", -] - -[[package]] -name = "rayon-core" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ab346ac5921dc62ffa9f89b7a773907511cdfa5490c572ae9be1be33e8afa4a" -dependencies = [ - "crossbeam-channel", - "crossbeam-deque", - "crossbeam-utils", - "lazy_static", - "num_cpus", -] - -[[package]] -name = "scopeguard" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" - -[[package]] -name = "scroll" -version = "0.11.0" -dependencies = [ - "byteorder", - "rayon", - "scroll_derive", -] - -[[package]] -name = "scroll_derive" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdbda6ac5cd1321e724fa9cee216f3a61885889b896f073b8f82322789c5250e" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "syn" -version = "1.0.60" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c700597eca8a5a762beb35753ef6b94df201c81cca676604f547495a0d7f0081" -dependencies = [ - "proc-macro2", - "quote", - "unicode-xid", -] - -[[package]] -name = "unicode-xid" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" diff --git a/third_party/rust/scroll/Cargo.toml b/third_party/rust/scroll/Cargo.toml index 548be72db9..70e9e8d1c3 100644 --- a/third_party/rust/scroll/Cargo.toml +++ b/third_party/rust/scroll/Cargo.toml @@ -11,26 +11,37 @@ [package] edition = "2021" +rust-version = "1.63" name = "scroll" -version = "0.11.0" -authors = ["m4b ", "Ted Mielczarek "] +version = "0.12.0" +authors = [ + "m4b ", + "Ted Mielczarek ", +] +include = [ + "src/**/*", + "Cargo.toml", + "LICENSE", + "README.md", +] description = "A suite of powerful, extensible, generic, endian-aware Read/Write traits for byte buffers" documentation = "https://docs.rs/scroll" readme = "README.md" -keywords = ["bytes", "endian", "immutable", "pread", "pwrite"] +keywords = [ + "bytes", + "endian", + "immutable", + "pread", + "pwrite", +] license = "MIT" repository = "https://github.com/m4b/scroll" -resolver = "2" + [dependencies.scroll_derive] -version = "0.11" +version = "0.12" optional = true -[dev-dependencies.byteorder] -version = "1" - -[dev-dependencies.rayon] -version = "1" [features] default = ["std"] -derive = ["scroll_derive"] +derive = ["dep:scroll_derive"] std = [] diff --git a/third_party/rust/scroll/README.md b/third_party/rust/scroll/README.md index 717fe6a234..50dde54d7e 100644 --- a/third_party/rust/scroll/README.md +++ b/third_party/rust/scroll/README.md @@ -1,4 +1,13 @@ - [![Build Status](https://travis-ci.org/m4b/scroll.svg?branch=master)](https://travis-ci.org/m4b/scroll) +[![Actions][actions-badge]][actions-url] +[![crates.io version][crates-scroll-badge]][crates-scroll] + + + +[actions-badge]: https://github.com/m4b/scroll/workflows/CI/badge.svg?branch=master +[actions-url]: https://github.com/m4b/scroll/actions +[crates-scroll-badge]: https://img.shields.io/crates/v/scroll.svg +[crates-scroll]: https://crates.io/crates/scroll + ## Scroll - cast some magic ```text @@ -23,7 +32,7 @@ Add to your `Cargo.toml` ```toml, no_test [dependencies] -scroll = "0.10" +scroll = "0.11" ``` ### Overview diff --git a/third_party/rust/scroll/benches/bench.rs b/third_party/rust/scroll/benches/bench.rs deleted file mode 100644 index 0787dbe14b..0000000000 --- a/third_party/rust/scroll/benches/bench.rs +++ /dev/null @@ -1,157 +0,0 @@ -#![feature(test)] -extern crate test; - -use scroll::{Cread, Pread, LE}; -use test::black_box; - -#[bench] -fn bench_parallel_cread_with(b: &mut test::Bencher) { - use rayon::prelude::*; - let vec = vec![0u8; 1_000_000]; - let nums = vec![0usize; 500_000]; - b.iter(|| { - let data = black_box(&vec[..]); - nums.par_iter().for_each(|offset| { - let _: u16 = black_box(data.cread_with(*offset, LE)); - }); - }); - b.bytes = vec.len() as u64; -} - -#[bench] -fn bench_cread_vec(b: &mut test::Bencher) { - let vec = vec![0u8; 1_000_000]; - b.iter(|| { - let data = black_box(&vec[..]); - for val in data.chunks(2) { - let _: u16 = black_box(val.cread_with(0, LE)); - } - }); - b.bytes = vec.len() as u64; -} - -#[bench] -fn bench_cread(b: &mut test::Bencher) { - const NITER: i32 = 100_000; - b.iter(|| { - for _ in 1..NITER { - let data = black_box([1, 2]); - let _: u16 = black_box(data.cread(0)); - } - }); - b.bytes = 2 * NITER as u64; -} - -#[bench] -fn bench_pread_ctx_vec(b: &mut test::Bencher) { - let vec = vec![0u8; 1_000_000]; - b.iter(|| { - let data = black_box(&vec[..]); - for val in data.chunks(2) { - let _: Result = black_box(val.pread(0)); - } - }); - b.bytes = vec.len() as u64; -} - -#[bench] -fn bench_pread_with_unwrap(b: &mut test::Bencher) { - const NITER: i32 = 100_000; - b.iter(|| { - for _ in 1..NITER { - let data: &[u8] = &black_box([1, 2]); - let _: u16 = black_box(data.pread_with(0, LE).unwrap()); - } - }); - b.bytes = 2 * NITER as u64; -} - -#[bench] -fn bench_pread_vec(b: &mut test::Bencher) { - let vec = vec![0u8; 1_000_000]; - b.iter(|| { - let data = black_box(&vec[..]); - for val in data.chunks(2) { - let _: Result = black_box(val.pread_with(0, LE)); - } - }); - b.bytes = vec.len() as u64; -} - -#[bench] -fn bench_pread_unwrap(b: &mut test::Bencher) { - const NITER: i32 = 100_000; - b.iter(|| { - for _ in 1..NITER { - let data = black_box([1, 2]); - let _: u16 = black_box(data.pread(0)).unwrap(); - } - }); - b.bytes = 2 * NITER as u64; -} - -#[bench] -fn bench_gread_vec(b: &mut test::Bencher) { - let vec = vec![0u8; 1_000_000]; - b.iter(|| { - let data = black_box(&vec[..]); - for val in data.chunks(2) { - let mut offset = 0; - let _: Result = black_box(val.gread(&mut offset)); - } - }); - b.bytes = vec.len() as u64; -} - -#[bench] -fn bench_gread_unwrap(b: &mut test::Bencher) { - const NITER: i32 = 100_000; - b.iter(|| { - for _ in 1..NITER { - let data = black_box([1, 2]); - let mut offset = 0; - let _: u16 = black_box(data.gread_with(&mut offset, LE).unwrap()); - } - }); - b.bytes = 2 * NITER as u64; -} - -#[bench] -fn bench_parallel_pread_with(b: &mut test::Bencher) { - use rayon::prelude::*; - let vec = vec![0u8; 1_000_000]; - let nums = vec![0usize; 500_000]; - b.iter(|| { - let data = black_box(&vec[..]); - nums.par_iter().for_each(|offset| { - let _: Result = black_box(data.pread_with(*offset, LE)); - }); - }); - b.bytes = vec.len() as u64; -} - -#[bench] -fn bench_byteorder_vec(b: &mut test::Bencher) { - use byteorder::ReadBytesExt; - let vec = vec![0u8; 1_000_000]; - b.iter(|| { - let data = black_box(&vec[..]); - for mut val in data.chunks(2) { - let _: Result = black_box(val.read_u16::()); - } - }); - b.bytes = vec.len() as u64; -} - -#[bench] -fn bench_byteorder(b: &mut test::Bencher) { - use byteorder::ByteOrder; - const NITER: i32 = 100_000; - b.iter(|| { - for _ in 1..NITER { - let data = black_box([1, 2]); - let _: u16 = black_box(byteorder::LittleEndian::read_u16(&data)); - } - }); - b.bytes = 2 * NITER as u64; -} diff --git a/third_party/rust/scroll/examples/data_ctx.rs b/third_party/rust/scroll/examples/data_ctx.rs deleted file mode 100644 index 667f4b18f0..0000000000 --- a/third_party/rust/scroll/examples/data_ctx.rs +++ /dev/null @@ -1,24 +0,0 @@ -use scroll::{ctx, Endian, Pread, BE}; - -#[derive(Debug)] -struct Data<'a> { - name: &'a str, - id: u32, -} - -impl<'a> ctx::TryFromCtx<'a, Endian> for Data<'a> { - type Error = scroll::Error; - fn try_from_ctx(src: &'a [u8], endian: Endian) -> Result<(Self, usize), Self::Error> { - let name = src.pread::<&'a str>(0)?; - let id = src.pread_with(name.len() + 1, endian)?; - Ok((Data { name: name, id: id }, name.len() + 4)) - } -} - -fn main() { - let bytes = b"UserName\x00\x01\x02\x03\x04"; - let data = bytes.pread_with::(0, BE).unwrap(); - assert_eq!(data.id, 0x01020304); - assert_eq!(data.name.to_string(), "UserName".to_string()); - println!("Data: {:?}", &data); -} diff --git a/third_party/rust/scroll/src/ctx.rs b/third_party/rust/scroll/src/ctx.rs index 1f982b82fa..e24d2dc506 100644 --- a/third_party/rust/scroll/src/ctx.rs +++ b/third_party/rust/scroll/src/ctx.rs @@ -180,17 +180,14 @@ //! } //! ``` -use core::mem::size_of; -use core::mem::transmute; +use core::mem::{size_of, MaybeUninit}; use core::ptr::copy_nonoverlapping; -use core::result; -use core::str; - +use core::{result, str}; #[cfg(feature = "std")] use std::ffi::{CStr, CString}; use crate::endian::Endian; -use crate::error; +use crate::{error, Pread, Pwrite}; /// A trait for measuring how large something is; for a byte sequence, it will be its length. pub trait MeasureWith { @@ -240,18 +237,14 @@ impl Default for StrCtx { impl StrCtx { pub fn len(&self) -> usize { - match *self { + match self { StrCtx::Delimiter(_) | StrCtx::DelimiterUntil(_, _) => 1, StrCtx::Length(_) => 0, } } pub fn is_empty(&self) -> bool { - if let StrCtx::Length(_) = *self { - true - } else { - false - } + matches!(self, StrCtx::Length(_)) } } @@ -267,6 +260,7 @@ pub trait FromCtx { /// `[u8]`), then you need to implement this trait /// /// ```rust +/// ##[cfg(feature = "std")] { /// use scroll::{self, ctx, Pread}; /// #[derive(Debug, PartialEq, Eq)] /// pub struct Foo(u16); @@ -286,6 +280,7 @@ pub trait FromCtx { /// /// let foo2 = bytes.pread_with::(0, scroll::BE).unwrap(); /// assert_eq!(Foo(0xdeadu16), foo2); +/// # } /// ``` /// /// # Advanced: Using Your Own Error in `TryFromCtx` @@ -350,6 +345,7 @@ pub trait IntoCtx: Sized { /// To implement writing into an arbitrary byte buffer, implement `TryIntoCtx` /// # Example /// ```rust +/// ##[cfg(feature = "std")] { /// use scroll::{self, ctx, LE, Endian, Pwrite}; /// #[derive(Debug, PartialEq, Eq)] /// pub struct Foo(u16); @@ -369,6 +365,7 @@ pub trait IntoCtx: Sized { /// /// let mut bytes: [u8; 4] = [0, 0, 0, 0]; /// bytes.pwrite_with(Foo(0x7f), 1, LE).unwrap(); +/// # } /// ``` pub trait TryIntoCtx: Sized { type Error; @@ -403,13 +400,14 @@ macro_rules! signed_to_unsigned { macro_rules! write_into { ($typ:ty, $size:expr, $n:expr, $dst:expr, $endian:expr) => {{ + assert!($dst.len() >= $size); + let bytes = if $endian.is_little() { + $n.to_le() + } else { + $n.to_be() + } + .to_ne_bytes(); unsafe { - assert!($dst.len() >= $size); - let bytes = transmute::<$typ, [u8; $size]>(if $endian.is_little() { - $n.to_le() - } else { - $n.to_be() - }); copy_nonoverlapping((&bytes).as_ptr(), $dst.as_mut_ptr(), $size); } }}; @@ -570,12 +568,12 @@ macro_rules! from_ctx_float_impl { &mut data as *mut signed_to_unsigned!($typ) as *mut u8, $size, ); - transmute(if le.is_little() { - data.to_le() - } else { - data.to_be() - }) } + $typ::from_bits(if le.is_little() { + data.to_le() + } else { + data.to_be() + }) } } impl<'a> TryFromCtx<'a, Endian> for $typ @@ -621,13 +619,7 @@ macro_rules! into_ctx_float_impl { #[inline] fn into_ctx(self, dst: &mut [u8], le: Endian) { assert!(dst.len() >= $size); - write_into!( - signed_to_unsigned!($typ), - $size, - transmute::<$typ, signed_to_unsigned!($typ)>(self), - dst, - le - ); + write_into!(signed_to_unsigned!($typ), $size, self.to_bits(), dst, le); } } impl<'a> IntoCtx for &'a $typ { @@ -725,7 +717,7 @@ impl<'a> TryIntoCtx for &'a [u8] { let src_len = self.len() as isize; let dst_len = dst.len() as isize; // if src_len < 0 || dst_len < 0 || offset < 0 { - // return Err(error::Error::BadOffset(format!("requested operation has negative casts: src len: {} dst len: {} offset: {}", src_len, dst_len, offset)).into()) + // return Err(error::Error::BadOffset(format!("requested operation has negative casts: src len: {src_len} dst len: {dst_len} offset: {offset}")).into()) // } if src_len > dst_len { Err(error::Error::TooBig { @@ -789,6 +781,56 @@ impl<'a> TryFromCtx<'a, usize> for &'a [u8] { } } +impl<'a, Ctx: Copy, T: TryFromCtx<'a, Ctx, Error = error::Error>, const N: usize> + TryFromCtx<'a, Ctx> for [T; N] +{ + type Error = error::Error; + fn try_from_ctx(src: &'a [u8], ctx: Ctx) -> Result<(Self, usize), Self::Error> { + let mut offset = 0; + + let mut buf: [MaybeUninit; N] = core::array::from_fn(|_| MaybeUninit::uninit()); + + let mut error_ctx = None; + for (idx, element) in buf.iter_mut().enumerate() { + match src.gread_with::(&mut offset, ctx) { + Ok(val) => { + *element = MaybeUninit::new(val); + } + Err(e) => { + error_ctx = Some((e, idx)); + break; + } + } + } + if let Some((e, idx)) = error_ctx { + for element in &mut buf[0..idx].iter_mut() { + // SAFETY: Any element upto idx must have already been initialized, since + // we iterate until we encounter an error. + unsafe { + element.assume_init_drop(); + } + } + Err(e) + } else { + // SAFETY: we initialized each element above by preading them out, correctness + // of the initialized element is guaranted by pread itself + Ok((buf.map(|element| unsafe { element.assume_init() }), offset)) + } + } +} +impl, const N: usize> TryIntoCtx + for [T; N] +{ + type Error = error::Error; + fn try_into_ctx(self, buf: &mut [u8], ctx: Ctx) -> Result { + let mut offset = 0; + for element in self { + buf.gwrite_with(element, &mut offset, ctx)?; + } + Ok(offset) + } +} + #[cfg(feature = "std")] impl<'a> TryFromCtx<'a> for &'a CStr { type Error = error::Error; @@ -863,11 +905,11 @@ impl TryIntoCtx for CString { // } #[cfg(test)] +#[cfg(feature = "std")] mod tests { use super::*; #[test] - #[cfg(feature = "std")] fn parse_a_cstr() { let src = CString::new("Hello World").unwrap(); let as_bytes = src.as_bytes_with_nul(); @@ -879,7 +921,6 @@ mod tests { } #[test] - #[cfg(feature = "std")] fn round_trip_a_c_str() { let src = CString::new("Hello World").unwrap(); let src = src.as_c_str(); diff --git a/third_party/rust/scroll/src/endian.rs b/third_party/rust/scroll/src/endian.rs index 06d7a1dc1c..7b83c348d5 100644 --- a/third_party/rust/scroll/src/endian.rs +++ b/third_party/rust/scroll/src/endian.rs @@ -43,9 +43,6 @@ impl Endian { } #[inline] pub fn is_little(&self) -> bool { - match *self { - LE => true, - _ => false, - } + *self == LE } } diff --git a/third_party/rust/scroll/src/error.rs b/third_party/rust/scroll/src/error.rs index 7740254774..1b68c2e4c7 100644 --- a/third_party/rust/scroll/src/error.rs +++ b/third_party/rust/scroll/src/error.rs @@ -1,10 +1,7 @@ use core::fmt::{self, Display}; use core::result; - -#[cfg(feature = "std")] -use std::error; #[cfg(feature = "std")] -use std::io; +use std::{error, io}; #[derive(Debug)] /// A custom Scroll error @@ -20,18 +17,19 @@ pub enum Error { size: usize, msg: &'static str, }, + /// A custom Scroll error for reporting messages to clients. + /// For no-std, use [`Error::BadInput`] with a static string. #[cfg(feature = "std")] - /// A custom Scroll error for reporting messages to clients Custom(String), - #[cfg(feature = "std")] /// Returned when IO based errors are encountered + #[cfg(feature = "std")] IO(io::Error), } #[cfg(feature = "std")] impl error::Error for Error { fn description(&self) -> &str { - match *self { + match self { Error::TooBig { .. } => "TooBig", Error::BadOffset(_) => "BadOffset", Error::BadInput { .. } => "BadInput", @@ -40,7 +38,7 @@ impl error::Error for Error { } } fn cause(&self) -> Option<&dyn error::Error> { - match *self { + match self { Error::TooBig { .. } => None, Error::BadOffset(_) => None, Error::BadInput { .. } => None, @@ -59,23 +57,23 @@ impl From for Error { impl Display for Error { fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { - match *self { + match self { Error::TooBig { ref size, ref len } => { - write!(fmt, "type is too big ({}) for {}", size, len) + write!(fmt, "type is too big ({size}) for {len}") } Error::BadOffset(ref offset) => { - write!(fmt, "bad offset {}", offset) + write!(fmt, "bad offset {offset}") } Error::BadInput { ref msg, ref size } => { - write!(fmt, "bad input {} ({})", msg, size) + write!(fmt, "bad input {msg} ({size})") } #[cfg(feature = "std")] Error::Custom(ref msg) => { - write!(fmt, "{}", msg) + write!(fmt, "{msg}") } #[cfg(feature = "std")] Error::IO(ref err) => { - write!(fmt, "{}", err) + write!(fmt, "{err}") } } } diff --git a/third_party/rust/scroll/src/leb128.rs b/third_party/rust/scroll/src/leb128.rs index 43f50b95f1..eceec6d166 100644 --- a/third_party/rust/scroll/src/leb128.rs +++ b/third_party/rust/scroll/src/leb128.rs @@ -1,9 +1,8 @@ -use crate::ctx::TryFromCtx; -use crate::error; -use crate::Pread; use core::convert::{AsRef, From}; -use core::result; -use core::u8; +use core::{result, u8}; + +use crate::ctx::TryFromCtx; +use crate::{error, Pread}; #[derive(Debug, PartialEq, Copy, Clone)] /// An unsigned leb128 integer @@ -184,21 +183,24 @@ mod tests { let buf = [2u8 | CONTINUATION_BIT, 1]; let bytes = &buf[..]; let num = bytes.pread::(0).unwrap(); - println!("num: {:?}", &num); + #[cfg(feature = "std")] + println!("num: {num:?}"); assert_eq!(130u64, num.into()); assert_eq!(num.size(), 2); let buf = [0x00, 0x01]; let bytes = &buf[..]; let num = bytes.pread::(0).unwrap(); - println!("num: {:?}", &num); + #[cfg(feature = "std")] + println!("num: {num:?}"); assert_eq!(0u64, num.into()); assert_eq!(num.size(), 1); let buf = [0x21]; let bytes = &buf[..]; let num = bytes.pread::(0).unwrap(); - println!("num: {:?}", &num); + #[cfg(feature = "std")] + println!("num: {num:?}"); assert_eq!(0x21u64, num.into()); assert_eq!(num.size(), 1); } diff --git a/third_party/rust/scroll/src/lesser.rs b/third_party/rust/scroll/src/lesser.rs index 46ef4c5b11..636bf2553e 100644 --- a/third_party/rust/scroll/src/lesser.rs +++ b/third_party/rust/scroll/src/lesser.rs @@ -1,6 +1,7 @@ -use crate::ctx::{FromCtx, IntoCtx, SizeWith}; use std::io::{Read, Result, Write}; +use crate::ctx::{FromCtx, IntoCtx, SizeWith}; + /// An extension trait to `std::io::Read` streams; mainly targeted at reading primitive types with /// a known size. /// @@ -104,8 +105,8 @@ pub trait IOread: Read { fn ioread_with + SizeWith>(&mut self, ctx: Ctx) -> Result { let mut scratch = [0u8; 256]; let size = N::size_with(&ctx); - let mut buf = &mut scratch[0..size]; - self.read_exact(&mut buf)?; + let buf = &mut scratch[0..size]; + self.read_exact(buf)?; Ok(N::from_ctx(buf, ctx)) } } diff --git a/third_party/rust/scroll/src/lib.rs b/third_party/rust/scroll/src/lib.rs index dcb58e7564..2740648517 100644 --- a/third_party/rust/scroll/src/lib.rs +++ b/third_party/rust/scroll/src/lib.rs @@ -119,6 +119,7 @@ //! [FromCtx](ctx/trait.FromCtx.html) and [SizeWith](ctx/trait.SizeWith.html). //! //! ```rust +//! ##[cfg(feature = "std")] { //! use std::io::Cursor; //! use scroll::{IOread, ctx, Endian}; //! let bytes = [0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0xef,0xbe,0x00,0x00,]; @@ -139,12 +140,14 @@ //! // read/written, e.g. switching between ELF32 or ELF64 at runtime. //! let size = >::size_with(&Endian::Little) as u64; //! assert_eq!(prev + size, after); +//! # } //! ``` //! //! In the same vein as IOread we can use IOwrite to write a type to anything implementing //! `std::io::Write`: //! //! ```rust +//! ##[cfg(feature = "std")] { //! use std::io::Cursor; //! use scroll::{IOwrite}; //! @@ -155,6 +158,7 @@ //! cursor.iowrite_with(0xdeadbeef as u32, scroll::BE).unwrap(); //! //! assert_eq!(cursor.into_inner(), [0xde, 0xad, 0xbe, 0xef, 0x0]); +//! # } //! ``` //! //! ## Complex use cases @@ -249,8 +253,7 @@ pub use crate::pwrite::*; #[doc(hidden)] pub mod export { - pub use ::core::mem; - pub use ::core::result; + pub use ::core::{mem, result}; } #[allow(unused)] @@ -267,7 +270,6 @@ doc_comment!(include_str!("../README.md")); #[cfg(test)] mod tests { - #[allow(overflowing_literals)] use super::LE; #[test] @@ -355,36 +357,48 @@ mod tests { let bytes: [u8; 2] = [0x2e, 0x0]; let b = &bytes[..]; let s: &str = b.pread(0).unwrap(); - println!("str: {}", s); + #[cfg(feature = "std")] + println!("str: {s}"); assert_eq!(s.len(), bytes[..].len() - 1); let bytes: &[u8] = b"hello, world!\0some_other_things"; let hello_world: &str = bytes.pread_with(0, StrCtx::Delimiter(NULL)).unwrap(); - println!("{:?}", &hello_world); + #[cfg(feature = "std")] + println!("{hello_world:?}"); assert_eq!(hello_world.len(), 13); let hello: &str = bytes.pread_with(0, StrCtx::Delimiter(SPACE)).unwrap(); - println!("{:?}", &hello); + #[cfg(feature = "std")] + println!("{hello:?}"); assert_eq!(hello.len(), 6); // this could result in underflow so we just try it let _error = bytes.pread_with::<&str>(6, StrCtx::Delimiter(SPACE)); let error = bytes.pread_with::<&str>(7, StrCtx::Delimiter(SPACE)); - println!("{:?}", &error); + #[cfg(feature = "std")] + println!("{error:?}"); assert!(error.is_ok()); } + /// In this test, we are testing preading + /// at length boundaries. + /// In the past, this test was supposed to test failures for `hello_world`. + /// Since PR#94, this test is unwrapping as we exploit + /// the fact that if you do &x[x.len()..] you get an empty slice. #[test] fn pread_str_weird() { use super::ctx::*; use super::Pread; let bytes: &[u8] = b""; let hello_world = bytes.pread_with::<&str>(0, StrCtx::Delimiter(NULL)); - println!("1 {:?}", &hello_world); - assert_eq!(hello_world.is_err(), true); + #[cfg(feature = "std")] + println!("1 {hello_world:?}"); + assert!(hello_world.unwrap().is_empty()); let error = bytes.pread_with::<&str>(7, StrCtx::Delimiter(SPACE)); - println!("2 {:?}", &error); + #[cfg(feature = "std")] + println!("2 {error:?}"); assert!(error.is_err()); let bytes: &[u8] = b"\0"; let null = bytes.pread::<&str>(0).unwrap(); - println!("3 {:?}", &null); + #[cfg(feature = "std")] + println!("3 {null:?}"); assert_eq!(null.len(), 0); } @@ -413,8 +427,7 @@ mod tests { assert_eq!(bytes, "bytes"); } - use std::error; - use std::fmt::{self, Display}; + use core::fmt::{self, Display}; #[derive(Debug)] pub struct ExternalError {} @@ -425,11 +438,12 @@ mod tests { } } - impl error::Error for ExternalError { + #[cfg(feature = "std")] + impl std::error::Error for ExternalError { fn description(&self) -> &str { "ExternalError" } - fn cause(&self) -> Option<&dyn error::Error> { + fn cause(&self) -> Option<&dyn std::error::Error> { None } } @@ -451,7 +465,7 @@ mod tests { fn try_into_ctx(self, this: &mut [u8], le: super::Endian) -> Result { use super::Pwrite; if this.len() < 2 { - return Err((ExternalError {}).into()); + return Err(ExternalError {}); } this.pwrite_with(self.0, 0, le)?; Ok(2) @@ -463,7 +477,7 @@ mod tests { fn try_from_ctx(this: &'a [u8], le: super::Endian) -> Result<(Self, usize), Self::Error> { use super::Pread; if this.len() > 2 { - return Err((ExternalError {}).into()); + return Err(ExternalError {}); } let n = this.pread_with(0, le)?; Ok((Foo(n), 2)) @@ -499,7 +513,7 @@ mod tests { let mut offset = 0; let deadbeef: $typ = bytes.gread_with(&mut offset, LE).unwrap(); assert_eq!(deadbeef, $deadbeef as $typ); - assert_eq!(offset, ::std::mem::size_of::<$typ>()); + assert_eq!(offset, ::core::mem::size_of::<$typ>()); } }; } @@ -518,7 +532,7 @@ mod tests { let mut offset = 0; let deadbeef: $typ = bytes.gread_with(&mut offset, LE).unwrap(); assert_eq!(deadbeef, $deadbeef as $typ); - assert_eq!(offset, ::std::mem::size_of::<$typ>()); + assert_eq!(offset, ::core::mem::size_of::<$typ>()); } }; } @@ -537,8 +551,8 @@ mod tests { let o2 = &mut 0; let val: $typ = buffer.gread_with(o2, LE).unwrap(); assert_eq!(val, $val); - assert_eq!(*offset, ::std::mem::size_of::<$typ>()); - assert_eq!(*o2, ::std::mem::size_of::<$typ>()); + assert_eq!(*offset, ::core::mem::size_of::<$typ>()); + assert_eq!(*o2, ::core::mem::size_of::<$typ>()); assert_eq!(*o2, *offset); buffer.gwrite_with($val.clone(), offset, BE).unwrap(); let val: $typ = buffer.gread_with(o2, BE).unwrap(); @@ -612,16 +626,17 @@ mod tests { let res = b.gread_with::<&str>(offset, StrCtx::Length(3)); assert!(res.is_err()); *offset = 0; - let astring: [u8; 3] = [0x45, 042, 0x44]; + let astring: [u8; 3] = [0x45, 0x42, 0x44]; let string = astring.gread_with::<&str>(offset, StrCtx::Length(2)); match &string { - &Ok(_) => {} - &Err(ref err) => { - println!("{}", &err); + Ok(_) => {} + Err(_err) => { + #[cfg(feature = "std")] + println!("{_err}"); panic!(); } } - assert_eq!(string.unwrap(), "E*"); + assert_eq!(string.unwrap(), "EB"); *offset = 0; let bytes2: &[u8] = b.gread_with(offset, 2).unwrap(); assert_eq!(*offset, 2); diff --git a/third_party/rust/scroll/src/pread.rs b/third_party/rust/scroll/src/pread.rs index 72ba877054..15bf1426be 100644 --- a/third_party/rust/scroll/src/pread.rs +++ b/third_party/rust/scroll/src/pread.rs @@ -20,6 +20,11 @@ use crate::error; /// over chunks of memory or any other indexable type — but scroll does come with a set of powerful /// blanket implementations for data being a continous block of byte-addressable memory. /// +/// Note that in the particular case of the implementation of `Pread` for `[u8]`, +/// reading it at the length boundary of that slice will cause to read from an empty slice. +/// i.e. we make use of the fact that `&bytes[bytes.len()..]` will return an empty slice, rather +/// than returning an error. In the past, scroll returned an offset error. +/// /// Pread provides two main groups of functions: pread and gread. /// /// `pread` is the basic function that simply extracts a given type from a given data store - either @@ -167,7 +172,7 @@ impl> Pread for [u8] { ctx: Ctx, ) -> result::Result { let start = *offset; - if start >= self.len() { + if start > self.len() { return Err(error::Error::BadOffset(start).into()); } N::try_from_ctx(&self[start..], ctx).map(|(n, size)| { diff --git a/third_party/rust/scroll/src/pwrite.rs b/third_party/rust/scroll/src/pwrite.rs index ab6d96157d..7a07f2daba 100644 --- a/third_party/rust/scroll/src/pwrite.rs +++ b/third_party/rust/scroll/src/pwrite.rs @@ -19,6 +19,13 @@ use crate::error; /// with 'read' switched for 'write' and 'From' switched with 'Into' so if you haven't yet you /// should read the documentation of `Pread` first. /// +/// As with `Pread`, note that in the particular case of the implementation of `Pwrite` for `[u8]`, +/// writing it at the length boundary of that slice will cause to write in an empty slice. +/// i.e. we make use of the fact that `&bytes[bytes.len()..]` will return an empty slice, rather +/// than returning an error. In the past, scroll returned an offset error. +/// In this case, this is relevant if you are writing an empty slice inside an empty slice and +/// expected this to work. +/// /// Unless you need to implement your own data store — that is either can't convert to `&[u8]` or /// have a data that does not expose a `&mut [u8]` — you will probably want to implement /// [TryIntoCtx](ctx/trait.TryIntoCtx.html) on your Rust types to be written. @@ -87,7 +94,7 @@ impl> Pwrite for [u8] { offset: usize, ctx: Ctx, ) -> result::Result { - if offset >= self.len() { + if offset > self.len() { return Err(error::Error::BadOffset(offset).into()); } let dst = &mut self[offset..]; diff --git a/third_party/rust/scroll/tests/api.rs b/third_party/rust/scroll/tests/api.rs deleted file mode 100644 index e10726f22a..0000000000 --- a/third_party/rust/scroll/tests/api.rs +++ /dev/null @@ -1,292 +0,0 @@ -// this exists primarily to test various API usages of scroll; e.g., must compile - -// guard against potential undefined behaviour when borrowing from -// packed structs. See https://github.com/rust-lang/rust/issues/46043 -#![deny(unaligned_references)] - -// #[macro_use] extern crate scroll_derive; - -use scroll::ctx::SizeWith; -use scroll::{ctx, Cread, Pread, Result}; -use std::ops::{Deref, DerefMut}; - -#[derive(Default)] -pub struct Section<'a> { - pub sectname: [u8; 16], - pub segname: [u8; 16], - pub addr: u64, - pub size: u64, - pub offset: u32, - pub align: u32, - pub reloff: u32, - pub nreloc: u32, - pub flags: u32, - pub data: &'a [u8], -} - -impl<'a> Section<'a> { - pub fn name(&self) -> Result<&str> { - self.sectname.pread::<&str>(0) - } - pub fn segname(&self) -> Result<&str> { - self.segname.pread::<&str>(0) - } -} - -impl<'a> ctx::SizeWith for Section<'a> { - fn size_with(_ctx: &()) -> usize { - 4 - } -} - -#[repr(C)] -// renable when scroll_derive Pread/Pwrite matches -//#[derive(Debug, Clone, Copy, Pread, Pwrite)] -#[derive(Debug, Clone, Copy)] -pub struct Section32 { - pub sectname: [u8; 16], - pub segname: [u8; 16], - pub addr: u32, - pub size: u32, - pub offset: u32, - pub align: u32, - pub reloff: u32, - pub nreloc: u32, - pub flags: u32, - pub reserved1: u32, - pub reserved2: u32, -} - -impl<'a> ctx::TryFromCtx<'a, ()> for Section<'a> { - type Error = scroll::Error; - fn try_from_ctx( - _bytes: &'a [u8], - _ctx: (), - ) -> ::std::result::Result<(Self, usize), Self::Error> { - let section = Section::default(); - Ok((section, ::std::mem::size_of::
())) - } -} - -pub struct Segment<'a> { - pub cmd: u32, - pub cmdsize: u32, - pub segname: [u8; 16], - pub vmaddr: u64, - pub vmsize: u64, - pub fileoff: u64, - pub filesize: u64, - pub maxprot: u32, - pub initprot: u32, - pub nsects: u32, - pub flags: u32, - pub data: &'a [u8], - offset: usize, - raw_data: &'a [u8], -} - -impl<'a> Segment<'a> { - pub fn name(&self) -> Result<&str> { - Ok(self.segname.pread::<&str>(0)?) - } - pub fn sections(&self) -> Result>> { - let nsects = self.nsects as usize; - let mut sections = Vec::with_capacity(nsects); - let offset = &mut (self.offset + Self::size_with(&())); - let _size = Section::size_with(&()); - let raw_data: &'a [u8] = self.raw_data; - for _ in 0..nsects { - let section = raw_data.gread_with::>(offset, ())?; - sections.push(section); - //offset += size; - } - Ok(sections) - } -} - -impl<'a> ctx::SizeWith for Segment<'a> { - fn size_with(_ctx: &()) -> usize { - 4 - } -} - -pub struct Segments<'a> { - pub segments: Vec>, -} - -impl<'a> Deref for Segments<'a> { - type Target = Vec>; - fn deref(&self) -> &Self::Target { - &self.segments - } -} - -impl<'a> DerefMut for Segments<'a> { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.segments - } -} - -impl<'a> Segments<'a> { - pub fn new() -> Self { - Segments { - segments: Vec::new(), - } - } - pub fn sections(&self) -> Result>>> { - let mut sections = Vec::new(); - for segment in &self.segments { - sections.push(segment.sections()?); - } - Ok(sections) - } -} - -fn lifetime_passthrough_<'a>(segments: &Segments<'a>, section_name: &str) -> Option<&'a [u8]> { - let segment_name = "__TEXT"; - for segment in &segments.segments { - if let Ok(name) = segment.name() { - println!("segment.name: {}", name); - if name == segment_name { - if let Ok(sections) = segment.sections() { - for section in sections { - let sname = section.name().unwrap(); - println!("section.name: {}", sname); - if section_name == sname { - return Some(section.data); - } - } - } - } - } - } - None -} - -#[test] -fn lifetime_passthrough() { - let segments = Segments::new(); - let _res = lifetime_passthrough_(&segments, "__text"); - assert!(true) -} - -#[derive(Default)] -#[repr(packed)] -struct Foo { - foo: i64, - bar: u32, -} - -impl scroll::ctx::FromCtx for Foo { - fn from_ctx(bytes: &[u8], ctx: scroll::Endian) -> Self { - Foo { - foo: bytes.cread_with::(0, ctx), - bar: bytes.cread_with::(8, ctx), - } - } -} - -impl scroll::ctx::SizeWith for Foo { - fn size_with(_: &scroll::Endian) -> usize { - ::std::mem::size_of::() - } -} - -#[test] -fn ioread_api() { - use scroll::{IOread, LE}; - use std::io::Cursor; - let bytes_ = [ - 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xef, 0xbe, 0x00, 0x00, - ]; - let mut bytes = Cursor::new(bytes_); - let foo = bytes.ioread_with::(LE).unwrap(); - let bar = bytes.ioread_with::(LE).unwrap(); - assert_eq!(foo, 1); - assert_eq!(bar, 0xbeef); - let error = bytes.ioread_with::(LE); - assert!(error.is_err()); - let mut bytes = Cursor::new(bytes_); - let foo_ = bytes.ioread_with::(LE).unwrap(); - assert_eq!({ foo_.foo }, foo); - assert_eq!({ foo_.bar }, bar); -} - -#[repr(packed)] -struct Bar { - foo: i32, - bar: u32, -} - -impl scroll::ctx::FromCtx for Bar { - fn from_ctx(bytes: &[u8], ctx: scroll::Endian) -> Self { - Bar { - foo: bytes.cread_with(0, ctx), - bar: bytes.cread_with(4, ctx), - } - } -} - -#[test] -fn cread_api() { - use scroll::{Cread, LE}; - let bytes = [ - 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xef, 0xbe, 0x00, 0x00, - ]; - let foo = bytes.cread_with::(0, LE); - let bar = bytes.cread_with::(8, LE); - assert_eq!(foo, 1); - assert_eq!(bar, 0xbeef); -} - -#[test] -fn cread_api_customtype() { - use scroll::{Cread, LE}; - let bytes = [0xff, 0xff, 0xff, 0xff, 0xef, 0xbe, 0xad, 0xde]; - let bar = &bytes[..].cread_with::(0, LE); - assert_eq!({ bar.foo }, -1); - assert_eq!({ bar.bar }, 0xdeadbeef); -} - -#[test] -#[should_panic] -fn cread_api_badindex() { - use scroll::Cread; - let bytes = [ - 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xef, 0xbe, 0xad, 0xde, - ]; - let _foo = bytes.cread::(1_000_000); -} - -#[test] -fn cwrite_api() { - use scroll::Cread; - use scroll::Cwrite; - let mut bytes = [0x0; 16]; - bytes.cwrite::(42, 0); - bytes.cwrite::(0xdeadbeef, 8); - assert_eq!(bytes.cread::(0), 42); - assert_eq!(bytes.cread::(8), 0xdeadbeef); -} - -impl scroll::ctx::IntoCtx for Bar { - fn into_ctx(self, bytes: &mut [u8], ctx: scroll::Endian) { - use scroll::Cwrite; - bytes.cwrite_with(self.foo, 0, ctx); - bytes.cwrite_with(self.bar, 4, ctx); - } -} - -#[test] -fn cwrite_api_customtype() { - use scroll::{Cread, Cwrite}; - let bar = Bar { - foo: -1, - bar: 0xdeadbeef, - }; - let mut bytes = [0x0; 16]; - let _ = &bytes[..].cwrite::(bar, 0); - let bar = bytes.cread::(0); - assert_eq!({ bar.foo }, -1); - assert_eq!({ bar.bar }, 0xdeadbeef); -} diff --git a/third_party/rust/scroll_derive/.cargo-checksum.json b/third_party/rust/scroll_derive/.cargo-checksum.json index 8c6b3b87c4..b881da6c6a 100644 --- a/third_party/rust/scroll_derive/.cargo-checksum.json +++ b/third_party/rust/scroll_derive/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"Cargo.toml":"9fbb5068c3ffbf2c357f4068f854f439bae4999e04527e2dedc6758fa37a9807","LICENSE":"afb11426e09da40a1ae4f8fa17ddcc6b6a52d14df04c29bc5bcd06eb8730624d","README.md":"f89c7768454b0d2b9db816afe05db3a4cea1125bef87f08ed3eefd65e9e2b180","src/lib.rs":"a9cabe3c0b373f352357745b817f188ab841e9445056014dee9cc83c4d167483"},"package":"1db149f81d46d2deba7cd3c50772474707729550221e69588478ebf9ada425ae"} \ No newline at end of file +{"files":{"Cargo.toml":"57ee02784903ef6f506e87164230e0bf543cf9f9bcd1546e123158c7ab98b648","LICENSE":"afb11426e09da40a1ae4f8fa17ddcc6b6a52d14df04c29bc5bcd06eb8730624d","README.md":"0ed9b8c8ec7dd75f14aab9b7e54769f81b86e68960658356e260e5ec8ccac206","src/lib.rs":"a9cabe3c0b373f352357745b817f188ab841e9445056014dee9cc83c4d167483"},"package":"7f81c2fde025af7e69b1d1420531c8a8811ca898919db177141a85313b1cb932"} \ No newline at end of file diff --git a/third_party/rust/scroll_derive/Cargo.toml b/third_party/rust/scroll_derive/Cargo.toml index 71f40a7e6c..9e2e33bd78 100644 --- a/third_party/rust/scroll_derive/Cargo.toml +++ b/third_party/rust/scroll_derive/Cargo.toml @@ -12,7 +12,7 @@ [package] edition = "2018" name = "scroll_derive" -version = "0.11.1" +version = "0.12.0" authors = [ "m4b ", "Ted Mielczarek ", diff --git a/third_party/rust/scroll_derive/README.md b/third_party/rust/scroll_derive/README.md index a7f7e85f0e..675b6d7803 100644 --- a/third_party/rust/scroll_derive/README.md +++ b/third_party/rust/scroll_derive/README.md @@ -21,7 +21,7 @@ use scroll::{Pread, Pwrite, Cread, LE}; fn main (){ let bytes = [0xefu8, 0xbe, 0xad, 0xde, 0, 0, 0, 0, 0, 0, 224, 63, 0xad, 0xde, 0xef, 0xbe]; let data: Data = bytes.pread_with(0, LE).unwrap(); - println!("data: {:?}", &data); + println!("data: {data:?}"); assert_eq!(data.id, 0xdeadbeefu32); let mut bytes2 = vec![0; ::std::mem::size_of::()]; bytes2.pwrite_with(data, 0, LE).unwrap(); diff --git a/third_party/rust/smawk/.cargo-checksum.json b/third_party/rust/smawk/.cargo-checksum.json new file mode 100644 index 0000000000..c553a4a2c6 --- /dev/null +++ b/third_party/rust/smawk/.cargo-checksum.json @@ -0,0 +1 @@ +{"files":{"Cargo.toml":"4581638d3c628d22826bde37114048c825ffb354f17f21645d8d49f9ebd64689","LICENSE":"0173035e025d60b1d19197840a93a887f6da8b075c01dd10601fcb6414a0043b","README.md":"c27297df61be8dd14e47dc30a80ae1d443f5acea82932139637543bc6d860631","dprint.json":"aacd5ec32db8741fbdea4ac916e61f0011485a51e8ec7a660f849be60cc7b512","rustfmt.toml":"6819baea67831b8a8b2a7ad33af1128dd2774a900c804635c912bb6545a4e922","src/brute_force.rs":"02edda18441ea5d6cc89d2fdfb9ab32a361e2598de74a71fb930fb630288ce35","src/lib.rs":"b312e4855945cfe27f4b1e9949b1c6ffea8f248ad80ac8fc49e72f0cc38df219","src/monge.rs":"f6c475f4d094b70b5e45d0c8a94112d42eaafa0ab41b2d3d96d06a38f1bac32d","src/recursive.rs":"e585286fe6c885dcac8001d0f484718aa8f73f3f85a452f8b4c1cb36d4fbfcf6","tests/agreement.rs":"764406a5d8c9a322bab8787764d780832cfc3962722ed01efda99684a619d543","tests/complexity.rs":"e2e850d38529f171eb6005807c2a86a3f95a907052253eaa8e24a834200cda0b","tests/monge.rs":"fe418373f89904cd40e2ed1d539bccd2d9be50c1f3f9ab2d93806ff3bce6b7ea","tests/random_monge/mod.rs":"83cf1dd0c7b0b511ad754c19857a5d830ed54e8fef3c31235cd70b709687534b","tests/version-numbers.rs":"73301b7bfe500eada5ede66f0dce89bd3e354af50a8e7a123b02931cd5eb8e16"},"package":"b7c388c1b5e93756d0c740965c41e8822f866621d41acbdf6336a6a168f8840c"} \ No newline at end of file diff --git a/third_party/rust/smawk/Cargo.toml b/third_party/rust/smawk/Cargo.toml new file mode 100644 index 0000000000..44fc03df30 --- /dev/null +++ b/third_party/rust/smawk/Cargo.toml @@ -0,0 +1,53 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2021" +name = "smawk" +version = "0.3.2" +authors = ["Martin Geisler "] +exclude = [ + ".github/", + ".gitignore", + "benches/", + "examples/", +] +description = "Functions for finding row-minima in a totally monotone matrix." +readme = "README.md" +keywords = [ + "smawk", + "matrix", + "optimization", + "dynamic-programming", +] +categories = [ + "algorithms", + "mathematics", + "science", +] +license = "MIT" +repository = "https://github.com/mgeisler/smawk" + +[dependencies.ndarray] +version = "0.15.4" +optional = true + +[dev-dependencies.num-traits] +version = "0.2.14" + +[dev-dependencies.rand] +version = "0.8.4" + +[dev-dependencies.rand_chacha] +version = "0.3.1" + +[dev-dependencies.version-sync] +version = "0.9.4" diff --git a/third_party/rust/smawk/LICENSE b/third_party/rust/smawk/LICENSE new file mode 100644 index 0000000000..124067f7ae --- /dev/null +++ b/third_party/rust/smawk/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2017 Martin Geisler + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/third_party/rust/smawk/README.md b/third_party/rust/smawk/README.md new file mode 100644 index 0000000000..7d45acfeef --- /dev/null +++ b/third_party/rust/smawk/README.md @@ -0,0 +1,151 @@ +# SMAWK Algorithm in Rust + +[![](https://github.com/mgeisler/smawk/workflows/build/badge.svg)][build-status] +[![](https://codecov.io/gh/mgeisler/smawk/branch/master/graph/badge.svg)][codecov] +[![](https://img.shields.io/crates/v/smawk.svg)][crates-io] +[![](https://docs.rs/smawk/badge.svg)][api-docs] + +This crate contains an implementation of the [SMAWK algorithm][smawk] for +finding the smallest element per row in a totally monotone matrix. + +The SMAWK algorithm allows you to lower the running time of some algorithms from +O(_n_²) to just O(_n_). In other words, you can turn a quadratic time complexity +(which is often too expensive) into linear time complexity. + +Finding optimal line breaks in a paragraph of text is an example of an algorithm +which would normally take O(_n_²) time for _n_ words. With this crate, the +running time becomes linear. Please see the [textwrap crate][textwrap] for an +example of this. + +## Usage + +Add this to your `Cargo.toml`: + +```toml +[dependencies] +smawk = "0.3" +``` + +You can now efficiently find row and column minima. Here is an example where we +find the column minima: + +```rust +use smawk::Matrix; + +let matrix = vec![ + vec![3, 2, 4, 5, 6], + vec![2, 1, 3, 3, 4], + vec![2, 1, 3, 3, 4], + vec![3, 2, 4, 3, 4], + vec![4, 3, 2, 1, 1], +]; +let minima = vec![1, 1, 4, 4, 4]; +assert_eq!(smawk::column_minima(&matrix), minima); +``` + +The `minima` vector gives the index of the minimum value per column, so +`minima[0] == 1` since the minimum value in the first column is 2 (row 1). Note +that the smallest row index is returned. + +### Cargo Features + +This crate has an optional dependency on the +[`ndarray` crate](https://docs.rs/ndarray/), which provides an efficient matrix +implementation. Enable the `ndarray` Cargo feature to use it. + +## Documentation + +**[API documentation][api-docs]** + +## Changelog + +### Version 0.3.2 (2023-09-17) + +This release adds more documentation and renames the top-level SMAWK functions. +The old names have been kept for now to ensure backwards compatibility, but they +will be removed in a future release. + +- [#65](https://github.com/mgeisler/smawk/pull/65): Forbid the use of unsafe + code. +- [#69](https://github.com/mgeisler/smawk/pull/69): Migrate to the Rust 2021 + edition. +- [#73](https://github.com/mgeisler/smawk/pull/73): Add examples to all + functions. +- [#74](https://github.com/mgeisler/smawk/pull/74): Add “mathematics” as a crate + category. +- [#75](https://github.com/mgeisler/smawk/pull/75): Remove `smawk_` prefix from + optimized functions. + +### Version 0.3.1 (2021-01-30) + +This release relaxes the bounds on the `smawk_row_minima`, +`smawk_column_minima`, and `online_column_minima` functions so that they work on +matrices containing floating point numbers. + +- [#55](https://github.com/mgeisler/smawk/pull/55): Relax bounds to `PartialOrd` + instead of `Ord`. +- [#56](https://github.com/mgeisler/smawk/pull/56): Update dependencies to their + latest versions. +- [#59](https://github.com/mgeisler/smawk/pull/59): Give an example of what + SMAWK does in the README. + +### Version 0.3.0 (2020-09-02) + +This release slims down the crate significantly by making `ndarray` an optional +dependency. + +- [#45](https://github.com/mgeisler/smawk/pull/45): Move non-SMAWK code and unit + tests out of lib and into separate modules. +- [#46](https://github.com/mgeisler/smawk/pull/46): Switch `smawk_row_minima` + and `smawk_column_minima` functions to a new `Matrix` trait. +- [#47](https://github.com/mgeisler/smawk/pull/47): Make the dependency on the + `ndarray` crate optional. +- [#48](https://github.com/mgeisler/smawk/pull/48): Let `is_monge` take a + `Matrix` argument instead of `ndarray::Array2`. +- [#50](https://github.com/mgeisler/smawk/pull/50): Remove mandatory + dependencies on `rand` and `num-traits` crates. + +### Version 0.2.0 (2020-07-29) + +This release updates the code to Rust 2018. + +- [#18](https://github.com/mgeisler/smawk/pull/18): Make `online_column_minima` + generic in matrix type. +- [#23](https://github.com/mgeisler/smawk/pull/23): Switch to the + [Rust 2018][rust-2018] edition. We test against the latest stable and nightly + version of Rust. +- [#29](https://github.com/mgeisler/smawk/pull/29): Drop strict Rust 2018 + compatibility by not testing with Rust 1.31.0. +- [#32](https://github.com/mgeisler/smawk/pull/32): Fix crash on overflow in + `is_monge`. +- [#33](https://github.com/mgeisler/smawk/pull/33): Update `rand` dependency to + latest version and get rid of `rand_derive`. +- [#34](https://github.com/mgeisler/smawk/pull/34): Bump `num-traits` and + `version-sync` dependencies to latest versions. +- [#35](https://github.com/mgeisler/smawk/pull/35): Drop unnecessary Windows + tests. The assumption is that the numeric computations we do are + cross-platform. +- [#36](https://github.com/mgeisler/smawk/pull/36): Update `ndarray` dependency + to the latest version. +- [#37](https://github.com/mgeisler/smawk/pull/37): Automate publishing new + releases to crates.io. + +### Version 0.1.0 — August 7th, 2018 + +First release with the classical offline SMAWK algorithm as well as a newer +online version where the matrix entries can depend on previously computed column +minima. + +## License + +SMAWK can be distributed according to the [MIT license][mit]. Contributions will +be accepted under the same license. + +[build-status]: https://github.com/mgeisler/smawk/actions?query=branch%3Amaster+workflow%3Abuild +[crates-io]: https://crates.io/crates/smawk +[codecov]: https://codecov.io/gh/mgeisler/smawk +[textwrap]: https://crates.io/crates/textwrap +[smawk]: https://en.wikipedia.org/wiki/SMAWK_algorithm +[api-docs]: https://docs.rs/smawk/ +[rust-2018]: https://doc.rust-lang.org/edition-guide/rust-2018/ +[mit]: LICENSE diff --git a/third_party/rust/smawk/dprint.json b/third_party/rust/smawk/dprint.json new file mode 100644 index 0000000000..e48af5fe9d --- /dev/null +++ b/third_party/rust/smawk/dprint.json @@ -0,0 +1,19 @@ +{ + "markdown": { + "textWrap": "always" + }, + "exec": { + "commands": [{ + "command": "rustfmt", + "exts": ["rs"] + }] + }, + "excludes": ["target/"], + "plugins": [ + "https://plugins.dprint.dev/json-0.17.4.wasm", + "https://plugins.dprint.dev/markdown-0.16.1.wasm", + "https://plugins.dprint.dev/toml-0.5.4.wasm", + "https://plugins.dprint.dev/exec-0.4.3.json@42343548b8022c99b1d750be6b894fe6b6c7ee25f72ae9f9082226dd2e515072", + "https://plugins.dprint.dev/prettier-0.27.0.json@3557a62b4507c55a47d8cde0683195b14d13c41dda66d0f0b0e111aed107e2fe" + ] +} diff --git a/third_party/rust/smawk/rustfmt.toml b/third_party/rust/smawk/rustfmt.toml new file mode 100644 index 0000000000..24e4ea784e --- /dev/null +++ b/third_party/rust/smawk/rustfmt.toml @@ -0,0 +1,2 @@ +# Use rustfmt from the nightly channel for this: +imports_granularity = "Module" diff --git a/third_party/rust/smawk/src/brute_force.rs b/third_party/rust/smawk/src/brute_force.rs new file mode 100644 index 0000000000..1ec0ca35a7 --- /dev/null +++ b/third_party/rust/smawk/src/brute_force.rs @@ -0,0 +1,150 @@ +//! Brute-force algorithm for finding column minima. +//! +//! The functions here are mostly meant to be used for testing +//! correctness of the SMAWK implementation. +//! +//! **Note: this module is only available if you enable the `ndarray` +//! Cargo feature.** + +use ndarray::{Array2, ArrayView1}; + +/// Compute lane minimum by brute force. +/// +/// This does a simple scan through the lane (row or column). +#[inline] +pub fn lane_minimum(lane: ArrayView1<'_, T>) -> usize { + lane.iter() + .enumerate() + .min_by_key(|&(idx, elem)| (elem, idx)) + .map(|(idx, _)| idx) + .expect("empty lane in matrix") +} + +/// Compute row minima by brute force in O(*mn*) time. +/// +/// This function implements a simple brute-force approach where each +/// matrix row is scanned completely. This means that the function +/// works on all matrices, not just Monge matrices. +/// +/// # Examples +/// +/// ``` +/// let matrix = ndarray::arr2(&[[4, 2, 4, 3], +/// [5, 3, 5, 3], +/// [5, 3, 3, 1]]); +/// assert_eq!(smawk::brute_force::row_minima(&matrix), +/// vec![1, 1, 3]); +/// ``` +/// +/// # Panics +/// +/// It is an error to call this on a matrix with zero columns. +pub fn row_minima(matrix: &Array2) -> Vec { + matrix.rows().into_iter().map(lane_minimum).collect() +} + +/// Compute column minima by brute force in O(*mn*) time. +/// +/// This function implements a simple brute-force approach where each +/// matrix column is scanned completely. This means that the function +/// works on all matrices, not just Monge matrices. +/// +/// # Examples +/// +/// ``` +/// let matrix = ndarray::arr2(&[[4, 2, 4, 3], +/// [5, 3, 5, 3], +/// [5, 3, 3, 1]]); +/// assert_eq!(smawk::brute_force::column_minima(&matrix), +/// vec![0, 0, 2, 2]); +/// ``` +/// +/// # Panics +/// +/// It is an error to call this on a matrix with zero rows. +pub fn column_minima(matrix: &Array2) -> Vec { + matrix.columns().into_iter().map(lane_minimum).collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use ndarray::arr2; + + #[test] + fn brute_force_1x1() { + let matrix = arr2(&[[2]]); + let minima = vec![0]; + assert_eq!(row_minima(&matrix), minima); + assert_eq!(column_minima(&matrix.reversed_axes()), minima); + } + + #[test] + fn brute_force_2x1() { + let matrix = arr2(&[ + [3], // + [2], + ]); + let minima = vec![0, 0]; + assert_eq!(row_minima(&matrix), minima); + assert_eq!(column_minima(&matrix.reversed_axes()), minima); + } + + #[test] + fn brute_force_1x2() { + let matrix = arr2(&[[2, 1]]); + let minima = vec![1]; + assert_eq!(row_minima(&matrix), minima); + assert_eq!(column_minima(&matrix.reversed_axes()), minima); + } + + #[test] + fn brute_force_2x2() { + let matrix = arr2(&[ + [3, 2], // + [2, 1], + ]); + let minima = vec![1, 1]; + assert_eq!(row_minima(&matrix), minima); + assert_eq!(column_minima(&matrix.reversed_axes()), minima); + } + + #[test] + fn brute_force_3x3() { + let matrix = arr2(&[ + [3, 4, 4], // + [3, 4, 4], + [2, 3, 3], + ]); + let minima = vec![0, 0, 0]; + assert_eq!(row_minima(&matrix), minima); + assert_eq!(column_minima(&matrix.reversed_axes()), minima); + } + + #[test] + fn brute_force_4x4() { + let matrix = arr2(&[ + [4, 5, 5, 5], // + [2, 3, 3, 3], + [2, 3, 3, 3], + [2, 2, 2, 2], + ]); + let minima = vec![0, 0, 0, 0]; + assert_eq!(row_minima(&matrix), minima); + assert_eq!(column_minima(&matrix.reversed_axes()), minima); + } + + #[test] + fn brute_force_5x5() { + let matrix = arr2(&[ + [3, 2, 4, 5, 6], + [2, 1, 3, 3, 4], + [2, 1, 3, 3, 4], + [3, 2, 4, 3, 4], + [4, 3, 2, 1, 1], + ]); + let minima = vec![1, 1, 1, 1, 3]; + assert_eq!(row_minima(&matrix), minima); + assert_eq!(column_minima(&matrix.reversed_axes()), minima); + } +} diff --git a/third_party/rust/smawk/src/lib.rs b/third_party/rust/smawk/src/lib.rs new file mode 100644 index 0000000000..367d0337dc --- /dev/null +++ b/third_party/rust/smawk/src/lib.rs @@ -0,0 +1,570 @@ +//! This crate implements various functions that help speed up dynamic +//! programming, most importantly the SMAWK algorithm for finding row +//! or column minima in a totally monotone matrix with *m* rows and +//! *n* columns in time O(*m* + *n*). This is much better than the +//! brute force solution which would take O(*mn*). When *m* and *n* +//! are of the same order, this turns a quadratic function into a +//! linear function. +//! +//! # Examples +//! +//! Computing the column minima of an *m* ✕ *n* Monge matrix can be +//! done efficiently with `smawk::column_minima`: +//! +//! ``` +//! use smawk::Matrix; +//! +//! let matrix = vec![ +//! vec![3, 2, 4, 5, 6], +//! vec![2, 1, 3, 3, 4], +//! vec![2, 1, 3, 3, 4], +//! vec![3, 2, 4, 3, 4], +//! vec![4, 3, 2, 1, 1], +//! ]; +//! let minima = vec![1, 1, 4, 4, 4]; +//! assert_eq!(smawk::column_minima(&matrix), minima); +//! ``` +//! +//! The `minima` vector gives the index of the minimum value per +//! column, so `minima[0] == 1` since the minimum value in the first +//! column is 2 (row 1). Note that the smallest row index is returned. +//! +//! # Definitions +//! +//! Some of the functions in this crate only work on matrices that are +//! *totally monotone*, which we will define below. +//! +//! ## Monotone Matrices +//! +//! We start with a helper definition. Given an *m* ✕ *n* matrix `M`, +//! we say that `M` is *monotone* when the minimum value of row `i` is +//! found to the left of the minimum value in row `i'` where `i < i'`. +//! +//! More formally, if we let `rm(i)` denote the column index of the +//! left-most minimum value in row `i`, then we have +//! +//! ```text +//! rm(0) ≤ rm(1) ≤ ... ≤ rm(m - 1) +//! ``` +//! +//! This means that as you go down the rows from top to bottom, the +//! row-minima proceed from left to right. +//! +//! The algorithms in this crate deal with finding such row- and +//! column-minima. +//! +//! ## Totally Monotone Matrices +//! +//! We say that a matrix `M` is *totally monotone* when every +//! sub-matrix is monotone. A sub-matrix is formed by the intersection +//! of any two rows `i < i'` and any two columns `j < j'`. +//! +//! This is often expressed as via this equivalent condition: +//! +//! ```text +//! M[i, j] > M[i, j'] => M[i', j] > M[i', j'] +//! ``` +//! +//! for all `i < i'` and `j < j'`. +//! +//! ## Monge Property for Matrices +//! +//! A matrix `M` is said to fulfill the *Monge property* if +//! +//! ```text +//! M[i, j] + M[i', j'] ≤ M[i, j'] + M[i', j] +//! ``` +//! +//! for all `i < i'` and `j < j'`. This says that given any rectangle +//! in the matrix, the sum of the top-left and bottom-right corners is +//! less than or equal to the sum of the bottom-left and upper-right +//! corners. +//! +//! All Monge matrices are totally monotone, so it is enough to +//! establish that the Monge property holds in order to use a matrix +//! with the functions in this crate. If your program is dealing with +//! unknown inputs, it can use [`monge::is_monge`] to verify that a +//! matrix is a Monge matrix. + +#![doc(html_root_url = "https://docs.rs/smawk/0.3.2")] +// The s! macro from ndarray uses unsafe internally, so we can only +// forbid unsafe code when building with the default features. +#![cfg_attr(not(feature = "ndarray"), forbid(unsafe_code))] + +#[cfg(feature = "ndarray")] +pub mod brute_force; +pub mod monge; +#[cfg(feature = "ndarray")] +pub mod recursive; + +/// Minimal matrix trait for two-dimensional arrays. +/// +/// This provides the functionality needed to represent a read-only +/// numeric matrix. You can query the size of the matrix and access +/// elements. Modeled after [`ndarray::Array2`] from the [ndarray +/// crate](https://crates.io/crates/ndarray). +/// +/// Enable the `ndarray` Cargo feature if you want to use it with +/// `ndarray::Array2`. +pub trait Matrix { + /// Return the number of rows. + fn nrows(&self) -> usize; + /// Return the number of columns. + fn ncols(&self) -> usize; + /// Return a matrix element. + fn index(&self, row: usize, column: usize) -> T; +} + +/// Simple and inefficient matrix representation used for doctest +/// examples and simple unit tests. +/// +/// You should prefer implementing it yourself, or you can enable the +/// `ndarray` Cargo feature and use the provided implementation for +/// [`ndarray::Array2`]. +impl Matrix for Vec> { + fn nrows(&self) -> usize { + self.len() + } + fn ncols(&self) -> usize { + self[0].len() + } + fn index(&self, row: usize, column: usize) -> T { + self[row][column] + } +} + +/// Adapting [`ndarray::Array2`] to the `Matrix` trait. +/// +/// **Note: this implementation is only available if you enable the +/// `ndarray` Cargo feature.** +#[cfg(feature = "ndarray")] +impl Matrix for ndarray::Array2 { + #[inline] + fn nrows(&self) -> usize { + self.nrows() + } + #[inline] + fn ncols(&self) -> usize { + self.ncols() + } + #[inline] + fn index(&self, row: usize, column: usize) -> T { + self[[row, column]] + } +} + +/// Compute row minima in O(*m* + *n*) time. +/// +/// This implements the [SMAWK algorithm] for efficiently finding row +/// minima in a totally monotone matrix. +/// +/// The SMAWK algorithm is from Agarwal, Klawe, Moran, Shor, and +/// Wilbur, *Geometric applications of a matrix searching algorithm*, +/// Algorithmica 2, pp. 195-208 (1987) and the code here is a +/// translation [David Eppstein's Python code][pads]. +/// +/// Running time on an *m* ✕ *n* matrix: O(*m* + *n*). +/// +/// # Examples +/// +/// ``` +/// use smawk::Matrix; +/// let matrix = vec![vec![4, 2, 4, 3], +/// vec![5, 3, 5, 3], +/// vec![5, 3, 3, 1]]; +/// assert_eq!(smawk::row_minima(&matrix), +/// vec![1, 1, 3]); +/// ``` +/// +/// # Panics +/// +/// It is an error to call this on a matrix with zero columns. +/// +/// [pads]: https://github.com/jfinkels/PADS/blob/master/pads/smawk.py +/// [SMAWK algorithm]: https://en.wikipedia.org/wiki/SMAWK_algorithm +pub fn row_minima>(matrix: &M) -> Vec { + // Benchmarking shows that SMAWK performs roughly the same on row- + // and column-major matrices. + let mut minima = vec![0; matrix.nrows()]; + smawk_inner( + &|j, i| matrix.index(i, j), + &(0..matrix.ncols()).collect::>(), + &(0..matrix.nrows()).collect::>(), + &mut minima, + ); + minima +} + +#[deprecated(since = "0.3.2", note = "Please use `row_minima` instead.")] +pub fn smawk_row_minima>(matrix: &M) -> Vec { + row_minima(matrix) +} + +/// Compute column minima in O(*m* + *n*) time. +/// +/// This implements the [SMAWK algorithm] for efficiently finding +/// column minima in a totally monotone matrix. +/// +/// The SMAWK algorithm is from Agarwal, Klawe, Moran, Shor, and +/// Wilbur, *Geometric applications of a matrix searching algorithm*, +/// Algorithmica 2, pp. 195-208 (1987) and the code here is a +/// translation [David Eppstein's Python code][pads]. +/// +/// Running time on an *m* ✕ *n* matrix: O(*m* + *n*). +/// +/// # Examples +/// +/// ``` +/// use smawk::Matrix; +/// let matrix = vec![vec![4, 2, 4, 3], +/// vec![5, 3, 5, 3], +/// vec![5, 3, 3, 1]]; +/// assert_eq!(smawk::column_minima(&matrix), +/// vec![0, 0, 2, 2]); +/// ``` +/// +/// # Panics +/// +/// It is an error to call this on a matrix with zero rows. +/// +/// [SMAWK algorithm]: https://en.wikipedia.org/wiki/SMAWK_algorithm +/// [pads]: https://github.com/jfinkels/PADS/blob/master/pads/smawk.py +pub fn column_minima>(matrix: &M) -> Vec { + let mut minima = vec![0; matrix.ncols()]; + smawk_inner( + &|i, j| matrix.index(i, j), + &(0..matrix.nrows()).collect::>(), + &(0..matrix.ncols()).collect::>(), + &mut minima, + ); + minima +} + +#[deprecated(since = "0.3.2", note = "Please use `column_minima` instead.")] +pub fn smawk_column_minima>(matrix: &M) -> Vec { + column_minima(matrix) +} + +/// Compute column minima in the given area of the matrix. The +/// `minima` slice is updated inplace. +fn smawk_inner T>( + matrix: &M, + rows: &[usize], + cols: &[usize], + minima: &mut [usize], +) { + if cols.is_empty() { + return; + } + + let mut stack = Vec::with_capacity(cols.len()); + for r in rows { + // TODO: use stack.last() instead of stack.is_empty() etc + while !stack.is_empty() + && matrix(stack[stack.len() - 1], cols[stack.len() - 1]) + > matrix(*r, cols[stack.len() - 1]) + { + stack.pop(); + } + if stack.len() != cols.len() { + stack.push(*r); + } + } + let rows = &stack; + + let mut odd_cols = Vec::with_capacity(1 + cols.len() / 2); + for (idx, c) in cols.iter().enumerate() { + if idx % 2 == 1 { + odd_cols.push(*c); + } + } + + smawk_inner(matrix, rows, &odd_cols, minima); + + let mut r = 0; + for (c, &col) in cols.iter().enumerate().filter(|(c, _)| c % 2 == 0) { + let mut row = rows[r]; + let last_row = if c == cols.len() - 1 { + rows[rows.len() - 1] + } else { + minima[cols[c + 1]] + }; + let mut pair = (matrix(row, col), row); + while row != last_row { + r += 1; + row = rows[r]; + if (matrix(row, col), row) < pair { + pair = (matrix(row, col), row); + } + } + minima[col] = pair.1; + } +} + +/// Compute upper-right column minima in O(*m* + *n*) time. +/// +/// The input matrix must be totally monotone. +/// +/// The function returns a vector of `(usize, T)`. The `usize` in the +/// tuple at index `j` tells you the row of the minimum value in +/// column `j` and the `T` value is minimum value itself. +/// +/// The algorithm only considers values above the main diagonal, which +/// means that it computes values `v(j)` where: +/// +/// ```text +/// v(0) = initial +/// v(j) = min { M[i, j] | i < j } for j > 0 +/// ``` +/// +/// If we let `r(j)` denote the row index of the minimum value in +/// column `j`, the tuples in the result vector become `(r(j), M[r(j), +/// j])`. +/// +/// The algorithm is an *online* algorithm, in the sense that `matrix` +/// function can refer back to previously computed column minima when +/// determining an entry in the matrix. The guarantee is that we only +/// call `matrix(i, j)` after having computed `v(i)`. This is +/// reflected in the `&[(usize, T)]` argument to `matrix`, which grows +/// as more and more values are computed. +pub fn online_column_minima T>( + initial: T, + size: usize, + matrix: M, +) -> Vec<(usize, T)> { + let mut result = vec![(0, initial)]; + + // State used by the algorithm. + let mut finished = 0; + let mut base = 0; + let mut tentative = 0; + + // Shorthand for evaluating the matrix. We need a macro here since + // we don't want to borrow the result vector. + macro_rules! m { + ($i:expr, $j:expr) => {{ + assert!($i < $j, "(i, j) not above diagonal: ({}, {})", $i, $j); + assert!( + $i < size && $j < size, + "(i, j) out of bounds: ({}, {}), size: {}", + $i, + $j, + size + ); + matrix(&result[..finished + 1], $i, $j) + }}; + } + + // Keep going until we have finished all size columns. Since the + // columns are zero-indexed, we're done when finished == size - 1. + while finished < size - 1 { + // First case: we have already advanced past the previous + // tentative value. We make a new tentative value by applying + // smawk_inner to the largest square submatrix that fits under + // the base. + let i = finished + 1; + if i > tentative { + let rows = (base..finished + 1).collect::>(); + tentative = std::cmp::min(finished + rows.len(), size - 1); + let cols = (finished + 1..tentative + 1).collect::>(); + let mut minima = vec![0; tentative + 1]; + smawk_inner(&|i, j| m![i, j], &rows, &cols, &mut minima); + for col in cols { + let row = minima[col]; + let v = m![row, col]; + if col >= result.len() { + result.push((row, v)); + } else if v < result[col].1 { + result[col] = (row, v); + } + } + finished = i; + continue; + } + + // Second case: the new column minimum is on the diagonal. All + // subsequent ones will be at least as low, so we can clear + // out all our work from higher rows. As in the fourth case, + // the loss of tentative is amortized against the increase in + // base. + let diag = m![i - 1, i]; + if diag < result[i].1 { + result[i] = (i - 1, diag); + base = i - 1; + tentative = i; + finished = i; + continue; + } + + // Third case: row i-1 does not supply a column minimum in any + // column up to tentative. We simply advance finished while + // maintaining the invariant. + if m![i - 1, tentative] >= result[tentative].1 { + finished = i; + continue; + } + + // Fourth and final case: a new column minimum at tentative. + // This allows us to make progress by incorporating rows prior + // to finished into the base. The base invariant holds because + // these rows cannot supply any later column minima. The work + // done when we last advanced tentative (and undone by this + // step) can be amortized against the increase in base. + base = i - 1; + tentative = i; + finished = i; + } + + result +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn smawk_1x1() { + let matrix = vec![vec![2]]; + assert_eq!(row_minima(&matrix), vec![0]); + assert_eq!(column_minima(&matrix), vec![0]); + } + + #[test] + fn smawk_2x1() { + let matrix = vec![ + vec![3], // + vec![2], + ]; + assert_eq!(row_minima(&matrix), vec![0, 0]); + assert_eq!(column_minima(&matrix), vec![1]); + } + + #[test] + fn smawk_1x2() { + let matrix = vec![vec![2, 1]]; + assert_eq!(row_minima(&matrix), vec![1]); + assert_eq!(column_minima(&matrix), vec![0, 0]); + } + + #[test] + fn smawk_2x2() { + let matrix = vec![ + vec![3, 2], // + vec![2, 1], + ]; + assert_eq!(row_minima(&matrix), vec![1, 1]); + assert_eq!(column_minima(&matrix), vec![1, 1]); + } + + #[test] + fn smawk_3x3() { + let matrix = vec![ + vec![3, 4, 4], // + vec![3, 4, 4], + vec![2, 3, 3], + ]; + assert_eq!(row_minima(&matrix), vec![0, 0, 0]); + assert_eq!(column_minima(&matrix), vec![2, 2, 2]); + } + + #[test] + fn smawk_4x4() { + let matrix = vec![ + vec![4, 5, 5, 5], // + vec![2, 3, 3, 3], + vec![2, 3, 3, 3], + vec![2, 2, 2, 2], + ]; + assert_eq!(row_minima(&matrix), vec![0, 0, 0, 0]); + assert_eq!(column_minima(&matrix), vec![1, 3, 3, 3]); + } + + #[test] + fn smawk_5x5() { + let matrix = vec![ + vec![3, 2, 4, 5, 6], + vec![2, 1, 3, 3, 4], + vec![2, 1, 3, 3, 4], + vec![3, 2, 4, 3, 4], + vec![4, 3, 2, 1, 1], + ]; + assert_eq!(row_minima(&matrix), vec![1, 1, 1, 1, 3]); + assert_eq!(column_minima(&matrix), vec![1, 1, 4, 4, 4]); + } + + #[test] + fn online_1x1() { + let matrix = vec![vec![0]]; + let minima = vec![(0, 0)]; + assert_eq!(online_column_minima(0, 1, |_, i, j| matrix[i][j]), minima); + } + + #[test] + fn online_2x2() { + let matrix = vec![ + vec![0, 2], // + vec![0, 0], + ]; + let minima = vec![(0, 0), (0, 2)]; + assert_eq!(online_column_minima(0, 2, |_, i, j| matrix[i][j]), minima); + } + + #[test] + fn online_3x3() { + let matrix = vec![ + vec![0, 4, 4], // + vec![0, 0, 4], + vec![0, 0, 0], + ]; + let minima = vec![(0, 0), (0, 4), (0, 4)]; + assert_eq!(online_column_minima(0, 3, |_, i, j| matrix[i][j]), minima); + } + + #[test] + fn online_4x4() { + let matrix = vec![ + vec![0, 5, 5, 5], // + vec![0, 0, 3, 3], + vec![0, 0, 0, 3], + vec![0, 0, 0, 0], + ]; + let minima = vec![(0, 0), (0, 5), (1, 3), (1, 3)]; + assert_eq!(online_column_minima(0, 4, |_, i, j| matrix[i][j]), minima); + } + + #[test] + fn online_5x5() { + let matrix = vec![ + vec![0, 2, 4, 6, 7], + vec![0, 0, 3, 4, 5], + vec![0, 0, 0, 3, 4], + vec![0, 0, 0, 0, 4], + vec![0, 0, 0, 0, 0], + ]; + let minima = vec![(0, 0), (0, 2), (1, 3), (2, 3), (2, 4)]; + assert_eq!(online_column_minima(0, 5, |_, i, j| matrix[i][j]), minima); + } + + #[test] + fn smawk_works_with_partial_ord() { + let matrix = vec![ + vec![3.0, 2.0], // + vec![2.0, 1.0], + ]; + assert_eq!(row_minima(&matrix), vec![1, 1]); + assert_eq!(column_minima(&matrix), vec![1, 1]); + } + + #[test] + fn online_works_with_partial_ord() { + let matrix = vec![ + vec![0.0, 2.0], // + vec![0.0, 0.0], + ]; + let minima = vec![(0, 0.0), (0, 2.0)]; + assert_eq!( + online_column_minima(0.0, 2, |_, i: usize, j: usize| matrix[i][j]), + minima + ); + } +} diff --git a/third_party/rust/smawk/src/monge.rs b/third_party/rust/smawk/src/monge.rs new file mode 100644 index 0000000000..dbc80e1517 --- /dev/null +++ b/third_party/rust/smawk/src/monge.rs @@ -0,0 +1,121 @@ +//! Functions for generating and checking Monge arrays. +//! +//! The functions here are mostly meant to be used for testing +//! correctness of the SMAWK implementation. + +use crate::Matrix; +use std::num::Wrapping; +use std::ops::Add; + +/// Verify that a matrix is a Monge matrix. +/// +/// A [Monge matrix] \(or array) is a matrix where the following +/// inequality holds: +/// +/// ```text +/// M[i, j] + M[i', j'] <= M[i, j'] + M[i', j] for all i < i', j < j' +/// ``` +/// +/// The inequality says that the sum of the main diagonal is less than +/// the sum of the antidiagonal. Checking this condition is done by +/// checking *n* ✕ *m* submatrices, so the running time is O(*mn*). +/// +/// [Monge matrix]: https://en.wikipedia.org/wiki/Monge_array +pub fn is_monge>(matrix: &M) -> bool +where + Wrapping: Add>, +{ + /// Returns `Ok(a + b)` if the computation can be done without + /// overflow, otherwise `Err(a + b - T::MAX - 1)` is returned. + fn checked_add(a: Wrapping, b: Wrapping) -> Result + where + Wrapping: Add>, + { + let sum = a + b; + if sum < a { + Err(sum.0) + } else { + Ok(sum.0) + } + } + + (0..matrix.nrows() - 1) + .flat_map(|row| (0..matrix.ncols() - 1).map(move |col| (row, col))) + .all(|(row, col)| { + let top_left = Wrapping(matrix.index(row, col)); + let top_right = Wrapping(matrix.index(row, col + 1)); + let bot_left = Wrapping(matrix.index(row + 1, col)); + let bot_right = Wrapping(matrix.index(row + 1, col + 1)); + + match ( + checked_add(top_left, bot_right), + checked_add(bot_left, top_right), + ) { + (Ok(a), Ok(b)) => a <= b, // No overflow. + (Err(a), Err(b)) => a <= b, // Double overflow. + (Ok(_), Err(_)) => true, // Anti-diagonal overflow. + (Err(_), Ok(_)) => false, // Main diagonal overflow. + } + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn is_monge_handles_overflow() { + // The x + y <= z + w computations will overflow for an u8 + // matrix unless is_monge is careful. + let matrix: Vec> = vec![ + vec![200, 200, 200, 200], + vec![200, 200, 200, 200], + vec![200, 200, 200, 200], + ]; + assert!(is_monge(&matrix)); + } + + #[test] + fn monge_constant_rows() { + let matrix = vec![ + vec![42, 42, 42, 42], + vec![0, 0, 0, 0], + vec![100, 100, 100, 100], + vec![1000, 1000, 1000, 1000], + ]; + assert!(is_monge(&matrix)); + } + + #[test] + fn monge_constant_cols() { + let matrix = vec![ + vec![42, 0, 100, 1000], + vec![42, 0, 100, 1000], + vec![42, 0, 100, 1000], + vec![42, 0, 100, 1000], + ]; + assert!(is_monge(&matrix)); + } + + #[test] + fn monge_upper_right() { + let matrix = vec![ + vec![10, 10, 42, 42, 42], + vec![10, 10, 42, 42, 42], + vec![10, 10, 10, 10, 10], + vec![10, 10, 10, 10, 10], + ]; + assert!(is_monge(&matrix)); + } + + #[test] + fn monge_lower_left() { + let matrix = vec![ + vec![10, 10, 10, 10, 10], + vec![10, 10, 10, 10, 10], + vec![42, 42, 42, 10, 10], + vec![42, 42, 42, 10, 10], + ]; + assert!(is_monge(&matrix)); + } +} diff --git a/third_party/rust/smawk/src/recursive.rs b/third_party/rust/smawk/src/recursive.rs new file mode 100644 index 0000000000..9df8b9c824 --- /dev/null +++ b/third_party/rust/smawk/src/recursive.rs @@ -0,0 +1,191 @@ +//! Recursive algorithm for finding column minima. +//! +//! The functions here are mostly meant to be used for testing +//! correctness of the SMAWK implementation. +//! +//! **Note: this module is only available if you enable the `ndarray` +//! Cargo feature.** + +use ndarray::{s, Array2, ArrayView2, Axis}; + +/// Compute row minima in O(*m* + *n* log *m*) time. +/// +/// This function computes row minima in a totally monotone matrix +/// using a recursive algorithm. +/// +/// Running time on an *m* ✕ *n* matrix: O(*m* + *n* log *m*). +/// +/// # Examples +/// +/// ``` +/// let matrix = ndarray::arr2(&[[4, 2, 4, 3], +/// [5, 3, 5, 3], +/// [5, 3, 3, 1]]); +/// assert_eq!(smawk::recursive::row_minima(&matrix), +/// vec![1, 1, 3]); +/// ``` +/// +/// # Panics +/// +/// It is an error to call this on a matrix with zero columns. +pub fn row_minima(matrix: &Array2) -> Vec { + let mut minima = vec![0; matrix.nrows()]; + recursive_inner(matrix.view(), &|| Direction::Row, 0, &mut minima); + minima +} + +/// Compute column minima in O(*n* + *m* log *n*) time. +/// +/// This function computes column minima in a totally monotone matrix +/// using a recursive algorithm. +/// +/// Running time on an *m* ✕ *n* matrix: O(*n* + *m* log *n*). +/// +/// # Examples +/// +/// ``` +/// let matrix = ndarray::arr2(&[[4, 2, 4, 3], +/// [5, 3, 5, 3], +/// [5, 3, 3, 1]]); +/// assert_eq!(smawk::recursive::column_minima(&matrix), +/// vec![0, 0, 2, 2]); +/// ``` +/// +/// # Panics +/// +/// It is an error to call this on a matrix with zero rows. +pub fn column_minima(matrix: &Array2) -> Vec { + let mut minima = vec![0; matrix.ncols()]; + recursive_inner(matrix.view(), &|| Direction::Column, 0, &mut minima); + minima +} + +/// The type of minima (row or column) we compute. +enum Direction { + Row, + Column, +} + +/// Compute the minima along the given direction (`Direction::Row` for +/// row minima and `Direction::Column` for column minima). +/// +/// The direction is given as a generic function argument to allow +/// monomorphization to kick in. The function calls will be inlined +/// and optimized away and the result is that the compiler generates +/// differnet code for finding row and column minima. +fn recursive_inner Direction>( + matrix: ArrayView2<'_, T>, + dir: &F, + offset: usize, + minima: &mut [usize], +) { + if matrix.is_empty() { + return; + } + + let axis = match dir() { + Direction::Row => Axis(0), + Direction::Column => Axis(1), + }; + let mid = matrix.len_of(axis) / 2; + let min_idx = crate::brute_force::lane_minimum(matrix.index_axis(axis, mid)); + minima[mid] = offset + min_idx; + + if mid == 0 { + return; // Matrix has a single row or column, so we're done. + } + + let top_left = match dir() { + Direction::Row => matrix.slice(s![..mid, ..(min_idx + 1)]), + Direction::Column => matrix.slice(s![..(min_idx + 1), ..mid]), + }; + let bot_right = match dir() { + Direction::Row => matrix.slice(s![(mid + 1).., min_idx..]), + Direction::Column => matrix.slice(s![min_idx.., (mid + 1)..]), + }; + recursive_inner(top_left, dir, offset, &mut minima[..mid]); + recursive_inner(bot_right, dir, offset + min_idx, &mut minima[mid + 1..]); +} + +#[cfg(test)] +mod tests { + use super::*; + use ndarray::arr2; + + #[test] + fn recursive_1x1() { + let matrix = arr2(&[[2]]); + let minima = vec![0]; + assert_eq!(row_minima(&matrix), minima); + assert_eq!(column_minima(&matrix.reversed_axes()), minima); + } + + #[test] + fn recursive_2x1() { + let matrix = arr2(&[ + [3], // + [2], + ]); + let minima = vec![0, 0]; + assert_eq!(row_minima(&matrix), minima); + assert_eq!(column_minima(&matrix.reversed_axes()), minima); + } + + #[test] + fn recursive_1x2() { + let matrix = arr2(&[[2, 1]]); + let minima = vec![1]; + assert_eq!(row_minima(&matrix), minima); + assert_eq!(column_minima(&matrix.reversed_axes()), minima); + } + + #[test] + fn recursive_2x2() { + let matrix = arr2(&[ + [3, 2], // + [2, 1], + ]); + let minima = vec![1, 1]; + assert_eq!(row_minima(&matrix), minima); + assert_eq!(column_minima(&matrix.reversed_axes()), minima); + } + + #[test] + fn recursive_3x3() { + let matrix = arr2(&[ + [3, 4, 4], // + [3, 4, 4], + [2, 3, 3], + ]); + let minima = vec![0, 0, 0]; + assert_eq!(row_minima(&matrix), minima); + assert_eq!(column_minima(&matrix.reversed_axes()), minima); + } + + #[test] + fn recursive_4x4() { + let matrix = arr2(&[ + [4, 5, 5, 5], // + [2, 3, 3, 3], + [2, 3, 3, 3], + [2, 2, 2, 2], + ]); + let minima = vec![0, 0, 0, 0]; + assert_eq!(row_minima(&matrix), minima); + assert_eq!(column_minima(&matrix.reversed_axes()), minima); + } + + #[test] + fn recursive_5x5() { + let matrix = arr2(&[ + [3, 2, 4, 5, 6], + [2, 1, 3, 3, 4], + [2, 1, 3, 3, 4], + [3, 2, 4, 3, 4], + [4, 3, 2, 1, 1], + ]); + let minima = vec![1, 1, 1, 1, 3]; + assert_eq!(row_minima(&matrix), minima); + assert_eq!(column_minima(&matrix.reversed_axes()), minima); + } +} diff --git a/third_party/rust/smawk/tests/agreement.rs b/third_party/rust/smawk/tests/agreement.rs new file mode 100644 index 0000000000..2e0343a59a --- /dev/null +++ b/third_party/rust/smawk/tests/agreement.rs @@ -0,0 +1,104 @@ +#![cfg(feature = "ndarray")] + +use ndarray::{s, Array2}; +use rand::SeedableRng; +use rand_chacha::ChaCha20Rng; +use smawk::{brute_force, online_column_minima, recursive}; + +mod random_monge; +use random_monge::random_monge_matrix; + +/// Check that the brute force, recursive, and SMAWK functions +/// give identical results on a large number of randomly generated +/// Monge matrices. +#[test] +fn column_minima_agree() { + let sizes = vec![1, 2, 3, 4, 5, 10, 15, 20, 30]; + let mut rng = ChaCha20Rng::seed_from_u64(0); + for _ in 0..4 { + for m in sizes.clone().iter() { + for n in sizes.clone().iter() { + let matrix: Array2 = random_monge_matrix(*m, *n, &mut rng); + + // Compute and test row minima. + let brute_force = brute_force::row_minima(&matrix); + let recursive = recursive::row_minima(&matrix); + let smawk = smawk::row_minima(&matrix); + assert_eq!( + brute_force, recursive, + "recursive and brute force differs on:\n{:?}", + matrix + ); + assert_eq!( + brute_force, smawk, + "SMAWK and brute force differs on:\n{:?}", + matrix + ); + + // Do the same for the column minima. + let brute_force = brute_force::column_minima(&matrix); + let recursive = recursive::column_minima(&matrix); + let smawk = smawk::column_minima(&matrix); + assert_eq!( + brute_force, recursive, + "recursive and brute force differs on:\n{:?}", + matrix + ); + assert_eq!( + brute_force, smawk, + "SMAWK and brute force differs on:\n{:?}", + matrix + ); + } + } + } +} + +/// Check that the brute force and online SMAWK functions give +/// identical results on a large number of randomly generated +/// Monge matrices. +#[test] +fn online_agree() { + let sizes = vec![1, 2, 3, 4, 5, 10, 15, 20, 30, 50]; + let mut rng = ChaCha20Rng::seed_from_u64(0); + for _ in 0..5 { + for &size in &sizes { + // Random totally monotone square matrix of the + // desired size. + let mut matrix: Array2 = random_monge_matrix(size, size, &mut rng); + + // Adjust matrix so the column minima are above the + // diagonal. The brute_force::column_minima will still + // work just fine on such a mangled Monge matrix. + let max = *matrix.iter().max().unwrap_or(&0); + for idx in 0..(size as isize) { + // Using the maximum value of the matrix instead + // of i32::max_value() makes for prettier matrices + // in case we want to print them. + matrix.slice_mut(s![idx..idx + 1, ..idx + 1]).fill(max); + } + + // The online algorithm always returns the initial + // value for the left-most column -- without + // inspecting the column at all. So we fill the + // left-most column with this value to have the brute + // force algorithm do the same. + let initial = 42; + matrix.slice_mut(s![0.., ..1]).fill(initial); + + // Brute-force computation of column minima, returned + // in the same form as online_column_minima. + let brute_force = brute_force::column_minima(&matrix) + .iter() + .enumerate() + .map(|(j, &i)| (i, matrix[[i, j]])) + .collect::>(); + let online = online_column_minima(initial, size, |_, i, j| matrix[[i, j]]); + assert_eq!( + brute_force, online, + "brute force and online differ on:\n{:3?}", + matrix + ); + } + } +} diff --git a/third_party/rust/smawk/tests/complexity.rs b/third_party/rust/smawk/tests/complexity.rs new file mode 100644 index 0000000000..c9881eaeac --- /dev/null +++ b/third_party/rust/smawk/tests/complexity.rs @@ -0,0 +1,83 @@ +#![cfg(feature = "ndarray")] + +use ndarray::{Array1, Array2}; +use rand::SeedableRng; +use rand_chacha::ChaCha20Rng; +use smawk::online_column_minima; + +mod random_monge; +use random_monge::random_monge_matrix; + +#[derive(Debug)] +struct LinRegression { + alpha: f64, + beta: f64, + r_squared: f64, +} + +/// Square an expression. Works equally well for floats and matrices. +macro_rules! squared { + ($x:expr) => { + $x * $x + }; +} + +/// Compute the mean of a 1-dimensional array. +macro_rules! mean { + ($a:expr) => { + $a.mean().expect("Mean of empty array") + }; +} + +/// Compute a simple linear regression from the list of values. +/// +/// See . +fn linear_regression(values: &[(usize, i32)]) -> LinRegression { + let xs = values.iter().map(|&(x, _)| x as f64).collect::>(); + let ys = values.iter().map(|&(_, y)| y as f64).collect::>(); + + let xs_mean = mean!(&xs); + let ys_mean = mean!(&ys); + let xs_ys_mean = mean!(&xs * &ys); + + let cov_xs_ys = ((&xs - xs_mean) * (&ys - ys_mean)).sum(); + let var_xs = squared!(&xs - xs_mean).sum(); + + let beta = cov_xs_ys / var_xs; + let alpha = ys_mean - beta * xs_mean; + let r_squared = squared!(xs_ys_mean - xs_mean * ys_mean) + / ((mean!(&xs * &xs) - squared!(xs_mean)) * (mean!(&ys * &ys) - squared!(ys_mean))); + + LinRegression { + alpha: alpha, + beta: beta, + r_squared: r_squared, + } +} + +/// Check that the number of matrix accesses in `online_column_minima` +/// grows as O(*n*) for *n* ✕ *n* matrix. +#[test] +fn online_linear_complexity() { + let mut rng = ChaCha20Rng::seed_from_u64(0); + let mut data = vec![]; + + for &size in &[1, 2, 3, 4, 5, 10, 15, 20, 30, 40, 50, 60, 70, 80, 90, 100] { + let matrix: Array2 = random_monge_matrix(size, size, &mut rng); + let count = std::cell::RefCell::new(0); + online_column_minima(0, size, |_, i, j| { + *count.borrow_mut() += 1; + matrix[[i, j]] + }); + data.push((size, count.into_inner())); + } + + let lin_reg = linear_regression(&data); + assert!( + lin_reg.r_squared > 0.95, + "r² = {:.4} is lower than expected for a linear fit\nData points: {:?}\n{:?}", + lin_reg.r_squared, + data, + lin_reg + ); +} diff --git a/third_party/rust/smawk/tests/monge.rs b/third_party/rust/smawk/tests/monge.rs new file mode 100644 index 0000000000..67058a75a5 --- /dev/null +++ b/third_party/rust/smawk/tests/monge.rs @@ -0,0 +1,83 @@ +#![cfg(feature = "ndarray")] + +use ndarray::{arr2, Array, Array2}; +use rand::SeedableRng; +use rand_chacha::ChaCha20Rng; +use smawk::monge::is_monge; + +mod random_monge; +use random_monge::{random_monge_matrix, MongePrim}; + +#[test] +fn random_monge() { + let mut rng = ChaCha20Rng::seed_from_u64(0); + let matrix: Array2 = random_monge_matrix(5, 5, &mut rng); + + assert!(is_monge(&matrix)); + assert_eq!( + matrix, + arr2(&[ + [2, 3, 4, 4, 5], + [5, 5, 6, 6, 7], + [3, 3, 4, 4, 5], + [5, 2, 3, 3, 4], + [5, 2, 3, 3, 4] + ]) + ); +} + +#[test] +fn monge_constant_rows() { + let mut rng = ChaCha20Rng::seed_from_u64(0); + let matrix: Array2 = MongePrim::ConstantRows.to_matrix(5, 4, &mut rng); + assert!(is_monge(&matrix)); + for row in matrix.rows() { + let elem = row[0]; + assert_eq!(row, Array::from_elem(matrix.ncols(), elem)); + } +} + +#[test] +fn monge_constant_cols() { + let mut rng = ChaCha20Rng::seed_from_u64(0); + let matrix: Array2 = MongePrim::ConstantCols.to_matrix(5, 4, &mut rng); + assert!(is_monge(&matrix)); + for column in matrix.columns() { + let elem = column[0]; + assert_eq!(column, Array::from_elem(matrix.nrows(), elem)); + } +} + +#[test] +fn monge_upper_right_ones() { + let mut rng = ChaCha20Rng::seed_from_u64(1); + let matrix: Array2 = MongePrim::UpperRightOnes.to_matrix(5, 4, &mut rng); + assert!(is_monge(&matrix)); + assert_eq!( + matrix, + arr2(&[ + [0, 0, 1, 1], + [0, 0, 1, 1], + [0, 0, 1, 1], + [0, 0, 0, 0], + [0, 0, 0, 0] + ]) + ); +} + +#[test] +fn monge_lower_left_ones() { + let mut rng = ChaCha20Rng::seed_from_u64(1); + let matrix: Array2 = MongePrim::LowerLeftOnes.to_matrix(5, 4, &mut rng); + assert!(is_monge(&matrix)); + assert_eq!( + matrix, + arr2(&[ + [0, 0, 0, 0], + [0, 0, 0, 0], + [1, 1, 0, 0], + [1, 1, 0, 0], + [1, 1, 0, 0] + ]) + ); +} diff --git a/third_party/rust/smawk/tests/random_monge/mod.rs b/third_party/rust/smawk/tests/random_monge/mod.rs new file mode 100644 index 0000000000..50a932fbeb --- /dev/null +++ b/third_party/rust/smawk/tests/random_monge/mod.rs @@ -0,0 +1,83 @@ +//! Test functionality for generating random Monge matrices. + +// The code is put here so we can reuse it in different integration +// tests, without Cargo finding it when `cargo test` is run. See the +// section on "Submodules in Integration Tests" in +// https://doc.rust-lang.org/book/ch11-03-test-organization.html + +use ndarray::{s, Array2}; +use num_traits::PrimInt; +use rand::distributions::{Distribution, Standard}; +use rand::Rng; + +/// A Monge matrix can be decomposed into one of these primitive +/// building blocks. +#[derive(Copy, Clone)] +pub enum MongePrim { + ConstantRows, + ConstantCols, + UpperRightOnes, + LowerLeftOnes, +} + +impl MongePrim { + /// Generate a Monge matrix from a primitive. + pub fn to_matrix(&self, m: usize, n: usize, rng: &mut R) -> Array2 + where + Standard: Distribution, + { + let mut matrix = Array2::from_elem((m, n), T::zero()); + // Avoid panic in UpperRightOnes and LowerLeftOnes below. + if m == 0 || n == 0 { + return matrix; + } + + match *self { + MongePrim::ConstantRows => { + for mut row in matrix.rows_mut() { + if rng.gen::() { + row.fill(T::one()) + } + } + } + MongePrim::ConstantCols => { + for mut col in matrix.columns_mut() { + if rng.gen::() { + col.fill(T::one()) + } + } + } + MongePrim::UpperRightOnes => { + let i = rng.gen_range(0..(m + 1) as isize); + let j = rng.gen_range(0..(n + 1) as isize); + matrix.slice_mut(s![..i, -j..]).fill(T::one()); + } + MongePrim::LowerLeftOnes => { + let i = rng.gen_range(0..(m + 1) as isize); + let j = rng.gen_range(0..(n + 1) as isize); + matrix.slice_mut(s![-i.., ..j]).fill(T::one()); + } + } + + matrix + } +} + +/// Generate a random Monge matrix. +pub fn random_monge_matrix(m: usize, n: usize, rng: &mut R) -> Array2 +where + Standard: Distribution, +{ + let monge_primitives = [ + MongePrim::ConstantRows, + MongePrim::ConstantCols, + MongePrim::LowerLeftOnes, + MongePrim::UpperRightOnes, + ]; + let mut matrix = Array2::from_elem((m, n), T::zero()); + for _ in 0..(m + n) { + let monge = monge_primitives[rng.gen_range(0..monge_primitives.len())]; + matrix = matrix + monge.to_matrix(m, n, rng); + } + matrix +} diff --git a/third_party/rust/smawk/tests/version-numbers.rs b/third_party/rust/smawk/tests/version-numbers.rs new file mode 100644 index 0000000000..288592d02f --- /dev/null +++ b/third_party/rust/smawk/tests/version-numbers.rs @@ -0,0 +1,9 @@ +#[test] +fn test_readme_deps() { + version_sync::assert_markdown_deps_updated!("README.md"); +} + +#[test] +fn test_html_root_url() { + version_sync::assert_html_root_url_updated!("src/lib.rs"); +} diff --git a/third_party/rust/sql-support/.cargo-checksum.json b/third_party/rust/sql-support/.cargo-checksum.json index bbfb487f7f..93cef6e94d 100644 --- a/third_party/rust/sql-support/.cargo-checksum.json +++ b/third_party/rust/sql-support/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"Cargo.toml":"812811e5a8e00abe3ec345cd8fd435e27fec7cb8f2e45a0e93e5becf564c46ad","src/conn_ext.rs":"e48e862e47c000c545dcc766fc1889498a8709bee00e240ed68d247b0fbef577","src/debug_tools.rs":"bece2bc3d35379b81ea2f942a0a3e909e0ab0553656505904745548eacaf402a","src/each_chunk.rs":"8aaba842e43b002fbc0fee95d14ce08faa7187b1979c765b2e270cd4802607a5","src/lib.rs":"af704ec04beb6c2c388d4566710e1167b18fb64acb248ccf37a67679daffddb6","src/maybe_cached.rs":"0b18425595055883a98807fbd62ff27a79c18af34e7cb3439f8c3438463ef2dd","src/open_database.rs":"40ad2da7d5559f0e5180e35d403c307ce230fe9d0d2a3fec7c9481ce13acda64","src/repeat.rs":"b4c5ff5d083afba7f9f153f54aba2e6859b78b85c82d48dbd6bd58f67da9e6b9"},"package":null} \ No newline at end of file +{"files":{"Cargo.toml":"812811e5a8e00abe3ec345cd8fd435e27fec7cb8f2e45a0e93e5becf564c46ad","src/conn_ext.rs":"e48e862e47c000c545dcc766fc1889498a8709bee00e240ed68d247b0fbef577","src/debug_tools.rs":"bece2bc3d35379b81ea2f942a0a3e909e0ab0553656505904745548eacaf402a","src/each_chunk.rs":"8aaba842e43b002fbc0fee95d14ce08faa7187b1979c765b2e270cd4802607a5","src/lib.rs":"af704ec04beb6c2c388d4566710e1167b18fb64acb248ccf37a67679daffddb6","src/maybe_cached.rs":"0b18425595055883a98807fbd62ff27a79c18af34e7cb3439f8c3438463ef2dd","src/open_database.rs":"ba290bfb39468e96f9b3ea865e0c13c2cc5a731ea8877a9feb6b1de4f7d666c4","src/repeat.rs":"b4c5ff5d083afba7f9f153f54aba2e6859b78b85c82d48dbd6bd58f67da9e6b9"},"package":null} \ No newline at end of file diff --git a/third_party/rust/sql-support/src/open_database.rs b/third_party/rust/sql-support/src/open_database.rs index 6c49471d29..d92a94a9ed 100644 --- a/third_party/rust/sql-support/src/open_database.rs +++ b/third_party/rust/sql-support/src/open_database.rs @@ -278,6 +278,9 @@ pub mod test_utils { } } + /// Attempt to run all upgrades up to a specific version. + /// + /// This will result in a panic if an upgrade fails to run. pub fn upgrade_to(&self, version: u32) { let mut conn = self.open(); let tx = conn.transaction().unwrap(); @@ -293,6 +296,9 @@ pub mod test_utils { tx.commit().unwrap(); } + /// Attempt to run all upgrades + /// + /// This will result in a panic if an upgrade fails to run. pub fn run_all_upgrades(&self) { let current_version = get_schema_version(&self.open()).unwrap(); for version in current_version..CI::END_VERSION { diff --git a/third_party/rust/suggest/.cargo-checksum.json b/third_party/rust/suggest/.cargo-checksum.json index c8c5fa2566..120a503f29 100644 --- a/third_party/rust/suggest/.cargo-checksum.json +++ b/third_party/rust/suggest/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"Cargo.toml":"4aa81cff67e67b08ba3348c1acddaa5aee887df3c35006754c9cda4273a94458","README.md":"8d7457893194e255b87e5a2667ee25c87bd470f5338d7078506f866a67a3fdbd","build.rs":"78780c5cccfe22c3ff4198624b9e188559c437c3e6fa1c8bb66548eee6aa66bf","src/config.rs":"03630b2219b6674e332a1f96f44db74def17f985c850a800299b815fa72241c2","src/db.rs":"d373ad097edac2bbcc6e1b14f51c21b6e2cab2289d27667332798c9cde4dcbef","src/error.rs":"f563210a6c050d98ec85e0f6d9401e7373bfb816e865e8edabbabb23d848ba13","src/keyword.rs":"988d0ab021c0df19cfd3c519df7d37f606bf984cd14d0efca4e5a7aff88344dd","src/lib.rs":"65a035dbfb17e2d2d9f237ad52dc03982ae28c70e3dcf3d96cc9f2d7af79efe3","src/pocket.rs":"c4dda43390d1c39dc795933596b3c1e4e282932cac6c69da53c6e05d39e9ef29","src/provider.rs":"4fe662587efc5a80d000c217ce124506c6800293c50ff460ef95e9e659c764b9","src/rs.rs":"0910368f9e7c4703b00d0de86902d647d70c1f75a256fbeb2126c91f0499a083","src/schema.rs":"8fad4cc624f48946676adbc3de7d061f05fe82531523008f417d6130a2132e34","src/store.rs":"a869971d5593bec2dd40822ba63d0e5a5def96a870ff5a7c33afbcbf5869946b","src/suggest.udl":"d941662596d48793d1570e5b8432b7fd7b4fb1b4550fb38d4e14224fcf4195bc","src/suggestion.rs":"7ee407949f40d88e5d3d4c0da400b987e85ace9f34c648f010cd7f5f2aba0506","src/yelp.rs":"37e77900c12c68cca292a84c6dd6c67d16628c68f4612d8d9bedb1bddf985229","uniffi.toml":"f26317442ddb5b3281245bef6e60ffcb78bb95d29fe4a351a56dbb88d4ec8aab"},"package":null} \ No newline at end of file +{"files":{"Cargo.toml":"05e4d7f7b3649a3e3fa441c4af53a633d18f20bb04fd761ed33fc9d461fd0dee","README.md":"fb72d0028586cab1421b853ef529d7ce78ad7316818b7733a4f3488b0fba67f7","benches/benchmark_all.rs":"c2343c9197b6d9ccb0798d7701b1b0d2569d494dd31a975d21d7ec6f26e32879","build.rs":"78780c5cccfe22c3ff4198624b9e188559c437c3e6fa1c8bb66548eee6aa66bf","src/benchmarks/README.md":"ee6d50df2c31cfd80a5bc047011b518dcf57f1ef928a811bb770f1a09f41b3de","src/benchmarks/client.rs":"5d5db3f6e132654c06532feba15f98576122f6b9572ab5fa27b0c67d5b39ecb6","src/benchmarks/ingest.rs":"1ffdc403fb945ea0b58353df9773ba45ab0e9082d61dd5330ad49fad8cbb5d9f","src/benchmarks/mod.rs":"fe1898ba4d783213525da10d92858ee84cebfd22749bad7aeb461d338fe5504a","src/bin/debug_ingestion_sizes.rs":"ce6e810be7b3fc19e826d75b622b82cfab5a1a99397a6d0833c2c4eebff2d364","src/config.rs":"206ae9dc768c755649cb0c88a7b1fc3c926c715441784f61e9dc06a8a02fc568","src/db.rs":"734f5fd9f36f03c07a508a9a353872b81107f5fe09f27294ba27d7e1249e3988","src/error.rs":"f563210a6c050d98ec85e0f6d9401e7373bfb816e865e8edabbabb23d848ba13","src/keyword.rs":"988d0ab021c0df19cfd3c519df7d37f606bf984cd14d0efca4e5a7aff88344dd","src/lib.rs":"91ebbe0e1ffb99eefde204f81bc6bb199b4941976347baf1f132fd0ede20479c","src/pocket.rs":"1316668840ec9b4ea886223921dc9d3b5a1731d1a5206c0b1089f2a6c45c1b7b","src/provider.rs":"fe76f19a223f5cac056c7d48525087ca2c26bf0629b0e11b1f8dc98d165c8bb2","src/rs.rs":"e3eabde58c859ebe1154bf8da56ca134ace135934e3f280acc8186b4204399b3","src/schema.rs":"8b21006940e872658d722b52ba171280c96789eecf614b837d8cdbc9153ab576","src/store.rs":"413779074db3ce4589c31cd4fb0a050d44d1cbad1df3c94101d03e98efdf09cb","src/suggest.udl":"de50ea5c7ece0ae0ff4798979e0e12a5227b42bf024d48b6f585ea30a5133eb3","src/suggestion.rs":"f31227779d13d1b03a622e08a417ceba4afb161885a01c2bc87a6a652b5e8be5","src/yelp.rs":"9c0dc02a994cc05df524aa4ef337d10f575d1891259193b6419fed6fe279cb54","uniffi.toml":"f26317442ddb5b3281245bef6e60ffcb78bb95d29fe4a351a56dbb88d4ec8aab"},"package":null} \ No newline at end of file diff --git a/third_party/rust/suggest/Cargo.toml b/third_party/rust/suggest/Cargo.toml index 17ce1af26d..cec02ceadf 100644 --- a/third_party/rust/suggest/Cargo.toml +++ b/third_party/rust/suggest/Cargo.toml @@ -21,6 +21,15 @@ description = "Manages sponsored and web suggestions for Firefox Suggest" readme = "README.md" license = "MPL-2.0" +[[bin]] +name = "debug_ingestion_sizes" +required-features = ["benchmark_api"] + +[[bench]] +name = "benchmark_all" +harness = false +required-features = ["benchmark_api"] + [dependencies] anyhow = "1.0" chrono = "0.4" @@ -28,7 +37,7 @@ once_cell = "1.5" parking_lot = ">=0.11,<=0.12" serde_json = "1" thiserror = "1" -uniffi = "0.25.2" +uniffi = "0.27.1" [dependencies.error-support] path = "../support/error" @@ -53,6 +62,10 @@ features = ["derive"] [dependencies.sql-support] path = "../support/sql" +[dependencies.tempfile] +version = "3.2.0" +optional = true + [dependencies.url] version = "2.1" features = ["serde"] @@ -60,7 +73,12 @@ features = ["serde"] [dependencies.viaduct] path = "../viaduct" +[dependencies.viaduct-reqwest] +path = "../support/viaduct-reqwest" +optional = true + [dev-dependencies] +criterion = "0.5" expect-test = "1.4" hex = "0.4" @@ -72,5 +90,11 @@ default-features = false path = "../support/rc_crypto" [build-dependencies.uniffi] -version = "0.25.2" +version = "0.27.1" features = ["build"] + +[features] +benchmark_api = [ + "tempfile", + "viaduct-reqwest", +] diff --git a/third_party/rust/suggest/README.md b/third_party/rust/suggest/README.md index 74716d2ebb..bd68f0d7c4 100644 --- a/third_party/rust/suggest/README.md +++ b/third_party/rust/suggest/README.md @@ -1,7 +1,111 @@ # Suggest -The **Suggest Rust component** powers the [Firefox Suggest](https://support.mozilla.org/en-US/kb/firefox-suggest-faq) feature. +The **Suggest Rust component** provides address bar search suggestions from Mozilla. This includes suggestions from sponsors, as well as non-sponsored suggestions for other web destinations. These suggestions are part of the [Firefox Suggest](https://support.mozilla.org/en-US/kb/firefox-suggest-faq) feature. -This component currently supports the basic Suggest experience only. The basic experience shows suggestions for sponsored and web content from a canned dataset. The component downloads the dataset from [Remote Settings](https://remote-settings.readthedocs.io/en/latest/), stores the suggestions in a local database, and makes them available to the Firefox address bar. Because matching is done locally, Mozilla never sees the user's query. +This component is integrated into Firefox Desktop, Android, and iOS. -The opt-in "Improved Firefox Suggest Experience", which sends user queries to a [Mozilla-owned proxy server](https://mozilla-services.github.io/merino/intro.html) for server-side matching, is not currently supported. +## Architecture + +Search suggestions from Mozilla are stored in a [Remote Settings](https://remote-settings.readthedocs.io/en/latest/) collection. The Suggest component downloads these suggestions from Remote Settings, stores them in a local SQLite database, and makes them available to the Firefox address bar. Because these suggestions are stored and matched locally, Mozilla never sees the user's search queries. + +This component follows the architecture of the other [Application Services Rust components](https://mozilla.github.io/application-services/book/index.html): a cross-platform Rust core, and platform-specific bindings for Firefox Desktop, Android, and iOS. These bindings are generated automatically using the [UniFFI](https://mozilla.github.io/uniffi-rs/) tool. + +### For consumers + +This section is for application developers. It describes how Firefox Desktop, Android, and iOS consume the Suggest Rust component. + +The cornerstone of the component is the `SuggestStore` interface, which is the **store**. The store _ingests_ (downloads and persists) suggestions from Remote Settings, and returns matching suggestions as the user types. This is the main interface that applications use to interact with the component. + +While the store provides most of the functionality, the application has a few responsibilities: + +**1. Create and manage a `SuggestStore` as a singleton.** Under the hood, the store holds multiple connections to the database: a read-write connection for ingestion, and a read-only connection for queries. The store uses the right connection for each operation, so applications shouldn't create multiple stores. The application is responsible for specifying the correct platform-specific storage directory for the database. The database contains _cached data_, like suggestions, and _user data_, like which suggestions have been dismissed. For this reason, applications should persist the database in their durable storage or "profile" directory. Applications specify the storage directory, and create a store, using the `SuggestStoreBuilder` interface. + +**2. Periodically call the store's `ingest()` method to ingest new suggestions.** While the store takes care of efficiently downloading and persisting suggestions from Remote Settings, the application is still responsible for scheduling this work. This is because the Suggest component doesn't have access to the various platform-specific background work scheduling mechanisms, like `nsIUpdateTimerManager` on Desktop, `WorkManager` on Android, or `BGTaskScheduler` on iOS. These are three different APIs with different constraints, written in three different languages. Instead of trying to bind to these different mechanisms, the component leaves it up to the application to use the right one on each platform. Ingestion is network- and disk I/O-bound, and should be done in the background. + +**3. Use the store's `query()` and `interrupt()` methods to query for fresh suggestions as the user types.** The application passes the user's input, and additional options like which suggestion types to return, to `query()`. Querying the database is disk I/O-bound, so applications should use their respective platforms' facilities for asynchronous work—Kotlin coroutines on Android, and Swift actors on iOS; the UniFFI bindings for Desktop take care of dispatching work to a background thread pool—to avoid blocking the main thread. Running `query()` off-main-thread also lets applications `interrupt()` those queries from the main thread when the user's input changes. This avoids waiting for a query to return suggestions that are now stale. + +### For contributors + +This section is a primer for engineers contributing code to the Suggest Rust component. + +`suggest.udl` describes the component's interface for foreign language consumers. UniFFI uses this file to generate the language bindings for each platform. If you're adding a new suggestion type, you'll want to update the declarations of `Suggestion` and `SuggestionProvider` in this file, as well as the definitions of those types in `suggestion.rs` and `provider.rs`, respectively. + +`store.rs` contains the implementation of `SuggestStore` and most of the Suggest component's tests. + +`schema.rs` manages the database schema. **Remember to bump the schema version and add a migration whenever you change the schema.** + +`db.rs` interacts with the Suggest database. The `SuggestDao` type in this file is a ["data access object"](https://en.wikipedia.org/wiki/Data_access_object) (DAO) that contains all the SQL statements for querying and updating the SQLite database. By convention, `SuggestDao` methods that can write to the database take `&mut self`, and methods that only read take `&self`. The `SuggestDb::read()` and `SuggestDb::write()` methods take a closure that receives either `&SuggestDao` or `&mut SuggestDao`; this is how the DAO enforces that all writes are done in a transaction. If you're curious to learn about how we use SQLite, or you're diagnosing a slow query or adding a new suggestion type, you'll almost certainly want to look at this file. + +`rs.rs` defines all the Remote Settings [record](https://docs.kinto-storage.org/en/stable/concepts.html#buckets-collections-and-records) and [attachment](https://docs.kinto-storage.org/en/stable/faq.html#can-i-store-files-inside-kinto) types. The records in the `quicksuggest` Remote Settings collection don't store the suggestions themselves. Instead, each record has a type, and a pointer to a JSON attachment that contains multiple suggestions of that type. This file defines [Serde](https://serde.rs/)-compatible types for all these records and attachments. + +`errors.rs` contains all the errors that this component returns. We use the crate-internal `Error` type for all fallible operations within the component, and the public `SuggestApiError` type for errors that applications should handle. + +There are other suggestion provider-specific files, like `yelp.rs`, `pocket.rs`, and `keyword.rs`, that aren't covered in this primer. If you're new to the component, we recommend starting with the highest-level interface in `store.rs` first, and jumping to the other files and types as you encounter them in the code. + +## Documentation + +Each Rust file contains [inline documentation](https://doc.rust-lang.org/rustdoc/what-is-rustdoc.html) for types, traits, functions, and methods. + +Documentation for `pub` symbols is written with application developers in mind: even if you're a Desktop, Android or iOS developer, the Rust documentation is meant to give you an understanding of how the Gecko, Kotlin and Swift bindings work. + +You can see the documentation for all public symbols by running from the command line: + +```shell +cargo doc --open +``` + +By convention, symbols that are exported as part of the component's foreign language interface have `pub` visibility, and symbols that are only used within the component have `pub(crate)` or private visibility. As an exception to this convention, symbols with [doctests](https://doc.rust-lang.org/rustdoc/write-documentation/documentation-tests.html) are also `pub`, because doctests [can only link against public symbols](https://doc.rust-lang.org/rustdoc/write-documentation/documentation-tests.html#include-items-only-when-collecting-doctests). + +If you're working on the component, you can see the documentation for `pub(crate)` and private symbols using: + +```shell +cargo doc --document-private-items --open +``` + +💡 If you're adding a new suggestion type, the documentation for the `rs` module is a great place to start. + +Please help us keep our documentation useful for everyone, and feel free to file bugs for anything that looks unclear or out-of-date! + +## Tests + +We use a technique called ["snapshot testing"](https://notlaura.com/what-is-a-snapshot-test/) with the [`expect-test`](https://docs.rs/expect-test/latest/expect_test/) crate for the Suggest component's tests. This technique makes it easier to compare and update all the expected outputs when adding, removing, and changing suggestion types. + +The snapshot tests in `store.rs` look like this: + +```rs +expect![["{expected-output}"]].assert_debug_eq(&actual_output); +``` + +The `expect-test` crate generates the `{expected-output}` string, and can update it automatically. If you add, remove, or change a suggestion type, or update the schema, and run `cargo test`, you'll likely see a few failures. `expect-test` will print a readable diff in the `cargo test` output, which you can audit for accuracy. + +If the diff looks good, you can update the expectations in-place from the command line using: + +```shell +env UPDATE_EXPECT=1 cargo test +``` + +Most of the tests in `store.rs` are integration-style tests that use a fake Remote Settings interface. + +## ⚠️ Breaking Changes ⚠️ + +A "breaking change" is any code change that breaks the build, tests, or behavior of a consuming application. + +These are some common changes that can break consumers: + +* **Changing the signature of a method or function that's currently in use.** Adding, removing, or changing the type of an argument or a return value, or reordering arguments, is a breaking change on Desktop, Android, and iOS. +* **Removing or renaming a method, function, or type that's currently in use.** +* **Adding or removing a `[Throws]` attribute**. Changing a non-throwing function to throw an error, or vice versa, is a build breaking change on iOS. On Desktop and Android, changing a non-throwing function to a throwing function won't break the build, but can cause crashes if the consumer doesn't handle the new error. +* **Changing the fields of a dictionary or an enum.** Adding, removing, reordering, or changing the type of a dictionary or an `[Enum]` interface field is a guaranteed build breaking change on iOS. It may be a breaking change on Desktop and Android if the consumer creates new instances of the dictionary or enum. + +When working on the Suggest component, this means: + +* Adding a new suggestion type is generally _not_ breaking. +* Adding a new method to `SuggestStore` is _not_ breaking. +* Renaming or removing a `SuggestStore` method that's currently in use _is_ breaking. +* Adding a new field to an existing suggestion type _is_ breaking. + +If you need to make a breaking change, don't panic! We have a [process for landing them in Application Services](https://mozilla.github.io/application-services/book/howtos/breaking-changes.html), and you can use a [branch build](https://mozilla.github.io/application-services/book/howtos/branch-builds.html) to verify that Android and iOS build and run their tests with your change. + +## Bugs + +We use Bugzilla to track bugs and feature work. You can use [this link](https://bugzilla.mozilla.org/enter_bug.cgi?product=Application+Services&component=Suggest) to file bugs in the `Application Services :: Suggest` bug component. diff --git a/third_party/rust/suggest/benches/benchmark_all.rs b/third_party/rust/suggest/benches/benchmark_all.rs new file mode 100644 index 0000000000..2e328e5804 --- /dev/null +++ b/third_party/rust/suggest/benches/benchmark_all.rs @@ -0,0 +1,25 @@ +use criterion::{criterion_group, criterion_main, BatchSize, Criterion}; +use suggest::benchmarks::{ingest, BenchmarkWithInput}; + +pub fn ingest_single_provider(c: &mut Criterion) { + let mut group = c.benchmark_group("ingest"); + viaduct_reqwest::use_reqwest_backend(); + // This needs to be 10 for now, or else the `ingest-amp-wikipedia` benchmark would take around + // 100s to run which feels like too long. `ingest-amp-mobile` also would take a around 50s. + group.sample_size(10); + for (name, benchmark) in ingest::all_benchmarks() { + group.bench_function(format!("ingest-{name}"), |b| { + b.iter_batched( + || benchmark.generate_input(), + |input| benchmark.benchmarked_code(input), + // See https://docs.rs/criterion/latest/criterion/enum.BatchSize.html#variants for + // a discussion of this. PerIteration is chosen for these benchmarks because the + // input holds a database file handle + BatchSize::PerIteration, + ); + }); + } +} + +criterion_group!(benches, ingest_single_provider); +criterion_main!(benches); diff --git a/third_party/rust/suggest/src/benchmarks/README.md b/third_party/rust/suggest/src/benchmarks/README.md new file mode 100644 index 0000000000..45150d8413 --- /dev/null +++ b/third_party/rust/suggest/src/benchmarks/README.md @@ -0,0 +1,28 @@ +# Suggest benchmarking code + +Use `cargo suggest-bench` to run these benchmarks. + +The main benchmarking code lives here, while the criterion integration code lives in the `benches/` +directory. + +## Benchmarks + +### ingest-[provider-type] + +Time it takes to ingest all suggestions for a provider type on an empty database. +The bechmark downloads network resources in advance in order to exclude the network request time +from these measurements. + +### Benchmarks it would be nice to have + +- Ingestion with synthetic data. This would isolate the benchmark from changes to the RS database. +- Fetching suggestions + +## cargo suggest-debug-ingestion-sizes + +Run this to get row counts for all database tables. This can be very useful for improving +benchmarks, since targeting the tables with the largest number of rows will usually lead to the +largest improvements. + +The command also prints out the size of all remote-settings attachments, which can be good to +optimize on its own since it represents the amount of data user's need to download. diff --git a/third_party/rust/suggest/src/benchmarks/client.rs b/third_party/rust/suggest/src/benchmarks/client.rs new file mode 100644 index 0000000000..f5a21fd9cc --- /dev/null +++ b/third_party/rust/suggest/src/benchmarks/client.rs @@ -0,0 +1,97 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use crate::{rs::SuggestRemoteSettingsClient, Result}; +use parking_lot::Mutex; +use remote_settings::{Client, GetItemsOptions, RemoteSettingsConfig, RemoteSettingsResponse}; +use std::collections::HashMap; + +/// Remotes settings client that runs during the benchmark warm-up phase. +/// +/// This should be used to run a full ingestion. +/// Then it can be converted into a [RemoteSettingsBenchmarkClient], which allows benchmark code to exclude the network request time. +/// [RemoteSettingsBenchmarkClient] implements [SuggestRemoteSettingsClient] by getting data from a HashMap rather than hitting the network. +pub struct RemoteSettingsWarmUpClient { + client: Client, + pub get_records_responses: Mutex>, + pub get_attachment_responses: Mutex>>, +} + +impl RemoteSettingsWarmUpClient { + pub fn new() -> Self { + Self { + client: Client::new(RemoteSettingsConfig { + server_url: None, + bucket_name: None, + collection_name: crate::rs::REMOTE_SETTINGS_COLLECTION.into(), + }) + .unwrap(), + get_records_responses: Mutex::new(HashMap::new()), + get_attachment_responses: Mutex::new(HashMap::new()), + } + } +} + +impl Default for RemoteSettingsWarmUpClient { + fn default() -> Self { + Self::new() + } +} + +impl SuggestRemoteSettingsClient for RemoteSettingsWarmUpClient { + fn get_records_with_options( + &self, + options: &GetItemsOptions, + ) -> Result { + let response = self.client.get_records_with_options(options)?; + self.get_records_responses + .lock() + .insert(options.clone(), response.clone()); + Ok(response) + } + + fn get_attachment(&self, location: &str) -> Result> { + let response = self.client.get_attachment(location)?; + self.get_attachment_responses + .lock() + .insert(location.to_string(), response.clone()); + Ok(response) + } +} + +#[derive(Clone)] +pub struct RemoteSettingsBenchmarkClient { + pub get_records_responses: HashMap, + pub get_attachment_responses: HashMap>, +} + +impl SuggestRemoteSettingsClient for RemoteSettingsBenchmarkClient { + fn get_records_with_options( + &self, + options: &GetItemsOptions, + ) -> Result { + Ok(self + .get_records_responses + .get(options) + .unwrap_or_else(|| panic!("options not found: {options:?}")) + .clone()) + } + + fn get_attachment(&self, location: &str) -> Result> { + Ok(self + .get_attachment_responses + .get(location) + .unwrap_or_else(|| panic!("location not found: {location:?}")) + .clone()) + } +} + +impl From for RemoteSettingsBenchmarkClient { + fn from(warm_up_client: RemoteSettingsWarmUpClient) -> Self { + Self { + get_records_responses: warm_up_client.get_records_responses.into_inner(), + get_attachment_responses: warm_up_client.get_attachment_responses.into_inner(), + } + } +} diff --git a/third_party/rust/suggest/src/benchmarks/ingest.rs b/third_party/rust/suggest/src/benchmarks/ingest.rs new file mode 100644 index 0000000000..bbefc6a00a --- /dev/null +++ b/third_party/rust/suggest/src/benchmarks/ingest.rs @@ -0,0 +1,116 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use crate::{ + benchmarks::{ + client::{RemoteSettingsBenchmarkClient, RemoteSettingsWarmUpClient}, + BenchmarkWithInput, + }, + rs::SuggestRecordType, + store::SuggestStoreInner, + SuggestIngestionConstraints, +}; +use std::sync::atomic::{AtomicU32, Ordering}; + +static DB_FILE_COUNTER: AtomicU32 = AtomicU32::new(0); + +pub struct IngestBenchmark { + temp_dir: tempfile::TempDir, + client: RemoteSettingsBenchmarkClient, + record_type: SuggestRecordType, +} + +impl IngestBenchmark { + pub fn new(record_type: SuggestRecordType) -> Self { + let temp_dir = tempfile::tempdir().unwrap(); + let store = SuggestStoreInner::new( + temp_dir.path().join("warmup.sqlite"), + RemoteSettingsWarmUpClient::new(), + ); + store.benchmark_ingest_records_by_type(record_type); + Self { + client: RemoteSettingsBenchmarkClient::from(store.into_settings_client()), + temp_dir, + record_type, + } + } +} + +// The input for each benchmark is `SuggestStoreInner` with a fresh database. +// +// This is wrapped in a newtype so that it can be exposed in the public trait +pub struct InputType(SuggestStoreInner); + +impl BenchmarkWithInput for IngestBenchmark { + type Input = InputType; + + fn generate_input(&self) -> Self::Input { + let data_path = self.temp_dir.path().join(format!( + "db{}.sqlite", + DB_FILE_COUNTER.fetch_add(1, Ordering::Relaxed) + )); + let store = SuggestStoreInner::new(data_path, self.client.clone()); + store.ensure_db_initialized(); + InputType(store) + } + + fn benchmarked_code(&self, input: Self::Input) { + let InputType(store) = input; + store.benchmark_ingest_records_by_type(self.record_type); + } +} + +/// Get IngestBenchmark instances for all record types +pub fn all_benchmarks() -> Vec<(&'static str, IngestBenchmark)> { + vec![ + ("icon", IngestBenchmark::new(SuggestRecordType::Icon)), + ( + "amp-wikipedia", + IngestBenchmark::new(SuggestRecordType::AmpWikipedia), + ), + ("amo", IngestBenchmark::new(SuggestRecordType::Amo)), + ("pocket", IngestBenchmark::new(SuggestRecordType::Pocket)), + ("yelp", IngestBenchmark::new(SuggestRecordType::Yelp)), + ("mdn", IngestBenchmark::new(SuggestRecordType::Mdn)), + ("weather", IngestBenchmark::new(SuggestRecordType::Weather)), + ( + "global-config", + IngestBenchmark::new(SuggestRecordType::GlobalConfig), + ), + ( + "amp-mobile", + IngestBenchmark::new(SuggestRecordType::AmpMobile), + ), + ] +} + +pub fn print_debug_ingestion_sizes() { + viaduct_reqwest::use_reqwest_backend(); + let store = SuggestStoreInner::new( + "file:debug_ingestion_sizes?mode=memory&cache=shared", + RemoteSettingsWarmUpClient::new(), + ); + store + .ingest(SuggestIngestionConstraints::default()) + .unwrap(); + let table_row_counts = store.table_row_counts(); + let client = store.into_settings_client(); + let total_attachment_size: usize = client + .get_attachment_responses + .lock() + .values() + .map(|data| data.len()) + .sum(); + + println!( + "Total attachment size: {}kb", + (total_attachment_size + 500) / 1000 + ); + println!(); + println!("Database table row counts"); + println!("-------------------------"); + for (name, count) in table_row_counts { + println!("{name:30}: {count}"); + } +} diff --git a/third_party/rust/suggest/src/benchmarks/mod.rs b/third_party/rust/suggest/src/benchmarks/mod.rs new file mode 100644 index 0000000000..eb3b2e8abe --- /dev/null +++ b/third_party/rust/suggest/src/benchmarks/mod.rs @@ -0,0 +1,40 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +//! Benchmarking support +//! +//! Benchmarks are split up into two parts: the functions to be benchmarked live here, which the benchmarking code itself lives in `benches/bench.rs`. +//! It's easier to write benchmarking code inside the main crate, where we have access to private items. +//! However, it's easier to integrate with Cargo and criterion if benchmarks live in a separate crate. +//! +//! All benchmarks are defined as structs that implement either the [Benchmark] or [BenchmarkWithInput] + +pub mod client; +pub mod ingest; + +/// Trait for simple benchmarks +/// +/// This supports simple benchmarks that don't require any input. Note: global setup can be done +/// in the `new()` method for the struct. +pub trait Benchmark { + /// Perform the operations that we're benchmarking. + fn benchmarked_code(&self); +} + +/// Trait for benchmarks that require input +/// +/// This will run using Criterion's `iter_batched` function. Criterion will create a batch of +/// inputs, then pass each one to benchmark. +/// +/// This supports simple benchmarks that don't require any input. Note: global setup can be done +/// in the `new()` method for the struct. +pub trait BenchmarkWithInput { + type Input; + + /// Generate the input (this is not included in the benchmark time) + fn generate_input(&self) -> Self::Input; + + /// Perform the operations that we're benchmarking. + fn benchmarked_code(&self, input: Self::Input); +} diff --git a/third_party/rust/suggest/src/bin/debug_ingestion_sizes.rs b/third_party/rust/suggest/src/bin/debug_ingestion_sizes.rs new file mode 100644 index 0000000000..14ca3d9462 --- /dev/null +++ b/third_party/rust/suggest/src/bin/debug_ingestion_sizes.rs @@ -0,0 +1,9 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use suggest::benchmarks::ingest; + +fn main() { + ingest::print_debug_ingestion_sizes() +} diff --git a/third_party/rust/suggest/src/config.rs b/third_party/rust/suggest/src/config.rs index fcb3c2e256..532ac5b504 100644 --- a/third_party/rust/suggest/src/config.rs +++ b/third_party/rust/suggest/src/config.rs @@ -1,3 +1,8 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + use serde::{Deserialize, Serialize}; use crate::rs::{DownloadedGlobalConfig, DownloadedWeatherData}; diff --git a/third_party/rust/suggest/src/db.rs b/third_party/rust/suggest/src/db.rs index 07fc3ab4a2..0412c50d8f 100644 --- a/third_party/rust/suggest/src/db.rs +++ b/third_party/rust/suggest/src/db.rs @@ -23,17 +23,17 @@ use crate::{ rs::{ DownloadedAmoSuggestion, DownloadedAmpSuggestion, DownloadedAmpWikipediaSuggestion, DownloadedMdnSuggestion, DownloadedPocketSuggestion, DownloadedWeatherData, - SuggestRecordId, + DownloadedWikipediaSuggestion, SuggestRecordId, }, - schema::{SuggestConnectionInitializer, VERSION}, + schema::{clear_database, SuggestConnectionInitializer, VERSION}, store::{UnparsableRecord, UnparsableRecords}, suggestion::{cook_raw_suggestion_url, AmpSuggestionType, Suggestion}, Result, SuggestionQuery, }; -/// The metadata key whose value is the timestamp of the last record ingested -/// from the Suggest Remote Settings collection. -pub const LAST_INGEST_META_KEY: &str = "last_quicksuggest_ingest"; +/// The metadata key prefix for the last ingested unparsable record. These are +/// records that were not parsed properly, or were not of the "approved" types. +pub const LAST_INGEST_META_UNPARSABLE: &str = "last_quicksuggest_ingest_unparsable"; /// The metadata key whose value keeps track of records of suggestions /// that aren't parsable and which schema version it was first seen in. pub const UNPARSABLE_RECORDS_META_KEY: &str = "unparsable_records"; @@ -148,20 +148,28 @@ impl<'a> SuggestDao<'a> { self.put_unparsable_record_id(&record_id)?; // Advance the last fetch time, so that we can resume // fetching after this record if we're interrupted. - self.put_last_ingest_if_newer(record.last_modified) + self.put_last_ingest_if_newer(LAST_INGEST_META_UNPARSABLE, record.last_modified) } - pub fn handle_ingested_record(&mut self, record: &RemoteSettingsRecord) -> Result<()> { + pub fn handle_ingested_record( + &mut self, + last_ingest_key: &str, + record: &RemoteSettingsRecord, + ) -> Result<()> { let record_id = SuggestRecordId::from(&record.id); // Remove this record's ID from the list of unparsable // records, since we understand it now. self.drop_unparsable_record_id(&record_id)?; // Advance the last fetch time, so that we can resume // fetching after this record if we're interrupted. - self.put_last_ingest_if_newer(record.last_modified) + self.put_last_ingest_if_newer(last_ingest_key, record.last_modified) } - pub fn handle_deleted_record(&mut self, record: &RemoteSettingsRecord) -> Result<()> { + pub fn handle_deleted_record( + &mut self, + last_ingest_key: &str, + record: &RemoteSettingsRecord, + ) -> Result<()> { let record_id = SuggestRecordId::from(&record.id); // Drop either the icon or suggestions, records only contain one or the other match record_id.as_icon_id() { @@ -173,7 +181,7 @@ impl<'a> SuggestDao<'a> { self.drop_unparsable_record_id(&record_id)?; // Advance the last fetch time, so that we can resume // fetching after this record if we're interrupted. - self.put_last_ingest_if_newer(record.last_modified) + self.put_last_ingest_if_newer(last_ingest_key, record.last_modified) } // =============== Low level API =============== @@ -231,15 +239,20 @@ impl<'a> SuggestDao<'a> { s.title, s.url, s.provider, - s.score + s.score, + fk.full_keyword FROM suggestions s JOIN keywords k ON k.suggestion_id = s.id + LEFT JOIN + full_keywords fk + ON k.full_keyword_id = fk.id WHERE s.provider = :provider AND k.keyword = :keyword + AND NOT EXISTS (SELECT 1 FROM dismissed_suggestions WHERE url=s.url) "#, named_params! { ":keyword": keyword_lowercased, @@ -248,8 +261,9 @@ impl<'a> SuggestDao<'a> { |row| -> Result { let suggestion_id: i64 = row.get("id")?; let title = row.get("title")?; - let raw_url = row.get::<_, String>("url")?; - let score = row.get::<_, f64>("score")?; + let raw_url: String = row.get("url")?; + let score: f64 = row.get("score")?; + let full_keyword_from_db: Option = row.get("full_keyword")?; let keywords: Vec = self.conn.query_rows_and_then_cached( r#" @@ -277,9 +291,12 @@ impl<'a> SuggestDao<'a> { amp.iab_category, amp.impression_url, amp.click_url, - (SELECT i.data FROM icons i WHERE i.id = amp.icon_id) AS icon + i.data AS icon, + i.mimetype AS icon_mimetype FROM amp_custom_details amp + LEFT JOIN + icons i ON amp.icon_id = i.id WHERE amp.suggestion_id = :suggestion_id "#, @@ -298,8 +315,10 @@ impl<'a> SuggestDao<'a> { title, url: cooked_url, raw_url, - full_keyword: full_keyword(keyword_lowercased, &keywords), + full_keyword: full_keyword_from_db + .unwrap_or_else(|| full_keyword(keyword_lowercased, &keywords)), icon: row.get("icon")?, + icon_mimetype: row.get("icon_mimetype")?, impression_url: row.get("impression_url")?, click_url: cooked_click_url, raw_click_url, @@ -330,6 +349,7 @@ impl<'a> SuggestDao<'a> { WHERE s.provider = :provider AND k.keyword = :keyword + AND NOT EXISTS (SELECT 1 FROM dismissed_suggestions WHERE url=s.url) "#, named_params! { ":keyword": keyword_lowercased, @@ -350,36 +370,51 @@ impl<'a> SuggestDao<'a> { }, |row| row.get(0), )?; - let icon = self.conn.try_query_one( - "SELECT i.data + let (icon, icon_mimetype) = self + .conn + .try_query_row( + "SELECT i.data, i.mimetype FROM icons i JOIN wikipedia_custom_details s ON s.icon_id = i.id - WHERE s.suggestion_id = :suggestion_id", - named_params! { - ":suggestion_id": suggestion_id - }, - true, - )?; + WHERE s.suggestion_id = :suggestion_id + LIMIT 1", + named_params! { + ":suggestion_id": suggestion_id + }, + |row| -> Result<_> { + Ok(( + row.get::<_, Option>>(0)?, + row.get::<_, Option>(1)?, + )) + }, + true, + )? + .unwrap_or((None, None)); + Ok(Suggestion::Wikipedia { title, url: raw_url, full_keyword: full_keyword(keyword_lowercased, &keywords), icon, + icon_mimetype, }) }, )?; Ok(suggestions) } - /// Fetches Suggestions of type Amo provider that match the given query - pub fn fetch_amo_suggestions(&self, query: &SuggestionQuery) -> Result> { + /// Query for suggestions using the keyword prefix and provider + fn map_prefix_keywords( + &self, + query: &SuggestionQuery, + provider: &SuggestionProvider, + mut mapper: impl FnMut(&rusqlite::Row, &str) -> Result, + ) -> Result> { let keyword_lowercased = &query.keyword.to_lowercase(); let (keyword_prefix, keyword_suffix) = split_keyword(keyword_lowercased); - let suggestions_limit = &query.limit.unwrap_or(-1); - let suggestions = self - .conn - .query_rows_and_then_cached( - r#" + let suggestions_limit = query.limit.unwrap_or(-1); + self.conn.query_rows_and_then_cached( + r#" SELECT s.id, MAX(k.rank) AS rank, @@ -397,6 +432,7 @@ impl<'a> SuggestDao<'a> { k.keyword_prefix = :keyword_prefix AND (k.keyword_suffix BETWEEN :keyword_suffix AND :keyword_suffix || x'FFFF') AND s.provider = :provider + AND NOT EXISTS (SELECT 1 FROM dismissed_suggestions WHERE url=s.url) GROUP BY s.id ORDER BY @@ -405,13 +441,23 @@ impl<'a> SuggestDao<'a> { LIMIT :suggestions_limit "#, - named_params! { - ":keyword_prefix": keyword_prefix, - ":keyword_suffix": keyword_suffix, - ":provider": SuggestionProvider::Amo, - ":suggestions_limit": suggestions_limit, - }, - |row| -> Result> { + &[ + (":keyword_prefix", &keyword_prefix as &dyn ToSql), + (":keyword_suffix", &keyword_suffix as &dyn ToSql), + (":provider", provider as &dyn ToSql), + (":suggestions_limit", &suggestions_limit as &dyn ToSql), + ], + |row| mapper(row, keyword_suffix), + ) + } + + /// Fetches Suggestions of type Amo provider that match the given query + pub fn fetch_amo_suggestions(&self, query: &SuggestionQuery) -> Result> { + let suggestions = self + .map_prefix_keywords( + query, + &SuggestionProvider::Amo, + |row, keyword_suffix| -> Result> { let suggestion_id: i64 = row.get("id")?; let title = row.get("title")?; let raw_url = row.get::<_, String>("url")?; @@ -486,6 +532,7 @@ impl<'a> SuggestDao<'a> { k.keyword_prefix = :keyword_prefix AND (k.keyword_suffix BETWEEN :keyword_suffix AND :keyword_suffix || x'FFFF') AND s.provider = :provider + AND NOT EXISTS (SELECT 1 FROM dismissed_suggestions WHERE url=s.url) GROUP BY s.id, k.confidence @@ -534,45 +581,11 @@ impl<'a> SuggestDao<'a> { /// Fetches suggestions for MDN pub fn fetch_mdn_suggestions(&self, query: &SuggestionQuery) -> Result> { - let keyword_lowercased = &query.keyword.to_lowercase(); - let (keyword_prefix, keyword_suffix) = split_keyword(keyword_lowercased); - let suggestions_limit = &query.limit.unwrap_or(-1); let suggestions = self - .conn - .query_rows_and_then_cached( - r#" - SELECT - s.id, - MAX(k.rank) AS rank, - s.title, - s.url, - s.provider, - s.score, - k.keyword_suffix - FROM - suggestions s - JOIN - prefix_keywords k - ON k.suggestion_id = s.id - WHERE - k.keyword_prefix = :keyword_prefix - AND (k.keyword_suffix BETWEEN :keyword_suffix AND :keyword_suffix || x'FFFF') - AND s.provider = :provider - GROUP BY - s.id - ORDER BY - s.score DESC, - rank DESC - LIMIT - :suggestions_limit - "#, - named_params! { - ":keyword_prefix": keyword_prefix, - ":keyword_suffix": keyword_suffix, - ":provider": SuggestionProvider::Mdn, - ":suggestions_limit": suggestions_limit, - }, - |row| -> Result> { + .map_prefix_keywords( + query, + &SuggestionProvider::Mdn, + |row, keyword_suffix| -> Result> { let suggestion_id: i64 = row.get("id")?; let title = row.get("title")?; let raw_url = row.get::<_, String>("url")?; @@ -657,35 +670,15 @@ impl<'a> SuggestDao<'a> { record_id: &SuggestRecordId, suggestions: &[DownloadedAmoSuggestion], ) -> Result<()> { + let mut suggestion_insert = SuggestionInsertStatement::new(self.conn)?; for suggestion in suggestions { self.scope.err_if_interrupted()?; - let suggestion_id: i64 = self.conn.query_row_and_then_cachable( - &format!( - "INSERT INTO suggestions( - record_id, - provider, - title, - url, - score - ) - VALUES( - :record_id, - {}, - :title, - :url, - :score - ) - RETURNING id", - SuggestionProvider::Amo as u8 - ), - named_params! { - ":record_id": record_id.as_str(), - ":title": suggestion.title, - ":url": suggestion.url, - ":score": suggestion.score, - }, - |row| row.get(0), - true, + let suggestion_id = suggestion_insert.execute( + record_id, + &suggestion.title, + &suggestion.url, + suggestion.score, + SuggestionProvider::Amo, )?; self.conn.execute( "INSERT INTO amo_custom_details( @@ -747,105 +740,48 @@ impl<'a> SuggestDao<'a> { record_id: &SuggestRecordId, suggestions: &[DownloadedAmpWikipediaSuggestion], ) -> Result<()> { + // Prepare statements outside of the loop. This results in a large performance + // improvement on a fresh ingest, since there are so many rows. + let mut suggestion_insert = SuggestionInsertStatement::new(self.conn)?; + let mut amp_insert = AmpInsertStatement::new(self.conn)?; + let mut wiki_insert = WikipediaInsertStatement::new(self.conn)?; + let mut keyword_insert = KeywordInsertStatement::new(self.conn)?; for suggestion in suggestions { self.scope.err_if_interrupted()?; let common_details = suggestion.common_details(); let provider = suggestion.provider(); - let suggestion_id: i64 = self.conn.query_row_and_then_cachable( - &format!( - "INSERT INTO suggestions( - record_id, - provider, - title, - url, - score - ) - VALUES( - :record_id, - {}, - :title, - :url, - :score - ) - RETURNING id", - provider as u8 - ), - named_params! { - ":record_id": record_id.as_str(), - ":title": common_details.title, - ":url": common_details.url, - ":score": common_details.score.unwrap_or(DEFAULT_SUGGESTION_SCORE) - }, - |row| row.get(0), - true, + let suggestion_id = suggestion_insert.execute( + record_id, + &common_details.title, + &common_details.url, + common_details.score.unwrap_or(DEFAULT_SUGGESTION_SCORE), + provider, )?; match suggestion { DownloadedAmpWikipediaSuggestion::Amp(amp) => { - self.conn.execute( - "INSERT INTO amp_custom_details( - suggestion_id, - advertiser, - block_id, - iab_category, - impression_url, - click_url, - icon_id - ) - VALUES( - :suggestion_id, - :advertiser, - :block_id, - :iab_category, - :impression_url, - :click_url, - :icon_id - )", - named_params! { - ":suggestion_id": suggestion_id, - ":advertiser": amp.advertiser, - ":block_id": amp.block_id, - ":iab_category": amp.iab_category, - ":impression_url": amp.impression_url, - ":click_url": amp.click_url, - ":icon_id": amp.icon_id, - }, - )?; + amp_insert.execute(suggestion_id, amp)?; } DownloadedAmpWikipediaSuggestion::Wikipedia(wikipedia) => { - self.conn.execute( - "INSERT INTO wikipedia_custom_details( - suggestion_id, - icon_id - ) - VALUES( - :suggestion_id, - :icon_id - )", - named_params! { - ":suggestion_id": suggestion_id, - ":icon_id": wikipedia.icon_id, - }, - )?; + wiki_insert.execute(suggestion_id, wikipedia)?; } } - for (index, keyword) in common_details.keywords.iter().enumerate() { - self.conn.execute( - "INSERT INTO keywords( - keyword, - suggestion_id, - rank - ) - VALUES( - :keyword, - :suggestion_id, - :rank - )", - named_params! { - ":keyword": keyword, - ":rank": index, - ":suggestion_id": suggestion_id, - }, + let mut full_keyword_inserter = FullKeywordInserter::new(self.conn, suggestion_id); + for keyword in common_details.keywords() { + let full_keyword_id = match (suggestion, keyword.full_keyword) { + // Try to associate full keyword data. Only do this for AMP, we decided to + // skip it for Wikipedia in https://bugzilla.mozilla.org/show_bug.cgi?id=1876217 + (DownloadedAmpWikipediaSuggestion::Amp(_), Some(full_keyword)) => { + Some(full_keyword_inserter.maybe_insert(full_keyword)?) + } + _ => None, + }; + + keyword_insert.execute( + suggestion_id, + keyword.keyword, + full_keyword_id, + keyword.rank, )?; } } @@ -859,84 +795,32 @@ impl<'a> SuggestDao<'a> { record_id: &SuggestRecordId, suggestions: &[DownloadedAmpSuggestion], ) -> Result<()> { + let mut suggestion_insert = SuggestionInsertStatement::new(self.conn)?; + let mut amp_insert = AmpInsertStatement::new(self.conn)?; + let mut keyword_insert = KeywordInsertStatement::new(self.conn)?; for suggestion in suggestions { self.scope.err_if_interrupted()?; let common_details = &suggestion.common_details; - let suggestion_id: i64 = self.conn.query_row_and_then_cachable( - &format!( - "INSERT INTO suggestions( - record_id, - provider, - title, - url, - score - ) - VALUES( - :record_id, - {}, - :title, - :url, - :score - ) - RETURNING id", - SuggestionProvider::AmpMobile as u8 - ), - named_params! { - ":record_id": record_id.as_str(), - ":title": common_details.title, - ":url": common_details.url, - ":score": common_details.score.unwrap_or(DEFAULT_SUGGESTION_SCORE) - }, - |row| row.get(0), - true, - )?; - self.conn.execute( - "INSERT INTO amp_custom_details( - suggestion_id, - advertiser, - block_id, - iab_category, - impression_url, - click_url, - icon_id - ) - VALUES( - :suggestion_id, - :advertiser, - :block_id, - :iab_category, - :impression_url, - :click_url, - :icon_id - )", - named_params! { - ":suggestion_id": suggestion_id, - ":advertiser": suggestion.advertiser, - ":block_id": suggestion.block_id, - ":iab_category": suggestion.iab_category, - ":impression_url": suggestion.impression_url, - ":click_url": suggestion.click_url, - ":icon_id": suggestion.icon_id, - }, + let suggestion_id = suggestion_insert.execute( + record_id, + &common_details.title, + &common_details.url, + common_details.score.unwrap_or(DEFAULT_SUGGESTION_SCORE), + SuggestionProvider::AmpMobile, )?; + amp_insert.execute(suggestion_id, suggestion)?; - for (index, keyword) in common_details.keywords.iter().enumerate() { - self.conn.execute( - "INSERT INTO keywords( - keyword, - suggestion_id, - rank - ) - VALUES( - :keyword, - :suggestion_id, - :rank - )", - named_params! { - ":keyword": keyword, - ":rank": index, - ":suggestion_id": suggestion_id, - }, + let mut full_keyword_inserter = FullKeywordInserter::new(self.conn, suggestion_id); + for keyword in common_details.keywords() { + let full_keyword_id = keyword + .full_keyword + .map(|full_keyword| full_keyword_inserter.maybe_insert(full_keyword)) + .transpose()?; + keyword_insert.execute( + suggestion_id, + keyword.keyword, + full_keyword_id, + keyword.rank, )?; } } @@ -950,37 +834,16 @@ impl<'a> SuggestDao<'a> { record_id: &SuggestRecordId, suggestions: &[DownloadedPocketSuggestion], ) -> Result<()> { + let mut suggestion_insert = SuggestionInsertStatement::new(self.conn)?; for suggestion in suggestions { self.scope.err_if_interrupted()?; - let suggestion_id: i64 = self.conn.query_row_and_then_cachable( - &format!( - "INSERT INTO suggestions( - record_id, - provider, - title, - url, - score - ) - VALUES( - :record_id, - {}, - :title, - :url, - :score - ) - RETURNING id", - SuggestionProvider::Pocket as u8 - ), - named_params! { - ":record_id": record_id.as_str(), - ":title": suggestion.title, - ":url": suggestion.url, - ":score": suggestion.score, - }, - |row| row.get(0), - true, + let suggestion_id = suggestion_insert.execute( + record_id, + &suggestion.title, + &suggestion.url, + suggestion.score, + SuggestionProvider::Pocket, )?; - for ((rank, keyword), confidence) in suggestion .high_confidence_keywords .iter() @@ -1030,35 +893,15 @@ impl<'a> SuggestDao<'a> { record_id: &SuggestRecordId, suggestions: &[DownloadedMdnSuggestion], ) -> Result<()> { + let mut suggestion_insert = SuggestionInsertStatement::new(self.conn)?; for suggestion in suggestions { self.scope.err_if_interrupted()?; - let suggestion_id: i64 = self.conn.query_row_and_then_cachable( - &format!( - "INSERT INTO suggestions( - record_id, - provider, - title, - url, - score - ) - VALUES( - :record_id, - {}, - :title, - :url, - :score - ) - RETURNING id", - SuggestionProvider::Mdn as u8 - ), - named_params! { - ":record_id": record_id.as_str(), - ":title": suggestion.title, - ":url": suggestion.url, - ":score": suggestion.score, - }, - |row| row.get(0), - true, + let suggestion_id = suggestion_insert.execute( + record_id, + &suggestion.title, + &suggestion.url, + suggestion.score, + SuggestionProvider::Mdn, )?; self.conn.execute_cached( "INSERT INTO mdn_custom_details( @@ -1107,20 +950,14 @@ impl<'a> SuggestDao<'a> { record_id: &SuggestRecordId, data: &DownloadedWeatherData, ) -> Result<()> { + let mut suggestion_insert = SuggestionInsertStatement::new(self.conn)?; self.scope.err_if_interrupted()?; - let suggestion_id: i64 = self.conn.query_row_and_then_cachable( - &format!( - "INSERT INTO suggestions(record_id, provider, title, url, score) - VALUES(:record_id, {}, '', '', :score) - RETURNING id", - SuggestionProvider::Weather as u8 - ), - named_params! { - ":record_id": record_id.as_str(), - ":score": data.weather.score.unwrap_or(DEFAULT_SUGGESTION_SCORE), - }, - |row| row.get(0), - true, + let suggestion_id = suggestion_insert.execute( + record_id, + "", + "", + data.weather.score.unwrap_or(DEFAULT_SUGGESTION_SCORE), + SuggestionProvider::Weather, )?; for (index, keyword) in data.weather.keywords.iter().enumerate() { self.conn.execute( @@ -1141,24 +978,43 @@ impl<'a> SuggestDao<'a> { } /// Inserts or replaces an icon for a suggestion into the database. - pub fn put_icon(&mut self, icon_id: &str, data: &[u8]) -> Result<()> { + pub fn put_icon(&mut self, icon_id: &str, data: &[u8], mimetype: &str) -> Result<()> { self.conn.execute( "INSERT OR REPLACE INTO icons( id, - data + data, + mimetype ) VALUES( :id, - :data + :data, + :mimetype )", named_params! { ":id": icon_id, ":data": data, + ":mimetype": mimetype, }, )?; Ok(()) } + pub fn insert_dismissal(&self, url: &str) -> Result<()> { + self.conn.execute( + "INSERT OR IGNORE INTO dismissed_suggestions(url) + VALUES(:url)", + named_params! { + ":url": url, + }, + )?; + Ok(()) + } + + pub fn clear_dismissals(&self) -> Result<()> { + self.conn.execute("DELETE FROM dismissed_suggestions", ())?; + Ok(()) + } + /// Deletes all suggestions associated with a Remote Settings record from /// the database. pub fn drop_suggestions(&mut self, record_id: &SuggestRecordId) -> Result<()> { @@ -1196,12 +1052,7 @@ impl<'a> SuggestDao<'a> { /// Clears the database, removing all suggestions, icons, and metadata. pub fn clear(&mut self) -> Result<()> { - self.conn.execute_batch( - "DELETE FROM suggestions; - DELETE FROM icons; - DELETE FROM meta;", - )?; - Ok(()) + Ok(clear_database(self.conn)?) } /// Returns the value associated with a metadata key. @@ -1224,12 +1075,14 @@ impl<'a> SuggestDao<'a> { /// Updates the last ingest timestamp if the given last modified time is /// newer than the existing one recorded. - pub fn put_last_ingest_if_newer(&mut self, record_last_modified: u64) -> Result<()> { - let last_ingest = self - .get_meta::(LAST_INGEST_META_KEY)? - .unwrap_or_default(); + pub fn put_last_ingest_if_newer( + &mut self, + last_ingest_key: &str, + record_last_modified: u64, + ) -> Result<()> { + let last_ingest = self.get_meta::(last_ingest_key)?.unwrap_or_default(); if record_last_modified > last_ingest { - self.put_meta(LAST_INGEST_META_KEY, record_last_modified)?; + self.put_meta(last_ingest_key, record_last_modified)?; } Ok(()) @@ -1310,6 +1163,185 @@ impl<'a> SuggestDao<'a> { } } +/// Helper struct to get full_keyword_ids for a suggestion +/// +/// `FullKeywordInserter` handles repeated full keywords efficiently. The first instance will +/// cause a row to be inserted into the database. Subsequent instances will return the same +/// full_keyword_id. +struct FullKeywordInserter<'a> { + conn: &'a Connection, + suggestion_id: i64, + last_inserted: Option<(&'a str, i64)>, +} + +impl<'a> FullKeywordInserter<'a> { + fn new(conn: &'a Connection, suggestion_id: i64) -> Self { + Self { + conn, + suggestion_id, + last_inserted: None, + } + } + + fn maybe_insert(&mut self, full_keyword: &'a str) -> rusqlite::Result { + match self.last_inserted { + Some((s, id)) if s == full_keyword => Ok(id), + _ => { + let full_keyword_id = self.conn.query_row_and_then( + "INSERT INTO full_keywords( + suggestion_id, + full_keyword + ) + VALUES( + :suggestion_id, + :keyword + ) + RETURNING id", + named_params! { + ":keyword": full_keyword, + ":suggestion_id": self.suggestion_id, + }, + |row| row.get(0), + )?; + self.last_inserted = Some((full_keyword, full_keyword_id)); + Ok(full_keyword_id) + } + } + } +} + +// ======================== Statement types ======================== +// +// During ingestion we can insert hundreds of thousands of rows. These types enable speedups by +// allowing us to prepare a statement outside a loop and use it many times inside the loop. +// +// Each type wraps [Connection::prepare] and [Statement] to provide a simplified interface, +// tailored to a specific query. +// +// This pattern is applicable for whenever we execute the same query repeatedly in a loop. +// The impact scales with the number of loop iterations, which is why we currently don't do this +// for providers like Mdn, Pocket, and Weather, which have relatively small number of records +// compared to Amp/Wikipedia. + +struct SuggestionInsertStatement<'conn>(rusqlite::Statement<'conn>); + +impl<'conn> SuggestionInsertStatement<'conn> { + fn new(conn: &'conn Connection) -> Result { + Ok(Self(conn.prepare( + "INSERT INTO suggestions( + record_id, + title, + url, + score, + provider + ) + VALUES(?, ?, ?, ?, ?) + RETURNING id", + )?)) + } + + /// Execute the insert and return the `suggestion_id` for the new row + fn execute( + &mut self, + record_id: &SuggestRecordId, + title: &str, + url: &str, + score: f64, + provider: SuggestionProvider, + ) -> Result { + Ok(self.0.query_row( + (record_id.as_str(), title, url, score, provider as u8), + |row| row.get(0), + )?) + } +} + +struct AmpInsertStatement<'conn>(rusqlite::Statement<'conn>); + +impl<'conn> AmpInsertStatement<'conn> { + fn new(conn: &'conn Connection) -> Result { + Ok(Self(conn.prepare( + "INSERT INTO amp_custom_details( + suggestion_id, + advertiser, + block_id, + iab_category, + impression_url, + click_url, + icon_id + ) + VALUES(?, ?, ?, ?, ?, ?, ?) + ", + )?)) + } + + fn execute(&mut self, suggestion_id: i64, amp: &DownloadedAmpSuggestion) -> Result<()> { + self.0.execute(( + suggestion_id, + &.advertiser, + amp.block_id, + &.iab_category, + &.impression_url, + &.click_url, + &.icon_id, + ))?; + Ok(()) + } +} + +struct WikipediaInsertStatement<'conn>(rusqlite::Statement<'conn>); + +impl<'conn> WikipediaInsertStatement<'conn> { + fn new(conn: &'conn Connection) -> Result { + Ok(Self(conn.prepare( + "INSERT INTO wikipedia_custom_details( + suggestion_id, + icon_id + ) + VALUES(?, ?) + ", + )?)) + } + + fn execute( + &mut self, + suggestion_id: i64, + wikipedia: &DownloadedWikipediaSuggestion, + ) -> Result<()> { + self.0.execute((suggestion_id, &wikipedia.icon_id))?; + Ok(()) + } +} + +struct KeywordInsertStatement<'conn>(rusqlite::Statement<'conn>); + +impl<'conn> KeywordInsertStatement<'conn> { + fn new(conn: &'conn Connection) -> Result { + Ok(Self(conn.prepare( + "INSERT INTO keywords( + suggestion_id, + keyword, + full_keyword_id, + rank + ) + VALUES(?, ?, ?, ?) + ", + )?)) + } + + fn execute( + &mut self, + suggestion_id: i64, + keyword: &str, + full_keyword_id: Option, + rank: usize, + ) -> Result<()> { + self.0 + .execute((suggestion_id, keyword, full_keyword_id, rank))?; + Ok(()) + } +} + fn provider_config_meta_key(provider: SuggestionProvider) -> String { format!("{}{}", PROVIDER_CONFIG_META_KEY_PREFIX, provider as u8) } diff --git a/third_party/rust/suggest/src/lib.rs b/third_party/rust/suggest/src/lib.rs index 23775b7dec..15746614d0 100644 --- a/third_party/rust/suggest/src/lib.rs +++ b/third_party/rust/suggest/src/lib.rs @@ -4,6 +4,8 @@ */ use remote_settings::RemoteSettingsConfig; +#[cfg(feature = "benchmark_api")] +pub mod benchmarks; mod config; mod db; mod error; @@ -26,7 +28,7 @@ pub(crate) type Result = std::result::Result; pub type SuggestApiResult = std::result::Result; /// A query for suggestions to show in the address bar. -#[derive(Debug, Default)] +#[derive(Clone, Debug, Default)] pub struct SuggestionQuery { pub keyword: String, pub providers: Vec, diff --git a/third_party/rust/suggest/src/pocket.rs b/third_party/rust/suggest/src/pocket.rs index cf7070c62a..06fd4b012d 100644 --- a/third_party/rust/suggest/src/pocket.rs +++ b/third_party/rust/suggest/src/pocket.rs @@ -12,7 +12,7 @@ use rusqlite::{Result as RusqliteResult, ToSql}; /// substring for the suffix. #[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] #[repr(u8)] -pub enum KeywordConfidence { +pub(crate) enum KeywordConfidence { Low = 0, High = 1, } diff --git a/third_party/rust/suggest/src/provider.rs b/third_party/rust/suggest/src/provider.rs index 1449c35c8a..2c0b6674cb 100644 --- a/third_party/rust/suggest/src/provider.rs +++ b/third_party/rust/suggest/src/provider.rs @@ -8,6 +8,8 @@ use rusqlite::{ Result as RusqliteResult, }; +use crate::rs::SuggestRecordType; + /// A provider is a source of search suggestions. #[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] #[repr(u8)] @@ -46,6 +48,52 @@ impl SuggestionProvider { _ => None, } } + + pub(crate) fn records_for_provider(&self) -> Vec { + match self { + SuggestionProvider::Amp => { + vec![ + SuggestRecordType::AmpWikipedia, + SuggestRecordType::Icon, + SuggestRecordType::GlobalConfig, + ] + } + SuggestionProvider::Wikipedia => { + vec![ + SuggestRecordType::AmpWikipedia, + SuggestRecordType::Icon, + SuggestRecordType::GlobalConfig, + ] + } + SuggestionProvider::Amo => { + vec![SuggestRecordType::Amo, SuggestRecordType::GlobalConfig] + } + SuggestionProvider::Pocket => { + vec![SuggestRecordType::Pocket, SuggestRecordType::GlobalConfig] + } + SuggestionProvider::Yelp => { + vec![ + SuggestRecordType::Yelp, + SuggestRecordType::Icon, + SuggestRecordType::GlobalConfig, + ] + } + SuggestionProvider::Mdn => { + vec![SuggestRecordType::Mdn, SuggestRecordType::GlobalConfig] + } + SuggestionProvider::Weather => { + vec![SuggestRecordType::Weather, SuggestRecordType::GlobalConfig] + } + SuggestionProvider::AmpMobile => { + vec![ + SuggestRecordType::AmpMobile, + SuggestRecordType::AmpWikipedia, + SuggestRecordType::Icon, + SuggestRecordType::GlobalConfig, + ] + } + } + } } impl ToSql for SuggestionProvider { diff --git a/third_party/rust/suggest/src/rs.rs b/third_party/rust/suggest/src/rs.rs index 198a8c43f6..4a733ece9d 100644 --- a/third_party/rust/suggest/src/rs.rs +++ b/third_party/rust/suggest/src/rs.rs @@ -31,7 +31,7 @@ //! the new suggestion in their results, and return `Suggestion::T` variants //! as needed. -use std::borrow::Cow; +use std::{borrow::Cow, fmt}; use remote_settings::{GetItemsOptions, RemoteSettingsResponse}; use serde::{Deserialize, Deserializer}; @@ -47,6 +47,20 @@ pub(crate) const REMOTE_SETTINGS_COLLECTION: &str = "quicksuggest"; /// `mozilla-services/quicksuggest-rs` repo. pub(crate) const SUGGESTIONS_PER_ATTACHMENT: u64 = 200; +/// A list of default record types to download if nothing is specified. +/// This currently defaults to all of the record types. +pub(crate) const DEFAULT_RECORDS_TYPES: [SuggestRecordType; 9] = [ + SuggestRecordType::Icon, + SuggestRecordType::AmpWikipedia, + SuggestRecordType::Amo, + SuggestRecordType::Pocket, + SuggestRecordType::Yelp, + SuggestRecordType::Mdn, + SuggestRecordType::Weather, + SuggestRecordType::GlobalConfig, + SuggestRecordType::AmpMobile, +]; + /// A trait for a client that downloads suggestions from Remote Settings. /// /// This trait lets tests use a mock client. @@ -102,6 +116,61 @@ pub(crate) enum SuggestRecord { AmpMobile, } +/// Enum for the different record types that can be consumed. +/// Extracting this from the serialization enum so that we can +/// extend it to get type metadata. +#[derive(Copy, Clone, PartialEq, PartialOrd, Eq, Ord)] +pub enum SuggestRecordType { + Icon, + AmpWikipedia, + Amo, + Pocket, + Yelp, + Mdn, + Weather, + GlobalConfig, + AmpMobile, +} + +impl From for SuggestRecordType { + fn from(suggest_record: SuggestRecord) -> Self { + match suggest_record { + SuggestRecord::Amo => Self::Amo, + SuggestRecord::AmpWikipedia => Self::AmpWikipedia, + SuggestRecord::Icon => Self::Icon, + SuggestRecord::Mdn => Self::Mdn, + SuggestRecord::Pocket => Self::Pocket, + SuggestRecord::Weather(_) => Self::Weather, + SuggestRecord::Yelp => Self::Yelp, + SuggestRecord::GlobalConfig(_) => Self::GlobalConfig, + SuggestRecord::AmpMobile => Self::AmpMobile, + } + } +} + +impl fmt::Display for SuggestRecordType { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Icon => write!(f, "icon"), + Self::AmpWikipedia => write!(f, "data"), + Self::Amo => write!(f, "amo-suggestions"), + Self::Pocket => write!(f, "pocket-suggestions"), + Self::Yelp => write!(f, "yelp-suggestions"), + Self::Mdn => write!(f, "mdn-suggestions"), + Self::Weather => write!(f, "weather"), + Self::GlobalConfig => write!(f, "configuration"), + Self::AmpMobile => write!(f, "amp-mobile-suggestions"), + } + } +} + +impl SuggestRecordType { + /// Return the meta key for the last ingested record. + pub fn last_ingest_meta_key(&self) -> String { + format!("last_quicksuggest_ingest_{}", self) + } +} + /// Represents either a single value, or a list of values. This is used to /// deserialize downloaded attachments. #[derive(Clone, Debug, Deserialize)] @@ -156,16 +225,18 @@ where } /// Fields that are common to all downloaded suggestions. -#[derive(Clone, Debug, Deserialize)] +#[derive(Clone, Debug, Default, Deserialize)] pub(crate) struct DownloadedSuggestionCommonDetails { pub keywords: Vec, pub title: String, pub url: String, pub score: Option, + #[serde(default)] + pub full_keywords: Vec<(String, usize)>, } /// An AMP suggestion to ingest from an AMP-Wikipedia attachment. -#[derive(Clone, Debug, Deserialize)] +#[derive(Clone, Debug, Default, Deserialize)] pub(crate) struct DownloadedAmpSuggestion { #[serde(flatten)] pub common_details: DownloadedSuggestionCommonDetails, @@ -180,7 +251,7 @@ pub(crate) struct DownloadedAmpSuggestion { } /// A Wikipedia suggestion to ingest from an AMP-Wikipedia attachment. -#[derive(Clone, Debug, Deserialize)] +#[derive(Clone, Debug, Default, Deserialize)] pub(crate) struct DownloadedWikipediaSuggestion { #[serde(flatten)] pub common_details: DownloadedSuggestionCommonDetails, @@ -214,6 +285,34 @@ impl DownloadedAmpWikipediaSuggestion { } } +impl DownloadedSuggestionCommonDetails { + /// Iterate over all keywords for this suggestion + pub fn keywords(&self) -> impl Iterator> { + let full_keywords = self + .full_keywords + .iter() + .flat_map(|(full_keyword, repeat_for)| { + std::iter::repeat(Some(full_keyword.as_str())).take(*repeat_for) + }) + .chain(std::iter::repeat(None)); // In case of insufficient full keywords, just fill in with infinite `None`s + // + self.keywords.iter().zip(full_keywords).enumerate().map( + move |(i, (keyword, full_keyword))| AmpKeyword { + rank: i, + keyword, + full_keyword, + }, + ) + } +} + +#[derive(Debug, PartialEq, Eq)] +pub(crate) struct AmpKeyword<'a> { + pub rank: usize, + pub keyword: &'a str, + pub full_keyword: Option<&'a str>, +} + impl<'de> Deserialize<'de> for DownloadedAmpWikipediaSuggestion { fn deserialize( deserializer: D, @@ -344,3 +443,119 @@ where { String::deserialize(deserializer).map(|s| s.parse().ok()) } + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_full_keywords() { + let suggestion = DownloadedAmpWikipediaSuggestion::Amp(DownloadedAmpSuggestion { + common_details: DownloadedSuggestionCommonDetails { + keywords: vec![ + String::from("f"), + String::from("fo"), + String::from("foo"), + String::from("foo b"), + String::from("foo ba"), + String::from("foo bar"), + ], + full_keywords: vec![(String::from("foo"), 3), (String::from("foo bar"), 3)], + ..DownloadedSuggestionCommonDetails::default() + }, + ..DownloadedAmpSuggestion::default() + }); + + assert_eq!( + Vec::from_iter(suggestion.common_details().keywords()), + vec![ + AmpKeyword { + rank: 0, + keyword: "f", + full_keyword: Some("foo"), + }, + AmpKeyword { + rank: 1, + keyword: "fo", + full_keyword: Some("foo"), + }, + AmpKeyword { + rank: 2, + keyword: "foo", + full_keyword: Some("foo"), + }, + AmpKeyword { + rank: 3, + keyword: "foo b", + full_keyword: Some("foo bar"), + }, + AmpKeyword { + rank: 4, + keyword: "foo ba", + full_keyword: Some("foo bar"), + }, + AmpKeyword { + rank: 5, + keyword: "foo bar", + full_keyword: Some("foo bar"), + }, + ], + ); + } + + #[test] + fn test_missing_full_keywords() { + let suggestion = DownloadedAmpWikipediaSuggestion::Amp(DownloadedAmpSuggestion { + common_details: DownloadedSuggestionCommonDetails { + keywords: vec![ + String::from("f"), + String::from("fo"), + String::from("foo"), + String::from("foo b"), + String::from("foo ba"), + String::from("foo bar"), + ], + // Only the first 3 keywords have full keywords associated with them + full_keywords: vec![(String::from("foo"), 3)], + ..DownloadedSuggestionCommonDetails::default() + }, + ..DownloadedAmpSuggestion::default() + }); + + assert_eq!( + Vec::from_iter(suggestion.common_details().keywords()), + vec![ + AmpKeyword { + rank: 0, + keyword: "f", + full_keyword: Some("foo"), + }, + AmpKeyword { + rank: 1, + keyword: "fo", + full_keyword: Some("foo"), + }, + AmpKeyword { + rank: 2, + keyword: "foo", + full_keyword: Some("foo"), + }, + AmpKeyword { + rank: 3, + keyword: "foo b", + full_keyword: None, + }, + AmpKeyword { + rank: 4, + keyword: "foo ba", + full_keyword: None, + }, + AmpKeyword { + rank: 5, + keyword: "foo bar", + full_keyword: None, + }, + ], + ); + } +} diff --git a/third_party/rust/suggest/src/schema.rs b/third_party/rust/suggest/src/schema.rs index 95d987c09e..b304363de5 100644 --- a/third_party/rust/suggest/src/schema.rs +++ b/third_party/rust/suggest/src/schema.rs @@ -13,7 +13,9 @@ use sql_support::open_database::{self, ConnectionInitializer}; /// 1. Bump this version. /// 2. Add a migration from the old version to the new version in /// [`SuggestConnectionInitializer::upgrade_from`]. -pub const VERSION: u32 = 14; +/// a. If suggestions should be re-ingested after the migration, call `clear_database()` inside +/// the migration. +pub const VERSION: u32 = 18; /// The current Suggest database schema. pub const SQL: &str = " @@ -25,10 +27,19 @@ pub const SQL: &str = " CREATE TABLE keywords( keyword TEXT NOT NULL, suggestion_id INTEGER NOT NULL REFERENCES suggestions(id) ON DELETE CASCADE, + full_keyword_id INTEGER NULL REFERENCES full_keywords(id) ON DELETE SET NULL, rank INTEGER NOT NULL, PRIMARY KEY (keyword, suggestion_id) ) WITHOUT ROWID; + -- full keywords are what we display to the user when a (partial) keyword matches + -- The FK to suggestion_id makes it so full keywords get deleted when the parent suggestion is deleted. + CREATE TABLE full_keywords( + id INTEGER PRIMARY KEY, + suggestion_id INTEGER NOT NULL REFERENCES suggestions(id) ON DELETE CASCADE, + full_keyword TEXT NOT NULL + ); + CREATE TABLE prefix_keywords( keyword_prefix TEXT NOT NULL, keyword_suffix TEXT NOT NULL DEFAULT '', @@ -79,7 +90,8 @@ pub const SQL: &str = " CREATE TABLE icons( id TEXT PRIMARY KEY, - data BLOB NOT NULL + data BLOB NOT NULL, + mimetype TEXT NOT NULL ) WITHOUT ROWID; CREATE TABLE yelp_subjects( @@ -111,6 +123,10 @@ pub const SQL: &str = " description TEXT NOT NULL, FOREIGN KEY(suggestion_id) REFERENCES suggestions(id) ON DELETE CASCADE ); + + CREATE TABLE dismissed_suggestions ( + url TEXT PRIMARY KEY + ) WITHOUT ROWID; "; /// Initializes an SQLite connection to the Suggest database, performing @@ -139,15 +155,178 @@ impl ConnectionInitializer for SuggestConnectionInitializer { Ok(db.execute_batch(SQL)?) } - fn upgrade_from(&self, _db: &Transaction<'_>, version: u32) -> open_database::Result<()> { + fn upgrade_from(&self, tx: &Transaction<'_>, version: u32) -> open_database::Result<()> { match version { - 1..=13 => { + 1..=15 => { // Treat databases with these older schema versions as corrupt, // so that they'll be replaced by a fresh, empty database with // the current schema. Err(open_database::Error::Corrupt) } + 16 => { + tx.execute( + " + CREATE TABLE dismissed_suggestions ( + url_hash INTEGER PRIMARY KEY + ) WITHOUT ROWID;", + (), + )?; + Ok(()) + } + 17 => { + tx.execute( + " + DROP TABLE dismissed_suggestions; + CREATE TABLE dismissed_suggestions ( + url TEXT PRIMARY KEY + ) WITHOUT ROWID;", + (), + )?; + Ok(()) + } _ => Err(open_database::Error::IncompatibleVersion(version)), } } } + +/// Clears the database, removing all suggestions, icons, and metadata. +pub fn clear_database(db: &Connection) -> rusqlite::Result<()> { + db.execute_batch( + " + DELETE FROM meta; + DELETE FROM suggestions; + DELETE FROM icons; + DELETE FROM yelp_subjects; + DELETE FROM yelp_modifiers; + DELETE FROM yelp_location_signs; + DELETE FROM yelp_custom_details; + ", + ) +} + +#[cfg(test)] +mod test { + use super::*; + use sql_support::open_database::test_utils::MigratedDatabaseFile; + + // Snapshot of the v16 schema. We use this to test that we can migrate from there to the + // current schema. + const V16_SCHEMA: &str = r#" + CREATE TABLE meta( + key TEXT PRIMARY KEY, + value NOT NULL + ) WITHOUT ROWID; + + CREATE TABLE keywords( + keyword TEXT NOT NULL, + suggestion_id INTEGER NOT NULL REFERENCES suggestions(id) ON DELETE CASCADE, + full_keyword_id INTEGER NULL REFERENCES full_keywords(id) ON DELETE SET NULL, + rank INTEGER NOT NULL, + PRIMARY KEY (keyword, suggestion_id) + ) WITHOUT ROWID; + + -- full keywords are what we display to the user when a (partial) keyword matches + -- The FK to suggestion_id makes it so full keywords get deleted when the parent suggestion is deleted. + CREATE TABLE full_keywords( + id INTEGER PRIMARY KEY, + suggestion_id INTEGER NOT NULL REFERENCES suggestions(id) ON DELETE CASCADE, + full_keyword TEXT NOT NULL + ); + + CREATE TABLE prefix_keywords( + keyword_prefix TEXT NOT NULL, + keyword_suffix TEXT NOT NULL DEFAULT '', + confidence INTEGER NOT NULL DEFAULT 0, + rank INTEGER NOT NULL, + suggestion_id INTEGER NOT NULL REFERENCES suggestions(id) ON DELETE CASCADE, + PRIMARY KEY (keyword_prefix, keyword_suffix, suggestion_id) + ) WITHOUT ROWID; + + CREATE UNIQUE INDEX keywords_suggestion_id_rank ON keywords(suggestion_id, rank); + + CREATE TABLE suggestions( + id INTEGER PRIMARY KEY, + record_id TEXT NOT NULL, + provider INTEGER NOT NULL, + title TEXT NOT NULL, + url TEXT NOT NULL, + score REAL NOT NULL + ); + + CREATE TABLE amp_custom_details( + suggestion_id INTEGER PRIMARY KEY, + advertiser TEXT NOT NULL, + block_id INTEGER NOT NULL, + iab_category TEXT NOT NULL, + impression_url TEXT NOT NULL, + click_url TEXT NOT NULL, + icon_id TEXT NOT NULL, + FOREIGN KEY(suggestion_id) REFERENCES suggestions(id) ON DELETE CASCADE + ); + + CREATE TABLE wikipedia_custom_details( + suggestion_id INTEGER PRIMARY KEY REFERENCES suggestions(id) ON DELETE CASCADE, + icon_id TEXT NOT NULL + ); + + CREATE TABLE amo_custom_details( + suggestion_id INTEGER PRIMARY KEY, + description TEXT NOT NULL, + guid TEXT NOT NULL, + icon_url TEXT NOT NULL, + rating TEXT, + number_of_ratings INTEGER NOT NULL, + FOREIGN KEY(suggestion_id) REFERENCES suggestions(id) ON DELETE CASCADE + ); + + CREATE INDEX suggestions_record_id ON suggestions(record_id); + + CREATE TABLE icons( + id TEXT PRIMARY KEY, + data BLOB NOT NULL, + mimetype TEXT NOT NULL + ) WITHOUT ROWID; + + CREATE TABLE yelp_subjects( + keyword TEXT PRIMARY KEY, + record_id TEXT NOT NULL + ) WITHOUT ROWID; + + CREATE TABLE yelp_modifiers( + type INTEGER NOT NULL, + keyword TEXT NOT NULL, + record_id TEXT NOT NULL, + PRIMARY KEY (type, keyword) + ) WITHOUT ROWID; + + CREATE TABLE yelp_location_signs( + keyword TEXT PRIMARY KEY, + need_location INTEGER NOT NULL, + record_id TEXT NOT NULL + ) WITHOUT ROWID; + + CREATE TABLE yelp_custom_details( + icon_id TEXT PRIMARY KEY, + score REAL NOT NULL, + record_id TEXT NOT NULL + ) WITHOUT ROWID; + + CREATE TABLE mdn_custom_details( + suggestion_id INTEGER PRIMARY KEY, + description TEXT NOT NULL, + FOREIGN KEY(suggestion_id) REFERENCES suggestions(id) ON DELETE CASCADE + ); + + PRAGMA user_version=16; +"#; + + /// Test running all schema upgrades from V16, which was the first schema with a "real" + /// migration. + /// + /// If an upgrade fails, then this test will fail with a panic. + #[test] + fn test_all_upgrades() { + let db_file = MigratedDatabaseFile::new(SuggestConnectionInitializer, V16_SCHEMA); + db_file.run_all_upgrades(); + } +} diff --git a/third_party/rust/suggest/src/store.rs b/third_party/rust/suggest/src/store.rs index e1f437e8c5..c55cffc7f5 100644 --- a/third_party/rust/suggest/src/store.rs +++ b/third_party/rust/suggest/src/store.rs @@ -4,7 +4,7 @@ */ use std::{ - collections::BTreeMap, + collections::{BTreeMap, BTreeSet}, path::{Path, PathBuf}, sync::Arc, }; @@ -24,13 +24,15 @@ use serde::{de::DeserializeOwned, Deserialize, Serialize}; use crate::{ config::{SuggestGlobalConfig, SuggestProviderConfig}, db::{ - ConnectionType, SuggestDao, SuggestDb, LAST_INGEST_META_KEY, UNPARSABLE_RECORDS_META_KEY, + ConnectionType, SuggestDao, SuggestDb, LAST_INGEST_META_UNPARSABLE, + UNPARSABLE_RECORDS_META_KEY, }, error::Error, provider::SuggestionProvider, rs::{ - SuggestAttachment, SuggestRecord, SuggestRecordId, SuggestRemoteSettingsClient, - REMOTE_SETTINGS_COLLECTION, SUGGESTIONS_PER_ATTACHMENT, + SuggestAttachment, SuggestRecord, SuggestRecordId, SuggestRecordType, + SuggestRemoteSettingsClient, DEFAULT_RECORDS_TYPES, REMOTE_SETTINGS_COLLECTION, + SUGGESTIONS_PER_ATTACHMENT, }, schema::VERSION, Result, SuggestApiResult, Suggestion, SuggestionQuery, @@ -48,7 +50,6 @@ pub struct SuggestStoreBuilder(Mutex); #[derive(Default)] struct SuggestStoreBuilderInner { data_path: Option, - cache_path: Option, remote_settings_config: Option, } @@ -68,8 +69,8 @@ impl SuggestStoreBuilder { self } - pub fn cache_path(self: Arc, path: String) -> Arc { - self.0.lock().cache_path = Some(path); + pub fn cache_path(self: Arc, _path: String) -> Arc { + // We used to use this, but we're not using it anymore, just ignore the call self } @@ -85,10 +86,6 @@ impl SuggestStoreBuilder { .data_path .clone() .ok_or_else(|| Error::SuggestStoreBuilder("data_path not specified".to_owned()))?; - let cache_path = inner - .cache_path - .clone() - .ok_or_else(|| Error::SuggestStoreBuilder("cache_path not specified".to_owned()))?; let settings_client = remote_settings::Client::new(inner.remote_settings_config.clone().unwrap_or_else( || RemoteSettingsConfig { @@ -98,7 +95,7 @@ impl SuggestStoreBuilder { }, ))?; Ok(Arc::new(SuggestStore { - inner: SuggestStoreInner::new(data_path, cache_path, settings_client), + inner: SuggestStoreInner::new(data_path, settings_client), })) } } @@ -182,7 +179,7 @@ impl SuggestStore { )?) }()?; Ok(Self { - inner: SuggestStoreInner::new("".to_owned(), path.to_owned(), settings_client), + inner: SuggestStoreInner::new(path.to_owned(), settings_client), }) } @@ -192,6 +189,22 @@ impl SuggestStore { self.inner.query(query) } + /// Dismiss a suggestion + /// + /// Dismissed suggestions will not be returned again + /// + /// In the case of AMP suggestions this should be the raw URL. + #[handle_error(Error)] + pub fn dismiss_suggestion(&self, suggestion_url: String) -> SuggestApiResult<()> { + self.inner.dismiss_suggestion(suggestion_url) + } + + /// Clear dismissed suggestions + #[handle_error(Error)] + pub fn clear_dismissed_suggestions(&self) -> SuggestApiResult<()> { + self.inner.clear_dismissed_suggestions() + } + /// Interrupts any ongoing queries. /// /// This should be called when the user types new input into the address @@ -238,6 +251,7 @@ pub struct SuggestIngestionConstraints { /// Because of how suggestions are partitioned in Remote Settings, this is a /// soft limit, and the store might ingest more than requested. pub max_suggestions: Option, + pub providers: Option>, } /// The implementation of the store. This is generic over the Remote Settings @@ -250,23 +264,14 @@ pub(crate) struct SuggestStoreInner { /// It's not currently used because not all consumers pass this in yet. #[allow(unused)] data_path: PathBuf, - /// Path to the temporary SQL database. - /// - /// This stores things that should be deleted when the user clears their cache. - cache_path: PathBuf, dbs: OnceCell, settings_client: S, } impl SuggestStoreInner { - fn new( - data_path: impl Into, - cache_path: impl Into, - settings_client: S, - ) -> Self { + pub fn new(data_path: impl Into, settings_client: S) -> Self { Self { data_path: data_path.into(), - cache_path: cache_path.into(), dbs: OnceCell::new(), settings_client, } @@ -276,7 +281,7 @@ impl SuggestStoreInner { /// they're not already open. fn dbs(&self) -> Result<&SuggestStoreDbs> { self.dbs - .get_or_try_init(|| SuggestStoreDbs::open(&self.cache_path)) + .get_or_try_init(|| SuggestStoreDbs::open(&self.data_path)) } fn query(&self, query: SuggestionQuery) -> Result> { @@ -286,6 +291,17 @@ impl SuggestStoreInner { self.dbs()?.reader.read(|dao| dao.fetch_suggestions(&query)) } + fn dismiss_suggestion(&self, suggestion_url: String) -> Result<()> { + self.dbs()? + .writer + .write(|dao| dao.insert_dismissal(&suggestion_url)) + } + + fn clear_dismissed_suggestions(&self) -> Result<()> { + self.dbs()?.writer.write(|dao| dao.clear_dismissals())?; + Ok(()) + } + fn interrupt(&self) { if let Some(dbs) = self.dbs.get() { // Only interrupt if the databases are already open. @@ -315,7 +331,7 @@ impl SuggestStoreInner where S: SuggestRemoteSettingsClient, { - fn ingest(&self, constraints: SuggestIngestionConstraints) -> Result<()> { + pub fn ingest(&self, constraints: SuggestIngestionConstraints) -> Result<()> { let writer = &self.dbs()?.writer; if let Some(unparsable_records) = @@ -330,26 +346,58 @@ where for unparsable_ids in all_unparsable_ids.chunks(UNPARSABLE_IDS_PER_REQUEST) { let mut options = GetItemsOptions::new(); for unparsable_id in unparsable_ids { - options.eq("id", *unparsable_id); + options.filter_eq("id", *unparsable_id); } let records_chunk = self .settings_client .get_records_with_options(&options)? .records; - self.ingest_records(writer, &records_chunk)?; + self.ingest_records(LAST_INGEST_META_UNPARSABLE, writer, &records_chunk)?; } } + // use std::collections::BTreeSet; + let ingest_record_types = if let Some(rt) = &constraints.providers { + rt.iter() + .flat_map(|x| x.records_for_provider()) + .collect::>() + .into_iter() + .collect() + } else { + DEFAULT_RECORDS_TYPES.to_vec() + }; + + for ingest_record_type in ingest_record_types { + self.ingest_records_by_type(ingest_record_type, writer, &constraints)?; + } + + Ok(()) + } + + fn ingest_records_by_type( + &self, + ingest_record_type: SuggestRecordType, + writer: &SuggestDb, + constraints: &SuggestIngestionConstraints, + ) -> Result<()> { let mut options = GetItemsOptions::new(); + // Remote Settings returns records in descending modification order // (newest first), but we want them in ascending order (oldest first), // so that we can eventually resume downloading where we left off. options.sort("last_modified", SortOrder::Ascending); - if let Some(last_ingest) = writer.read(|dao| dao.get_meta::(LAST_INGEST_META_KEY))? { + + options.filter_eq("type", ingest_record_type.to_string()); + + // Get the last ingest value. This is the max of the last_ingest_keys + // that are in the database. + if let Some(last_ingest) = writer + .read(|dao| dao.get_meta::(ingest_record_type.last_ingest_meta_key().as_str()))? + { // Only download changes since our last ingest. If our last ingest // was interrupted, we'll pick up where we left off. - options.gt("last_modified", last_ingest.to_string()); + options.filter_gt("last_modified", last_ingest.to_string()); } if let Some(max_suggestions) = constraints.max_suggestions { @@ -363,39 +411,56 @@ where .settings_client .get_records_with_options(&options)? .records; - self.ingest_records(writer, &records)?; - + self.ingest_records(&ingest_record_type.last_ingest_meta_key(), writer, &records)?; Ok(()) } - fn ingest_records(&self, writer: &SuggestDb, records: &[RemoteSettingsRecord]) -> Result<()> { + fn ingest_records( + &self, + last_ingest_key: &str, + writer: &SuggestDb, + records: &[RemoteSettingsRecord], + ) -> Result<()> { for record in records { let record_id = SuggestRecordId::from(&record.id); if record.deleted { // If the entire record was deleted, drop all its suggestions // and advance the last ingest time. - writer.write(|dao| dao.handle_deleted_record(record))?; + writer.write(|dao| dao.handle_deleted_record(last_ingest_key, record))?; continue; } let Ok(fields) = serde_json::from_value(serde_json::Value::Object(record.fields.clone())) else { // We don't recognize this record's type, so we don't know how - // to ingest its suggestions. Record this in the meta table. + // to ingest its suggestions. Skip processing this record. writer.write(|dao| dao.handle_unparsable_record(record))?; continue; }; match fields { SuggestRecord::AmpWikipedia => { - self.ingest_attachment(writer, record, |dao, record_id, suggestions| { - dao.insert_amp_wikipedia_suggestions(record_id, suggestions) - })?; + self.ingest_attachment( + // TODO: Currently re-creating the last_ingest_key because using last_ingest_meta + // breaks the tests (particularly the unparsable functionality). So, keeping + // a direct reference until we remove the "unparsable" functionality. + &SuggestRecordType::AmpWikipedia.last_ingest_meta_key(), + writer, + record, + |dao, record_id, suggestions| { + dao.insert_amp_wikipedia_suggestions(record_id, suggestions) + }, + )?; } SuggestRecord::AmpMobile => { - self.ingest_attachment(writer, record, |dao, record_id, suggestions| { - dao.insert_amp_mobile_suggestions(record_id, suggestions) - })?; + self.ingest_attachment( + &SuggestRecordType::AmpMobile.last_ingest_meta_key(), + writer, + record, + |dao, record_id, suggestions| { + dao.insert_amp_mobile_suggestions(record_id, suggestions) + }, + )?; } SuggestRecord::Icon => { let (Some(icon_id), Some(attachment)) = @@ -404,47 +469,79 @@ where // An icon record should have an icon ID and an // attachment. Icons that don't have these are // malformed, so skip to the next record. - writer.write(|dao| dao.put_last_ingest_if_newer(record.last_modified))?; + writer.write(|dao| { + dao.put_last_ingest_if_newer( + &SuggestRecordType::Icon.last_ingest_meta_key(), + record.last_modified, + ) + })?; continue; }; let data = self.settings_client.get_attachment(&attachment.location)?; writer.write(|dao| { - dao.put_icon(icon_id, &data)?; - dao.handle_ingested_record(record) + dao.put_icon(icon_id, &data, &attachment.mimetype)?; + dao.handle_ingested_record( + &SuggestRecordType::Icon.last_ingest_meta_key(), + record, + ) })?; } SuggestRecord::Amo => { - self.ingest_attachment(writer, record, |dao, record_id, suggestions| { - dao.insert_amo_suggestions(record_id, suggestions) - })?; + self.ingest_attachment( + &SuggestRecordType::Amo.last_ingest_meta_key(), + writer, + record, + |dao, record_id, suggestions| { + dao.insert_amo_suggestions(record_id, suggestions) + }, + )?; } SuggestRecord::Pocket => { - self.ingest_attachment(writer, record, |dao, record_id, suggestions| { - dao.insert_pocket_suggestions(record_id, suggestions) - })?; + self.ingest_attachment( + &SuggestRecordType::Pocket.last_ingest_meta_key(), + writer, + record, + |dao, record_id, suggestions| { + dao.insert_pocket_suggestions(record_id, suggestions) + }, + )?; } SuggestRecord::Yelp => { - self.ingest_attachment(writer, record, |dao, record_id, suggestions| { - match suggestions.first() { + self.ingest_attachment( + &SuggestRecordType::Yelp.last_ingest_meta_key(), + writer, + record, + |dao, record_id, suggestions| match suggestions.first() { Some(suggestion) => dao.insert_yelp_suggestions(record_id, suggestion), None => Ok(()), - } - })?; + }, + )?; } SuggestRecord::Mdn => { - self.ingest_attachment(writer, record, |dao, record_id, suggestions| { - dao.insert_mdn_suggestions(record_id, suggestions) - })?; + self.ingest_attachment( + &SuggestRecordType::Mdn.last_ingest_meta_key(), + writer, + record, + |dao, record_id, suggestions| { + dao.insert_mdn_suggestions(record_id, suggestions) + }, + )?; } SuggestRecord::Weather(data) => { - self.ingest_record(writer, record, |dao, record_id| { - dao.insert_weather_data(record_id, &data) - })?; + self.ingest_record( + &SuggestRecordType::Weather.last_ingest_meta_key(), + writer, + record, + |dao, record_id| dao.insert_weather_data(record_id, &data), + )?; } SuggestRecord::GlobalConfig(config) => { - self.ingest_record(writer, record, |dao, _| { - dao.put_global_config(&SuggestGlobalConfig::from(&config)) - })?; + self.ingest_record( + &SuggestRecordType::GlobalConfig.last_ingest_meta_key(), + writer, + record, + |dao, _| dao.put_global_config(&SuggestGlobalConfig::from(&config)), + )?; } } } @@ -453,6 +550,7 @@ where fn ingest_record( &self, + last_ingest_key: &str, writer: &SuggestDb, record: &RemoteSettingsRecord, ingestion_handler: impl FnOnce(&mut SuggestDao<'_>, &SuggestRecordId) -> Result<()>, @@ -469,12 +567,13 @@ where // Ingest (or re-ingest) all data in the record. ingestion_handler(dao, &record_id)?; - dao.handle_ingested_record(record) + dao.handle_ingested_record(last_ingest_key, record) }) } fn ingest_attachment( &self, + last_ingest_key: &str, writer: &SuggestDb, record: &RemoteSettingsRecord, ingestion_handler: impl FnOnce(&mut SuggestDao<'_>, &SuggestRecordId, &[T]) -> Result<()>, @@ -486,20 +585,72 @@ where // This method should be called only when a record is expected to // have an attachment. If it doesn't have one, it's malformed, so // skip to the next record. - writer.write(|dao| dao.put_last_ingest_if_newer(record.last_modified))?; + writer + .write(|dao| dao.put_last_ingest_if_newer(last_ingest_key, record.last_modified))?; return Ok(()); }; let attachment_data = self.settings_client.get_attachment(&attachment.location)?; match serde_json::from_slice::>(&attachment_data) { - Ok(attachment) => self.ingest_record(writer, record, |dao, record_id| { - ingestion_handler(dao, record_id, attachment.suggestions()) - }), + Ok(attachment) => { + self.ingest_record(last_ingest_key, writer, record, |dao, record_id| { + ingestion_handler(dao, record_id, attachment.suggestions()) + }) + } Err(_) => writer.write(|dao| dao.handle_unparsable_record(record)), } } } +#[cfg(feature = "benchmark_api")] +impl SuggestStoreInner +where + S: SuggestRemoteSettingsClient, +{ + pub fn into_settings_client(self) -> S { + self.settings_client + } + + pub fn ensure_db_initialized(&self) { + self.dbs().unwrap(); + } + + pub fn benchmark_ingest_records_by_type(&self, ingest_record_type: SuggestRecordType) { + self.ingest_records_by_type( + ingest_record_type, + &self.dbs().unwrap().writer, + &SuggestIngestionConstraints::default(), + ) + .unwrap() + } + + pub fn table_row_counts(&self) -> Vec<(String, u32)> { + use sql_support::ConnExt; + + // Note: since this is just used for debugging, use unwrap to simplify the error handling. + let reader = &self.dbs().unwrap().reader; + let conn = reader.conn.lock(); + let table_names: Vec = conn + .query_rows_and_then( + "SELECT name FROM sqlite_master where type = 'table'", + (), + |row| row.get(0), + ) + .unwrap(); + let mut table_names_with_counts: Vec<(String, u32)> = table_names + .into_iter() + .map(|name| { + let count: u32 = conn + .query_one(&format!("SELECT COUNT(*) FROM {name}")) + .unwrap(); + (name, count) + }) + .collect(); + table_names_with_counts.sort_by(|a, b| (b.1.cmp(&a.1))); + table_names_with_counts + } +} + /// Holds a store's open connections to the Suggest database. struct SuggestStoreDbs { /// A read-write connection used to update the database with new data. @@ -549,10 +700,6 @@ mod tests { "file:test_store_data_{}?mode=memory&cache=shared", hex::encode(unique_suffix), ), - format!( - "file:test_store_cache_{}?mode=memory&cache=shared", - hex::encode(unique_suffix), - ), settings_client, ) } @@ -721,7 +868,14 @@ mod tests { store.ingest(SuggestIngestionConstraints::default())?; store.dbs()?.reader.read(|dao| { - assert_eq!(dao.get_meta::(LAST_INGEST_META_KEY)?, Some(15)); + assert_eq!( + dao.get_meta::( + SuggestRecordType::AmpWikipedia + .last_ingest_meta_key() + .as_str() + )?, + Some(15) + ); expect![[r#" [ Amp { @@ -729,6 +883,7 @@ mod tests { url: "https://www.lph-nm.biz", raw_url: "https://www.lph-nm.biz", icon: None, + icon_mimetype: None, full_keyword: "los", block_id: 0, advertiser: "Los Pollos Hermanos", @@ -834,6 +989,9 @@ mod tests { 110, ], ), + icon_mimetype: Some( + "image/png", + ), full_keyword: "lasagna", block_id: 0, advertiser: "Good Place Eats", @@ -872,6 +1030,9 @@ mod tests { 110, ], ), + icon_mimetype: Some( + "image/png", + ), full_keyword: "penne", block_id: 0, advertiser: "Good Place Eats", @@ -895,6 +1056,267 @@ mod tests { Ok(()) } + #[test] + fn ingest_full_keywords() -> anyhow::Result<()> { + before_each(); + + let snapshot = Snapshot::with_records(json!([{ + "id": "1", + "type": "data", + "last_modified": 15, + "attachment": { + "filename": "data-1.json", + "mimetype": "application/json", + "location": "data-1.json", + "hash": "", + "size": 0, + }, + }, { + "id": "2", + "type": "data", + "last_modified": 15, + "attachment": { + "filename": "data-2.json", + "mimetype": "application/json", + "location": "data-2.json", + "hash": "", + "size": 0, + }, + }, { + "id": "3", + "type": "data", + "last_modified": 15, + "attachment": { + "filename": "data-3.json", + "mimetype": "application/json", + "location": "data-3.json", + "hash": "", + "size": 0, + }, + }, { + "id": "4", + "type": "amp-mobile-suggestions", + "last_modified": 15, + "attachment": { + "filename": "data-4.json", + "mimetype": "application/json", + "location": "data-4.json", + "hash": "", + "size": 0, + }, + }]))? + // AMP attachment with full keyword data + .with_data( + "data-1.json", + json!([{ + "id": 0, + "advertiser": "Los Pollos Hermanos", + "iab_category": "8 - Food & Drink", + "keywords": ["lo", "los", "los p", "los pollos", "los pollos h", "los pollos hermanos"], + "full_keywords": [ + // Full keyword for the first 4 keywords + ("los pollos", 4), + // Full keyword for the next 2 keywords + ("los pollos hermanos (restaurant)", 2), + ], + "title": "Los Pollos Hermanos - Albuquerque - 1", + "url": "https://www.lph-nm.biz", + "icon": "5678", + "impression_url": "https://example.com/impression_url", + "click_url": "https://example.com/click_url", + "score": 0.3 + }]), + )? + // AMP attachment without a full keyword + .with_data( + "data-2.json", + json!([{ + "id": 1, + "advertiser": "Los Pollos Hermanos", + "iab_category": "8 - Food & Drink", + "keywords": ["lo", "los", "los p", "los pollos", "los pollos h", "los pollos hermanos"], + "title": "Los Pollos Hermanos - Albuquerque - 2", + "url": "https://www.lph-nm.biz", + "icon": "5678", + "impression_url": "https://example.com/impression_url", + "click_url": "https://example.com/click_url", + "score": 0.3 + }]), + )? + // Wikipedia attachment with full keyword data. We should ignore the full + // keyword data for Wikipedia suggestions + .with_data( + "data-3.json", + json!([{ + "id": 2, + "advertiser": "Wikipedia", + "keywords": ["lo", "los", "los p", "los pollos", "los pollos h", "los pollos hermanos"], + "title": "Los Pollos Hermanos - Albuquerque - Wiki", + "full_keywords": [ + ("Los Pollos Hermanos - Albuquerque", 6), + ], + "url": "https://www.lph-nm.biz", + "icon": "5678", + "score": 0.3, + }]), + )? + // Amp mobile suggestion, this is essentially the same as 1, except for the SuggestionProvider + .with_data( + "data-4.json", + json!([{ + "id": 0, + "advertiser": "Los Pollos Hermanos", + "iab_category": "8 - Food & Drink", + "keywords": ["lo", "los", "los p", "los pollos", "los pollos h", "los pollos hermanos"], + "full_keywords": [ + // Full keyword for the first 4 keywords + ("los pollos", 4), + // Full keyword for the next 2 keywords + ("los pollos hermanos (restaurant)", 2), + ], + "title": "Los Pollos Hermanos - Albuquerque - 4", + "url": "https://www.lph-nm.biz", + "icon": "5678", + "impression_url": "https://example.com/impression_url", + "click_url": "https://example.com/click_url", + "score": 0.3 + }]), + )?; + + let store = unique_test_store(SnapshotSettingsClient::with_snapshot(snapshot)); + + store.ingest(SuggestIngestionConstraints::default())?; + + store.dbs()?.reader.read(|dao| { + // This one should match the first full keyword for the first AMP item. + expect![[r#" + [ + Amp { + title: "Los Pollos Hermanos - Albuquerque - 1", + url: "https://www.lph-nm.biz", + raw_url: "https://www.lph-nm.biz", + icon: None, + icon_mimetype: None, + full_keyword: "los pollos", + block_id: 0, + advertiser: "Los Pollos Hermanos", + iab_category: "8 - Food & Drink", + impression_url: "https://example.com/impression_url", + click_url: "https://example.com/click_url", + raw_click_url: "https://example.com/click_url", + score: 0.3, + }, + Amp { + title: "Los Pollos Hermanos - Albuquerque - 2", + url: "https://www.lph-nm.biz", + raw_url: "https://www.lph-nm.biz", + icon: None, + icon_mimetype: None, + full_keyword: "los", + block_id: 1, + advertiser: "Los Pollos Hermanos", + iab_category: "8 - Food & Drink", + impression_url: "https://example.com/impression_url", + click_url: "https://example.com/click_url", + raw_click_url: "https://example.com/click_url", + score: 0.3, + }, + ] + "#]] + .assert_debug_eq(&dao.fetch_suggestions(&SuggestionQuery { + keyword: "lo".into(), + providers: vec![SuggestionProvider::Amp], + limit: None, + })?); + // This one should match the second full keyword for the first AMP item. + expect![[r#" + [ + Amp { + title: "Los Pollos Hermanos - Albuquerque - 1", + url: "https://www.lph-nm.biz", + raw_url: "https://www.lph-nm.biz", + icon: None, + icon_mimetype: None, + full_keyword: "los pollos hermanos (restaurant)", + block_id: 0, + advertiser: "Los Pollos Hermanos", + iab_category: "8 - Food & Drink", + impression_url: "https://example.com/impression_url", + click_url: "https://example.com/click_url", + raw_click_url: "https://example.com/click_url", + score: 0.3, + }, + Amp { + title: "Los Pollos Hermanos - Albuquerque - 2", + url: "https://www.lph-nm.biz", + raw_url: "https://www.lph-nm.biz", + icon: None, + icon_mimetype: None, + full_keyword: "los pollos hermanos", + block_id: 1, + advertiser: "Los Pollos Hermanos", + iab_category: "8 - Food & Drink", + impression_url: "https://example.com/impression_url", + click_url: "https://example.com/click_url", + raw_click_url: "https://example.com/click_url", + score: 0.3, + }, + ] + "#]] + .assert_debug_eq(&dao.fetch_suggestions(&SuggestionQuery { + keyword: "los pollos h".into(), + providers: vec![SuggestionProvider::Amp], + limit: None, + })?); + // This one matches a Wikipedia suggestion, so the full keyword should be ignored + expect![[r#" + [ + Wikipedia { + title: "Los Pollos Hermanos - Albuquerque - Wiki", + url: "https://www.lph-nm.biz", + icon: None, + icon_mimetype: None, + full_keyword: "los", + }, + ] + "#]] + .assert_debug_eq(&dao.fetch_suggestions(&SuggestionQuery { + keyword: "los".into(), + providers: vec![SuggestionProvider::Wikipedia], + limit: None, + })?); + // This one matches a Wikipedia suggestion, so the full keyword should be ignored + expect![[r#" + [ + Amp { + title: "Los Pollos Hermanos - Albuquerque - 4", + url: "https://www.lph-nm.biz", + raw_url: "https://www.lph-nm.biz", + icon: None, + icon_mimetype: None, + full_keyword: "los pollos hermanos (restaurant)", + block_id: 0, + advertiser: "Los Pollos Hermanos", + iab_category: "8 - Food & Drink", + impression_url: "https://example.com/impression_url", + click_url: "https://example.com/click_url", + raw_click_url: "https://example.com/click_url", + score: 0.3, + }, + ] + "#]] + .assert_debug_eq(&dao.fetch_suggestions(&SuggestionQuery { + keyword: "los pollos h".into(), + providers: vec![SuggestionProvider::AmpMobile], + limit: None, + })?); + + Ok(()) + })?; + + Ok(()) + } + /// Tests ingesting a data attachment containing a single suggestion, /// instead of an array of suggestions. #[test] @@ -941,6 +1363,7 @@ mod tests { url: "https://www.lasagna.restaurant", raw_url: "https://www.lasagna.restaurant", icon: None, + icon_mimetype: None, full_keyword: "lasagna", block_id: 0, advertiser: "Good Place Eats", @@ -1014,7 +1437,14 @@ mod tests { store.ingest(SuggestIngestionConstraints::default())?; store.dbs()?.reader.read(|dao| { - assert_eq!(dao.get_meta(LAST_INGEST_META_KEY)?, Some(15u64)); + assert_eq!( + dao.get_meta( + SuggestRecordType::AmpWikipedia + .last_ingest_meta_key() + .as_str() + )?, + Some(15u64) + ); expect![[r#" [ Amp { @@ -1022,6 +1452,7 @@ mod tests { url: "https://www.lasagna.restaurant", raw_url: "https://www.lasagna.restaurant", icon: None, + icon_mimetype: None, full_keyword: "lasagna", block_id: 0, advertiser: "Good Place Eats", @@ -1084,8 +1515,15 @@ mod tests { store.ingest(SuggestIngestionConstraints::default())?; - store.dbs()?.reader.read(|dao| { - assert_eq!(dao.get_meta(LAST_INGEST_META_KEY)?, Some(30u64)); + store.dbs()?.reader.read(|dao: &SuggestDao<'_>| { + assert_eq!( + dao.get_meta( + SuggestRecordType::AmpWikipedia + .last_ingest_meta_key() + .as_str() + )?, + Some(30u64) + ); assert!(dao .fetch_suggestions(&SuggestionQuery { keyword: "la".into(), @@ -1100,6 +1538,7 @@ mod tests { url: "https://www.lph-nm.biz", raw_url: "https://www.lph-nm.biz", icon: None, + icon_mimetype: None, full_keyword: "los pollos", block_id: 0, advertiser: "Los Pollos Hermanos", @@ -1123,6 +1562,7 @@ mod tests { url: "https://penne.biz", raw_url: "https://penne.biz", icon: None, + icon_mimetype: None, full_keyword: "penne", block_id: 0, advertiser: "Good Place Eats", @@ -1219,7 +1659,10 @@ mod tests { store.ingest(SuggestIngestionConstraints::default())?; store.dbs()?.reader.read(|dao| { - assert_eq!(dao.get_meta(LAST_INGEST_META_KEY)?, Some(25u64)); + assert_eq!( + dao.get_meta(SuggestRecordType::Icon.last_ingest_meta_key().as_str())?, + Some(25u64) + ); assert_eq!( dao.conn .query_one::("SELECT count(*) FROM suggestions")?, @@ -1259,7 +1702,10 @@ mod tests { store.ingest(SuggestIngestionConstraints::default())?; store.dbs()?.reader.read(|dao| { - assert_eq!(dao.get_meta(LAST_INGEST_META_KEY)?, Some(35u64)); + assert_eq!( + dao.get_meta(SuggestRecordType::Icon.last_ingest_meta_key().as_str())?, + Some(35u64) + ); expect![[r#" [ Amp { @@ -1286,6 +1732,9 @@ mod tests { 110, ], ), + icon_mimetype: Some( + "image/png", + ), full_keyword: "lasagna", block_id: 0, advertiser: "Good Place Eats", @@ -1327,6 +1776,9 @@ mod tests { 110, ], ), + icon_mimetype: Some( + "image/png", + ), full_keyword: "los", block_id: 0, advertiser: "Los Pollos Hermanos", @@ -1422,7 +1874,10 @@ mod tests { store.ingest(SuggestIngestionConstraints::default())?; store.dbs()?.reader.read(|dao| { - assert_eq!(dao.get_meta(LAST_INGEST_META_KEY)?, Some(15u64)); + assert_eq!( + dao.get_meta(SuggestRecordType::Amo.last_ingest_meta_key().as_str())?, + Some(15u64) + ); expect![[r#" [ @@ -1513,7 +1968,10 @@ mod tests { store.ingest(SuggestIngestionConstraints::default())?; store.dbs()?.reader.read(|dao| { - assert_eq!(dao.get_meta(LAST_INGEST_META_KEY)?, Some(30u64)); + assert_eq!( + dao.get_meta(SuggestRecordType::Amo.last_ingest_meta_key().as_str())?, + Some(30u64) + ); expect![[r#" [ @@ -1648,13 +2106,24 @@ mod tests { store.ingest(SuggestIngestionConstraints::default())?; store.dbs()?.reader.read(|dao| { - assert_eq!(dao.get_meta::(LAST_INGEST_META_KEY)?, Some(20)); assert_eq!( dao.conn .query_one::("SELECT count(*) FROM suggestions")?, 1 ); assert_eq!(dao.conn.query_one::("SELECT count(*) FROM icons")?, 1); + assert_eq!( + dao.get_meta( + SuggestRecordType::AmpWikipedia + .last_ingest_meta_key() + .as_str() + )?, + Some(15) + ); + assert_eq!( + dao.get_meta(SuggestRecordType::Icon.last_ingest_meta_key().as_str())?, + Some(20) + ); Ok(()) })?; @@ -1674,14 +2143,16 @@ mod tests { store.ingest(SuggestIngestionConstraints::default())?; store.dbs()?.reader.read(|dao| { - assert_eq!(dao.get_meta::(LAST_INGEST_META_KEY)?, Some(30)); assert_eq!( dao.conn .query_one::("SELECT count(*) FROM suggestions")?, 0 ); assert_eq!(dao.conn.query_one::("SELECT count(*) FROM icons")?, 0); - + assert_eq!( + dao.get_meta(SuggestRecordType::Icon.last_ingest_meta_key().as_str())?, + Some(30) + ); Ok(()) })?; @@ -1717,6 +2188,7 @@ mod tests { for (max_suggestions, expected_limit) in table { store.ingest(SuggestIngestionConstraints { max_suggestions: Some(max_suggestions), + providers: Some(vec![SuggestionProvider::Amp]), })?; let actual_limit = store .settings_client @@ -1772,7 +2244,14 @@ mod tests { store.ingest(SuggestIngestionConstraints::default())?; store.dbs()?.reader.read(|dao| { - assert_eq!(dao.get_meta::(LAST_INGEST_META_KEY)?, Some(15)); + assert_eq!( + dao.get_meta::( + SuggestRecordType::AmpWikipedia + .last_ingest_meta_key() + .as_str() + )?, + Some(15) + ); assert_eq!( dao.conn .query_one::("SELECT count(*) FROM suggestions")?, @@ -1789,7 +2268,14 @@ mod tests { store.clear()?; store.dbs()?.reader.read(|dao| { - assert_eq!(dao.get_meta::(LAST_INGEST_META_KEY)?, None); + assert_eq!( + dao.get_meta::( + SuggestRecordType::AmpWikipedia + .last_ingest_meta_key() + .as_str() + )?, + None + ); assert_eq!( dao.conn .query_one::("SELECT count(*) FROM suggestions")?, @@ -2080,6 +2566,9 @@ mod tests { 110, ], ), + icon_mimetype: Some( + "image/png", + ), full_keyword: "lasagna", block_id: 0, advertiser: "Good Place Eats", @@ -2143,6 +2632,9 @@ mod tests { 110, ], ), + icon_mimetype: Some( + "image/png", + ), full_keyword: "multimatch", }, ] @@ -2199,6 +2691,9 @@ mod tests { 110, ], ), + icon_mimetype: Some( + "image/png", + ), full_keyword: "multimatch", }, ] @@ -2268,6 +2763,9 @@ mod tests { 110, ], ), + icon_mimetype: Some( + "image/png", + ), full_keyword: "lasagna", block_id: 0, advertiser: "Good Place Eats", @@ -2349,6 +2847,9 @@ mod tests { 110, ], ), + icon_mimetype: Some( + "image/png", + ), full_keyword: "california", }, Wikipedia { @@ -2370,6 +2871,9 @@ mod tests { 110, ], ), + icon_mimetype: Some( + "image/png", + ), full_keyword: "california", }, ] @@ -2403,6 +2907,9 @@ mod tests { 110, ], ), + icon_mimetype: Some( + "image/png", + ), full_keyword: "california", }, ] @@ -2614,6 +3121,9 @@ mod tests { 110, ], ), + icon_mimetype: Some( + "image/svg+xml", + ), score: 0.5, has_location_sign: true, subject_exact_match: true, @@ -2647,6 +3157,9 @@ mod tests { 110, ], ), + icon_mimetype: Some( + "image/svg+xml", + ), score: 0.5, has_location_sign: true, subject_exact_match: true, @@ -2680,6 +3193,9 @@ mod tests { 110, ], ), + icon_mimetype: Some( + "image/svg+xml", + ), score: 0.5, has_location_sign: true, subject_exact_match: true, @@ -2735,6 +3251,9 @@ mod tests { 110, ], ), + icon_mimetype: Some( + "image/svg+xml", + ), score: 0.5, has_location_sign: true, subject_exact_match: true, @@ -2779,6 +3298,9 @@ mod tests { 110, ], ), + icon_mimetype: Some( + "image/svg+xml", + ), score: 0.5, has_location_sign: true, subject_exact_match: true, @@ -2812,6 +3334,9 @@ mod tests { 110, ], ), + icon_mimetype: Some( + "image/svg+xml", + ), score: 0.5, has_location_sign: true, subject_exact_match: true, @@ -2856,6 +3381,9 @@ mod tests { 110, ], ), + icon_mimetype: Some( + "image/svg+xml", + ), score: 0.5, has_location_sign: true, subject_exact_match: true, @@ -2889,6 +3417,9 @@ mod tests { 110, ], ), + icon_mimetype: Some( + "image/svg+xml", + ), score: 0.5, has_location_sign: true, subject_exact_match: true, @@ -2933,6 +3464,9 @@ mod tests { 110, ], ), + icon_mimetype: Some( + "image/svg+xml", + ), score: 0.5, has_location_sign: true, subject_exact_match: true, @@ -2966,6 +3500,9 @@ mod tests { 110, ], ), + icon_mimetype: Some( + "image/svg+xml", + ), score: 0.5, has_location_sign: true, subject_exact_match: true, @@ -2999,6 +3536,9 @@ mod tests { 110, ], ), + icon_mimetype: Some( + "image/svg+xml", + ), score: 0.5, has_location_sign: false, subject_exact_match: true, @@ -3032,6 +3572,9 @@ mod tests { 110, ], ), + icon_mimetype: Some( + "image/svg+xml", + ), score: 0.5, has_location_sign: false, subject_exact_match: true, @@ -3076,6 +3619,9 @@ mod tests { 110, ], ), + icon_mimetype: Some( + "image/svg+xml", + ), score: 0.5, has_location_sign: false, subject_exact_match: true, @@ -3109,6 +3655,9 @@ mod tests { 110, ], ), + icon_mimetype: Some( + "image/svg+xml", + ), score: 0.5, has_location_sign: false, subject_exact_match: true, @@ -3186,6 +3735,9 @@ mod tests { 110, ], ), + icon_mimetype: Some( + "image/svg+xml", + ), score: 0.5, has_location_sign: false, subject_exact_match: true, @@ -3219,6 +3771,9 @@ mod tests { 110, ], ), + icon_mimetype: Some( + "image/svg+xml", + ), score: 0.5, has_location_sign: false, subject_exact_match: true, @@ -3252,6 +3807,9 @@ mod tests { 110, ], ), + icon_mimetype: Some( + "image/svg+xml", + ), score: 0.5, has_location_sign: true, subject_exact_match: true, @@ -3285,6 +3843,9 @@ mod tests { 110, ], ), + icon_mimetype: Some( + "image/svg+xml", + ), score: 0.5, has_location_sign: true, subject_exact_match: true, @@ -3318,6 +3879,9 @@ mod tests { 110, ], ), + icon_mimetype: Some( + "image/svg+xml", + ), score: 0.5, has_location_sign: false, subject_exact_match: true, @@ -3362,6 +3926,9 @@ mod tests { 110, ], ), + icon_mimetype: Some( + "image/svg+xml", + ), score: 0.5, has_location_sign: false, subject_exact_match: false, @@ -3395,6 +3962,9 @@ mod tests { 110, ], ), + icon_mimetype: Some( + "image/svg+xml", + ), score: 0.5, has_location_sign: false, subject_exact_match: true, @@ -3428,6 +3998,9 @@ mod tests { 110, ], ), + icon_mimetype: Some( + "image/svg+xml", + ), score: 0.5, has_location_sign: false, subject_exact_match: false, @@ -3483,6 +4056,9 @@ mod tests { 110, ], ), + icon_mimetype: Some( + "image/svg+xml", + ), score: 0.5, has_location_sign: false, subject_exact_match: false, @@ -3516,6 +4092,9 @@ mod tests { 110, ], ), + icon_mimetype: Some( + "image/svg+xml", + ), score: 0.5, has_location_sign: false, subject_exact_match: false, @@ -3549,6 +4128,9 @@ mod tests { 110, ], ), + icon_mimetype: Some( + "image/svg+xml", + ), score: 0.5, has_location_sign: false, subject_exact_match: false, @@ -3593,6 +4175,9 @@ mod tests { 110, ], ), + icon_mimetype: Some( + "image/svg+xml", + ), score: 0.5, has_location_sign: false, subject_exact_match: false, @@ -3734,6 +4319,7 @@ mod tests { url: "https://www.lasagna.restaurant", raw_url: "https://www.lasagna.restaurant", icon: None, + icon_mimetype: None, full_keyword: "amp wiki match", block_id: 0, advertiser: "Good Place Eats", @@ -3762,6 +4348,9 @@ mod tests { 110, ], ), + icon_mimetype: Some( + "image/png", + ), full_keyword: "amp wiki match", }, Amp { @@ -3769,6 +4358,7 @@ mod tests { url: "https://penne.biz", raw_url: "https://penne.biz", icon: None, + icon_mimetype: None, full_keyword: "amp wiki match", block_id: 0, advertiser: "Good Place Eats", @@ -3801,6 +4391,7 @@ mod tests { url: "https://www.lasagna.restaurant", raw_url: "https://www.lasagna.restaurant", icon: None, + icon_mimetype: None, full_keyword: "amp wiki match", block_id: 0, advertiser: "Good Place Eats", @@ -3829,6 +4420,9 @@ mod tests { 110, ], ), + icon_mimetype: Some( + "image/png", + ), full_keyword: "amp wiki match", }, ] @@ -3873,6 +4467,9 @@ mod tests { 110, ], ), + icon_mimetype: Some( + "image/png", + ), full_keyword: "pocket wiki match", }, Pocket { @@ -4193,6 +4790,9 @@ mod tests { 110, ], ), + icon_mimetype: Some( + "image/png", + ), full_keyword: "lasagna", block_id: 0, advertiser: "Good Place Eats", @@ -4234,6 +4834,9 @@ mod tests { 110, ], ), + icon_mimetype: Some( + "image/png", + ), full_keyword: "lasagna", block_id: 0, advertiser: "Good Place Eats", @@ -4275,6 +4878,9 @@ mod tests { 110, ], ), + icon_mimetype: Some( + "image/png", + ), full_keyword: "lasagna", block_id: 0, advertiser: "Good Place Eats", @@ -4304,6 +4910,9 @@ mod tests { 110, ], ), + icon_mimetype: Some( + "image/png", + ), full_keyword: "lasagna", block_id: 0, advertiser: "Good Place Eats", @@ -4365,7 +4974,10 @@ mod tests { store.ingest(SuggestIngestionConstraints::default())?; store.dbs()?.reader.read(|dao| { - assert_eq!(dao.get_meta::(LAST_INGEST_META_KEY)?, Some(45)); + assert_eq!( + dao.get_meta::(SuggestRecordType::Icon.last_ingest_meta_key().as_str())?, + Some(45) + ); assert_eq!( dao.conn .query_one::("SELECT count(*) FROM suggestions")?, @@ -4400,16 +5012,19 @@ mod tests { store.ingest(SuggestIngestionConstraints::default())?; store.dbs()?.reader.read(|dao| { - assert_eq!(dao.get_meta::(LAST_INGEST_META_KEY)?, Some(30)); + assert_eq!( + dao.get_meta("last_quicksuggest_ingest_unparsable")?, + Some(30) + ); expect![[r#" Some( UnparsableRecords( { "clippy-2": UnparsableRecord { - schema_version: 14, + schema_version: 18, }, "fancy-new-suggestions-1": UnparsableRecord { - schema_version: 14, + schema_version: 18, }, }, ), @@ -4469,16 +5084,19 @@ mod tests { store.ingest(SuggestIngestionConstraints::default())?; store.dbs()?.reader.read(|dao| { - assert_eq!(dao.get_meta::(LAST_INGEST_META_KEY)?, Some(30)); + assert_eq!( + dao.get_meta("last_quicksuggest_ingest_unparsable")?, + Some(30) + ); expect![[r#" Some( UnparsableRecords( { "clippy-2": UnparsableRecord { - schema_version: 14, + schema_version: 18, }, "fancy-new-suggestions-1": UnparsableRecord { - schema_version: 14, + schema_version: 18, }, }, ), @@ -4505,13 +5123,105 @@ mod tests { let store = unique_test_store(SnapshotSettingsClient::with_snapshot(snapshot)); store.dbs()?.writer.write(|dao| { - dao.put_meta(LAST_INGEST_META_KEY, 30)?; + dao.put_meta( + SuggestRecordType::AmpWikipedia + .last_ingest_meta_key() + .as_str(), + 30, + )?; Ok(()) })?; store.ingest(SuggestIngestionConstraints::default())?; store.dbs()?.reader.read(|dao| { - assert_eq!(dao.get_meta::(LAST_INGEST_META_KEY)?, Some(30)); + assert_eq!( + dao.get_meta::( + SuggestRecordType::AmpWikipedia + .last_ingest_meta_key() + .as_str() + )?, + Some(30) + ); + Ok(()) + })?; + + Ok(()) + } + + /// Tests that we only ingest providers that we're concerned with. + #[test] + fn ingest_constraints_provider() -> anyhow::Result<()> { + before_each(); + + let snapshot = Snapshot::with_records(json!([{ + "id": "data-1", + "type": "data", + "last_modified": 15, + }, { + "id": "icon-1", + "type": "icon", + "last_modified": 30, + }]))?; + + let store = unique_test_store(SnapshotSettingsClient::with_snapshot(snapshot)); + store.dbs()?.writer.write(|dao| { + // Check that existing data is updated properly. + dao.put_meta( + SuggestRecordType::AmpWikipedia + .last_ingest_meta_key() + .as_str(), + 10, + )?; + Ok(()) + })?; + + let constraints = SuggestIngestionConstraints { + max_suggestions: Some(100), + providers: Some(vec![SuggestionProvider::Amp, SuggestionProvider::Pocket]), + }; + store.ingest(constraints)?; + + store.dbs()?.reader.read(|dao| { + assert_eq!( + dao.get_meta::( + SuggestRecordType::AmpWikipedia + .last_ingest_meta_key() + .as_str() + )?, + Some(15) + ); + assert_eq!( + dao.get_meta::(SuggestRecordType::Icon.last_ingest_meta_key().as_str())?, + Some(30) + ); + assert_eq!( + dao.get_meta::(SuggestRecordType::Pocket.last_ingest_meta_key().as_str())?, + None + ); + assert_eq!( + dao.get_meta::(SuggestRecordType::Amo.last_ingest_meta_key().as_str())?, + None + ); + assert_eq!( + dao.get_meta::(SuggestRecordType::Yelp.last_ingest_meta_key().as_str())?, + None + ); + assert_eq!( + dao.get_meta::(SuggestRecordType::Mdn.last_ingest_meta_key().as_str())?, + None + ); + assert_eq!( + dao.get_meta::(SuggestRecordType::AmpMobile.last_ingest_meta_key().as_str())?, + None + ); + assert_eq!( + dao.get_meta::( + SuggestRecordType::GlobalConfig + .last_ingest_meta_key() + .as_str() + )?, + None + ); Ok(()) })?; @@ -4582,10 +5292,10 @@ mod tests { UnparsableRecords( { "clippy-2": UnparsableRecord { - schema_version: 14, + schema_version: 18, }, "fancy-new-suggestions-1": UnparsableRecord { - schema_version: 14, + schema_version: 18, }, }, ), @@ -4671,7 +5381,7 @@ mod tests { UnparsableRecords( { "invalid-attachment": UnparsableRecord { - schema_version: 14, + schema_version: 18, }, }, ), @@ -4683,7 +5393,14 @@ mod tests { // Test that the valid record was read store.dbs()?.reader.read(|dao| { - assert_eq!(dao.get_meta::(LAST_INGEST_META_KEY)?, Some(15)); + assert_eq!( + dao.get_meta( + SuggestRecordType::AmpWikipedia + .last_ingest_meta_key() + .as_str() + )?, + Some(15) + ); expect![[r#" [ Amp { @@ -4691,6 +5408,7 @@ mod tests { url: "https://www.lph-nm.biz", raw_url: "https://www.lph-nm.biz", icon: None, + icon_mimetype: None, full_keyword: "los", block_id: 0, advertiser: "Los Pollos Hermanos", @@ -4899,6 +5617,7 @@ mod tests { url: "https://www.yelp.com/search?find_desc=ramen", title: "ramen", icon: None, + icon_mimetype: None, score: 0.5, has_location_sign: false, subject_exact_match: true, @@ -5313,4 +6032,204 @@ mod tests { Ok(()) } + + #[test] + fn remove_dismissed_suggestions() -> anyhow::Result<()> { + before_each(); + + let snapshot = Snapshot::with_records(json!([{ + "id": "data-1", + "type": "data", + "last_modified": 15, + "attachment": { + "filename": "data-1.json", + "mimetype": "application/json", + "location": "data-1.json", + "hash": "", + "size": 0, + }, + + }, { + "id": "data-2", + "type": "amo-suggestions", + "last_modified": 15, + "attachment": { + "filename": "data-2.json", + "mimetype": "application/json", + "location": "data-2.json", + "hash": "", + "size": 0, + }, + }, { + "id": "data-3", + "type": "pocket-suggestions", + "last_modified": 15, + "attachment": { + "filename": "data-3.json", + "mimetype": "application/json", + "location": "data-3.json", + "hash": "", + "size": 0, + }, + }, { + "id": "data-5", + "type": "mdn-suggestions", + "last_modified": 15, + "attachment": { + "filename": "data-5.json", + "mimetype": "application/json", + "location": "data-5.json", + "hash": "", + "size": 0, + }, + }, { + "id": "data-6", + "type": "amp-mobile-suggestions", + "last_modified": 15, + "attachment": { + "filename": "data-6.json", + "mimetype": "application/json", + "location": "data-6.json", + "hash": "", + "size": 0, + }, + }, { + "id": "icon-2", + "type": "icon", + "last_modified": 20, + "attachment": { + "filename": "icon-2.png", + "mimetype": "image/png", + "location": "icon-2.png", + "hash": "", + "size": 0, + }, + }, { + "id": "icon-3", + "type": "icon", + "last_modified": 25, + "attachment": { + "filename": "icon-3.png", + "mimetype": "image/png", + "location": "icon-3.png", + "hash": "", + "size": 0, + }, + }]))? + .with_data( + "data-1.json", + json!([{ + "id": 0, + "advertiser": "Good Place Eats", + "iab_category": "8 - Food & Drink", + "keywords": ["cats"], + "title": "Lasagna Come Out Tomorrow", + "url": "https://www.lasagna.restaurant", + "icon": "2", + "impression_url": "https://example.com/impression_url", + "click_url": "https://example.com/click_url", + "score": 0.31 + }, { + "id": 0, + "advertiser": "Wikipedia", + "iab_category": "5 - Education", + "keywords": ["cats"], + "title": "California", + "url": "https://wikipedia.org/California", + "icon": "3" + }]), + )? + .with_data( + "data-2.json", + json!([ + { + "description": "amo suggestion", + "url": "https://addons.mozilla.org/en-US/firefox/addon/example", + "guid": "{b9db16a4-6edc-47ec-a1f4-b86292ed211d}", + "keywords": ["cats"], + "title": "Firefox Relay", + "icon": "https://addons.mozilla.org/user-media/addon_icons/2633/2633704-64.png?modified=2c11a80b", + "rating": "4.9", + "number_of_ratings": 888, + "score": 0.32 + }, + ]), + )? + .with_data( + "data-3.json", + json!([ + { + "description": "pocket suggestion", + "url": "https://getpocket.com/collections/its-not-just-burnout-how-grind-culture-failed-women", + "lowConfidenceKeywords": [], + "highConfidenceKeywords": ["cats"], + "title": "‘It’s Not Just Burnout:’ How Grind Culture Fails Women", + "score": 0.33 + }, + ]), + )? + .with_data( + "data-5.json", + json!([ + { + "description": "Javascript Array", + "url": "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array", + "keywords": ["cats"], + "title": "Array", + "score": 0.24 + }, + ]), + )? + .with_data( + "data-6.json", + json!([ + { + "id": 0, + "advertiser": "Good Place Eats", + "iab_category": "8 - Food & Drink", + "keywords": ["cats"], + "title": "Mobile - Lasagna Come Out Tomorrow", + "url": "https://www.lasagna.restaurant", + "icon": "3", + "impression_url": "https://example.com/impression_url", + "click_url": "https://example.com/click_url", + "score": 0.26 + } + ]), + )? + .with_icon("icon-2.png", "i-am-an-icon".as_bytes().into()) + .with_icon("icon-3.png", "also-an-icon".as_bytes().into()); + + let store = unique_test_store(SnapshotSettingsClient::with_snapshot(snapshot)); + store.ingest(SuggestIngestionConstraints::default())?; + + // A query for cats should return all suggestions + let query = SuggestionQuery { + keyword: "cats".into(), + providers: vec![ + SuggestionProvider::Amp, + SuggestionProvider::Wikipedia, + SuggestionProvider::Amo, + SuggestionProvider::Pocket, + SuggestionProvider::Mdn, + SuggestionProvider::AmpMobile, + ], + limit: None, + }; + let results = store.query(query.clone())?; + assert_eq!(results.len(), 6); + + for result in results { + store.dismiss_suggestion(result.raw_url().unwrap().to_string())?; + } + + // After dismissing the suggestions, the next query shouldn't return them + assert_eq!(store.query(query.clone())?.len(), 0); + + // Clearing the dismissals should cause them to be returned again + store.clear_dismissed_suggestions()?; + assert_eq!(store.query(query.clone())?.len(), 6); + + Ok(()) + } } diff --git a/third_party/rust/suggest/src/suggest.udl b/third_party/rust/suggest/src/suggest.udl index 1cd8911a48..4a4e3fe9a0 100644 --- a/third_party/rust/suggest/src/suggest.udl +++ b/third_party/rust/suggest/src/suggest.udl @@ -40,6 +40,7 @@ interface Suggestion { string url, string raw_url, sequence? icon, + string? icon_mimetype, string full_keyword, i64 block_id, string advertiser, @@ -59,6 +60,7 @@ interface Suggestion { string title, string url, sequence? icon, + string? icon_mimetype, string full_keyword ); Amo( @@ -75,6 +77,7 @@ interface Suggestion { string url, string title, sequence? icon, + string? icon_mimetype, f64 score, boolean has_location_sign, boolean subject_exact_match, @@ -99,6 +102,7 @@ dictionary SuggestionQuery { dictionary SuggestIngestionConstraints { u64? max_suggestions = null; + sequence? providers = null; }; dictionary SuggestGlobalConfig { @@ -119,6 +123,12 @@ interface SuggestStore { [Throws=SuggestApiError] sequence query(SuggestionQuery query); + [Throws=SuggestApiError] + void dismiss_suggestion(string raw_suggestion_url); + + [Throws=SuggestApiError] + void clear_dismissed_suggestions(); + void interrupt(); [Throws=SuggestApiError] @@ -140,6 +150,7 @@ interface SuggestStoreBuilder { [Self=ByArc] SuggestStoreBuilder data_path(string path); + // Deprecated: this is no longer used by the suggest component. [Self=ByArc] SuggestStoreBuilder cache_path(string path); diff --git a/third_party/rust/suggest/src/suggestion.rs b/third_party/rust/suggest/src/suggestion.rs index f5425e3c73..c0b45524c7 100644 --- a/third_party/rust/suggest/src/suggestion.rs +++ b/third_party/rust/suggest/src/suggestion.rs @@ -29,6 +29,7 @@ pub enum Suggestion { url: String, raw_url: String, icon: Option>, + icon_mimetype: Option, full_keyword: String, block_id: i64, advertiser: String, @@ -48,6 +49,7 @@ pub enum Suggestion { title: String, url: String, icon: Option>, + icon_mimetype: Option, full_keyword: String, }, Amo { @@ -64,6 +66,7 @@ pub enum Suggestion { url: String, title: String, icon: Option>, + icon_mimetype: Option, score: f64, has_location_sign: bool, subject_exact_match: bool, @@ -106,6 +109,37 @@ impl Ord for Suggestion { } } +impl Suggestion { + /// Get the URL for this suggestion, if present + pub fn url(&self) -> Option<&str> { + match self { + Self::Amp { url, .. } + | Self::Pocket { url, .. } + | Self::Wikipedia { url, .. } + | Self::Amo { url, .. } + | Self::Yelp { url, .. } + | Self::Mdn { url, .. } => Some(url), + _ => None, + } + } + + /// Get the raw URL for this suggestion, if present + /// + /// This is the same as `url` except for Amp. In that case, `url` is the URL after being + /// "cooked" using template interpolation, while `raw_url` is the URL template. + pub fn raw_url(&self) -> Option<&str> { + match self { + Self::Amp { raw_url: url, .. } + | Self::Pocket { url, .. } + | Self::Wikipedia { url, .. } + | Self::Amo { url, .. } + | Self::Yelp { url, .. } + | Self::Mdn { url, .. } => Some(url), + _ => None, + } + } +} + impl Eq for Suggestion {} /// Replaces all template parameters in a "raw" sponsored suggestion URL, /// producing a "cooked" URL with real values. diff --git a/third_party/rust/suggest/src/yelp.rs b/third_party/rust/suggest/src/yelp.rs index 2413709c67..0eb6c10d56 100644 --- a/third_party/rust/suggest/src/yelp.rs +++ b/third_party/rust/suggest/src/yelp.rs @@ -52,7 +52,7 @@ const SUBJECT_PREFIX_MATCH_THRESHOLD: usize = 2; impl<'a> SuggestDao<'a> { /// Inserts the suggestions for Yelp attachment into the database. - pub fn insert_yelp_suggestions( + pub(crate) fn insert_yelp_suggestions( &mut self, record_id: &SuggestRecordId, suggestion: &DownloadedYelpSuggestion, @@ -130,7 +130,10 @@ impl<'a> SuggestDao<'a> { } /// Fetch Yelp suggestion from given user's query. - pub fn fetch_yelp_suggestions(&self, query: &SuggestionQuery) -> Result> { + pub(crate) fn fetch_yelp_suggestions( + &self, + query: &SuggestionQuery, + ) -> Result> { if !query.providers.contains(&SuggestionProvider::Yelp) { return Ok(vec![]); } @@ -144,7 +147,7 @@ impl<'a> SuggestDao<'a> { let Some((subject, subject_exact_match)) = self.find_subject(query_string)? else { return Ok(vec![]); }; - let (icon, score) = self.fetch_custom_details()?; + let (icon, icon_mimetype, score) = self.fetch_custom_details()?; let builder = SuggestionBuilder { subject: &subject, subject_exact_match, @@ -154,6 +157,7 @@ impl<'a> SuggestDao<'a> { location: None, need_location: false, icon, + icon_mimetype, score, }; return Ok(vec![builder.into()]); @@ -185,7 +189,7 @@ impl<'a> SuggestDao<'a> { return Ok(vec![]); }; - let (icon, score) = self.fetch_custom_details()?; + let (icon, icon_mimetype, score) = self.fetch_custom_details()?; let builder = SuggestionBuilder { subject: &subject, subject_exact_match, @@ -195,6 +199,7 @@ impl<'a> SuggestDao<'a> { location, need_location, icon, + icon_mimetype, score, }; Ok(vec![builder.into()]) @@ -204,6 +209,7 @@ impl<'a> SuggestDao<'a> { /// It returns the location tuple as follows: /// ( /// Option>: Icon data. If not found, returns None. + /// Option: Mimetype of the icon data. If not found, returns None. /// f64: Reflects score field in the yelp_custom_details table. /// ) /// @@ -212,11 +218,11 @@ impl<'a> SuggestDao<'a> { /// on Remote Settings. The following query will perform a table scan against /// `yelp_custom_details` followed by an index search against `icons`, /// which should be fine since there is only one record in the first table. - fn fetch_custom_details(&self) -> Result<(Option>, f64)> { + fn fetch_custom_details(&self) -> Result<(Option>, Option, f64)> { let result = self.conn.query_row_and_then_cachable( r#" SELECT - i.data, y.score + i.data, i.mimetype, y.score FROM yelp_custom_details y LEFT JOIN @@ -226,7 +232,13 @@ impl<'a> SuggestDao<'a> { 1 "#, (), - |row| -> Result<_> { Ok((row.get::<_, Option>>(0)?, row.get::<_, f64>(1)?)) }, + |row| -> Result<_> { + Ok(( + row.get::<_, Option>>(0)?, + row.get::<_, Option>(1)?, + row.get::<_, f64>(2)?, + )) + }, true, )?; @@ -439,6 +451,7 @@ struct SuggestionBuilder<'a> { location: Option, need_location: bool, icon: Option>, + icon_mimetype: Option, score: f64, } @@ -488,6 +501,7 @@ impl<'a> From> for Suggestion { url, title, icon: builder.icon, + icon_mimetype: builder.icon_mimetype, score: builder.score, has_location_sign: location_modifier.is_none() && builder.location_sign.is_some(), subject_exact_match: builder.subject_exact_match, diff --git a/third_party/rust/sync15/.cargo-checksum.json b/third_party/rust/sync15/.cargo-checksum.json index 5f995892d5..723c4c4384 100644 --- a/third_party/rust/sync15/.cargo-checksum.json +++ b/third_party/rust/sync15/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"Cargo.toml":"739abc68b38e8468c5d1eb3f7a66f01e638765f9d8714080e07817f0939a2b66","README.md":"6d4ff5b079ac5340d18fa127f583e7ad793c5a2328b8ecd12c3fc723939804f2","build.rs":"aa971160d67ce8626b26e15c04c34b730f594c45c817aae34cfc9f3ea14ae284","src/bso/content.rs":"92935258745bdf0c3915a555cb6884a7fa69faa1290ec2c1815f6e2f3c0f0562","src/bso/crypto.rs":"27602dcccb37d3a55620ee4e16b705da455d49af575de115c7c79c0178eb1d6d","src/bso/mod.rs":"09e723dc7e99295ecafdcadffaf604d66ea27cf2b7f1fd9ab3cac4f4698ff6a7","src/bso/test_utils.rs":"4ec5a2df5e1c0ec14dc770681e959bdcef6ef04f6fde435999197f46a8ae4831","src/client/coll_state.rs":"13e6ef55273baf5536acc369be522e34a803a32cabf19cce43e426aea9b6223e","src/client/coll_update.rs":"dac04a90c29dd969f8b4250414609c9b6d61daf2dfa4ae77d1c4a165ba970b05","src/client/collection_keys.rs":"c27b2277a3a52033b58ab01490fc2ea7007494195dd5e6dc2c6931a4ca96795a","src/client/mod.rs":"8f588d4a035cf79d96f2500f06d5651c1a7c566127c456ffa5429811ddce3fd6","src/client/request.rs":"8841524e37d8195867bdf6ba98c75f610cf47a4644adeebd6372cc6713f2260a","src/client/state.rs":"4e31193ef2471c1dfabf1c6a391bcb95e14ddb45855786a4194ff187d5c9347c","src/client/status.rs":"f445a8765dac9789444e23b5145148413407bb1d18a15ef56682243997f591bf","src/client/storage_client.rs":"8de72d4ba3ca4f68c8e1898466de83a2b543545a18679800cb4f7fbda2dc3183","src/client/sync.rs":"b29abb512ec9d163f7883b71f78c9202802dcb17cad1fc5dc08087fb0bb66704","src/client/sync_multiple.rs":"6e92571132f89744b553190c596be8aff9b2d031d8f79d82c94cdf78b1683f4a","src/client/token.rs":"b268759d31e0fe17e0e2a428694cd9a317fcfbdd52f023d5d8c7cc6f00f1a102","src/client/util.rs":"71cc70ee41f821f53078675e636e9fad9c6046fa1a989e37f5487e340a2277d6","src/client_types.rs":"3c3cac1540b92482f43660d9e43bdde8481c4cc1a98253a68c80e791231f5976","src/clients_engine/engine.rs":"9e11b47be81fc63214f31879af74075674aa50a8f8989afe20fefa7990fa99b9","src/clients_engine/mod.rs":"461729e6f89b66b2cbd89b041a03d4d6a8ba582284ed4f3015cb13e1a0c6da97","src/clients_engine/record.rs":"b0d84bf420743d7638a45e4836633a45e50257d5548fe7ecd04bff4d724439b8","src/clients_engine/ser.rs":"ef12daeb11faf618fe3cafe91f20a031fe5bb6751369b6ee5aee03f196efe88c","src/device_type.rs":"dc2d4296d25e31471c8e68488f1043ff239b902036cd6aea8a686cf79b4ed335","src/enc_payload.rs":"aa3eea7df49b24cd59831680a47c417b73a3e36e6b0f3f4baf14ca66bd68be6b","src/engine/bridged_engine.rs":"f70f1bfce6e0c04b0c72ec9cbfbb12c82d4009a23fb9768792107d41b2865a4f","src/engine/mod.rs":"90f1f9760f5f712a337aebb04e59c736e4b6fbd89d6a188d969210c7f3f321ae","src/engine/request.rs":"5923025fb9550178339f880a1bf8526d8e853e7a0b2bce6d9d687cc808ac0085","src/engine/sync_engine.rs":"531b35d72ce9e04c3e543c0468c1e450fba2c0dc3d33d68d9b1c0a5c1ad7dd34","src/error.rs":"a45cfe02e6301f473c34678b694943c1a04308b8c292c6e0448bf495194c3b5e","src/key_bundle.rs":"abd0781f3be8c8e7c691f18bb71f3433b633803c48da9794e15ac6301ed60d6c","src/lib.rs":"f59f8817978d943518dfa03ab31fc0f6b1fc72ee9943a97aef1537e2769649f5","src/record_types.rs":"02bb3d352fb808131d298f9b90d9c95b7e9e0138b97c5401f3b9fdacc5562f44","src/server_timestamp.rs":"6272299c92b05b9ec9dc2e18402ebe927b07ccf1dcab5082301a09e0ee56ce24","src/sync15.udl":"005b2b056b93c959a04670f6f489afecb8e17093d8e4be34765a3a4cc0faeb8c","src/telemetry.rs":"e3a7e13e85f5e336526ebf07db04c81b8f1ba89ae1db4159a3a570826cb8cfd2","uniffi.toml":"34488f947497a9b05007445dd816024ef02e6b1696f1056ee868f039722828ee"},"package":null} \ No newline at end of file +{"files":{"Cargo.toml":"6827fe696bd5e7ef806389493be837c7fa87f3301a7b243890259c5304fda21f","README.md":"6d4ff5b079ac5340d18fa127f583e7ad793c5a2328b8ecd12c3fc723939804f2","build.rs":"aa971160d67ce8626b26e15c04c34b730f594c45c817aae34cfc9f3ea14ae284","src/bso/content.rs":"92935258745bdf0c3915a555cb6884a7fa69faa1290ec2c1815f6e2f3c0f0562","src/bso/crypto.rs":"27602dcccb37d3a55620ee4e16b705da455d49af575de115c7c79c0178eb1d6d","src/bso/mod.rs":"09e723dc7e99295ecafdcadffaf604d66ea27cf2b7f1fd9ab3cac4f4698ff6a7","src/bso/test_utils.rs":"4ec5a2df5e1c0ec14dc770681e959bdcef6ef04f6fde435999197f46a8ae4831","src/client/coll_state.rs":"13e6ef55273baf5536acc369be522e34a803a32cabf19cce43e426aea9b6223e","src/client/coll_update.rs":"dac04a90c29dd969f8b4250414609c9b6d61daf2dfa4ae77d1c4a165ba970b05","src/client/collection_keys.rs":"c27b2277a3a52033b58ab01490fc2ea7007494195dd5e6dc2c6931a4ca96795a","src/client/mod.rs":"8f588d4a035cf79d96f2500f06d5651c1a7c566127c456ffa5429811ddce3fd6","src/client/request.rs":"8841524e37d8195867bdf6ba98c75f610cf47a4644adeebd6372cc6713f2260a","src/client/state.rs":"4e31193ef2471c1dfabf1c6a391bcb95e14ddb45855786a4194ff187d5c9347c","src/client/status.rs":"f445a8765dac9789444e23b5145148413407bb1d18a15ef56682243997f591bf","src/client/storage_client.rs":"8de72d4ba3ca4f68c8e1898466de83a2b543545a18679800cb4f7fbda2dc3183","src/client/sync.rs":"b29abb512ec9d163f7883b71f78c9202802dcb17cad1fc5dc08087fb0bb66704","src/client/sync_multiple.rs":"6e92571132f89744b553190c596be8aff9b2d031d8f79d82c94cdf78b1683f4a","src/client/token.rs":"b268759d31e0fe17e0e2a428694cd9a317fcfbdd52f023d5d8c7cc6f00f1a102","src/client/util.rs":"71cc70ee41f821f53078675e636e9fad9c6046fa1a989e37f5487e340a2277d6","src/client_types.rs":"3c3cac1540b92482f43660d9e43bdde8481c4cc1a98253a68c80e791231f5976","src/clients_engine/engine.rs":"9e11b47be81fc63214f31879af74075674aa50a8f8989afe20fefa7990fa99b9","src/clients_engine/mod.rs":"461729e6f89b66b2cbd89b041a03d4d6a8ba582284ed4f3015cb13e1a0c6da97","src/clients_engine/record.rs":"b0d84bf420743d7638a45e4836633a45e50257d5548fe7ecd04bff4d724439b8","src/clients_engine/ser.rs":"ef12daeb11faf618fe3cafe91f20a031fe5bb6751369b6ee5aee03f196efe88c","src/device_type.rs":"dc2d4296d25e31471c8e68488f1043ff239b902036cd6aea8a686cf79b4ed335","src/enc_payload.rs":"aa3eea7df49b24cd59831680a47c417b73a3e36e6b0f3f4baf14ca66bd68be6b","src/engine/bridged_engine.rs":"f70f1bfce6e0c04b0c72ec9cbfbb12c82d4009a23fb9768792107d41b2865a4f","src/engine/mod.rs":"90f1f9760f5f712a337aebb04e59c736e4b6fbd89d6a188d969210c7f3f321ae","src/engine/request.rs":"5923025fb9550178339f880a1bf8526d8e853e7a0b2bce6d9d687cc808ac0085","src/engine/sync_engine.rs":"531b35d72ce9e04c3e543c0468c1e450fba2c0dc3d33d68d9b1c0a5c1ad7dd34","src/error.rs":"a45cfe02e6301f473c34678b694943c1a04308b8c292c6e0448bf495194c3b5e","src/key_bundle.rs":"abd0781f3be8c8e7c691f18bb71f3433b633803c48da9794e15ac6301ed60d6c","src/lib.rs":"f59f8817978d943518dfa03ab31fc0f6b1fc72ee9943a97aef1537e2769649f5","src/record_types.rs":"02bb3d352fb808131d298f9b90d9c95b7e9e0138b97c5401f3b9fdacc5562f44","src/server_timestamp.rs":"6272299c92b05b9ec9dc2e18402ebe927b07ccf1dcab5082301a09e0ee56ce24","src/sync15.udl":"005b2b056b93c959a04670f6f489afecb8e17093d8e4be34765a3a4cc0faeb8c","src/telemetry.rs":"e3a7e13e85f5e336526ebf07db04c81b8f1ba89ae1db4159a3a570826cb8cfd2","uniffi.toml":"34488f947497a9b05007445dd816024ef02e6b1696f1056ee868f039722828ee"},"package":null} \ No newline at end of file diff --git a/third_party/rust/sync15/Cargo.toml b/third_party/rust/sync15/Cargo.toml index 86f7da91f8..943bee58ae 100644 --- a/third_party/rust/sync15/Cargo.toml +++ b/third_party/rust/sync15/Cargo.toml @@ -30,7 +30,7 @@ serde_derive = "1" serde_json = "1" serde_path_to_error = "0.1" thiserror = "1.0" -uniffi = "0.25.2" +uniffi = "0.27.1" [dependencies.base16] version = "0.2" @@ -72,7 +72,7 @@ version = "0.10" default-features = false [build-dependencies.uniffi] -version = "0.25.2" +version = "0.27.1" features = ["build"] [features] diff --git a/third_party/rust/tabs/.cargo-checksum.json b/third_party/rust/tabs/.cargo-checksum.json index 1f1fa20bfa..a9adbf9214 100644 --- a/third_party/rust/tabs/.cargo-checksum.json +++ b/third_party/rust/tabs/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"Cargo.toml":"c06aa10f3dfa7be35c4fb54cb629704826da5bd9d08eaf09211343bb2b62bf74","README.md":"c48b8f391ef822c4f3971b5f453a1e7b43bea232752d520460d2f04803aead1a","build.rs":"33e61b811b19ed2b58e319cc65d5988bed258d2c4fea2d706301184c59847a0f","src/error.rs":"2694657aeb12f99c4b2fe102ad2b08b79955d209201831b3e071129f0b7d7eda","src/lib.rs":"7208f78955e015ef8bab7916307e551cd3c1bd56d7fe14f8b53cd53bc4b38555","src/schema.rs":"2b7b51f3c2edc0ca603495c10b917603fd9ac791c4a366080e40d090b13b91f2","src/storage.rs":"18f449b6daf1641dc351be451311495b7c05e16c4e2d4eaf12c1fa02fa750b67","src/store.rs":"ab0b6214b30b0f0fa7c6a89098ff3db1a8f76264f6711c4481c0be460afe522b","src/sync/bridge.rs":"18d3a7913a030b598d4b6cbd5b7e2ab4cef4cc7ea964f5bc84d7fb2f28787529","src/sync/engine.rs":"2d14d899a38ac72b9141d505babd94ef7b6fbc5a95be70f324a40bf01935793d","src/sync/mod.rs":"09ba3c87f1174a243bf5aaa481effd18929d54359ceb9b23ccb2c32ee3482f34","src/sync/record.rs":"eef6751c209d039958afbe245ddb006cfdf6b8b6b47f925f69c552b832b87922","src/tabs.udl":"2cefc7f6a27b5619bc536d4a19608cf24153d745199fbeaf192e24b4381dedfb","uniffi.toml":"f9125e8d55b109e86076ee88bfd640372f06b142b7db557e41816c7227dd445c"},"package":null} \ No newline at end of file +{"files":{"Cargo.toml":"73ac434bafd302d09e8b61742cdb094db767985f0cbae7d00efb5096a33dea4b","README.md":"c48b8f391ef822c4f3971b5f453a1e7b43bea232752d520460d2f04803aead1a","build.rs":"33e61b811b19ed2b58e319cc65d5988bed258d2c4fea2d706301184c59847a0f","src/error.rs":"2694657aeb12f99c4b2fe102ad2b08b79955d209201831b3e071129f0b7d7eda","src/lib.rs":"7208f78955e015ef8bab7916307e551cd3c1bd56d7fe14f8b53cd53bc4b38555","src/schema.rs":"2b7b51f3c2edc0ca603495c10b917603fd9ac791c4a366080e40d090b13b91f2","src/storage.rs":"18f449b6daf1641dc351be451311495b7c05e16c4e2d4eaf12c1fa02fa750b67","src/store.rs":"ab0b6214b30b0f0fa7c6a89098ff3db1a8f76264f6711c4481c0be460afe522b","src/sync/bridge.rs":"18d3a7913a030b598d4b6cbd5b7e2ab4cef4cc7ea964f5bc84d7fb2f28787529","src/sync/engine.rs":"2d14d899a38ac72b9141d505babd94ef7b6fbc5a95be70f324a40bf01935793d","src/sync/mod.rs":"09ba3c87f1174a243bf5aaa481effd18929d54359ceb9b23ccb2c32ee3482f34","src/sync/record.rs":"eef6751c209d039958afbe245ddb006cfdf6b8b6b47f925f69c552b832b87922","src/tabs.udl":"2cefc7f6a27b5619bc536d4a19608cf24153d745199fbeaf192e24b4381dedfb","uniffi.toml":"f9125e8d55b109e86076ee88bfd640372f06b142b7db557e41816c7227dd445c"},"package":null} \ No newline at end of file diff --git a/third_party/rust/tabs/Cargo.toml b/third_party/rust/tabs/Cargo.toml index c06ad84ca9..84a5c7ad33 100644 --- a/third_party/rust/tabs/Cargo.toml +++ b/third_party/rust/tabs/Cargo.toml @@ -29,7 +29,7 @@ serde = "1" serde_derive = "1" serde_json = "1" thiserror = "1.0" -uniffi = "0.25.2" +uniffi = "0.27.1" url = "2.1" [dependencies.error-support] @@ -65,5 +65,5 @@ features = ["humantime"] default-features = false [build-dependencies.uniffi] -version = "0.25.2" +version = "0.27.1" features = ["build"] diff --git a/third_party/rust/textwrap/.cargo-checksum.json b/third_party/rust/textwrap/.cargo-checksum.json new file mode 100644 index 0000000000..0948683828 --- /dev/null +++ b/third_party/rust/textwrap/.cargo-checksum.json @@ -0,0 +1 @@ +{"files":{"CHANGELOG.md":"2b49672ca15da27844abb6643660faa3a517b1b731c7c5023a55de2e570f35fb","Cargo.lock":"c60631b9ccb2798984507f9e074ce15d9b6e59f34d4dc3418ef40c222ac55725","Cargo.toml":"bccd38e75da62b89a7bb2d741926688f1822ecd5165a703ffaadf9177a34e919","LICENSE":"ce93600c49fbb3e14df32efe752264644f6a2f8e08a735ba981725799e5309ef","README.md":"5dd8128a4e9057aeb6133a073d30a819230243907e717349101b41a11ec23234","rustfmt.toml":"02637ad90caa19885b25b1ce8230657b3703402775d9db83687a3f55de567509","src/columns.rs":"73432251f95ac0b84d5e971989ebc5f867d8b8ca82d5e3fc67fe3a66216fbc38","src/core.rs":"e2cc6b1e5978df0db9b6d0425e7d0ebf65dd188aff90df800f1f2dda7b1c53f2","src/fill.rs":"1fe773dad2d0bb67a7739b3931c1ee3269d677b71a0716dcdb5b01fe2539d7c2","src/fuzzing.rs":"0a77010a555a244ac5e907754b2104912299815009922cfdc0f6b48d92135295","src/indentation.rs":"f41ee8be41e01620c7d88b76f81a01ce6a619939505eaf3fcfe6c8021fae022b","src/lib.rs":"d5d39085faa4527bf6c16a91c5a44b9b894e3f3a2606763bceac22038528c28c","src/line_ending.rs":"bf416f683ab952d4df75d5dc3c199e7ae7740db2c5982ac1a20c3f4b186ded76","src/options.rs":"0d3aec6ab238f3aa14aa57e736384ec208cd3013373941c76d66c0125ca0630f","src/refill.rs":"33ce98ef31c4791893fc2136edd8f8d95cdd38fa54daa59aaf078b359c43d913","src/termwidth.rs":"2e7854e822c435341bc4d467d13614d417df4f2f530cea3c5e49e3b44e754943","src/word_separators.rs":"d3b2b5faf224bf414bf9da48be02eaffb41aec3a91674bedab02ad5748344143","src/word_splitters.rs":"8de2b92eff6d752e321f219136b45b9812267b5be7ace57602a3bb9d3b5cf332","src/wrap.rs":"52c48e2e5155100e4067363e56b180785684bca3109c95c3425ef8051738ff0e","src/wrap_algorithms.rs":"c99498f2e58634f707545ba73c3a99025086d1afb8c12aeceff2ced2887bb8ae","src/wrap_algorithms/optimal_fit.rs":"a9ce8bad61d4fa81df9e292a557fbf5303df78391d63610ec512b9b06f9193b1","tests/indent.rs":"51f977db11632a32fafecf86af88413d51238fe6efcf18ec52fac89133714278","tests/version-numbers.rs":"9e964f58dbdf051fc6fe0d6542ab312d3e95f26c3fd14bce84449bb625e45761"},"package":"23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9"} \ No newline at end of file diff --git a/third_party/rust/textwrap/CHANGELOG.md b/third_party/rust/textwrap/CHANGELOG.md new file mode 100644 index 0000000000..362c0c0b9d --- /dev/null +++ b/third_party/rust/textwrap/CHANGELOG.md @@ -0,0 +1,616 @@ +# Changelog + +This file lists the most important changes made in each release of +`textwrap`. + +## Version 0.16.1 (2024-02-17) + +This release fixes `display_width` to ignore inline-hyperlinks. The minimum +supported version of Rust is now documented to be 1.56. + +* [#526](https://github.com/mgeisler/textwrap/pull/526): Ignore ANSI hyperlinks + in `display_width`: calculations. +* [#521](https://github.com/mgeisler/textwrap/pull/521): Add `Options::width` + setter method. +* [#520](https://github.com/mgeisler/textwrap/pull/520): Clarify that + `WordSeparator` is an enum rather than a trait. +* [#518](https://github.com/mgeisler/textwrap/pull/518): Test with latest stable + and nightly Rust, but check that we can build with Rust 1.56. + +## Version 0.16.0 (2022-10-23) + +This release marks `Options` as `non_exhaustive` and extends it to +make line endings configurable, it adds new fast paths to `fill` and +`wrap`, and it fixes crashes in `unfill` and `refill`. + +* [#480](https://github.com/mgeisler/textwrap/pull/480): Mark + `Options` as `non_exhaustive`. This will allow us to extend the + struct in the future without breaking backwards compatibility. +* [#478](https://github.com/mgeisler/textwrap/pull/478): Add fast + paths to `fill` and `wrap`. This makes the functions 10-25 times + faster when the no wrapping is needed. +* [#468](https://github.com/mgeisler/textwrap/pull/468): Fix `refill` + to add back correct line ending. +* [#467](https://github.com/mgeisler/textwrap/pull/467): Fix crashes + in `unfill` and `refill`. +* [#458](https://github.com/mgeisler/textwrap/pull/458): Test with + Rust 1.56 (first compiler release with support for Rust 2021). +* [#454](https://github.com/mgeisler/textwrap/pull/454): Make line + endings configurable. +* [#448](https://github.com/mgeisler/textwrap/pull/448): Migrate to + the Rust 2021 edition. + +## Version 0.15.2 (2022-10-24) + +This release is identical to 0.15.0 and is only there to give people a +way to install crates which depend on the yanked 0.15.1 release. See +[#484](https://github.com/mgeisler/textwrap/issues/484) for details. + +## Version 0.15.1 (2022-09-15) + +This release was yanked since it accidentally broke backwards +compatibility with 0.15.0. + +## Version 0.15.0 (2022-02-27) + +This is a major feature release with two main changes: + +* [#421](https://github.com/mgeisler/textwrap/pull/421): Use `f64` + instead of `usize` for fragment widths. + + This fixes problems with overflows in the internal computations of + `wrap_optimal_fit` when fragments (words) or line lengths had + extreme values, such as `usize::MAX`. + +* [#438](https://github.com/mgeisler/textwrap/pull/438): Simplify + `Options` by removing generic type parameters. + + This change removes the new generic parameters introduced in version + 0.14, as well as the original `WrapSplitter` parameter which has + been present since very early versions. + + The result is a simplification of function and struct signatures + across the board. So what used to be + + ```rust + let options: Options< + wrap_algorithms::FirstFit, + word_separators::AsciiSpace, + word_splitters::HyphenSplitter, + > = Options::new(80); + ``` + + if types are fully written out, is now simply + + ```rust + let options: Options<'_> = Options::new(80); + ``` + + The anonymous lifetime represent the lifetime of the + `initial_indent` and `subsequent_indent` strings. The change is + nearly performance neutral (a 1-2% regression). + +Smaller improvements and changes: + +* [#404](https://github.com/mgeisler/textwrap/pull/404): Make + documentation for short last-line penalty more precise. +* [#405](https://github.com/mgeisler/textwrap/pull/405): Cleanup and + simplify `Options` docstring. +* [#411](https://github.com/mgeisler/textwrap/pull/411): Default to + `OptimalFit` in interactive example. +* [#415](https://github.com/mgeisler/textwrap/pull/415): Add demo + program to help compute binary sizes. +* [#423](https://github.com/mgeisler/textwrap/pull/423): Add fuzz + tests with fully arbitrary fragments. +* [#424](https://github.com/mgeisler/textwrap/pull/424): Change + `wrap_optimal_fit` penalties to non-negative numbers. +* [#430](https://github.com/mgeisler/textwrap/pull/430): Add + `debug-words` example. +* [#432](https://github.com/mgeisler/textwrap/pull/432): Use precise + dependency versions in Cargo.toml. + +## Version 0.14.2 (2021-06-27) + +The 0.14.1 release included more changes than intended and has been +yanked. The change intended for 0.14.1 is now included in 0.14.2. + +## Version 0.14.1 (2021-06-26) + +This release fixes a panic reported by @Makoto, thanks! + +* [#391](https://github.com/mgeisler/textwrap/pull/391): Fix panic in + `find_words` due to string access outside of a character boundary. + +## Version 0.14.0 (2021-06-05) + +This is a major feature release which makes Textwrap more configurable +and flexible. The high-level API of `textwrap::wrap` and +`textwrap::fill` remains unchanged, but low-level structs have moved +around. + +The biggest change is the introduction of new generic type parameters +to the `Options` struct. These parameters lets you statically +configure the wrapping algorithm, the word separator, and the word +splitter. If you previously spelled out the full type for `Options`, +you now need to take the extra type parameters into account. This +means that + +```rust +let options: Options = Options::new(80); +``` + +changes to + +```rust +let options: Options< + wrap_algorithms::FirstFit, + word_separators::AsciiSpace, + word_splitters::HyphenSplitter, +> = Options::new(80); +``` + +This is quite a mouthful, so we suggest using type inference where +possible. You won’t see any chance if you call `wrap` directly with a +width or with an `Options` value constructed on the fly. Please open +an issue if this causes problems for you! + +### New `WordSeparator` Trait + +* [#332](https://github.com/mgeisler/textwrap/pull/332): Add + `WordSeparator` trait to allow customizing how words are found in a + line of text. Until now, Textwrap would always assume that words are + separated by ASCII space characters. You can now customize this as + needed. + +* [#313](https://github.com/mgeisler/textwrap/pull/313): Add support + for using the Unicode line breaking algorithm to find words. This is + done by adding a second implementation of the new `WordSeparator` + trait. The implementation uses the unicode-linebreak crate, which is + a new optional dependency. + + With this, Textwrap can be used with East-Asian languages such as + Chinese or Japanese where there are no spaces between words. + Breaking a long sequence of emojis is another example where line + breaks might be wanted even if there are no whitespace to be found. + Feedback would be appreciated for this feature. + + +### Indent + +* [#353](https://github.com/mgeisler/textwrap/pull/353): Trim trailing + whitespace from `prefix` in `indent`. + + Before, empty lines would get no prefix added. Now, empty lines have + a trimmed prefix added. This little trick makes `indent` much more + useful since you can now safely indent with `"# "` without creating + trailing whitespace in the output due to the trailing whitespace in + your prefix. + +* [#354](https://github.com/mgeisler/textwrap/pull/354): Make `indent` + about 20% faster by preallocating the output string. + + +### Documentation + +* [#308](https://github.com/mgeisler/textwrap/pull/308): Document + handling of leading and trailing whitespace when wrapping text. + +### WebAssembly Demo + +* [#310](https://github.com/mgeisler/textwrap/pull/310): Thanks to + WebAssembly, you can now try out Textwrap directly in your browser. + Please try it out: https://mgeisler.github.io/textwrap/. + +### New Generic Parameters + +* [#331](https://github.com/mgeisler/textwrap/pull/331): Remove outer + boxing from `Options`. + +* [#357](https://github.com/mgeisler/textwrap/pull/357): Replace + `core::WrapAlgorithm` enum with a `wrap_algorithms::WrapAlgorithm` + trait. This allows for arbitrary wrapping algorithms to be plugged + into the library. + +* [#358](https://github.com/mgeisler/textwrap/pull/358): Switch + wrapping functions to use a slice for `line_widths`. + +* [#368](https://github.com/mgeisler/textwrap/pull/368): Move + `WordSeparator` and `WordSplitter` traits to separate modules. + Before, Textwrap had several top-level structs such as + `NoHyphenation` and `HyphenSplitter`. These implementations of + `WordSplitter` now lives in a dedicated `word_splitters` module. + Similarly, we have a new `word_separators` module for + implementations of `WordSeparator`. + +* [#369](https://github.com/mgeisler/textwrap/pull/369): Rename + `Options::splitter` to `Options::word_splitter` for consistency with + the other fields backed by traits. + +## Version 0.13.4 (2021-02-23) + +This release removes `println!` statements which was left behind in +`unfill` by mistake. + +* [#296](https://github.com/mgeisler/textwrap/pull/296): Improve house + building example with more comments. +* [#297](https://github.com/mgeisler/textwrap/pull/297): Remove debug + prints in the new `unfill` function. + +## Version 0.13.3 (2021-02-20) + +This release contains a bugfix for `indent` and improved handling of +emojis. We’ve also added a new function for formatting text in columns +and functions for reformatting already wrapped text. + +* [#276](https://github.com/mgeisler/textwrap/pull/276): Extend + `core::display_width` to handle emojis when the unicode-width Cargo + feature is disabled. +* [#279](https://github.com/mgeisler/textwrap/pull/279): Make `indent` + preserve existing newlines in the input string. Before, + `indent("foo", "")` would return `"foo\n"` by mistake. It now + returns `"foo"` instead. +* [#281](https://github.com/mgeisler/textwrap/pull/281): Ensure all + `Options` fields have examples. +* [#282](https://github.com/mgeisler/textwrap/pull/282): Add a + `wrap_columns` function. +* [#294](https://github.com/mgeisler/textwrap/pull/294): Add new + `unfill` and `refill` functions. + +## Version 0.13.2 (2020-12-30) + +This release primarily makes all dependencies optional. This makes it +possible to slim down textwrap as needed. + +* [#254](https://github.com/mgeisler/textwrap/pull/254): `impl + WordSplitter` for `Box where T: WordSplitter`. +* [#255](https://github.com/mgeisler/textwrap/pull/255): Use command + line arguments as initial text in interactive example. +* [#256](https://github.com/mgeisler/textwrap/pull/256): Introduce + fuzz tests for `wrap_optimal_fit` and `wrap_first_fit`. +* [#260](https://github.com/mgeisler/textwrap/pull/260): Make the + unicode-width dependency optional. +* [#261](https://github.com/mgeisler/textwrap/pull/261): Make the + smawk dependency optional. + +## Version 0.13.1 (2020-12-10) + +This is a bugfix release which fixes a regression in 0.13.0. The bug +meant that colored text was wrapped incorrectly. + +* [#245](https://github.com/mgeisler/textwrap/pull/245): Support + deleting a word with Ctrl-Backspace in the interactive demo. +* [#246](https://github.com/mgeisler/textwrap/pull/246): Show build + type (debug/release) in interactive demo. +* [#249](https://github.com/mgeisler/textwrap/pull/249): Correctly + compute width while skipping over ANSI escape sequences. + +## Version 0.13.0 (2020-12-05) + +This is a major release which rewrites the core logic, adds many new +features, and fixes a couple of bugs. Most programs which use +`textwrap` stays the same, incompatibilities and upgrade notes are +given below. + +Clone the repository and run the following to explore the new features +in an interactive demo (Linux only): + +```sh +$ cargo run --example interactive --all-features +``` + +### Bug Fixes + +#### Rewritten core wrapping algorithm + +* [#221](https://github.com/mgeisler/textwrap/pull/221): Reformulate + wrapping in terms of words with whitespace and penalties. + +The core wrapping algorithm has been completely rewritten. This fixed +bugs and simplified the code, while also making it possible to use +`textwrap` outside the context of the terminal. + +As part of this, trailing whitespace is now discarded consistently +from wrapped lines. Before we would inconsistently remove whitespace +at the end of wrapped lines, except for the last. Leading whitespace +is still preserved. + +### New Features + +#### Optimal-fit wrapping + +* [#234](https://github.com/mgeisler/textwrap/pull/234): Introduce + wrapping using an optimal-fit algorithm. + +This release adds support for new wrapping algorithm which finds a +globally optimal set of line breaks, taking certain penalties into +account. As an example, the old algorithm would produce + + "To be, or" + "not to be:" + "that is" + "the" + "question" + +Notice how the fourth line with “the” is very short. The new algorithm +shortens the previous lines slightly to produce fewer short lines: + + "To be," + "or not to" + "be: that" + "is the" + "question" + +Use the new `textwrap::core::WrapAlgorithm` enum to select between the +new and old algorithm. By default, the new algorithm is used. + +The optimal-fit algorithm is inspired by the line breaking algorithm +used in TeX, described in the 1981 article [_Breaking Paragraphs into +Lines_](http://www.eprg.org/G53DOC/pdfs/knuth-plass-breaking.pdf) by +Knuth and Plass. + +#### In-place wrapping + +* [#226](https://github.com/mgeisler/textwrap/pull/226): Add a + `fill_inplace` function. + +When the text you want to fill is already a temporary `String`, you +can now mutate it in-place with `fill_inplace`: + +```rust +let mut greeting = format!("Greetings {}, welcome to the game! You have {} lives left.", + player.name, player.lives); +fill_inplace(&mut greeting, line_width); +``` + +This is faster than calling `fill` and it will reuse the memory +already allocated for the string. + +### Changed Features + +#### `Wrapper` is replaced with `Options` + +* [#213](https://github.com/mgeisler/textwrap/pull/213): Simplify API + with only top-level functions. +* [#215](https://github.com/mgeisler/textwrap/pull/215): Reintroducing + the type parameter on `Options` (previously known as `Wrapper`). +* [#219](https://github.com/mgeisler/textwrap/pull/219): Allow using + trait objects with `fill` & `wrap`. +* [#227](https://github.com/mgeisler/textwrap/pull/227): Replace + `WrapOptions` with `Into`. + +The `Wrapper` struct held the options (line width, indentation, etc) +for wrapping text. It was also the entry point for actually wrapping +the text via its methods such as `wrap`, `wrap_iter`, +`into_wrap_iter`, and `fill` methods. + +The struct has been replaced by a simpler `Options` struct which only +holds options. The `Wrapper` methods are gone, their job has been +taken over by the top-level `wrap` and `fill` functions. The signature +of these functions have changed from + +```rust +fn fill(s: &str, width: usize) -> String; + +fn wrap(s: &str, width: usize) -> Vec>; +``` + +to the more general + +```rust +fn fill<'a, S, Opt>(text: &str, options: Opt) -> String +where + S: WordSplitter, + Opt: Into>; + +fn wrap<'a, S, Opt>(text: &str, options: Opt) -> Vec> +where + S: WordSplitter, + Opt: Into>; +``` + +The `Into` bound allows you to pass an `usize` (which +is interpreted as the line width) *and* a full `Options` object. This +allows the new functions to work like the old, plus you can now fully +customize the behavior of the wrapping via `Options` when needed. + +Code that call `textwrap::wrap` or `textwrap::fill` can remain +unchanged. Code that calls into `Wrapper::wrap` or `Wrapper::fill` +will need to be update. This is a mechanical change, please see +[#213](https://github.com/mgeisler/textwrap/pull/213) for examples. + +Thanks to @CryptJar and @Koxiat for their support in the PRs above! + +### Removed Features + +* The `wrap_iter` and `into_wrap_iter` methods are gone. This means + that lazy iteration is no longer supported: you always get all + wrapped lines back as a `Vec`. This was done to simplify the code + and to support the optimal-fit algorithm. + + The first-fit algorithm could still be implemented in an incremental + fashion. Please let us know if this is important to you. + +### Other Changes + +* [#206](https://github.com/mgeisler/textwrap/pull/206): Change + `Wrapper.splitter` from `T: WordSplitter` to `Box`. +* [#216](https://github.com/mgeisler/textwrap/pull/216): Forbid the + use of unsafe code. + +## Version 0.12.1 (2020-07-03) + +This is a bugfix release. + +* Fixed [#176][issue-176]: Mention compile-time wrapping by linking to + the [`textwrap-macros` crate]. +* Fixed [#193][issue-193]: Wrapping with `break_words(false)` was + broken and would cause extra whitespace to be inserted when words + were longer than the line width. + +## Version 0.12.0 (2020-06-26) + +The code has been updated to the [Rust 2018 edition][rust-2018] and +each new release of `textwrap` will only support the latest stable +version of Rust. Trying to support older Rust versions is a fool's +errand: our dependencies keep releasing new patch versions that +require newer and newer versions of Rust. + +The `term_size` feature has been replaced by `terminal_size`. The API +is unchanged, it is just the name of the Cargo feature that changed. + +The `hyphenation` feature now only embeds the hyphenation patterns for +US-English. This slims down the dependency. + +* Fixed [#140][issue-140]: Ignore ANSI escape sequences. +* Fixed [#158][issue-158]: Unintended wrapping when using external splitter. +* Fixed [#177][issue-177]: Update examples to the 2018 edition. + +## Version 0.11.0 (2018-12-09) + +Due to our dependencies bumping their minimum supported version of +Rust, the minimum version of Rust we test against is now 1.22.0. + +* Merged [#141][issue-141]: Fix `dedent` handling of empty lines and + trailing newlines. Thanks @bbqsrc! +* Fixed [#151][issue-151]: Release of version with hyphenation 0.7. + +## Version 0.10.0 (2018-04-28) + +Due to our dependencies bumping their minimum supported version of +Rust, the minimum version of Rust we test against is now 1.17.0. + +* Fixed [#99][issue-99]: Word broken even though it would fit on line. +* Fixed [#107][issue-107]: Automatic hyphenation is off by one. +* Fixed [#122][issue-122]: Take newlines into account when wrapping. +* Fixed [#129][issue-129]: Panic on string with em-dash. + +## Version 0.9.0 (2017-10-05) + +The dependency on `term_size` is now optional, and by default this +feature is not enabled. This is a *breaking change* for users of +`Wrapper::with_termwidth`. Enable the `term_size` feature to restore +the old functionality. + +Added a regression test for the case where `width` is set to +`usize::MAX`, thanks @Fraser999! All public structs now implement +`Debug`, thanks @hcpl! + +* Fixed [#101][issue-101]: Make `term_size` an optional dependency. + +## Version 0.8.0 (2017-09-04) + +The `Wrapper` struct is now generic over the type of word splitter +being used. This means less boxing and a nicer API. The +`Wrapper::word_splitter` method has been removed. This is a *breaking +API change* if you used the method to change the word splitter. + +The `Wrapper` struct has two new methods that will wrap the input text +lazily: `Wrapper::wrap_iter` and `Wrapper::into_wrap_iter`. Use those +if you will be iterating over the wrapped lines one by one. + +* Fixed [#59][issue-59]: `wrap` could return an iterator. Thanks + @hcpl! +* Fixed [#81][issue-81]: Set `html_root_url`. + +## Version 0.7.0 (2017-07-20) + +Version 0.7.0 changes the return type of `Wrapper::wrap` from +`Vec` to `Vec>`. This means that the output lines +borrow data from the input string. This is a *breaking API change* if +you relied on the exact return type of `Wrapper::wrap`. Callers of the +`textwrap::fill` convenience function will see no breakage. + +The above change and other optimizations makes version 0.7.0 roughly +15-30% faster than version 0.6.0. + +The `squeeze_whitespace` option has been removed since it was +complicating the above optimization. Let us know if this option is +important for you so we can provide a work around. + +* Fixed [#58][issue-58]: Add a "fast_wrap" function. +* Fixed [#61][issue-61]: Documentation errors. + +## Version 0.6.0 (2017-05-22) + +Version 0.6.0 adds builder methods to `Wrapper` for easy one-line +initialization and configuration: + +```rust +let wrapper = Wrapper::new(60).break_words(false); +``` + +It also add a new `NoHyphenation` word splitter that will never split +words, not even at existing hyphens. + +* Fixed [#28][issue-28]: Support not squeezing whitespace. + +## Version 0.5.0 (2017-05-15) + +Version 0.5.0 has *breaking API changes*. However, this only affects +code using the hyphenation feature. The feature is now optional, so +you will first need to enable the `hyphenation` feature as described +above. Afterwards, please change your code from +```rust +wrapper.corpus = Some(&corpus); +``` +to +```rust +wrapper.splitter = Box::new(corpus); +``` + +Other changes include optimizations, so version 0.5.0 is roughly +10-15% faster than version 0.4.0. + +* Fixed [#19][issue-19]: Add support for finding terminal size. +* Fixed [#25][issue-25]: Handle words longer than `self.width`. +* Fixed [#26][issue-26]: Support custom indentation. +* Fixed [#36][issue-36]: Support building without `hyphenation`. +* Fixed [#39][issue-39]: Respect non-breaking spaces. + +## Version 0.4.0 (2017-01-24) + +Documented complexities and tested these via `cargo bench`. + +* Fixed [#13][issue-13]: Immediatedly add word if it fits. +* Fixed [#14][issue-14]: Avoid splitting on initial hyphens. + +## Version 0.3.0 (2017-01-07) + +Added support for automatic hyphenation. + +## Version 0.2.0 (2016-12-28) + +Introduced `Wrapper` struct. Added support for wrapping on hyphens. + +## Version 0.1.0 (2016-12-17) + +First public release with support for wrapping strings on whitespace. + +[rust-2018]: https://doc.rust-lang.org/edition-guide/rust-2018/ +[`textwrap-macros` crate]: https://crates.io/crates/textwrap-macros + +[issue-13]: https://github.com/mgeisler/textwrap/issues/13 +[issue-14]: https://github.com/mgeisler/textwrap/issues/14 +[issue-19]: https://github.com/mgeisler/textwrap/issues/19 +[issue-25]: https://github.com/mgeisler/textwrap/issues/25 +[issue-26]: https://github.com/mgeisler/textwrap/issues/26 +[issue-28]: https://github.com/mgeisler/textwrap/issues/28 +[issue-36]: https://github.com/mgeisler/textwrap/issues/36 +[issue-39]: https://github.com/mgeisler/textwrap/issues/39 +[issue-58]: https://github.com/mgeisler/textwrap/issues/58 +[issue-59]: https://github.com/mgeisler/textwrap/issues/59 +[issue-61]: https://github.com/mgeisler/textwrap/issues/61 +[issue-81]: https://github.com/mgeisler/textwrap/issues/81 +[issue-99]: https://github.com/mgeisler/textwrap/issues/99 +[issue-101]: https://github.com/mgeisler/textwrap/issues/101 +[issue-107]: https://github.com/mgeisler/textwrap/issues/107 +[issue-122]: https://github.com/mgeisler/textwrap/issues/122 +[issue-129]: https://github.com/mgeisler/textwrap/issues/129 +[issue-140]: https://github.com/mgeisler/textwrap/issues/140 +[issue-141]: https://github.com/mgeisler/textwrap/issues/141 +[issue-151]: https://github.com/mgeisler/textwrap/issues/151 +[issue-158]: https://github.com/mgeisler/textwrap/issues/158 +[issue-176]: https://github.com/mgeisler/textwrap/issues/176 +[issue-177]: https://github.com/mgeisler/textwrap/issues/177 +[issue-193]: https://github.com/mgeisler/textwrap/issues/193 diff --git a/third_party/rust/textwrap/Cargo.lock b/third_party/rust/textwrap/Cargo.lock new file mode 100644 index 0000000000..98b642073e --- /dev/null +++ b/third_party/rust/textwrap/Cargo.lock @@ -0,0 +1,657 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "aho-corasick" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" +dependencies = [ + "memchr", +] + +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" + +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "errno" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "form_urlencoded" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "fst" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ab85b9b05e3978cc9a9cf8fea7f01b494e1a09ed3037e16ba39edc7a29eb61a" + +[[package]] +name = "hashbrown" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" + +[[package]] +name = "hermit-abi" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd5256b483761cd23699d0da46cc6fd2ee3be420bbe6d020ae4a091e70b7e9fd" + +[[package]] +name = "hyphenation" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcf4dd4c44ae85155502a52c48739c8a48185d1449fff1963cffee63c28a50f0" +dependencies = [ + "bincode", + "fst", + "hyphenation_commons", + "pocket-resources", + "serde", +] + +[[package]] +name = "hyphenation_commons" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5febe7a2ade5c7d98eb8b75f946c046b335324b06a14ea0998271504134c05bf" +dependencies = [ + "fst", + "serde", +] + +[[package]] +name = "idna" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "indexmap" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "233cf39063f058ea2caae4091bf4a3ef70a653afbc026f5c4a4135d114e3c177" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "io-lifetimes" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys 0.48.0", +] + +[[package]] +name = "libc" +version = "0.2.153" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" + +[[package]] +name = "libredox" +version = "0.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3af92c55d7d839293953fcd0fda5ecfe93297cfde6ffbdec13b41d99c0ba6607" +dependencies = [ + "bitflags 2.4.2", + "libc", + "redox_syscall", +] + +[[package]] +name = "linux-raw-sys" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" + +[[package]] +name = "memchr" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" + +[[package]] +name = "numtoa" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8f8bdf33df195859076e54ab11ee78a1b208382d3a26ec40d142ffc1ecc49ef" + +[[package]] +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" + +[[package]] +name = "pocket-resources" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c135f38778ad324d9e9ee68690bac2c1a51f340fdf96ca13e2ab3914eb2e51d8" + +[[package]] +name = "proc-macro2" +version = "1.0.78" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "pulldown-cmark" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57206b407293d2bcd3af849ce869d52068623f19e1b5ff8e8778e3309439682b" +dependencies = [ + "bitflags 2.4.2", + "memchr", + "unicase", +] + +[[package]] +name = "quote" +version = "1.0.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "redox_syscall" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "redox_termios" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20145670ba436b55d91fc92d25e71160fbfbdd57831631c8d7d36377a476f1cb" + +[[package]] +name = "regex" +version = "1.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bb987efffd3c6d0d8f5f89510bb458559eab11e4f869acb20bf845e016259cd" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" + +[[package]] +name = "rustix" +version = "0.37.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fea8ca367a3a01fe35e6943c400addf443c0f57670e6ec51196f71a4b8762dd2" +dependencies = [ + "bitflags 1.3.2", + "errno", + "io-lifetimes", + "libc", + "linux-raw-sys", + "windows-sys 0.48.0", +] + +[[package]] +name = "semver" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97ed7a9823b74f99c7742f5336af7be5ecd3eeafcb1507d1fa93347b1d589b0" + +[[package]] +name = "serde" +version = "1.0.196" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "870026e60fa08c69f064aa766c10f10b1d62db9ccd4d0abb206472bee0ce3b32" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.196" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33c85360c95e7d137454dc81d9a4ed2b8efd8fbe19cee57357b32b9771fccb67" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_spanned" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb3622f419d1296904700073ea6cc23ad690adbd66f13ea683df73298736f0c1" +dependencies = [ + "serde", +] + +[[package]] +name = "smawk" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c388c1b5e93756d0c740965c41e8822f866621d41acbdf6336a6a168f8840c" + +[[package]] +name = "syn" +version = "2.0.49" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "915aea9e586f80826ee59f8453c1101f9d1c4b3964cd2460185ee8e299ada496" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "terminal_size" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e6bf6f19e9f8ed8d4048dc22981458ebcf406d67e94cd422e5ecd73d63b3237" +dependencies = [ + "rustix", + "windows-sys 0.48.0", +] + +[[package]] +name = "termion" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4648c7def6f2043b2568617b9f9b75eae88ca185dbc1f1fda30e95a85d49d7d" +dependencies = [ + "libc", + "libredox", + "numtoa", + "redox_termios", +] + +[[package]] +name = "textwrap" +version = "0.16.1" +dependencies = [ + "hyphenation", + "smawk", + "terminal_size", + "termion", + "unic-emoji-char", + "unicode-linebreak", + "unicode-width", + "version-sync", +] + +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "toml" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd79e69d3b627db300ff956027cc6c3798cef26d22526befdfcd12feeb6d2257" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.19.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] + +[[package]] +name = "unic-char-property" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8c57a407d9b6fa02b4795eb81c5b6652060a15a7903ea981f3d723e6c0be221" +dependencies = [ + "unic-char-range", +] + +[[package]] +name = "unic-char-range" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0398022d5f700414f6b899e10b8348231abf9173fa93144cbc1a43b9793c1fbc" + +[[package]] +name = "unic-common" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80d7ff825a6a654ee85a63e80f92f054f904f21e7d12da4e22f9834a4aaa35bc" + +[[package]] +name = "unic-emoji-char" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b07221e68897210270a38bde4babb655869637af0f69407f96053a34f76494d" +dependencies = [ + "unic-char-property", + "unic-char-range", + "unic-ucd-version", +] + +[[package]] +name = "unic-ucd-version" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96bd2f2237fe450fcd0a1d2f5f4e91711124f7857ba2e964247776ebeeb7b0c4" +dependencies = [ + "unic-common", +] + +[[package]] +name = "unicase" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89" +dependencies = [ + "version_check", +] + +[[package]] +name = "unicode-bidi" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "unicode-linebreak" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f" + +[[package]] +name = "unicode-normalization" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-width" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" + +[[package]] +name = "url" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "version-sync" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "835169da0173ea373ddf5987632aac1f918967fbbe58195e304342282efa6089" +dependencies = [ + "proc-macro2", + "pulldown-cmark", + "regex", + "semver", + "syn", + "toml", + "url", +] + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.0", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd" +dependencies = [ + "windows_aarch64_gnullvm 0.52.0", + "windows_aarch64_msvc 0.52.0", + "windows_i686_gnu 0.52.0", + "windows_i686_msvc 0.52.0", + "windows_x86_64_gnu 0.52.0", + "windows_x86_64_gnullvm 0.52.0", + "windows_x86_64_msvc 0.52.0", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" + +[[package]] +name = "winnow" +version = "0.5.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f593a95398737aeed53e489c785df13f3618e41dbcd6718c6addbf1395aa6876" +dependencies = [ + "memchr", +] diff --git a/third_party/rust/textwrap/Cargo.toml b/third_party/rust/textwrap/Cargo.toml new file mode 100644 index 0000000000..1493f0a8a9 --- /dev/null +++ b/third_party/rust/textwrap/Cargo.toml @@ -0,0 +1,91 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2021" +rust-version = "1.56" +name = "textwrap" +version = "0.16.1" +authors = ["Martin Geisler "] +exclude = [ + ".github/", + ".gitignore", + "benchmarks/", + "examples/", + "fuzz/", + "images/", +] +description = "Library for word wrapping, indenting, and dedenting strings. Has optional support for Unicode and emojis as well as machine hyphenation." +documentation = "https://docs.rs/textwrap/" +readme = "README.md" +keywords = [ + "text", + "formatting", + "wrap", + "typesetting", + "hyphenation", +] +categories = [ + "text-processing", + "command-line-interface", +] +license = "MIT" +repository = "https://github.com/mgeisler/textwrap" + +[package.metadata.docs.rs] +all-features = true + +[[example]] +name = "hyphenation" +path = "examples/hyphenation.rs" +required-features = ["hyphenation"] + +[[example]] +name = "termwidth" +path = "examples/termwidth.rs" +required-features = ["terminal_size"] + +[dependencies.hyphenation] +version = "0.8.4" +features = ["embed_en-us"] +optional = true + +[dependencies.smawk] +version = "0.3.1" +optional = true + +[dependencies.terminal_size] +version = "0.2.1" +optional = true + +[dependencies.unicode-linebreak] +version = "0.1.4" +optional = true + +[dependencies.unicode-width] +version = "0.1.10" +optional = true + +[dev-dependencies.unic-emoji-char] +version = "0.9.0" + +[dev-dependencies.version-sync] +version = "0.9.4" + +[features] +default = [ + "unicode-linebreak", + "unicode-width", + "smawk", +] + +[target."cfg(unix)".dev-dependencies.termion] +version = "2.0.1" diff --git a/third_party/rust/textwrap/LICENSE b/third_party/rust/textwrap/LICENSE new file mode 100644 index 0000000000..0d37ec3891 --- /dev/null +++ b/third_party/rust/textwrap/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2016 Martin Geisler + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/third_party/rust/textwrap/README.md b/third_party/rust/textwrap/README.md new file mode 100644 index 0000000000..bf13cd0957 --- /dev/null +++ b/third_party/rust/textwrap/README.md @@ -0,0 +1,176 @@ +# Textwrap + +[![](https://github.com/mgeisler/textwrap/workflows/build/badge.svg)][build-status] +[![](https://codecov.io/gh/mgeisler/textwrap/branch/master/graph/badge.svg)][codecov] +[![](https://img.shields.io/crates/v/textwrap.svg)][crates-io] +[![](https://docs.rs/textwrap/badge.svg)][api-docs] + +Textwrap is a library for wrapping and indenting text. It is most +often used by command-line programs to format dynamic output nicely so +it looks good in a terminal. You can also use Textwrap to wrap text +set in a proportional font—such as text used to generate PDF files, or +drawn on a [HTML5 canvas using WebAssembly][wasm-demo]. + +## Usage + +To use the textwrap crate, add this to your `Cargo.toml` file: +```toml +[dependencies] +textwrap = "0.16" +``` + +By default, this enables word wrapping with support for Unicode +strings. Extra features can be enabled with Cargo features—and the +Unicode support can be disabled if needed. This allows you slim down +the library and so you will only pay for the features you actually +use. + +Please see the [_Cargo Features_ in the crate +documentation](https://docs.rs/textwrap/#cargo-features) for a full +list of the available features as well as their impact on the size of +your binary. + +## Documentation + +**[API documentation][api-docs]** + +## Getting Started + +Word wrapping is easy using the `wrap` and `fill` functions: + +```rust +#[cfg(feature = "smawk")] { +let text = "textwrap: an efficient and powerful library for wrapping text."; +assert_eq!( + textwrap::wrap(text, 28), + vec![ + "textwrap: an efficient", + "and powerful library for", + "wrapping text.", + ] +); +} +``` + +Sharp-eyed readers will notice that the first line is 22 columns wide. +So why is the word “and” put in the second line when there is space +for it in the first line? + +The explanation is that textwrap does not just wrap text one line at a +time. Instead, it uses an optimal-fit algorithm which looks ahead and +chooses line breaks which minimize the gaps left at ends of lines. +This is controlled with the `smawk` Cargo feature, which is why the +example is wrapped in the `cfg`-block. + +Without look-ahead, the first line would be longer and the text would +look like this: + +```rust +#[cfg(not(feature = "smawk"))] { +let text = "textwrap: an efficient and powerful library for wrapping text."; +assert_eq!( + textwrap::wrap(text, 28), + vec![ + "textwrap: an efficient and", + "powerful library for", + "wrapping text.", + ] +); +} +``` + +The second line is now shorter and the text is more ragged. The kind +of wrapping can be configured via `Options::wrap_algorithm`. + +If you enable the `hyphenation` Cargo feature, you get support for +automatic hyphenation for [about 70 languages][patterns] via +high-quality TeX hyphenation patterns. + +Your program must load the hyphenation pattern and configure +`Options::word_splitter` to use it: + +```rust +#[cfg(feature = "hyphenation")] { +use hyphenation::{Language, Load, Standard}; +use textwrap::{fill, Options, WordSplitter}; + +let dictionary = Standard::from_embedded(Language::EnglishUS).unwrap(); +let options = textwrap::Options::new(28).word_splitter(WordSplitter::Hyphenation(dictionary)); +let text = "textwrap: an efficient and powerful library for wrapping text."; + +assert_eq!( + textwrap::wrap(text, &options), + vec![ + "textwrap: an efficient and", + "powerful library for wrap-", + "ping text." + ] +); +} +``` + +The US-English hyphenation patterns are embedded when you enable the +`hyphenation` feature. They are licensed under a [permissive +license][en-us license] and take up about 88 KB in your binary. If you +need hyphenation for other languages, you need to download a +[precompiled `.bincode` file][bincode] and load it yourself. Please +see the [`hyphenation` documentation] for details. + +## Wrapping Strings at Compile Time + +If your strings are known at compile time, please take a look at the +procedural macros from the [`textwrap-macros` crate]. + +## Examples + +The library comes with [a +collection](https://github.com/mgeisler/textwrap/tree/master/examples) +of small example programs that shows various features. + +If you want to see Textwrap in action right away, then take a look at +[`examples/wasm/`], which shows how to wrap sans-serif, serif, and +monospace text. It uses WebAssembly and is automatically deployed to +https://mgeisler.github.io/textwrap/. + +For the command-line examples, you’re invited to clone the repository +and try them out for yourself! Of special note is +[`examples/interactive.rs`]. This is a demo program which demonstrates +most of the available features: you can enter text and adjust the +width at which it is wrapped interactively. You can also adjust the +`Options` used to see the effect of different `WordSplitter`s and wrap +algorithms. + +Run the demo with + +```sh +$ cargo run --example interactive +``` + +The demo needs a Linux terminal to function. + +## Release History + +Please see the [CHANGELOG file] for details on the changes made in +each release. + +## License + +Textwrap can be distributed according to the [MIT license][mit]. +Contributions will be accepted under the same license. + +[crates-io]: https://crates.io/crates/textwrap +[build-status]: https://github.com/mgeisler/textwrap/actions?query=workflow%3Abuild+branch%3Amaster +[codecov]: https://codecov.io/gh/mgeisler/textwrap +[wasm-demo]: https://mgeisler.github.io/textwrap/ +[`textwrap-macros` crate]: https://crates.io/crates/textwrap-macros +[`hyphenation` example]: https://github.com/mgeisler/textwrap/blob/master/examples/hyphenation.rs +[`termwidth` example]: https://github.com/mgeisler/textwrap/blob/master/examples/termwidth.rs +[patterns]: https://github.com/tapeinosyne/hyphenation/tree/master/patterns +[en-us license]: https://github.com/hyphenation/tex-hyphen/blob/master/hyph-utf8/tex/generic/hyph-utf8/patterns/tex/hyph-en-us.tex +[bincode]: https://github.com/tapeinosyne/hyphenation/tree/master/dictionaries +[`hyphenation` documentation]: http://docs.rs/hyphenation +[`examples/wasm/`]: https://github.com/mgeisler/textwrap/tree/master/examples/wasm +[`examples/interactive.rs`]: https://github.com/mgeisler/textwrap/tree/master/examples/interactive.rs +[api-docs]: https://docs.rs/textwrap/ +[CHANGELOG file]: https://github.com/mgeisler/textwrap/blob/master/CHANGELOG.md +[mit]: LICENSE diff --git a/third_party/rust/textwrap/rustfmt.toml b/third_party/rust/textwrap/rustfmt.toml new file mode 100644 index 0000000000..c1578aafbc --- /dev/null +++ b/third_party/rust/textwrap/rustfmt.toml @@ -0,0 +1 @@ +imports_granularity = "Module" diff --git a/third_party/rust/textwrap/src/columns.rs b/third_party/rust/textwrap/src/columns.rs new file mode 100644 index 0000000000..d14d5588fa --- /dev/null +++ b/third_party/rust/textwrap/src/columns.rs @@ -0,0 +1,193 @@ +//! Functionality for wrapping text into columns. + +use crate::core::display_width; +use crate::{wrap, Options}; + +/// Wrap text into columns with a given total width. +/// +/// The `left_gap`, `middle_gap` and `right_gap` arguments specify the +/// strings to insert before, between, and after the columns. The +/// total width of all columns and all gaps is specified using the +/// `total_width_or_options` argument. This argument can simply be an +/// integer if you want to use default settings when wrapping, or it +/// can be a [`Options`] value if you want to customize the wrapping. +/// +/// If the columns are narrow, it is recommended to set +/// [`Options::break_words`] to `true` to prevent words from +/// protruding into the margins. +/// +/// The per-column width is computed like this: +/// +/// ``` +/// # let (left_gap, middle_gap, right_gap) = ("", "", ""); +/// # let columns = 2; +/// # let options = textwrap::Options::new(80); +/// let inner_width = options.width +/// - textwrap::core::display_width(left_gap) +/// - textwrap::core::display_width(right_gap) +/// - textwrap::core::display_width(middle_gap) * (columns - 1); +/// let column_width = inner_width / columns; +/// ``` +/// +/// The `text` is wrapped using [`wrap()`] and the given `options` +/// argument, but the width is overwritten to the computed +/// `column_width`. +/// +/// # Panics +/// +/// Panics if `columns` is zero. +/// +/// # Examples +/// +/// ``` +/// use textwrap::wrap_columns; +/// +/// let text = "\ +/// This is an example text, which is wrapped into three columns. \ +/// Notice how the final column can be shorter than the others."; +/// +/// #[cfg(feature = "smawk")] +/// assert_eq!(wrap_columns(text, 3, 50, "| ", " | ", " |"), +/// vec!["| This is | into three | column can be |", +/// "| an example | columns. | shorter than |", +/// "| text, which | Notice how | the others. |", +/// "| is wrapped | the final | |"]); +/// +/// // Without the `smawk` feature, the middle column is a little more uneven: +/// #[cfg(not(feature = "smawk"))] +/// assert_eq!(wrap_columns(text, 3, 50, "| ", " | ", " |"), +/// vec!["| This is an | three | column can be |", +/// "| example text, | columns. | shorter than |", +/// "| which is | Notice how | the others. |", +/// "| wrapped into | the final | |"]); +pub fn wrap_columns<'a, Opt>( + text: &str, + columns: usize, + total_width_or_options: Opt, + left_gap: &str, + middle_gap: &str, + right_gap: &str, +) -> Vec +where + Opt: Into>, +{ + assert!(columns > 0); + + let mut options: Options = total_width_or_options.into(); + + let inner_width = options + .width + .saturating_sub(display_width(left_gap)) + .saturating_sub(display_width(right_gap)) + .saturating_sub(display_width(middle_gap) * (columns - 1)); + + let column_width = std::cmp::max(inner_width / columns, 1); + options.width = column_width; + let last_column_padding = " ".repeat(inner_width % column_width); + let wrapped_lines = wrap(text, options); + let lines_per_column = + wrapped_lines.len() / columns + usize::from(wrapped_lines.len() % columns > 0); + let mut lines = Vec::new(); + for line_no in 0..lines_per_column { + let mut line = String::from(left_gap); + for column_no in 0..columns { + match wrapped_lines.get(line_no + column_no * lines_per_column) { + Some(column_line) => { + line.push_str(column_line); + line.push_str(&" ".repeat(column_width - display_width(column_line))); + } + None => { + line.push_str(&" ".repeat(column_width)); + } + } + if column_no == columns - 1 { + line.push_str(&last_column_padding); + } else { + line.push_str(middle_gap); + } + } + line.push_str(right_gap); + lines.push(line); + } + + lines +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn wrap_columns_empty_text() { + assert_eq!(wrap_columns("", 1, 10, "| ", "", " |"), vec!["| |"]); + } + + #[test] + fn wrap_columns_single_column() { + assert_eq!( + wrap_columns("Foo", 3, 30, "| ", " | ", " |"), + vec!["| Foo | | |"] + ); + } + + #[test] + fn wrap_columns_uneven_columns() { + // The gaps take up a total of 5 columns, so the columns are + // (21 - 5)/4 = 4 columns wide: + assert_eq!( + wrap_columns("Foo Bar Baz Quux", 4, 21, "|", "|", "|"), + vec!["|Foo |Bar |Baz |Quux|"] + ); + // As the total width increases, the last column absorbs the + // excess width: + assert_eq!( + wrap_columns("Foo Bar Baz Quux", 4, 24, "|", "|", "|"), + vec!["|Foo |Bar |Baz |Quux |"] + ); + // Finally, when the width is 25, the columns can be resized + // to a width of (25 - 5)/4 = 5 columns: + assert_eq!( + wrap_columns("Foo Bar Baz Quux", 4, 25, "|", "|", "|"), + vec!["|Foo |Bar |Baz |Quux |"] + ); + } + + #[test] + #[cfg(feature = "unicode-width")] + fn wrap_columns_with_emojis() { + assert_eq!( + wrap_columns( + "Words and a few emojis 😍 wrapped in ⓶ columns", + 2, + 30, + "✨ ", + " ⚽ ", + " 👀" + ), + vec![ + "✨ Words ⚽ wrapped in 👀", + "✨ and a few ⚽ ⓶ columns 👀", + "✨ emojis 😍 ⚽ 👀" + ] + ); + } + + #[test] + fn wrap_columns_big_gaps() { + // The column width shrinks to 1 because the gaps take up all + // the space. + assert_eq!( + wrap_columns("xyz", 2, 10, "----> ", " !!! ", " <----"), + vec![ + "----> x !!! z <----", // + "----> y !!! <----" + ] + ); + } + + #[test] + #[should_panic] + fn wrap_columns_panic_with_zero_columns() { + wrap_columns("", 0, 10, "", "", ""); + } +} diff --git a/third_party/rust/textwrap/src/core.rs b/third_party/rust/textwrap/src/core.rs new file mode 100644 index 0000000000..6b07f763c8 --- /dev/null +++ b/third_party/rust/textwrap/src/core.rs @@ -0,0 +1,461 @@ +//! Building blocks for advanced wrapping functionality. +//! +//! The functions and structs in this module can be used to implement +//! advanced wrapping functionality when [`wrap()`](crate::wrap()) +//! [`fill()`](crate::fill()) don't do what you want. +//! +//! In general, you want to follow these steps when wrapping +//! something: +//! +//! 1. Split your input into [`Fragment`]s. These are abstract blocks +//! of text or content which can be wrapped into lines. See +//! [`WordSeparator`](crate::word_separators::WordSeparator) for +//! how to do this for text. +//! +//! 2. Potentially split your fragments into smaller pieces. This +//! allows you to implement things like hyphenation. If you use the +//! `Word` type, you can use [`WordSplitter`](crate::WordSplitter) +//! enum for this. +//! +//! 3. Potentially break apart fragments that are still too large to +//! fit on a single line. This is implemented in [`break_words`]. +//! +//! 4. Finally take your fragments and put them into lines. There are +//! two algorithms for this in the +//! [`wrap_algorithms`](crate::wrap_algorithms) module: +//! [`wrap_optimal_fit`](crate::wrap_algorithms::wrap_optimal_fit) +//! and [`wrap_first_fit`](crate::wrap_algorithms::wrap_first_fit). +//! The former produces better line breaks, the latter is faster. +//! +//! 5. Iterate through the slices returned by the wrapping functions +//! and construct your lines of output. +//! +//! Please [open an issue](https://github.com/mgeisler/textwrap/) if +//! the functionality here is not sufficient or if you have ideas for +//! improving it. We would love to hear from you! + +/// The CSI or “Control Sequence Introducer” introduces an ANSI escape +/// sequence. This is typically used for colored text and will be +/// ignored when computing the text width. +const CSI: (char, char) = ('\x1b', '['); +/// The final bytes of an ANSI escape sequence must be in this range. +const ANSI_FINAL_BYTE: std::ops::RangeInclusive = '\x40'..='\x7e'; + +/// Skip ANSI escape sequences. +/// +/// The `ch` is the current `char`, the `chars` provide the following +/// characters. The `chars` will be modified if `ch` is the start of +/// an ANSI escape sequence. +/// +/// Returns `true` if one or more chars were skipped. +#[inline] +pub(crate) fn skip_ansi_escape_sequence>(ch: char, chars: &mut I) -> bool { + if ch != CSI.0 { + return false; // Nothing to skip here. + } + + let next = chars.next(); + if next == Some(CSI.1) { + // We have found the start of an ANSI escape code, typically + // used for colored terminal text. We skip until we find a + // "final byte" in the range 0x40–0x7E. + for ch in chars { + if ANSI_FINAL_BYTE.contains(&ch) { + break; + } + } + } else if next == Some(']') { + // We have found the start of an Operating System Command, + // which extends until the next sequence "\x1b\\" (the String + // Terminator sequence) or the BEL character. The BEL + // character is non-standard, but it is still used quite + // often, for example, by GNU ls. + let mut last = ']'; + for new in chars { + if new == '\x07' || (new == '\\' && last == CSI.0) { + break; + } + last = new; + } + } + + true // Indicate that some chars were skipped. +} + +#[cfg(feature = "unicode-width")] +#[inline] +fn ch_width(ch: char) -> usize { + unicode_width::UnicodeWidthChar::width(ch).unwrap_or(0) +} + +/// First character which [`ch_width`] will classify as double-width. +/// Please see [`display_width`]. +#[cfg(not(feature = "unicode-width"))] +const DOUBLE_WIDTH_CUTOFF: char = '\u{1100}'; + +#[cfg(not(feature = "unicode-width"))] +#[inline] +fn ch_width(ch: char) -> usize { + if ch < DOUBLE_WIDTH_CUTOFF { + 1 + } else { + 2 + } +} + +/// Compute the display width of `text` while skipping over ANSI +/// escape sequences. +/// +/// # Examples +/// +/// ``` +/// use textwrap::core::display_width; +/// +/// assert_eq!(display_width("Café Plain"), 10); +/// assert_eq!(display_width("\u{1b}[31mCafé Rouge\u{1b}[0m"), 10); +/// assert_eq!(display_width("\x1b]8;;http://example.com\x1b\\This is a link\x1b]8;;\x1b\\"), 14); +/// ``` +/// +/// **Note:** When the `unicode-width` Cargo feature is disabled, the +/// width of a `char` is determined by a crude approximation which +/// simply counts chars below U+1100 as 1 column wide, and all other +/// characters as 2 columns wide. With the feature enabled, function +/// will correctly deal with [combining characters] in their +/// decomposed form (see [Unicode equivalence]). +/// +/// An example of a decomposed character is “é”, which can be +/// decomposed into: “e” followed by a combining acute accent: “◌́”. +/// Without the `unicode-width` Cargo feature, every `char` below +/// U+1100 has a width of 1. This includes the combining accent: +/// +/// ``` +/// use textwrap::core::display_width; +/// +/// assert_eq!(display_width("Cafe Plain"), 10); +/// #[cfg(feature = "unicode-width")] +/// assert_eq!(display_width("Cafe\u{301} Plain"), 10); +/// #[cfg(not(feature = "unicode-width"))] +/// assert_eq!(display_width("Cafe\u{301} Plain"), 11); +/// ``` +/// +/// ## Emojis and CJK Characters +/// +/// Characters such as emojis and [CJK characters] used in the +/// Chinese, Japanese, and Korean languages are seen as double-width, +/// even if the `unicode-width` feature is disabled: +/// +/// ``` +/// use textwrap::core::display_width; +/// +/// assert_eq!(display_width("😂😭🥺🤣✨😍🙏🥰😊🔥"), 20); +/// assert_eq!(display_width("你好"), 4); // “Nǐ hǎo” or “Hello” in Chinese +/// ``` +/// +/// # Limitations +/// +/// The displayed width of a string cannot always be computed from the +/// string alone. This is because the width depends on the rendering +/// engine used. This is particularly visible with [emoji modifier +/// sequences] where a base emoji is modified with, e.g., skin tone or +/// hair color modifiers. It is up to the rendering engine to detect +/// this and to produce a suitable emoji. +/// +/// A simple example is “❤️”, which consists of “❤” (U+2764: Black +/// Heart Symbol) followed by U+FE0F (Variation Selector-16). By +/// itself, “❤” is a black heart, but if you follow it with the +/// variant selector, you may get a wider red heart. +/// +/// A more complex example would be “👨‍🦰” which should depict a man +/// with red hair. Here the computed width is too large — and the +/// width differs depending on the use of the `unicode-width` feature: +/// +/// ``` +/// use textwrap::core::display_width; +/// +/// assert_eq!("👨‍🦰".chars().collect::>(), ['\u{1f468}', '\u{200d}', '\u{1f9b0}']); +/// #[cfg(feature = "unicode-width")] +/// assert_eq!(display_width("👨‍🦰"), 4); +/// #[cfg(not(feature = "unicode-width"))] +/// assert_eq!(display_width("👨‍🦰"), 6); +/// ``` +/// +/// This happens because the grapheme consists of three code points: +/// “👨” (U+1F468: Man), Zero Width Joiner (U+200D), and “🦰” +/// (U+1F9B0: Red Hair). You can see them above in the test. With +/// `unicode-width` enabled, the ZWJ is correctly seen as having zero +/// width, without it is counted as a double-width character. +/// +/// ## Terminal Support +/// +/// Modern browsers typically do a great job at combining characters +/// as shown above, but terminals often struggle more. As an example, +/// Gnome Terminal version 3.38.1, shows “❤️” as a big red heart, but +/// shows "👨‍🦰" as “👨🦰”. +/// +/// [combining characters]: https://en.wikipedia.org/wiki/Combining_character +/// [Unicode equivalence]: https://en.wikipedia.org/wiki/Unicode_equivalence +/// [CJK characters]: https://en.wikipedia.org/wiki/CJK_characters +/// [emoji modifier sequences]: https://unicode.org/emoji/charts/full-emoji-modifiers.html +pub fn display_width(text: &str) -> usize { + let mut chars = text.chars(); + let mut width = 0; + while let Some(ch) = chars.next() { + if skip_ansi_escape_sequence(ch, &mut chars) { + continue; + } + width += ch_width(ch); + } + width +} + +/// A (text) fragment denotes the unit which we wrap into lines. +/// +/// Fragments represent an abstract _word_ plus the _whitespace_ +/// following the word. In case the word falls at the end of the line, +/// the whitespace is dropped and a so-called _penalty_ is inserted +/// instead (typically `"-"` if the word was hyphenated). +/// +/// For wrapping purposes, the precise content of the word, the +/// whitespace, and the penalty is irrelevant. All we need to know is +/// the displayed width of each part, which this trait provides. +pub trait Fragment: std::fmt::Debug { + /// Displayed width of word represented by this fragment. + fn width(&self) -> f64; + + /// Displayed width of the whitespace that must follow the word + /// when the word is not at the end of a line. + fn whitespace_width(&self) -> f64; + + /// Displayed width of the penalty that must be inserted if the + /// word falls at the end of a line. + fn penalty_width(&self) -> f64; +} + +/// A piece of wrappable text, including any trailing whitespace. +/// +/// A `Word` is an example of a [`Fragment`], so it has a width, +/// trailing whitespace, and potentially a penalty item. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct Word<'a> { + /// Word content. + pub word: &'a str, + /// Whitespace to insert if the word does not fall at the end of a line. + pub whitespace: &'a str, + /// Penalty string to insert if the word falls at the end of a line. + pub penalty: &'a str, + // Cached width in columns. + pub(crate) width: usize, +} + +impl std::ops::Deref for Word<'_> { + type Target = str; + + fn deref(&self) -> &Self::Target { + self.word + } +} + +impl<'a> Word<'a> { + /// Construct a `Word` from a string. + /// + /// A trailing stretch of `' '` is automatically taken to be the + /// whitespace part of the word. + pub fn from(word: &str) -> Word<'_> { + let trimmed = word.trim_end_matches(' '); + Word { + word: trimmed, + width: display_width(trimmed), + whitespace: &word[trimmed.len()..], + penalty: "", + } + } + + /// Break this word into smaller words with a width of at most + /// `line_width`. The whitespace and penalty from this `Word` is + /// added to the last piece. + /// + /// # Examples + /// + /// ``` + /// use textwrap::core::Word; + /// assert_eq!( + /// Word::from("Hello! ").break_apart(3).collect::>(), + /// vec![Word::from("Hel"), Word::from("lo! ")] + /// ); + /// ``` + pub fn break_apart<'b>(&'b self, line_width: usize) -> impl Iterator> + 'b { + let mut char_indices = self.word.char_indices(); + let mut offset = 0; + let mut width = 0; + + std::iter::from_fn(move || { + while let Some((idx, ch)) = char_indices.next() { + if skip_ansi_escape_sequence(ch, &mut char_indices.by_ref().map(|(_, ch)| ch)) { + continue; + } + + if width > 0 && width + ch_width(ch) > line_width { + let word = Word { + word: &self.word[offset..idx], + width: width, + whitespace: "", + penalty: "", + }; + offset = idx; + width = ch_width(ch); + return Some(word); + } + + width += ch_width(ch); + } + + if offset < self.word.len() { + let word = Word { + word: &self.word[offset..], + width: width, + whitespace: self.whitespace, + penalty: self.penalty, + }; + offset = self.word.len(); + return Some(word); + } + + None + }) + } +} + +impl Fragment for Word<'_> { + #[inline] + fn width(&self) -> f64 { + self.width as f64 + } + + // We assume the whitespace consist of ' ' only. This allows us to + // compute the display width in constant time. + #[inline] + fn whitespace_width(&self) -> f64 { + self.whitespace.len() as f64 + } + + // We assume the penalty is `""` or `"-"`. This allows us to + // compute the display width in constant time. + #[inline] + fn penalty_width(&self) -> f64 { + self.penalty.len() as f64 + } +} + +/// Forcibly break words wider than `line_width` into smaller words. +/// +/// This simply calls [`Word::break_apart`] on words that are too +/// wide. This means that no extra `'-'` is inserted, the word is +/// simply broken into smaller pieces. +pub fn break_words<'a, I>(words: I, line_width: usize) -> Vec> +where + I: IntoIterator>, +{ + let mut shortened_words = Vec::new(); + for word in words { + if word.width() > line_width as f64 { + shortened_words.extend(word.break_apart(line_width)); + } else { + shortened_words.push(word); + } + } + shortened_words +} + +#[cfg(test)] +mod tests { + use super::*; + + #[cfg(feature = "unicode-width")] + use unicode_width::UnicodeWidthChar; + + #[test] + fn skip_ansi_escape_sequence_works() { + let blue_text = "\u{1b}[34mHello\u{1b}[0m"; + let mut chars = blue_text.chars(); + let ch = chars.next().unwrap(); + assert!(skip_ansi_escape_sequence(ch, &mut chars)); + assert_eq!(chars.next(), Some('H')); + } + + #[test] + fn emojis_have_correct_width() { + use unic_emoji_char::is_emoji; + + // Emojis in the Basic Latin (ASCII) and Latin-1 Supplement + // blocks all have a width of 1 column. This includes + // characters such as '#' and '©'. + for ch in '\u{1}'..'\u{FF}' { + if is_emoji(ch) { + let desc = format!("{:?} U+{:04X}", ch, ch as u32); + + #[cfg(feature = "unicode-width")] + assert_eq!(ch.width().unwrap(), 1, "char: {}", desc); + + #[cfg(not(feature = "unicode-width"))] + assert_eq!(ch_width(ch), 1, "char: {}", desc); + } + } + + // Emojis in the remaining blocks of the Basic Multilingual + // Plane (BMP), in the Supplementary Multilingual Plane (SMP), + // and in the Supplementary Ideographic Plane (SIP), are all 1 + // or 2 columns wide when unicode-width is used, and always 2 + // columns wide otherwise. This includes all of our favorite + // emojis such as 😊. + for ch in '\u{FF}'..'\u{2FFFF}' { + if is_emoji(ch) { + let desc = format!("{:?} U+{:04X}", ch, ch as u32); + + #[cfg(feature = "unicode-width")] + assert!(ch.width().unwrap() <= 2, "char: {}", desc); + + #[cfg(not(feature = "unicode-width"))] + assert_eq!(ch_width(ch), 2, "char: {}", desc); + } + } + + // The remaining planes contain almost no assigned code points + // and thus also no emojis. + } + + #[test] + fn display_width_works() { + assert_eq!("Café Plain".len(), 11); // “é” is two bytes + assert_eq!(display_width("Café Plain"), 10); + assert_eq!(display_width("\u{1b}[31mCafé Rouge\u{1b}[0m"), 10); + assert_eq!( + display_width("\x1b]8;;http://example.com\x1b\\This is a link\x1b]8;;\x1b\\"), + 14 + ); + } + + #[test] + fn display_width_narrow_emojis() { + #[cfg(feature = "unicode-width")] + assert_eq!(display_width("⁉"), 1); + + // The ⁉ character is above DOUBLE_WIDTH_CUTOFF. + #[cfg(not(feature = "unicode-width"))] + assert_eq!(display_width("⁉"), 2); + } + + #[test] + fn display_width_narrow_emojis_variant_selector() { + #[cfg(feature = "unicode-width")] + assert_eq!(display_width("⁉\u{fe0f}"), 1); + + // The variant selector-16 is also counted. + #[cfg(not(feature = "unicode-width"))] + assert_eq!(display_width("⁉\u{fe0f}"), 4); + } + + #[test] + fn display_width_emojis() { + assert_eq!(display_width("😂😭🥺🤣✨😍🙏🥰😊🔥"), 20); + } +} diff --git a/third_party/rust/textwrap/src/fill.rs b/third_party/rust/textwrap/src/fill.rs new file mode 100644 index 0000000000..fbcaab9e21 --- /dev/null +++ b/third_party/rust/textwrap/src/fill.rs @@ -0,0 +1,298 @@ +//! Functions for filling text. + +use crate::{wrap, wrap_algorithms, Options, WordSeparator}; + +/// Fill a line of text at a given width. +/// +/// The result is a [`String`], complete with newlines between each +/// line. Use [`wrap()`] if you need access to the individual lines. +/// +/// The easiest way to use this function is to pass an integer for +/// `width_or_options`: +/// +/// ``` +/// use textwrap::fill; +/// +/// assert_eq!( +/// fill("Memory safety without garbage collection.", 15), +/// "Memory safety\nwithout garbage\ncollection." +/// ); +/// ``` +/// +/// If you need to customize the wrapping, you can pass an [`Options`] +/// instead of an `usize`: +/// +/// ``` +/// use textwrap::{fill, Options}; +/// +/// let options = Options::new(15) +/// .initial_indent("- ") +/// .subsequent_indent(" "); +/// assert_eq!( +/// fill("Memory safety without garbage collection.", &options), +/// "- Memory safety\n without\n garbage\n collection." +/// ); +/// ``` +pub fn fill<'a, Opt>(text: &str, width_or_options: Opt) -> String +where + Opt: Into>, +{ + let options = width_or_options.into(); + + if text.len() < options.width && !text.contains('\n') && options.initial_indent.is_empty() { + String::from(text.trim_end_matches(' ')) + } else { + fill_slow_path(text, options) + } +} + +/// Slow path for fill. +/// +/// This is taken when `text` is longer than `options.width`. +pub(crate) fn fill_slow_path(text: &str, options: Options<'_>) -> String { + // This will avoid reallocation in simple cases (no + // indentation, no hyphenation). + let mut result = String::with_capacity(text.len()); + + let line_ending_str = options.line_ending.as_str(); + for (i, line) in wrap(text, options).iter().enumerate() { + if i > 0 { + result.push_str(line_ending_str); + } + result.push_str(line); + } + + result +} + +/// Fill `text` in-place without reallocating the input string. +/// +/// This function works by modifying the input string: some `' '` +/// characters will be replaced by `'\n'` characters. The rest of the +/// text remains untouched. +/// +/// Since we can only replace existing whitespace in the input with +/// `'\n'` (there is no space for `"\r\n"`), we cannot do hyphenation +/// nor can we split words longer than the line width. We also need to +/// use `AsciiSpace` as the word separator since we need `' '` +/// characters between words in order to replace some of them with a +/// `'\n'`. Indentation is also ruled out. In other words, +/// `fill_inplace(width)` behaves as if you had called [`fill()`] with +/// these options: +/// +/// ``` +/// # use textwrap::{core, LineEnding, Options, WordSplitter, WordSeparator, WrapAlgorithm}; +/// # let width = 80; +/// Options::new(width) +/// .break_words(false) +/// .line_ending(LineEnding::LF) +/// .word_separator(WordSeparator::AsciiSpace) +/// .wrap_algorithm(WrapAlgorithm::FirstFit) +/// .word_splitter(WordSplitter::NoHyphenation); +/// ``` +/// +/// The wrap algorithm is +/// [`WrapAlgorithm::FirstFit`](crate::WrapAlgorithm::FirstFit) since +/// this is the fastest algorithm — and the main reason to use +/// `fill_inplace` is to get the string broken into newlines as fast +/// as possible. +/// +/// A last difference is that (unlike [`fill()`]) `fill_inplace` can +/// leave trailing whitespace on lines. This is because we wrap by +/// inserting a `'\n'` at the final whitespace in the input string: +/// +/// ``` +/// let mut text = String::from("Hello World!"); +/// textwrap::fill_inplace(&mut text, 10); +/// assert_eq!(text, "Hello \nWorld!"); +/// ``` +/// +/// If we didn't do this, the word `World!` would end up being +/// indented. You can avoid this if you make sure that your input text +/// has no double spaces. +/// +/// # Performance +/// +/// In benchmarks, `fill_inplace` is about twice as fast as +/// [`fill()`]. Please see the [`linear` +/// benchmark](https://github.com/mgeisler/textwrap/blob/master/benchmarks/linear.rs) +/// for details. +pub fn fill_inplace(text: &mut String, width: usize) { + let mut indices = Vec::new(); + + let mut offset = 0; + for line in text.split('\n') { + let words = WordSeparator::AsciiSpace + .find_words(line) + .collect::>(); + let wrapped_words = wrap_algorithms::wrap_first_fit(&words, &[width as f64]); + + let mut line_offset = offset; + for words in &wrapped_words[..wrapped_words.len() - 1] { + let line_len = words + .iter() + .map(|word| word.len() + word.whitespace.len()) + .sum::(); + + line_offset += line_len; + // We've advanced past all ' ' characters -- want to move + // one ' ' backwards and insert our '\n' there. + indices.push(line_offset - 1); + } + + // Advance past entire line, plus the '\n' which was removed + // by the split call above. + offset += line.len() + 1; + } + + let mut bytes = std::mem::take(text).into_bytes(); + for idx in indices { + bytes[idx] = b'\n'; + } + *text = String::from_utf8(bytes).unwrap(); +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::WrapAlgorithm; + + #[test] + fn fill_simple() { + assert_eq!(fill("foo bar baz", 10), "foo bar\nbaz"); + } + + #[test] + fn fill_unicode_boundary() { + // https://github.com/mgeisler/textwrap/issues/390 + fill("\u{1b}!Ͽ", 10); + } + + #[test] + fn non_breaking_space() { + let options = Options::new(5).break_words(false); + assert_eq!(fill("foo bar baz", &options), "foo bar baz"); + } + + #[test] + fn non_breaking_hyphen() { + let options = Options::new(5).break_words(false); + assert_eq!(fill("foo‑bar‑baz", &options), "foo‑bar‑baz"); + } + + #[test] + fn fill_preserves_line_breaks_trims_whitespace() { + assert_eq!(fill(" ", 80), ""); + assert_eq!(fill(" \n ", 80), "\n"); + assert_eq!(fill(" \n \n \n ", 80), "\n\n\n"); + } + + #[test] + fn preserve_line_breaks() { + assert_eq!(fill("", 80), ""); + assert_eq!(fill("\n", 80), "\n"); + assert_eq!(fill("\n\n\n", 80), "\n\n\n"); + assert_eq!(fill("test\n", 80), "test\n"); + assert_eq!(fill("test\n\na\n\n", 80), "test\n\na\n\n"); + assert_eq!( + fill( + "1 3 5 7\n1 3 5 7", + Options::new(7).wrap_algorithm(WrapAlgorithm::FirstFit) + ), + "1 3 5 7\n1 3 5 7" + ); + assert_eq!( + fill( + "1 3 5 7\n1 3 5 7", + Options::new(5).wrap_algorithm(WrapAlgorithm::FirstFit) + ), + "1 3 5\n7\n1 3 5\n7" + ); + } + + #[test] + fn break_words_line_breaks() { + assert_eq!(fill("ab\ncdefghijkl", 5), "ab\ncdefg\nhijkl"); + assert_eq!(fill("abcdefgh\nijkl", 5), "abcde\nfgh\nijkl"); + } + + #[test] + fn break_words_empty_lines() { + assert_eq!( + fill("foo\nbar", &Options::new(2).break_words(false)), + "foo\nbar" + ); + } + + #[test] + fn fill_inplace_empty() { + let mut text = String::from(""); + fill_inplace(&mut text, 80); + assert_eq!(text, ""); + } + + #[test] + fn fill_inplace_simple() { + let mut text = String::from("foo bar baz"); + fill_inplace(&mut text, 10); + assert_eq!(text, "foo bar\nbaz"); + } + + #[test] + fn fill_inplace_multiple_lines() { + let mut text = String::from("Some text to wrap over multiple lines"); + fill_inplace(&mut text, 12); + assert_eq!(text, "Some text to\nwrap over\nmultiple\nlines"); + } + + #[test] + fn fill_inplace_long_word() { + let mut text = String::from("Internationalization is hard"); + fill_inplace(&mut text, 10); + assert_eq!(text, "Internationalization\nis hard"); + } + + #[test] + fn fill_inplace_no_hyphen_splitting() { + let mut text = String::from("A well-chosen example"); + fill_inplace(&mut text, 10); + assert_eq!(text, "A\nwell-chosen\nexample"); + } + + #[test] + fn fill_inplace_newlines() { + let mut text = String::from("foo bar\n\nbaz\n\n\n"); + fill_inplace(&mut text, 10); + assert_eq!(text, "foo bar\n\nbaz\n\n\n"); + } + + #[test] + fn fill_inplace_newlines_reset_line_width() { + let mut text = String::from("1 3 5\n1 3 5 7 9\n1 3 5 7 9 1 3"); + fill_inplace(&mut text, 10); + assert_eq!(text, "1 3 5\n1 3 5 7 9\n1 3 5 7 9\n1 3"); + } + + #[test] + fn fill_inplace_leading_whitespace() { + let mut text = String::from(" foo bar baz"); + fill_inplace(&mut text, 10); + assert_eq!(text, " foo bar\nbaz"); + } + + #[test] + fn fill_inplace_trailing_whitespace() { + let mut text = String::from("foo bar baz "); + fill_inplace(&mut text, 10); + assert_eq!(text, "foo bar\nbaz "); + } + + #[test] + fn fill_inplace_interior_whitespace() { + // To avoid an unwanted indentation of "baz", it is important + // to replace the final ' ' with '\n'. + let mut text = String::from("foo bar baz"); + fill_inplace(&mut text, 10); + assert_eq!(text, "foo bar \nbaz"); + } +} diff --git a/third_party/rust/textwrap/src/fuzzing.rs b/third_party/rust/textwrap/src/fuzzing.rs new file mode 100644 index 0000000000..b7ad4812a2 --- /dev/null +++ b/third_party/rust/textwrap/src/fuzzing.rs @@ -0,0 +1,23 @@ +//! Fuzzing helpers. + +use super::Options; +use std::borrow::Cow; + +/// Exposed for fuzzing so we can check the slow path is correct. +pub fn fill_slow_path<'a>(text: &str, options: Options<'_>) -> String { + crate::fill::fill_slow_path(text, options) +} + +/// Exposed for fuzzing so we can check the slow path is correct. +pub fn wrap_single_line<'a>(line: &'a str, options: &Options<'_>, lines: &mut Vec>) { + crate::wrap::wrap_single_line(line, options, lines); +} + +/// Exposed for fuzzing so we can check the slow path is correct. +pub fn wrap_single_line_slow_path<'a>( + line: &'a str, + options: &Options<'_>, + lines: &mut Vec>, +) { + crate::wrap::wrap_single_line_slow_path(line, options, lines) +} diff --git a/third_party/rust/textwrap/src/indentation.rs b/third_party/rust/textwrap/src/indentation.rs new file mode 100644 index 0000000000..2f3a853b3c --- /dev/null +++ b/third_party/rust/textwrap/src/indentation.rs @@ -0,0 +1,347 @@ +//! Functions related to adding and removing indentation from lines of +//! text. +//! +//! The functions here can be used to uniformly indent or dedent +//! (unindent) word wrapped lines of text. + +/// Indent each line by the given prefix. +/// +/// # Examples +/// +/// ``` +/// use textwrap::indent; +/// +/// assert_eq!(indent("First line.\nSecond line.\n", " "), +/// " First line.\n Second line.\n"); +/// ``` +/// +/// When indenting, trailing whitespace is stripped from the prefix. +/// This means that empty lines remain empty afterwards: +/// +/// ``` +/// use textwrap::indent; +/// +/// assert_eq!(indent("First line.\n\n\nSecond line.\n", " "), +/// " First line.\n\n\n Second line.\n"); +/// ``` +/// +/// Notice how `"\n\n\n"` remained as `"\n\n\n"`. +/// +/// This feature is useful when you want to indent text and have a +/// space between your prefix and the text. In this case, you _don't_ +/// want a trailing space on empty lines: +/// +/// ``` +/// use textwrap::indent; +/// +/// assert_eq!(indent("foo = 123\n\nprint(foo)\n", "# "), +/// "# foo = 123\n#\n# print(foo)\n"); +/// ``` +/// +/// Notice how `"\n\n"` became `"\n#\n"` instead of `"\n# \n"` which +/// would have trailing whitespace. +/// +/// Leading and trailing whitespace coming from the text itself is +/// kept unchanged: +/// +/// ``` +/// use textwrap::indent; +/// +/// assert_eq!(indent(" \t Foo ", "->"), "-> \t Foo "); +/// ``` +pub fn indent(s: &str, prefix: &str) -> String { + // We know we'll need more than s.len() bytes for the output, but + // without counting '\n' characters (which is somewhat slow), we + // don't know exactly how much. However, we can preemptively do + // the first doubling of the output size. + let mut result = String::with_capacity(2 * s.len()); + let trimmed_prefix = prefix.trim_end(); + for (idx, line) in s.split_terminator('\n').enumerate() { + if idx > 0 { + result.push('\n'); + } + if line.trim().is_empty() { + result.push_str(trimmed_prefix); + } else { + result.push_str(prefix); + } + result.push_str(line); + } + if s.ends_with('\n') { + // split_terminator will have eaten the final '\n'. + result.push('\n'); + } + result +} + +/// Removes common leading whitespace from each line. +/// +/// This function will look at each non-empty line and determine the +/// maximum amount of whitespace that can be removed from all lines: +/// +/// ``` +/// use textwrap::dedent; +/// +/// assert_eq!(dedent(" +/// 1st line +/// 2nd line +/// 3rd line +/// "), " +/// 1st line +/// 2nd line +/// 3rd line +/// "); +/// ``` +pub fn dedent(s: &str) -> String { + let mut prefix = ""; + let mut lines = s.lines(); + + // We first search for a non-empty line to find a prefix. + for line in &mut lines { + let mut whitespace_idx = line.len(); + for (idx, ch) in line.char_indices() { + if !ch.is_whitespace() { + whitespace_idx = idx; + break; + } + } + + // Check if the line had anything but whitespace + if whitespace_idx < line.len() { + prefix = &line[..whitespace_idx]; + break; + } + } + + // We then continue looking through the remaining lines to + // possibly shorten the prefix. + for line in &mut lines { + let mut whitespace_idx = line.len(); + for ((idx, a), b) in line.char_indices().zip(prefix.chars()) { + if a != b { + whitespace_idx = idx; + break; + } + } + + // Check if the line had anything but whitespace and if we + // have found a shorter prefix + if whitespace_idx < line.len() && whitespace_idx < prefix.len() { + prefix = &line[..whitespace_idx]; + } + } + + // We now go over the lines a second time to build the result. + let mut result = String::new(); + for line in s.lines() { + if line.starts_with(prefix) && line.chars().any(|c| !c.is_whitespace()) { + let (_, tail) = line.split_at(prefix.len()); + result.push_str(tail); + } + result.push('\n'); + } + + if result.ends_with('\n') && !s.ends_with('\n') { + let new_len = result.len() - 1; + result.truncate(new_len); + } + + result +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn indent_empty() { + assert_eq!(indent("\n", " "), "\n"); + } + + #[test] + #[rustfmt::skip] + fn indent_nonempty() { + let text = [ + " foo\n", + "bar\n", + " baz\n", + ].join(""); + let expected = [ + "// foo\n", + "// bar\n", + "// baz\n", + ].join(""); + assert_eq!(indent(&text, "// "), expected); + } + + #[test] + #[rustfmt::skip] + fn indent_empty_line() { + let text = [ + " foo", + "bar", + "", + " baz", + ].join("\n"); + let expected = [ + "// foo", + "// bar", + "//", + "// baz", + ].join("\n"); + assert_eq!(indent(&text, "// "), expected); + } + + #[test] + fn dedent_empty() { + assert_eq!(dedent(""), ""); + } + + #[test] + #[rustfmt::skip] + fn dedent_multi_line() { + let x = [ + " foo", + " bar", + " baz", + ].join("\n"); + let y = [ + " foo", + "bar", + " baz" + ].join("\n"); + assert_eq!(dedent(&x), y); + } + + #[test] + #[rustfmt::skip] + fn dedent_empty_line() { + let x = [ + " foo", + " bar", + " ", + " baz" + ].join("\n"); + let y = [ + " foo", + "bar", + "", + " baz" + ].join("\n"); + assert_eq!(dedent(&x), y); + } + + #[test] + #[rustfmt::skip] + fn dedent_blank_line() { + let x = [ + " foo", + "", + " bar", + " foo", + " bar", + " baz", + ].join("\n"); + let y = [ + "foo", + "", + " bar", + " foo", + " bar", + " baz", + ].join("\n"); + assert_eq!(dedent(&x), y); + } + + #[test] + #[rustfmt::skip] + fn dedent_whitespace_line() { + let x = [ + " foo", + " ", + " bar", + " foo", + " bar", + " baz", + ].join("\n"); + let y = [ + "foo", + "", + " bar", + " foo", + " bar", + " baz", + ].join("\n"); + assert_eq!(dedent(&x), y); + } + + #[test] + #[rustfmt::skip] + fn dedent_mixed_whitespace() { + let x = [ + "\tfoo", + " bar", + ].join("\n"); + let y = [ + "\tfoo", + " bar", + ].join("\n"); + assert_eq!(dedent(&x), y); + } + + #[test] + #[rustfmt::skip] + fn dedent_tabbed_whitespace() { + let x = [ + "\t\tfoo", + "\t\t\tbar", + ].join("\n"); + let y = [ + "foo", + "\tbar", + ].join("\n"); + assert_eq!(dedent(&x), y); + } + + #[test] + #[rustfmt::skip] + fn dedent_mixed_tabbed_whitespace() { + let x = [ + "\t \tfoo", + "\t \t\tbar", + ].join("\n"); + let y = [ + "foo", + "\tbar", + ].join("\n"); + assert_eq!(dedent(&x), y); + } + + #[test] + #[rustfmt::skip] + fn dedent_mixed_tabbed_whitespace2() { + let x = [ + "\t \tfoo", + "\t \tbar", + ].join("\n"); + let y = [ + "\tfoo", + " \tbar", + ].join("\n"); + assert_eq!(dedent(&x), y); + } + + #[test] + #[rustfmt::skip] + fn dedent_preserve_no_terminating_newline() { + let x = [ + " foo", + " bar", + ].join("\n"); + let y = [ + "foo", + " bar", + ].join("\n"); + assert_eq!(dedent(&x), y); + } +} diff --git a/third_party/rust/textwrap/src/lib.rs b/third_party/rust/textwrap/src/lib.rs new file mode 100644 index 0000000000..32611c0938 --- /dev/null +++ b/third_party/rust/textwrap/src/lib.rs @@ -0,0 +1,235 @@ +//! The textwrap library provides functions for word wrapping and +//! indenting text. +//! +//! # Wrapping Text +//! +//! Wrapping text can be very useful in command-line programs where +//! you want to format dynamic output nicely so it looks good in a +//! terminal. A quick example: +//! +//! ``` +//! # #[cfg(feature = "smawk")] { +//! let text = "textwrap: a small library for wrapping text."; +//! assert_eq!(textwrap::wrap(text, 18), +//! vec!["textwrap: a", +//! "small library for", +//! "wrapping text."]); +//! # } +//! ``` +//! +//! The [`wrap()`] function returns the individual lines, use +//! [`fill()`] is you want the lines joined with `'\n'` to form a +//! `String`. +//! +//! If you enable the `hyphenation` Cargo feature, you can get +//! automatic hyphenation for a number of languages: +//! +//! ``` +//! #[cfg(feature = "hyphenation")] { +//! use hyphenation::{Language, Load, Standard}; +//! use textwrap::{wrap, Options, WordSplitter}; +//! +//! let text = "textwrap: a small library for wrapping text."; +//! let dictionary = Standard::from_embedded(Language::EnglishUS).unwrap(); +//! let options = Options::new(18).word_splitter(WordSplitter::Hyphenation(dictionary)); +//! assert_eq!(wrap(text, &options), +//! vec!["textwrap: a small", +//! "library for wrap-", +//! "ping text."]); +//! } +//! ``` +//! +//! See also the [`unfill()`] and [`refill()`] functions which allow +//! you to manipulate already wrapped text. +//! +//! ## Wrapping Strings at Compile Time +//! +//! If your strings are known at compile time, please take a look at +//! the procedural macros from the [textwrap-macros] crate. +//! +//! ## Displayed Width vs Byte Size +//! +//! To word wrap text, one must know the width of each word so one can +//! know when to break lines. This library will by default measure the +//! width of text using the _displayed width_, not the size in bytes. +//! The `unicode-width` Cargo feature controls this. +//! +//! This is important for non-ASCII text. ASCII characters such as `a` +//! and `!` are simple and take up one column each. This means that +//! the displayed width is equal to the string length in bytes. +//! However, non-ASCII characters and symbols take up more than one +//! byte when UTF-8 encoded: `é` is `0xc3 0xa9` (two bytes) and `⚙` is +//! `0xe2 0x9a 0x99` (three bytes) in UTF-8, respectively. +//! +//! This is why we take care to use the displayed width instead of the +//! byte count when computing line lengths. All functions in this +//! library handle Unicode characters like this when the +//! `unicode-width` Cargo feature is enabled (it is enabled by +//! default). +//! +//! # Indentation and Dedentation +//! +//! The textwrap library also offers functions for adding a prefix to +//! every line of a string and to remove leading whitespace. As an +//! example, [`indent()`] allows you to turn lines of text into a +//! bullet list: +//! +//! ``` +//! let before = "\ +//! foo +//! bar +//! baz +//! "; +//! let after = "\ +//! * foo +//! * bar +//! * baz +//! "; +//! assert_eq!(textwrap::indent(before, "* "), after); +//! ``` +//! +//! Removing leading whitespace is done with [`dedent()`]: +//! +//! ``` +//! let before = " +//! Some +//! indented +//! text +//! "; +//! let after = " +//! Some +//! indented +//! text +//! "; +//! assert_eq!(textwrap::dedent(before), after); +//! ``` +//! +//! # Cargo Features +//! +//! The textwrap library can be slimmed down as needed via a number of +//! Cargo features. This means you only pay for the features you +//! actually use. +//! +//! The full dependency graph, where dashed lines indicate optional +//! dependencies, is shown below: +//! +//! +//! +//! ## Default Features +//! +//! These features are enabled by default: +//! +//! * `unicode-linebreak`: enables finding words using the +//! [unicode-linebreak] crate, which implements the line breaking +//! algorithm described in [Unicode Standard Annex +//! #14](https://www.unicode.org/reports/tr14/). +//! +//! This feature can be disabled if you are happy to find words +//! separated by ASCII space characters only. People wrapping text +//! with emojis or East-Asian characters will want most likely want +//! to enable this feature. See [`WordSeparator`] for details. +//! +//! * `unicode-width`: enables correct width computation of non-ASCII +//! characters via the [unicode-width] crate. Without this feature, +//! every [`char`] is 1 column wide, except for emojis which are 2 +//! columns wide. See [`core::display_width()`] for details. +//! +//! This feature can be disabled if you only need to wrap ASCII +//! text, or if the functions in [`core`] are used directly with +//! [`core::Fragment`]s for which the widths have been computed in +//! other ways. +//! +//! * `smawk`: enables linear-time wrapping of the whole paragraph via +//! the [smawk] crate. See [`wrap_algorithms::wrap_optimal_fit()`] +//! for details on the optimal-fit algorithm. +//! +//! This feature can be disabled if you only ever intend to use +//! [`wrap_algorithms::wrap_first_fit()`]. +//! +//! +//! +//! With Rust 1.64.0, the size impact of the above features on your +//! binary is as follows: +//! +//! | Configuration | Binary Size | Delta | +//! | :--- | ---: | ---: | +//! | quick-and-dirty implementation | 289 KB | — KB | +//! | textwrap without default features | 305 KB | 16 KB | +//! | textwrap with smawk | 317 KB | 28 KB | +//! | textwrap with unicode-width | 309 KB | 20 KB | +//! | textwrap with unicode-linebreak | 342 KB | 53 KB | +//! +//! +//! +//! The above sizes are the stripped sizes and the binary is compiled +//! in release mode with this profile: +//! +//! ```toml +//! [profile.release] +//! lto = true +//! codegen-units = 1 +//! ``` +//! +//! See the [binary-sizes demo] if you want to reproduce these +//! results. +//! +//! ## Optional Features +//! +//! These Cargo features enable new functionality: +//! +//! * `terminal_size`: enables automatic detection of the terminal +//! width via the [terminal_size] crate. See +//! [`Options::with_termwidth()`] for details. +//! +//! * `hyphenation`: enables language-sensitive hyphenation via the +//! [hyphenation] crate. See the [`word_splitters::WordSplitter`] +//! trait for details. +//! +//! [unicode-linebreak]: https://docs.rs/unicode-linebreak/ +//! [unicode-width]: https://docs.rs/unicode-width/ +//! [smawk]: https://docs.rs/smawk/ +//! [binary-sizes demo]: https://github.com/mgeisler/textwrap/tree/master/examples/binary-sizes +//! [textwrap-macros]: https://docs.rs/textwrap-macros/ +//! [terminal_size]: https://docs.rs/terminal_size/ +//! [hyphenation]: https://docs.rs/hyphenation/ + +#![doc(html_root_url = "https://docs.rs/textwrap/0.16.1")] +#![forbid(unsafe_code)] // See https://github.com/mgeisler/textwrap/issues/210 +#![deny(missing_docs)] +#![deny(missing_debug_implementations)] +#![allow(clippy::redundant_field_names)] + +// Make `cargo test` execute the README doctests. +#[cfg(doctest)] +#[doc = include_str!("../README.md")] +mod readme_doctest {} + +pub mod core; +#[cfg(fuzzing)] +pub mod fuzzing; +pub mod word_splitters; +pub mod wrap_algorithms; + +mod columns; +mod fill; +mod indentation; +mod line_ending; +mod options; +mod refill; +#[cfg(feature = "terminal_size")] +mod termwidth; +mod word_separators; +mod wrap; + +pub use columns::wrap_columns; +pub use fill::{fill, fill_inplace}; +pub use indentation::{dedent, indent}; +pub use line_ending::LineEnding; +pub use options::Options; +pub use refill::{refill, unfill}; +#[cfg(feature = "terminal_size")] +pub use termwidth::termwidth; +pub use word_separators::WordSeparator; +pub use word_splitters::WordSplitter; +pub use wrap::wrap; +pub use wrap_algorithms::WrapAlgorithm; diff --git a/third_party/rust/textwrap/src/line_ending.rs b/third_party/rust/textwrap/src/line_ending.rs new file mode 100644 index 0000000000..0514fe5fc0 --- /dev/null +++ b/third_party/rust/textwrap/src/line_ending.rs @@ -0,0 +1,88 @@ +//! Line ending detection and conversion. + +use std::fmt::Debug; + +/// Supported line endings. Like in the Rust standard library, two line +/// endings are supported: `\r\n` and `\n` +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum LineEnding { + /// _Carriage return and line feed_ – a line ending sequence + /// historically used in Windows. Corresponds to the sequence + /// of ASCII control characters `0x0D 0x0A` or `\r\n` + CRLF, + /// _Line feed_ – a line ending historically used in Unix. + /// Corresponds to the ASCII control character `0x0A` or `\n` + LF, +} + +impl LineEnding { + /// Turns this [`LineEnding`] value into its ASCII representation. + #[inline] + pub const fn as_str(&self) -> &'static str { + match self { + Self::CRLF => "\r\n", + Self::LF => "\n", + } + } +} + +/// An iterator over the lines of a string, as tuples of string slice +/// and [`LineEnding`] value; it only emits non-empty lines (i.e. having +/// some content before the terminating `\r\n` or `\n`). +/// +/// This struct is used internally by the library. +#[derive(Debug, Clone, Copy)] +pub(crate) struct NonEmptyLines<'a>(pub &'a str); + +impl<'a> Iterator for NonEmptyLines<'a> { + type Item = (&'a str, Option); + + fn next(&mut self) -> Option { + while let Some(lf) = self.0.find('\n') { + if lf == 0 || (lf == 1 && self.0.as_bytes()[lf - 1] == b'\r') { + self.0 = &self.0[(lf + 1)..]; + continue; + } + let trimmed = match self.0.as_bytes()[lf - 1] { + b'\r' => (&self.0[..(lf - 1)], Some(LineEnding::CRLF)), + _ => (&self.0[..lf], Some(LineEnding::LF)), + }; + self.0 = &self.0[(lf + 1)..]; + return Some(trimmed); + } + if self.0.is_empty() { + None + } else { + let line = std::mem::take(&mut self.0); + Some((line, None)) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn non_empty_lines_full_case() { + assert_eq!( + NonEmptyLines("LF\nCRLF\r\n\r\n\nunterminated") + .collect::)>>(), + vec![ + ("LF", Some(LineEnding::LF)), + ("CRLF", Some(LineEnding::CRLF)), + ("unterminated", None), + ] + ); + } + + #[test] + fn non_empty_lines_new_lines_only() { + assert_eq!(NonEmptyLines("\r\n\n\n\r\n").next(), None); + } + + #[test] + fn non_empty_lines_no_input() { + assert_eq!(NonEmptyLines("").next(), None); + } +} diff --git a/third_party/rust/textwrap/src/options.rs b/third_party/rust/textwrap/src/options.rs new file mode 100644 index 0000000000..80e420d195 --- /dev/null +++ b/third_party/rust/textwrap/src/options.rs @@ -0,0 +1,300 @@ +//! Options for wrapping text. + +use crate::{LineEnding, WordSeparator, WordSplitter, WrapAlgorithm}; + +/// Holds configuration options for wrapping and filling text. +#[non_exhaustive] +#[derive(Debug, Clone)] +pub struct Options<'a> { + /// The width in columns at which the text will be wrapped. + pub width: usize, + /// Line ending used for breaking lines. + pub line_ending: LineEnding, + /// Indentation used for the first line of output. See the + /// [`Options::initial_indent`] method. + pub initial_indent: &'a str, + /// Indentation used for subsequent lines of output. See the + /// [`Options::subsequent_indent`] method. + pub subsequent_indent: &'a str, + /// Allow long words to be broken if they cannot fit on a line. + /// When set to `false`, some lines may be longer than + /// `self.width`. See the [`Options::break_words`] method. + pub break_words: bool, + /// Wrapping algorithm to use, see the implementations of the + /// [`WrapAlgorithm`] trait for details. + pub wrap_algorithm: WrapAlgorithm, + /// The line breaking algorithm to use, see the [`WordSeparator`] + /// trait for an overview and possible implementations. + pub word_separator: WordSeparator, + /// The method for splitting words. This can be used to prohibit + /// splitting words on hyphens, or it can be used to implement + /// language-aware machine hyphenation. + pub word_splitter: WordSplitter, +} + +impl<'a> From<&'a Options<'a>> for Options<'a> { + fn from(options: &'a Options<'a>) -> Self { + Self { + width: options.width, + line_ending: options.line_ending, + initial_indent: options.initial_indent, + subsequent_indent: options.subsequent_indent, + break_words: options.break_words, + word_separator: options.word_separator, + wrap_algorithm: options.wrap_algorithm, + word_splitter: options.word_splitter.clone(), + } + } +} + +impl<'a> From for Options<'a> { + fn from(width: usize) -> Self { + Options::new(width) + } +} + +impl<'a> Options<'a> { + /// Creates a new [`Options`] with the specified width. + /// + /// The other fields are given default values as follows: + /// + /// ``` + /// # use textwrap::{LineEnding, Options, WordSplitter, WordSeparator, WrapAlgorithm}; + /// # let width = 80; + /// let options = Options::new(width); + /// assert_eq!(options.line_ending, LineEnding::LF); + /// assert_eq!(options.initial_indent, ""); + /// assert_eq!(options.subsequent_indent, ""); + /// assert_eq!(options.break_words, true); + /// + /// #[cfg(feature = "unicode-linebreak")] + /// assert_eq!(options.word_separator, WordSeparator::UnicodeBreakProperties); + /// #[cfg(not(feature = "unicode-linebreak"))] + /// assert_eq!(options.word_separator, WordSeparator::AsciiSpace); + /// + /// #[cfg(feature = "smawk")] + /// assert_eq!(options.wrap_algorithm, WrapAlgorithm::new_optimal_fit()); + /// #[cfg(not(feature = "smawk"))] + /// assert_eq!(options.wrap_algorithm, WrapAlgorithm::FirstFit); + /// + /// assert_eq!(options.word_splitter, WordSplitter::HyphenSplitter); + /// ``` + /// + /// Note that the default word separator and wrap algorithms + /// changes based on the available Cargo features. The best + /// available algorithms are used by default. + pub const fn new(width: usize) -> Self { + Options { + width, + line_ending: LineEnding::LF, + initial_indent: "", + subsequent_indent: "", + break_words: true, + word_separator: WordSeparator::new(), + wrap_algorithm: WrapAlgorithm::new(), + word_splitter: WordSplitter::HyphenSplitter, + } + } + + /// Change [`self.line_ending`]. This specifies which of the + /// supported line endings should be used to break the lines of the + /// input text. + /// + /// # Examples + /// + /// ``` + /// use textwrap::{refill, LineEnding, Options}; + /// + /// let options = Options::new(15).line_ending(LineEnding::CRLF); + /// assert_eq!(refill("This is a little example.", options), + /// "This is a\r\nlittle example."); + /// ``` + /// + /// [`self.line_ending`]: #structfield.line_ending + pub fn line_ending(self, line_ending: LineEnding) -> Self { + Options { + line_ending, + ..self + } + } + + /// Set [`self.width`] to the given value. + /// + /// [`self.width`]: #structfield.width + pub fn width(self, width: usize) -> Self { + Options { width, ..self } + } + + /// Change [`self.initial_indent`]. The initial indentation is + /// used on the very first line of output. + /// + /// # Examples + /// + /// Classic paragraph indentation can be achieved by specifying an + /// initial indentation and wrapping each paragraph by itself: + /// + /// ``` + /// use textwrap::{wrap, Options}; + /// + /// let options = Options::new(16).initial_indent(" "); + /// assert_eq!(wrap("This is a little example.", options), + /// vec![" This is a", + /// "little example."]); + /// ``` + /// + /// [`self.initial_indent`]: #structfield.initial_indent + pub fn initial_indent(self, initial_indent: &'a str) -> Self { + Options { + initial_indent, + ..self + } + } + + /// Change [`self.subsequent_indent`]. The subsequent indentation + /// is used on lines following the first line of output. + /// + /// # Examples + /// + /// Combining initial and subsequent indentation lets you format a + /// single paragraph as a bullet list: + /// + /// ``` + /// use textwrap::{wrap, Options}; + /// + /// let options = Options::new(12) + /// .initial_indent("* ") + /// .subsequent_indent(" "); + /// #[cfg(feature = "smawk")] + /// assert_eq!(wrap("This is a little example.", options), + /// vec!["* This is", + /// " a little", + /// " example."]); + /// + /// // Without the `smawk` feature, the wrapping is a little different: + /// #[cfg(not(feature = "smawk"))] + /// assert_eq!(wrap("This is a little example.", options), + /// vec!["* This is a", + /// " little", + /// " example."]); + /// ``` + /// + /// [`self.subsequent_indent`]: #structfield.subsequent_indent + pub fn subsequent_indent(self, subsequent_indent: &'a str) -> Self { + Options { + subsequent_indent, + ..self + } + } + + /// Change [`self.break_words`]. This controls if words longer + /// than `self.width` can be broken, or if they will be left + /// sticking out into the right margin. + /// + /// See [`Options::word_splitter`] instead if you want to control + /// hyphenation. + /// + /// # Examples + /// + /// ``` + /// use textwrap::{wrap, Options}; + /// + /// let options = Options::new(4).break_words(true); + /// assert_eq!(wrap("This is a little example.", options), + /// vec!["This", + /// "is a", + /// "litt", + /// "le", + /// "exam", + /// "ple."]); + /// ``` + /// + /// [`self.break_words`]: #structfield.break_words + pub fn break_words(self, break_words: bool) -> Self { + Options { + break_words, + ..self + } + } + + /// Change [`self.word_separator`]. + /// + /// See the [`WordSeparator`] trait for details on the choices. + /// + /// [`self.word_separator`]: #structfield.word_separator + pub fn word_separator(self, word_separator: WordSeparator) -> Options<'a> { + Options { + word_separator, + ..self + } + } + + /// Change [`self.wrap_algorithm`]. + /// + /// See the [`WrapAlgorithm`] trait for details on the choices. + /// + /// [`self.wrap_algorithm`]: #structfield.wrap_algorithm + pub fn wrap_algorithm(self, wrap_algorithm: WrapAlgorithm) -> Options<'a> { + Options { + wrap_algorithm, + ..self + } + } + + /// Change [`self.word_splitter`]. The [`WordSplitter`] is used to + /// fit part of a word into the current line when wrapping text. + /// + /// See [`Options::break_words`] instead if you want to control the + /// handling of words longer than the line width. + /// + /// # Examples + /// + /// ``` + /// use textwrap::{wrap, Options, WordSplitter}; + /// + /// // The default is WordSplitter::HyphenSplitter. + /// let options = Options::new(5); + /// assert_eq!(wrap("foo-bar-baz", &options), + /// vec!["foo-", "bar-", "baz"]); + /// + /// // The word is now so long that break_words kick in: + /// let options = Options::new(5) + /// .word_splitter(WordSplitter::NoHyphenation); + /// assert_eq!(wrap("foo-bar-baz", &options), + /// vec!["foo-b", "ar-ba", "z"]); + /// + /// // If you want to breaks at all, disable both: + /// let options = Options::new(5) + /// .break_words(false) + /// .word_splitter(WordSplitter::NoHyphenation); + /// assert_eq!(wrap("foo-bar-baz", &options), + /// vec!["foo-bar-baz"]); + /// ``` + /// + /// [`self.word_splitter`]: #structfield.word_splitter + pub fn word_splitter(self, word_splitter: WordSplitter) -> Options<'a> { + Options { + word_splitter, + ..self + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn options_agree_with_usize() { + let opt_usize = Options::from(42_usize); + let opt_options = Options::new(42); + + assert_eq!(opt_usize.width, opt_options.width); + assert_eq!(opt_usize.initial_indent, opt_options.initial_indent); + assert_eq!(opt_usize.subsequent_indent, opt_options.subsequent_indent); + assert_eq!(opt_usize.break_words, opt_options.break_words); + assert_eq!( + opt_usize.word_splitter.split_points("hello-world"), + opt_options.word_splitter.split_points("hello-world") + ); + } +} diff --git a/third_party/rust/textwrap/src/refill.rs b/third_party/rust/textwrap/src/refill.rs new file mode 100644 index 0000000000..1be85f04eb --- /dev/null +++ b/third_party/rust/textwrap/src/refill.rs @@ -0,0 +1,352 @@ +//! Functionality for unfilling and refilling text. + +use crate::core::display_width; +use crate::line_ending::NonEmptyLines; +use crate::{fill, LineEnding, Options}; + +/// Unpack a paragraph of already-wrapped text. +/// +/// This function attempts to recover the original text from a single +/// paragraph of wrapped text, such as what [`fill()`] would produce. +/// This means that it turns +/// +/// ```text +/// textwrap: a small +/// library for +/// wrapping text. +/// ``` +/// +/// back into +/// +/// ```text +/// textwrap: a small library for wrapping text. +/// ``` +/// +/// In addition, it will recognize a common prefix and a common line +/// ending among the lines. +/// +/// The prefix of the first line is returned in +/// [`Options::initial_indent`] and the prefix (if any) of the the +/// other lines is returned in [`Options::subsequent_indent`]. +/// +/// Line ending is returned in [`Options::line_ending`]. If line ending +/// can not be confidently detected (mixed or no line endings in the +/// input), [`LineEnding::LF`] will be returned. +/// +/// In addition to `' '`, the prefixes can consist of characters used +/// for unordered lists (`'-'`, `'+'`, and `'*'`) and block quotes +/// (`'>'`) in Markdown as well as characters often used for inline +/// comments (`'#'` and `'/'`). +/// +/// The text must come from a single wrapped paragraph. This means +/// that there can be no empty lines (`"\n\n"` or `"\r\n\r\n"`) within +/// the text. It is unspecified what happens if `unfill` is called on +/// more than one paragraph of text. +/// +/// # Examples +/// +/// ``` +/// use textwrap::{LineEnding, unfill}; +/// +/// let (text, options) = unfill("\ +/// * This is an +/// example of +/// a list item. +/// "); +/// +/// assert_eq!(text, "This is an example of a list item.\n"); +/// assert_eq!(options.initial_indent, "* "); +/// assert_eq!(options.subsequent_indent, " "); +/// assert_eq!(options.line_ending, LineEnding::LF); +/// ``` +pub fn unfill(text: &str) -> (String, Options<'_>) { + let prefix_chars: &[_] = &[' ', '-', '+', '*', '>', '#', '/']; + + let mut options = Options::new(0); + for (idx, line) in text.lines().enumerate() { + options.width = std::cmp::max(options.width, display_width(line)); + let without_prefix = line.trim_start_matches(prefix_chars); + let prefix = &line[..line.len() - without_prefix.len()]; + + if idx == 0 { + options.initial_indent = prefix; + } else if idx == 1 { + options.subsequent_indent = prefix; + } else if idx > 1 { + for ((idx, x), y) in prefix.char_indices().zip(options.subsequent_indent.chars()) { + if x != y { + options.subsequent_indent = &prefix[..idx]; + break; + } + } + if prefix.len() < options.subsequent_indent.len() { + options.subsequent_indent = prefix; + } + } + } + + let mut unfilled = String::with_capacity(text.len()); + let mut detected_line_ending = None; + + for (idx, (line, ending)) in NonEmptyLines(text).enumerate() { + if idx == 0 { + unfilled.push_str(&line[options.initial_indent.len()..]); + } else { + unfilled.push(' '); + unfilled.push_str(&line[options.subsequent_indent.len()..]); + } + match (detected_line_ending, ending) { + (None, Some(_)) => detected_line_ending = ending, + (Some(LineEnding::CRLF), Some(LineEnding::LF)) => detected_line_ending = ending, + _ => (), + } + } + + // Add back a line ending if `text` ends with the one we detect. + if let Some(line_ending) = detected_line_ending { + if text.ends_with(line_ending.as_str()) { + unfilled.push_str(line_ending.as_str()); + } + } + + options.line_ending = detected_line_ending.unwrap_or(LineEnding::LF); + (unfilled, options) +} + +/// Refill a paragraph of wrapped text with a new width. +/// +/// This function will first use [`unfill()`] to remove newlines from +/// the text. Afterwards the text is filled again using [`fill()`]. +/// +/// The `new_width_or_options` argument specify the new width and can +/// specify other options as well — except for +/// [`Options::initial_indent`] and [`Options::subsequent_indent`], +/// which are deduced from `filled_text`. +/// +/// # Examples +/// +/// ``` +/// use textwrap::refill; +/// +/// // Some loosely wrapped text. The "> " prefix is recognized automatically. +/// let text = "\ +/// > Memory +/// > safety without garbage +/// > collection. +/// "; +/// +/// assert_eq!(refill(text, 20), "\ +/// > Memory safety +/// > without garbage +/// > collection. +/// "); +/// +/// assert_eq!(refill(text, 40), "\ +/// > Memory safety without garbage +/// > collection. +/// "); +/// +/// assert_eq!(refill(text, 60), "\ +/// > Memory safety without garbage collection. +/// "); +/// ``` +/// +/// You can also reshape bullet points: +/// +/// ``` +/// use textwrap::refill; +/// +/// let text = "\ +/// - This is my +/// list item. +/// "; +/// +/// assert_eq!(refill(text, 20), "\ +/// - This is my list +/// item. +/// "); +/// ``` +pub fn refill<'a, Opt>(filled_text: &str, new_width_or_options: Opt) -> String +where + Opt: Into>, +{ + let mut new_options = new_width_or_options.into(); + let (text, options) = unfill(filled_text); + // The original line ending is kept by `unfill`. + let stripped = text.strip_suffix(options.line_ending.as_str()); + let new_line_ending = new_options.line_ending.as_str(); + + new_options.initial_indent = options.initial_indent; + new_options.subsequent_indent = options.subsequent_indent; + let mut refilled = fill(stripped.unwrap_or(&text), new_options); + + // Add back right line ending if we stripped one off above. + if stripped.is_some() { + refilled.push_str(new_line_ending); + } + refilled +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn unfill_simple() { + let (text, options) = unfill("foo\nbar"); + assert_eq!(text, "foo bar"); + assert_eq!(options.width, 3); + assert_eq!(options.line_ending, LineEnding::LF); + } + + #[test] + fn unfill_no_new_line() { + let (text, options) = unfill("foo bar"); + assert_eq!(text, "foo bar"); + assert_eq!(options.width, 7); + assert_eq!(options.line_ending, LineEnding::LF); + } + + #[test] + fn unfill_simple_crlf() { + let (text, options) = unfill("foo\r\nbar"); + assert_eq!(text, "foo bar"); + assert_eq!(options.width, 3); + assert_eq!(options.line_ending, LineEnding::CRLF); + } + + #[test] + fn unfill_mixed_new_lines() { + let (text, options) = unfill("foo\r\nbar\nbaz"); + assert_eq!(text, "foo bar baz"); + assert_eq!(options.width, 3); + assert_eq!(options.line_ending, LineEnding::LF); + } + + #[test] + fn test_unfill_consecutive_different_prefix() { + let (text, options) = unfill("foo\n*\n/"); + assert_eq!(text, "foo * /"); + assert_eq!(options.width, 3); + assert_eq!(options.line_ending, LineEnding::LF); + } + + #[test] + fn unfill_trailing_newlines() { + let (text, options) = unfill("foo\nbar\n\n\n"); + assert_eq!(text, "foo bar\n"); + assert_eq!(options.width, 3); + } + + #[test] + fn unfill_mixed_trailing_newlines() { + let (text, options) = unfill("foo\r\nbar\n\r\n\n"); + assert_eq!(text, "foo bar\n"); + assert_eq!(options.width, 3); + assert_eq!(options.line_ending, LineEnding::LF); + } + + #[test] + fn unfill_trailing_crlf() { + let (text, options) = unfill("foo bar\r\n"); + assert_eq!(text, "foo bar\r\n"); + assert_eq!(options.width, 7); + assert_eq!(options.line_ending, LineEnding::CRLF); + } + + #[test] + fn unfill_initial_indent() { + let (text, options) = unfill(" foo\nbar\nbaz"); + assert_eq!(text, "foo bar baz"); + assert_eq!(options.width, 5); + assert_eq!(options.initial_indent, " "); + } + + #[test] + fn unfill_differing_indents() { + let (text, options) = unfill(" foo\n bar\n baz"); + assert_eq!(text, "foo bar baz"); + assert_eq!(options.width, 7); + assert_eq!(options.initial_indent, " "); + assert_eq!(options.subsequent_indent, " "); + } + + #[test] + fn unfill_list_item() { + let (text, options) = unfill("* foo\n bar\n baz"); + assert_eq!(text, "foo bar baz"); + assert_eq!(options.width, 5); + assert_eq!(options.initial_indent, "* "); + assert_eq!(options.subsequent_indent, " "); + } + + #[test] + fn unfill_multiple_char_prefix() { + let (text, options) = unfill(" // foo bar\n // baz\n // quux"); + assert_eq!(text, "foo bar baz quux"); + assert_eq!(options.width, 14); + assert_eq!(options.initial_indent, " // "); + assert_eq!(options.subsequent_indent, " // "); + } + + #[test] + fn unfill_block_quote() { + let (text, options) = unfill("> foo\n> bar\n> baz"); + assert_eq!(text, "foo bar baz"); + assert_eq!(options.width, 5); + assert_eq!(options.initial_indent, "> "); + assert_eq!(options.subsequent_indent, "> "); + } + + #[test] + fn unfill_only_prefixes_issue_466() { + // Test that we don't crash if the first line has only prefix + // chars *and* the second line is shorter than the first line. + let (text, options) = unfill("######\nfoo"); + assert_eq!(text, " foo"); + assert_eq!(options.width, 6); + assert_eq!(options.initial_indent, "######"); + assert_eq!(options.subsequent_indent, ""); + } + + #[test] + fn unfill_trailing_newlines_issue_466() { + // Test that we don't crash on a '\r' following a string of + // '\n'. The problem was that we removed both kinds of + // characters in one code path, but not in the other. + let (text, options) = unfill("foo\n##\n\n\r"); + // The \n\n changes subsequent_indent to "". + assert_eq!(text, "foo ## \r"); + assert_eq!(options.width, 3); + assert_eq!(options.initial_indent, ""); + assert_eq!(options.subsequent_indent, ""); + } + + #[test] + fn unfill_whitespace() { + assert_eq!(unfill("foo bar").0, "foo bar"); + } + + #[test] + fn refill_convert_lf_to_crlf() { + let options = Options::new(5).line_ending(LineEnding::CRLF); + assert_eq!(refill("foo\nbar\n", options), "foo\r\nbar\r\n",); + } + + #[test] + fn refill_convert_crlf_to_lf() { + let options = Options::new(5).line_ending(LineEnding::LF); + assert_eq!(refill("foo\r\nbar\r\n", options), "foo\nbar\n",); + } + + #[test] + fn refill_convert_mixed_newlines() { + let options = Options::new(5).line_ending(LineEnding::CRLF); + assert_eq!(refill("foo\r\nbar\n", options), "foo\r\nbar\r\n",); + } + + #[test] + fn refill_defaults_to_lf() { + assert_eq!(refill("foo bar baz", 5), "foo\nbar\nbaz"); + } +} diff --git a/third_party/rust/textwrap/src/termwidth.rs b/third_party/rust/textwrap/src/termwidth.rs new file mode 100644 index 0000000000..5c66191b77 --- /dev/null +++ b/third_party/rust/textwrap/src/termwidth.rs @@ -0,0 +1,52 @@ +//! Functions related to the terminal size. + +use crate::Options; + +/// Return the current terminal width. +/// +/// If the terminal width cannot be determined (typically because the +/// standard output is not connected to a terminal), a default width +/// of 80 characters will be used. +/// +/// # Examples +/// +/// Create an [`Options`] for wrapping at the current terminal width +/// with a two column margin to the left and the right: +/// +/// ```no_run +/// use textwrap::{termwidth, Options}; +/// +/// let width = termwidth() - 4; // Two columns on each side. +/// let options = Options::new(width) +/// .initial_indent(" ") +/// .subsequent_indent(" "); +/// ``` +/// +/// **Note:** Only available when the `terminal_size` Cargo feature is +/// enabled. +pub fn termwidth() -> usize { + terminal_size::terminal_size().map_or(80, |(terminal_size::Width(w), _)| w.into()) +} + +impl<'a> Options<'a> { + /// Creates a new [`Options`] with `width` set to the current + /// terminal width. If the terminal width cannot be determined + /// (typically because the standard input and output is not + /// connected to a terminal), a width of 80 characters will be + /// used. Other settings use the same defaults as + /// [`Options::new`]. + /// + /// Equivalent to: + /// + /// ```no_run + /// use textwrap::{termwidth, Options}; + /// + /// let options = Options::new(termwidth()); + /// ``` + /// + /// **Note:** Only available when the `terminal_size` feature is + /// enabled. + pub fn with_termwidth() -> Self { + Self::new(termwidth()) + } +} diff --git a/third_party/rust/textwrap/src/word_separators.rs b/third_party/rust/textwrap/src/word_separators.rs new file mode 100644 index 0000000000..e06e9b88aa --- /dev/null +++ b/third_party/rust/textwrap/src/word_separators.rs @@ -0,0 +1,481 @@ +//! Functionality for finding words. +//! +//! In order to wrap text, we need to know where the legal break +//! points are, i.e., where the words of the text are. This means that +//! we need to define what a "word" is. +//! +//! A simple approach is to simply split the text on whitespace, but +//! this does not work for East-Asian languages such as Chinese or +//! Japanese where there are no spaces between words. Breaking a long +//! sequence of emojis is another example where line breaks might be +//! wanted even if there are no whitespace to be found. +//! +//! The [`WordSeparator`] enum is responsible for determining where +//! there words are in a line of text. Please refer to the enum and +//! its variants for more information. + +#[cfg(feature = "unicode-linebreak")] +use crate::core::skip_ansi_escape_sequence; +use crate::core::Word; + +/// Describes where words occur in a line of text. +/// +/// The simplest approach is say that words are separated by one or +/// more ASCII spaces (`' '`). This works for Western languages +/// without emojis. A more complex approach is to use the Unicode line +/// breaking algorithm, which finds break points in non-ASCII text. +/// +/// The line breaks occur between words, please see +/// [`WordSplitter`](crate::WordSplitter) for options of how to handle +/// hyphenation of individual words. +/// +/// # Examples +/// +/// ``` +/// use textwrap::core::Word; +/// use textwrap::WordSeparator::AsciiSpace; +/// +/// let words = AsciiSpace.find_words("Hello World!").collect::>(); +/// assert_eq!(words, vec![Word::from("Hello "), Word::from("World!")]); +/// ``` +#[derive(Clone, Copy)] +pub enum WordSeparator { + /// Find words by splitting on runs of `' '` characters. + /// + /// # Examples + /// + /// ``` + /// use textwrap::core::Word; + /// use textwrap::WordSeparator::AsciiSpace; + /// + /// let words = AsciiSpace.find_words("Hello World!").collect::>(); + /// assert_eq!(words, vec![Word::from("Hello "), + /// Word::from("World!")]); + /// ``` + AsciiSpace, + + /// Split `line` into words using Unicode break properties. + /// + /// This word separator uses the Unicode line breaking algorithm + /// described in [Unicode Standard Annex + /// #14](https://www.unicode.org/reports/tr14/) to find legal places + /// to break lines. There is a small difference in that the U+002D + /// (Hyphen-Minus) and U+00AD (Soft Hyphen) don’t create a line break: + /// to allow a line break at a hyphen, use + /// [`WordSplitter::HyphenSplitter`](crate::WordSplitter::HyphenSplitter). + /// Soft hyphens are not currently supported. + /// + /// # Examples + /// + /// Unlike [`WordSeparator::AsciiSpace`], the Unicode line + /// breaking algorithm will find line break opportunities between + /// some characters with no intervening whitespace: + /// + /// ``` + /// #[cfg(feature = "unicode-linebreak")] { + /// use textwrap::core::Word; + /// use textwrap::WordSeparator::UnicodeBreakProperties; + /// + /// assert_eq!(UnicodeBreakProperties.find_words("Emojis: 😂😍").collect::>(), + /// vec![Word::from("Emojis: "), + /// Word::from("😂"), + /// Word::from("😍")]); + /// + /// assert_eq!(UnicodeBreakProperties.find_words("CJK: 你好").collect::>(), + /// vec![Word::from("CJK: "), + /// Word::from("你"), + /// Word::from("好")]); + /// } + /// ``` + /// + /// A U+2060 (Word Joiner) character can be inserted if you want to + /// manually override the defaults and keep the characters together: + /// + /// ``` + /// #[cfg(feature = "unicode-linebreak")] { + /// use textwrap::core::Word; + /// use textwrap::WordSeparator::UnicodeBreakProperties; + /// + /// assert_eq!(UnicodeBreakProperties.find_words("Emojis: 😂\u{2060}😍").collect::>(), + /// vec![Word::from("Emojis: "), + /// Word::from("😂\u{2060}😍")]); + /// } + /// ``` + /// + /// The Unicode line breaking algorithm will also automatically + /// suppress break breaks around certain punctuation characters:: + /// + /// ``` + /// #[cfg(feature = "unicode-linebreak")] { + /// use textwrap::core::Word; + /// use textwrap::WordSeparator::UnicodeBreakProperties; + /// + /// assert_eq!(UnicodeBreakProperties.find_words("[ foo ] bar !").collect::>(), + /// vec![Word::from("[ foo ] "), + /// Word::from("bar !")]); + /// } + /// ``` + #[cfg(feature = "unicode-linebreak")] + UnicodeBreakProperties, + + /// Find words using a custom word separator + Custom(fn(line: &str) -> Box> + '_>), +} + +impl PartialEq for WordSeparator { + /// Compare two word separators. + /// + /// ``` + /// use textwrap::WordSeparator; + /// + /// assert_eq!(WordSeparator::AsciiSpace, WordSeparator::AsciiSpace); + /// #[cfg(feature = "unicode-linebreak")] { + /// assert_eq!(WordSeparator::UnicodeBreakProperties, + /// WordSeparator::UnicodeBreakProperties); + /// } + /// ``` + /// + /// Note that `WordSeparator::Custom` values never compare equal: + /// + /// ``` + /// use textwrap::WordSeparator; + /// use textwrap::core::Word; + /// fn word_separator(line: &str) -> Box> + '_> { + /// Box::new(line.split_inclusive(' ').map(Word::from)) + /// } + /// assert_ne!(WordSeparator::Custom(word_separator), + /// WordSeparator::Custom(word_separator)); + /// ``` + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (WordSeparator::AsciiSpace, WordSeparator::AsciiSpace) => true, + #[cfg(feature = "unicode-linebreak")] + (WordSeparator::UnicodeBreakProperties, WordSeparator::UnicodeBreakProperties) => true, + (_, _) => false, + } + } +} + +impl std::fmt::Debug for WordSeparator { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + WordSeparator::AsciiSpace => f.write_str("AsciiSpace"), + #[cfg(feature = "unicode-linebreak")] + WordSeparator::UnicodeBreakProperties => f.write_str("UnicodeBreakProperties"), + WordSeparator::Custom(_) => f.write_str("Custom(...)"), + } + } +} + +impl WordSeparator { + /// Create a new word separator. + /// + /// The best available algorithm is used by default, i.e., + /// [`WordSeparator::UnicodeBreakProperties`] if available, + /// otherwise [`WordSeparator::AsciiSpace`]. + pub const fn new() -> Self { + #[cfg(feature = "unicode-linebreak")] + { + WordSeparator::UnicodeBreakProperties + } + + #[cfg(not(feature = "unicode-linebreak"))] + { + WordSeparator::AsciiSpace + } + } + + // This function should really return impl Iterator, but + // this isn't possible until Rust supports higher-kinded types: + // https://github.com/rust-lang/rfcs/blob/master/text/1522-conservative-impl-trait.md + /// Find all words in `line`. + pub fn find_words<'a>(&self, line: &'a str) -> Box> + 'a> { + match self { + WordSeparator::AsciiSpace => find_words_ascii_space(line), + #[cfg(feature = "unicode-linebreak")] + WordSeparator::UnicodeBreakProperties => find_words_unicode_break_properties(line), + WordSeparator::Custom(func) => func(line), + } + } +} + +fn find_words_ascii_space<'a>(line: &'a str) -> Box> + 'a> { + let mut start = 0; + let mut in_whitespace = false; + let mut char_indices = line.char_indices(); + + Box::new(std::iter::from_fn(move || { + for (idx, ch) in char_indices.by_ref() { + if in_whitespace && ch != ' ' { + let word = Word::from(&line[start..idx]); + start = idx; + in_whitespace = ch == ' '; + return Some(word); + } + + in_whitespace = ch == ' '; + } + + if start < line.len() { + let word = Word::from(&line[start..]); + start = line.len(); + return Some(word); + } + + None + })) +} + +// Strip all ANSI escape sequences from `text`. +#[cfg(feature = "unicode-linebreak")] +fn strip_ansi_escape_sequences(text: &str) -> String { + let mut result = String::with_capacity(text.len()); + + let mut chars = text.chars(); + while let Some(ch) = chars.next() { + if skip_ansi_escape_sequence(ch, &mut chars) { + continue; + } + result.push(ch); + } + + result +} + +/// Soft hyphen, also knows as a “shy hyphen”. Should show up as ‘-’ +/// if a line is broken at this point, and otherwise be invisible. +/// Textwrap does not currently support breaking words at soft +/// hyphens. +#[cfg(feature = "unicode-linebreak")] +const SHY: char = '\u{00ad}'; + +/// Find words in line. ANSI escape sequences are ignored in `line`. +#[cfg(feature = "unicode-linebreak")] +fn find_words_unicode_break_properties<'a>( + line: &'a str, +) -> Box> + 'a> { + // Construct an iterator over (original index, stripped index) + // tuples. We find the Unicode linebreaks on a stripped string, + // but we need the original indices so we can form words based on + // the original string. + let mut last_stripped_idx = 0; + let mut char_indices = line.char_indices(); + let mut idx_map = std::iter::from_fn(move || match char_indices.next() { + Some((orig_idx, ch)) => { + let stripped_idx = last_stripped_idx; + if !skip_ansi_escape_sequence(ch, &mut char_indices.by_ref().map(|(_, ch)| ch)) { + last_stripped_idx += ch.len_utf8(); + } + Some((orig_idx, stripped_idx)) + } + None => None, + }); + + let stripped = strip_ansi_escape_sequences(line); + let mut opportunities = unicode_linebreak::linebreaks(&stripped) + .filter(|(idx, _)| { + #[allow(clippy::match_like_matches_macro)] + match &stripped[..*idx].chars().next_back() { + // We suppress breaks at ‘-’ since we want to control + // this via the WordSplitter. + Some('-') => false, + // Soft hyphens are currently not supported since we + // require all `Word` fragments to be continuous in + // the input string. + Some(SHY) => false, + // Other breaks should be fine! + _ => true, + } + }) + .collect::>() + .into_iter(); + + // Remove final break opportunity, we will add it below using + // &line[start..]; This ensures that we correctly include a + // trailing ANSI escape sequence. + opportunities.next_back(); + + let mut start = 0; + Box::new(std::iter::from_fn(move || { + for (idx, _) in opportunities.by_ref() { + if let Some((orig_idx, _)) = idx_map.find(|&(_, stripped_idx)| stripped_idx == idx) { + let word = Word::from(&line[start..orig_idx]); + start = orig_idx; + return Some(word); + } + } + + if start < line.len() { + let word = Word::from(&line[start..]); + start = line.len(); + return Some(word); + } + + None + })) +} + +#[cfg(test)] +mod tests { + use super::WordSeparator::*; + use super::*; + + // Like assert_eq!, but the left expression is an iterator. + macro_rules! assert_iter_eq { + ($left:expr, $right:expr) => { + assert_eq!($left.collect::>(), $right); + }; + } + + fn to_words(words: Vec<&str>) -> Vec> { + words.into_iter().map(Word::from).collect() + } + + macro_rules! test_find_words { + ($ascii_name:ident, + $unicode_name:ident, + $([ $line:expr, $ascii_words:expr, $unicode_words:expr ]),+) => { + #[test] + fn $ascii_name() { + $( + let expected_words = to_words($ascii_words.to_vec()); + let actual_words = WordSeparator::AsciiSpace + .find_words($line) + .collect::>(); + assert_eq!(actual_words, expected_words, "Line: {:?}", $line); + )+ + } + + #[test] + #[cfg(feature = "unicode-linebreak")] + fn $unicode_name() { + $( + let expected_words = to_words($unicode_words.to_vec()); + let actual_words = WordSeparator::UnicodeBreakProperties + .find_words($line) + .collect::>(); + assert_eq!(actual_words, expected_words, "Line: {:?}", $line); + )+ + } + }; + } + + test_find_words!(ascii_space_empty, unicode_empty, ["", [], []]); + + test_find_words!( + ascii_single_word, + unicode_single_word, + ["foo", ["foo"], ["foo"]] + ); + + test_find_words!( + ascii_two_words, + unicode_two_words, + ["foo bar", ["foo ", "bar"], ["foo ", "bar"]] + ); + + test_find_words!( + ascii_multiple_words, + unicode_multiple_words, + ["foo bar", ["foo ", "bar"], ["foo ", "bar"]], + ["x y z", ["x ", "y ", "z"], ["x ", "y ", "z"]] + ); + + test_find_words!( + ascii_only_whitespace, + unicode_only_whitespace, + [" ", [" "], [" "]], + [" ", [" "], [" "]] + ); + + test_find_words!( + ascii_inter_word_whitespace, + unicode_inter_word_whitespace, + ["foo bar", ["foo ", "bar"], ["foo ", "bar"]] + ); + + test_find_words!( + ascii_trailing_whitespace, + unicode_trailing_whitespace, + ["foo ", ["foo "], ["foo "]] + ); + + test_find_words!( + ascii_leading_whitespace, + unicode_leading_whitespace, + [" foo", [" ", "foo"], [" ", "foo"]] + ); + + test_find_words!( + ascii_multi_column_char, + unicode_multi_column_char, + ["\u{1f920}", ["\u{1f920}"], ["\u{1f920}"]] // cowboy emoji 🤠 + ); + + test_find_words!( + ascii_hyphens, + unicode_hyphens, + ["foo-bar", ["foo-bar"], ["foo-bar"]], + ["foo- bar", ["foo- ", "bar"], ["foo- ", "bar"]], + ["foo - bar", ["foo ", "- ", "bar"], ["foo ", "- ", "bar"]], + ["foo -bar", ["foo ", "-bar"], ["foo ", "-bar"]] + ); + + test_find_words!( + ascii_newline, + unicode_newline, + ["foo\nbar", ["foo\nbar"], ["foo\n", "bar"]] + ); + + test_find_words!( + ascii_tab, + unicode_tab, + ["foo\tbar", ["foo\tbar"], ["foo\t", "bar"]] + ); + + test_find_words!( + ascii_non_breaking_space, + unicode_non_breaking_space, + ["foo\u{00A0}bar", ["foo\u{00A0}bar"], ["foo\u{00A0}bar"]] + ); + + #[test] + #[cfg(unix)] + fn find_words_colored_text() { + use termion::color::{Blue, Fg, Green, Reset}; + + let green_hello = format!("{}Hello{} ", Fg(Green), Fg(Reset)); + let blue_world = format!("{}World!{}", Fg(Blue), Fg(Reset)); + assert_iter_eq!( + AsciiSpace.find_words(&format!("{}{}", green_hello, blue_world)), + vec![Word::from(&green_hello), Word::from(&blue_world)] + ); + + #[cfg(feature = "unicode-linebreak")] + assert_iter_eq!( + UnicodeBreakProperties.find_words(&format!("{}{}", green_hello, blue_world)), + vec![Word::from(&green_hello), Word::from(&blue_world)] + ); + } + + #[test] + fn find_words_color_inside_word() { + let text = "foo\u{1b}[0m\u{1b}[32mbar\u{1b}[0mbaz"; + assert_iter_eq!(AsciiSpace.find_words(text), vec![Word::from(text)]); + + #[cfg(feature = "unicode-linebreak")] + assert_iter_eq!( + UnicodeBreakProperties.find_words(text), + vec![Word::from(text)] + ); + } + + #[test] + fn word_separator_new() { + #[cfg(feature = "unicode-linebreak")] + assert!(matches!(WordSeparator::new(), UnicodeBreakProperties)); + + #[cfg(not(feature = "unicode-linebreak"))] + assert!(matches!(WordSeparator::new(), AsciiSpace)); + } +} diff --git a/third_party/rust/textwrap/src/word_splitters.rs b/third_party/rust/textwrap/src/word_splitters.rs new file mode 100644 index 0000000000..e2dc6aa01f --- /dev/null +++ b/third_party/rust/textwrap/src/word_splitters.rs @@ -0,0 +1,314 @@ +//! Word splitting functionality. +//! +//! To wrap text into lines, long words sometimes need to be split +//! across lines. The [`WordSplitter`] enum defines this +//! functionality. + +use crate::core::{display_width, Word}; + +/// The `WordSplitter` enum describes where words can be split. +/// +/// If the textwrap crate has been compiled with the `hyphenation` +/// Cargo feature enabled, you will find a +/// [`WordSplitter::Hyphenation`] variant. Use this struct for +/// language-aware hyphenation: +/// +/// ``` +/// #[cfg(feature = "hyphenation")] { +/// use hyphenation::{Language, Load, Standard}; +/// use textwrap::{wrap, Options, WordSplitter}; +/// +/// let text = "Oxidation is the loss of electrons."; +/// let dictionary = Standard::from_embedded(Language::EnglishUS).unwrap(); +/// let options = Options::new(8).word_splitter(WordSplitter::Hyphenation(dictionary)); +/// assert_eq!(wrap(text, &options), vec!["Oxida-", +/// "tion is", +/// "the loss", +/// "of elec-", +/// "trons."]); +/// } +/// ``` +/// +/// Please see the documentation for the [hyphenation] crate for more +/// details. +/// +/// [hyphenation]: https://docs.rs/hyphenation/ +#[derive(Clone)] +pub enum WordSplitter { + /// Use this as a [`Options.word_splitter`] to avoid any kind of + /// hyphenation: + /// + /// ``` + /// use textwrap::{wrap, Options, WordSplitter}; + /// + /// let options = Options::new(8).word_splitter(WordSplitter::NoHyphenation); + /// assert_eq!(wrap("foo bar-baz", &options), + /// vec!["foo", "bar-baz"]); + /// ``` + /// + /// [`Options.word_splitter`]: super::Options::word_splitter + NoHyphenation, + + /// `HyphenSplitter` is the default `WordSplitter` used by + /// [`Options::new`](super::Options::new). It will split words on + /// existing hyphens in the word. + /// + /// It will only use hyphens that are surrounded by alphanumeric + /// characters, which prevents a word like `"--foo-bar"` from + /// being split into `"--"` and `"foo-bar"`. + /// + /// # Examples + /// + /// ``` + /// use textwrap::WordSplitter; + /// + /// assert_eq!(WordSplitter::HyphenSplitter.split_points("--foo-bar"), + /// vec![6]); + /// ``` + HyphenSplitter, + + /// Use a custom function as the word splitter. + /// + /// This variant lets you implement a custom word splitter using + /// your own function. + /// + /// # Examples + /// + /// ``` + /// use textwrap::WordSplitter; + /// + /// fn split_at_underscore(word: &str) -> Vec { + /// word.match_indices('_').map(|(idx, _)| idx + 1).collect() + /// } + /// + /// let word_splitter = WordSplitter::Custom(split_at_underscore); + /// assert_eq!(word_splitter.split_points("a_long_identifier"), + /// vec![2, 7]); + /// ``` + Custom(fn(word: &str) -> Vec), + + /// A hyphenation dictionary can be used to do language-specific + /// hyphenation using patterns from the [hyphenation] crate. + /// + /// **Note:** Only available when the `hyphenation` Cargo feature is + /// enabled. + /// + /// [hyphenation]: https://docs.rs/hyphenation/ + #[cfg(feature = "hyphenation")] + Hyphenation(hyphenation::Standard), +} + +impl std::fmt::Debug for WordSplitter { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + WordSplitter::NoHyphenation => f.write_str("NoHyphenation"), + WordSplitter::HyphenSplitter => f.write_str("HyphenSplitter"), + WordSplitter::Custom(_) => f.write_str("Custom(...)"), + #[cfg(feature = "hyphenation")] + WordSplitter::Hyphenation(dict) => write!(f, "Hyphenation({})", dict.language()), + } + } +} + +impl PartialEq for WordSplitter { + fn eq(&self, other: &WordSplitter) -> bool { + match (self, other) { + (WordSplitter::NoHyphenation, WordSplitter::NoHyphenation) => true, + (WordSplitter::HyphenSplitter, WordSplitter::HyphenSplitter) => true, + #[cfg(feature = "hyphenation")] + (WordSplitter::Hyphenation(this_dict), WordSplitter::Hyphenation(other_dict)) => { + this_dict.language() == other_dict.language() + } + (_, _) => false, + } + } +} + +impl WordSplitter { + /// Return all possible indices where `word` can be split. + /// + /// The indices are in the range `0..word.len()`. They point to + /// the index _after_ the split point, i.e., after `-` if + /// splitting on hyphens. This way, `word.split_at(idx)` will + /// break the word into two well-formed pieces. + /// + /// # Examples + /// + /// ``` + /// use textwrap::WordSplitter; + /// assert_eq!(WordSplitter::NoHyphenation.split_points("cannot-be-split"), vec![]); + /// assert_eq!(WordSplitter::HyphenSplitter.split_points("can-be-split"), vec![4, 7]); + /// assert_eq!(WordSplitter::Custom(|word| vec![word.len()/2]).split_points("middle"), vec![3]); + /// ``` + pub fn split_points(&self, word: &str) -> Vec { + match self { + WordSplitter::NoHyphenation => Vec::new(), + WordSplitter::HyphenSplitter => { + let mut splits = Vec::new(); + + for (idx, _) in word.match_indices('-') { + // We only use hyphens that are surrounded by alphanumeric + // characters. This is to avoid splitting on repeated hyphens, + // such as those found in --foo-bar. + let prev = word[..idx].chars().next_back(); + let next = word[idx + 1..].chars().next(); + + if prev.filter(|ch| ch.is_alphanumeric()).is_some() + && next.filter(|ch| ch.is_alphanumeric()).is_some() + { + splits.push(idx + 1); // +1 due to width of '-'. + } + } + + splits + } + WordSplitter::Custom(splitter_func) => splitter_func(word), + #[cfg(feature = "hyphenation")] + WordSplitter::Hyphenation(dictionary) => { + use hyphenation::Hyphenator; + dictionary.hyphenate(word).breaks + } + } + } +} + +/// Split words into smaller words according to the split points given +/// by `word_splitter`. +/// +/// Note that we split all words, regardless of their length. This is +/// to more cleanly separate the business of splitting (including +/// automatic hyphenation) from the business of word wrapping. +pub fn split_words<'a, I>( + words: I, + word_splitter: &'a WordSplitter, +) -> impl Iterator> +where + I: IntoIterator>, +{ + words.into_iter().flat_map(move |word| { + let mut prev = 0; + let mut split_points = word_splitter.split_points(&word).into_iter(); + std::iter::from_fn(move || { + if let Some(idx) = split_points.next() { + let need_hyphen = !word[..idx].ends_with('-'); + let w = Word { + word: &word.word[prev..idx], + width: display_width(&word[prev..idx]), + whitespace: "", + penalty: if need_hyphen { "-" } else { "" }, + }; + prev = idx; + return Some(w); + } + + if prev < word.word.len() || prev == 0 { + let w = Word { + word: &word.word[prev..], + width: display_width(&word[prev..]), + whitespace: word.whitespace, + penalty: word.penalty, + }; + prev = word.word.len() + 1; + return Some(w); + } + + None + }) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + // Like assert_eq!, but the left expression is an iterator. + macro_rules! assert_iter_eq { + ($left:expr, $right:expr) => { + assert_eq!($left.collect::>(), $right); + }; + } + + #[test] + fn split_words_no_words() { + assert_iter_eq!(split_words(vec![], &WordSplitter::HyphenSplitter), vec![]); + } + + #[test] + fn split_words_empty_word() { + assert_iter_eq!( + split_words(vec![Word::from(" ")], &WordSplitter::HyphenSplitter), + vec![Word::from(" ")] + ); + } + + #[test] + fn split_words_single_word() { + assert_iter_eq!( + split_words(vec![Word::from("foobar")], &WordSplitter::HyphenSplitter), + vec![Word::from("foobar")] + ); + } + + #[test] + fn split_words_hyphen_splitter() { + assert_iter_eq!( + split_words(vec![Word::from("foo-bar")], &WordSplitter::HyphenSplitter), + vec![Word::from("foo-"), Word::from("bar")] + ); + } + + #[test] + fn split_words_no_hyphenation() { + assert_iter_eq!( + split_words(vec![Word::from("foo-bar")], &WordSplitter::NoHyphenation), + vec![Word::from("foo-bar")] + ); + } + + #[test] + fn split_words_adds_penalty() { + let fixed_split_point = |_: &str| vec![3]; + + assert_iter_eq!( + split_words( + vec![Word::from("foobar")].into_iter(), + &WordSplitter::Custom(fixed_split_point) + ), + vec![ + Word { + word: "foo", + width: 3, + whitespace: "", + penalty: "-" + }, + Word { + word: "bar", + width: 3, + whitespace: "", + penalty: "" + } + ] + ); + + assert_iter_eq!( + split_words( + vec![Word::from("fo-bar")].into_iter(), + &WordSplitter::Custom(fixed_split_point) + ), + vec![ + Word { + word: "fo-", + width: 3, + whitespace: "", + penalty: "" + }, + Word { + word: "bar", + width: 3, + whitespace: "", + penalty: "" + } + ] + ); + } +} diff --git a/third_party/rust/textwrap/src/wrap.rs b/third_party/rust/textwrap/src/wrap.rs new file mode 100644 index 0000000000..a7f2ccf298 --- /dev/null +++ b/third_party/rust/textwrap/src/wrap.rs @@ -0,0 +1,686 @@ +//! Functions for wrapping text. + +use std::borrow::Cow; + +use crate::core::{break_words, display_width, Word}; +use crate::word_splitters::split_words; +use crate::Options; + +/// Wrap a line of text at a given width. +/// +/// The result is a vector of lines, each line is of type [`Cow<'_, +/// str>`](Cow), which means that the line will borrow from the input +/// `&str` if possible. The lines do not have trailing whitespace, +/// including a final `'\n'`. Please use [`fill()`](crate::fill()) if +/// you need a [`String`] instead. +/// +/// The easiest way to use this function is to pass an integer for +/// `width_or_options`: +/// +/// ``` +/// use textwrap::wrap; +/// +/// let lines = wrap("Memory safety without garbage collection.", 15); +/// assert_eq!(lines, &[ +/// "Memory safety", +/// "without garbage", +/// "collection.", +/// ]); +/// ``` +/// +/// If you need to customize the wrapping, you can pass an [`Options`] +/// instead of an `usize`: +/// +/// ``` +/// use textwrap::{wrap, Options}; +/// +/// let options = Options::new(15) +/// .initial_indent("- ") +/// .subsequent_indent(" "); +/// let lines = wrap("Memory safety without garbage collection.", &options); +/// assert_eq!(lines, &[ +/// "- Memory safety", +/// " without", +/// " garbage", +/// " collection.", +/// ]); +/// ``` +/// +/// # Optimal-Fit Wrapping +/// +/// By default, `wrap` will try to ensure an even right margin by +/// finding breaks which avoid short lines. We call this an +/// “optimal-fit algorithm” since the line breaks are computed by +/// considering all possible line breaks. The alternative is a +/// “first-fit algorithm” which simply accumulates words until they no +/// longer fit on the line. +/// +/// As an example, using the first-fit algorithm to wrap the famous +/// Hamlet quote “To be, or not to be: that is the question” in a +/// narrow column with room for only 10 characters looks like this: +/// +/// ``` +/// # use textwrap::{WrapAlgorithm::FirstFit, Options, wrap}; +/// # +/// # let lines = wrap("To be, or not to be: that is the question", +/// # Options::new(10).wrap_algorithm(FirstFit)); +/// # assert_eq!(lines.join("\n") + "\n", "\ +/// To be, or +/// not to be: +/// that is +/// the +/// question +/// # "); +/// ``` +/// +/// Notice how the second to last line is quite narrow because +/// “question” was too large to fit? The greedy first-fit algorithm +/// doesn’t look ahead, so it has no other option than to put +/// “question” onto its own line. +/// +/// With the optimal-fit wrapping algorithm, the previous lines are +/// shortened slightly in order to make the word “is” go into the +/// second last line: +/// +/// ``` +/// # #[cfg(feature = "smawk")] { +/// # use textwrap::{Options, WrapAlgorithm, wrap}; +/// # +/// # let lines = wrap( +/// # "To be, or not to be: that is the question", +/// # Options::new(10).wrap_algorithm(WrapAlgorithm::new_optimal_fit()) +/// # ); +/// # assert_eq!(lines.join("\n") + "\n", "\ +/// To be, +/// or not to +/// be: that +/// is the +/// question +/// # "); } +/// ``` +/// +/// Please see [`WrapAlgorithm`](crate::WrapAlgorithm) for details on +/// the choices. +/// +/// # Examples +/// +/// The returned iterator yields lines of type `Cow<'_, str>`. If +/// possible, the wrapped lines will borrow from the input string. As +/// an example, a hanging indentation, the first line can borrow from +/// the input, but the subsequent lines become owned strings: +/// +/// ``` +/// use std::borrow::Cow::{Borrowed, Owned}; +/// use textwrap::{wrap, Options}; +/// +/// let options = Options::new(15).subsequent_indent("...."); +/// let lines = wrap("Wrapping text all day long.", &options); +/// let annotated = lines +/// .iter() +/// .map(|line| match line { +/// Borrowed(text) => format!("[Borrowed] {}", text), +/// Owned(text) => format!("[Owned] {}", text), +/// }) +/// .collect::>(); +/// assert_eq!( +/// annotated, +/// &[ +/// "[Borrowed] Wrapping text", +/// "[Owned] ....all day", +/// "[Owned] ....long.", +/// ] +/// ); +/// ``` +/// +/// ## Leading and Trailing Whitespace +/// +/// As a rule, leading whitespace (indentation) is preserved and +/// trailing whitespace is discarded. +/// +/// In more details, when wrapping words into lines, words are found +/// by splitting the input text on space characters. One or more +/// spaces (shown here as “␣”) are attached to the end of each word: +/// +/// ```text +/// "Foo␣␣␣bar␣baz" -> ["Foo␣␣␣", "bar␣", "baz"] +/// ``` +/// +/// These words are then put into lines. The interword whitespace is +/// preserved, unless the lines are wrapped so that the `"Foo␣␣␣"` +/// word falls at the end of a line: +/// +/// ``` +/// use textwrap::wrap; +/// +/// assert_eq!(wrap("Foo bar baz", 10), vec!["Foo bar", "baz"]); +/// assert_eq!(wrap("Foo bar baz", 8), vec!["Foo", "bar baz"]); +/// ``` +/// +/// Notice how the trailing whitespace is removed in both case: in the +/// first example, `"bar␣"` becomes `"bar"` and in the second case +/// `"Foo␣␣␣"` becomes `"Foo"`. +/// +/// Leading whitespace is preserved when the following word fits on +/// the first line. To understand this, consider how words are found +/// in a text with leading spaces: +/// +/// ```text +/// "␣␣foo␣bar" -> ["␣␣", "foo␣", "bar"] +/// ``` +/// +/// When put into lines, the indentation is preserved if `"foo"` fits +/// on the first line, otherwise you end up with an empty line: +/// +/// ``` +/// use textwrap::wrap; +/// +/// assert_eq!(wrap(" foo bar", 8), vec![" foo", "bar"]); +/// assert_eq!(wrap(" foo bar", 4), vec!["", "foo", "bar"]); +/// ``` +pub fn wrap<'a, Opt>(text: &str, width_or_options: Opt) -> Vec> +where + Opt: Into>, +{ + let options: Options = width_or_options.into(); + let line_ending_str = options.line_ending.as_str(); + + let mut lines = Vec::new(); + for line in text.split(line_ending_str) { + wrap_single_line(line, &options, &mut lines); + } + + lines +} + +pub(crate) fn wrap_single_line<'a>( + line: &'a str, + options: &Options<'_>, + lines: &mut Vec>, +) { + let indent = if lines.is_empty() { + options.initial_indent + } else { + options.subsequent_indent + }; + if line.len() < options.width && indent.is_empty() { + lines.push(Cow::from(line.trim_end_matches(' '))); + } else { + wrap_single_line_slow_path(line, options, lines) + } +} + +/// Wrap a single line of text. +/// +/// This is taken when `line` is longer than `options.width`. +pub(crate) fn wrap_single_line_slow_path<'a>( + line: &'a str, + options: &Options<'_>, + lines: &mut Vec>, +) { + let initial_width = options + .width + .saturating_sub(display_width(options.initial_indent)); + let subsequent_width = options + .width + .saturating_sub(display_width(options.subsequent_indent)); + let line_widths = [initial_width, subsequent_width]; + + let words = options.word_separator.find_words(line); + let split_words = split_words(words, &options.word_splitter); + let broken_words = if options.break_words { + let mut broken_words = break_words(split_words, line_widths[1]); + if !options.initial_indent.is_empty() { + // Without this, the first word will always go into the + // first line. However, since we break words based on the + // _second_ line width, it can be wrong to unconditionally + // put the first word onto the first line. An empty + // zero-width word fixed this. + broken_words.insert(0, Word::from("")); + } + broken_words + } else { + split_words.collect::>() + }; + + let wrapped_words = options.wrap_algorithm.wrap(&broken_words, &line_widths); + + let mut idx = 0; + for words in wrapped_words { + let last_word = match words.last() { + None => { + lines.push(Cow::from("")); + continue; + } + Some(word) => word, + }; + + // We assume here that all words are contiguous in `line`. + // That is, the sum of their lengths should add up to the + // length of `line`. + let len = words + .iter() + .map(|word| word.len() + word.whitespace.len()) + .sum::() + - last_word.whitespace.len(); + + // The result is owned if we have indentation, otherwise we + // can simply borrow an empty string. + let mut result = if lines.is_empty() && !options.initial_indent.is_empty() { + Cow::Owned(options.initial_indent.to_owned()) + } else if !lines.is_empty() && !options.subsequent_indent.is_empty() { + Cow::Owned(options.subsequent_indent.to_owned()) + } else { + // We can use an empty string here since string + // concatenation for `Cow` preserves a borrowed value when + // either side is empty. + Cow::from("") + }; + + result += &line[idx..idx + len]; + + if !last_word.penalty.is_empty() { + result.to_mut().push_str(last_word.penalty); + } + + lines.push(result); + + // Advance by the length of `result`, plus the length of + // `last_word.whitespace` -- even if we had a penalty, we need + // to skip over the whitespace. + idx += len + last_word.whitespace.len(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{WordSeparator, WordSplitter, WrapAlgorithm}; + + #[cfg(feature = "hyphenation")] + use hyphenation::{Language, Load, Standard}; + + #[test] + fn no_wrap() { + assert_eq!(wrap("foo", 10), vec!["foo"]); + } + + #[test] + fn wrap_simple() { + assert_eq!(wrap("foo bar baz", 5), vec!["foo", "bar", "baz"]); + } + + #[test] + fn to_be_or_not() { + assert_eq!( + wrap( + "To be, or not to be, that is the question.", + Options::new(10).wrap_algorithm(WrapAlgorithm::FirstFit) + ), + vec!["To be, or", "not to be,", "that is", "the", "question."] + ); + } + + #[test] + fn multiple_words_on_first_line() { + assert_eq!(wrap("foo bar baz", 10), vec!["foo bar", "baz"]); + } + + #[test] + fn long_word() { + assert_eq!(wrap("foo", 0), vec!["f", "o", "o"]); + } + + #[test] + fn long_words() { + assert_eq!(wrap("foo bar", 0), vec!["f", "o", "o", "b", "a", "r"]); + } + + #[test] + fn max_width() { + assert_eq!(wrap("foo bar", usize::MAX), vec!["foo bar"]); + + let text = "Hello there! This is some English text. \ + It should not be wrapped given the extents below."; + assert_eq!(wrap(text, usize::MAX), vec![text]); + } + + #[test] + fn leading_whitespace() { + assert_eq!(wrap(" foo bar", 6), vec![" foo", "bar"]); + } + + #[test] + fn leading_whitespace_empty_first_line() { + // If there is no space for the first word, the first line + // will be empty. This is because the string is split into + // words like [" ", "foobar ", "baz"], which puts "foobar " on + // the second line. We never output trailing whitespace + assert_eq!(wrap(" foobar baz", 6), vec!["", "foobar", "baz"]); + } + + #[test] + fn trailing_whitespace() { + // Whitespace is only significant inside a line. After a line + // gets too long and is broken, the first word starts in + // column zero and is not indented. + assert_eq!(wrap("foo bar baz ", 5), vec!["foo", "bar", "baz"]); + } + + #[test] + fn issue_99() { + // We did not reset the in_whitespace flag correctly and did + // not handle single-character words after a line break. + assert_eq!( + wrap("aaabbbccc x yyyzzzwww", 9), + vec!["aaabbbccc", "x", "yyyzzzwww"] + ); + } + + #[test] + fn issue_129() { + // The dash is an em-dash which takes up four bytes. We used + // to panic since we tried to index into the character. + let options = Options::new(1).word_separator(WordSeparator::AsciiSpace); + assert_eq!(wrap("x – x", options), vec!["x", "–", "x"]); + } + + #[test] + fn wide_character_handling() { + assert_eq!(wrap("Hello, World!", 15), vec!["Hello, World!"]); + assert_eq!( + wrap( + "Ｈｅｌｌｏ, Ｗｏｒｌｄ!", + Options::new(15).word_separator(WordSeparator::AsciiSpace) + ), + vec!["Ｈｅｌｌｏ,", "Ｗｏｒｌｄ!"] + ); + + // Wide characters are allowed to break if the + // unicode-linebreak feature is enabled. + #[cfg(feature = "unicode-linebreak")] + assert_eq!( + wrap( + "Ｈｅｌｌｏ, Ｗｏｒｌｄ!", + Options::new(15).word_separator(WordSeparator::UnicodeBreakProperties), + ), + vec!["Ｈｅｌｌｏ, Ｗ", "ｏｒｌｄ!"] + ); + } + + #[test] + fn indent_empty_line() { + // Previously, indentation was not applied to empty lines. + // However, this is somewhat inconsistent and undesirable if + // the indentation is something like a border ("| ") which you + // want to apply to all lines, empty or not. + let options = Options::new(10).initial_indent("!!!"); + assert_eq!(wrap("", &options), vec!["!!!"]); + } + + #[test] + fn indent_single_line() { + let options = Options::new(10).initial_indent(">>>"); // No trailing space + assert_eq!(wrap("foo", &options), vec![">>>foo"]); + } + + #[test] + fn indent_first_emoji() { + let options = Options::new(10).initial_indent("👉👉"); + assert_eq!( + wrap("x x x x x x x x x x x x x", &options), + vec!["👉👉x x x", "x x x x x", "x x x x x"] + ); + } + + #[test] + fn indent_multiple_lines() { + let options = Options::new(6).initial_indent("* ").subsequent_indent(" "); + assert_eq!( + wrap("foo bar baz", &options), + vec!["* foo", " bar", " baz"] + ); + } + + #[test] + fn only_initial_indent_multiple_lines() { + let options = Options::new(10).initial_indent(" "); + assert_eq!(wrap("foo\nbar\nbaz", &options), vec![" foo", "bar", "baz"]); + } + + #[test] + fn only_subsequent_indent_multiple_lines() { + let options = Options::new(10).subsequent_indent(" "); + assert_eq!( + wrap("foo\nbar\nbaz", &options), + vec!["foo", " bar", " baz"] + ); + } + + #[test] + fn indent_break_words() { + let options = Options::new(5).initial_indent("* ").subsequent_indent(" "); + assert_eq!(wrap("foobarbaz", &options), vec!["* foo", " bar", " baz"]); + } + + #[test] + fn initial_indent_break_words() { + // This is a corner-case showing how the long word is broken + // according to the width of the subsequent lines. The first + // fragment of the word no longer fits on the first line, + // which ends up being pure indentation. + let options = Options::new(5).initial_indent("-->"); + assert_eq!(wrap("foobarbaz", &options), vec!["-->", "fooba", "rbaz"]); + } + + #[test] + fn hyphens() { + assert_eq!(wrap("foo-bar", 5), vec!["foo-", "bar"]); + } + + #[test] + fn trailing_hyphen() { + let options = Options::new(5).break_words(false); + assert_eq!(wrap("foobar-", &options), vec!["foobar-"]); + } + + #[test] + fn multiple_hyphens() { + assert_eq!(wrap("foo-bar-baz", 5), vec!["foo-", "bar-", "baz"]); + } + + #[test] + fn hyphens_flag() { + let options = Options::new(5).break_words(false); + assert_eq!( + wrap("The --foo-bar flag.", &options), + vec!["The", "--foo-", "bar", "flag."] + ); + } + + #[test] + fn repeated_hyphens() { + let options = Options::new(4).break_words(false); + assert_eq!(wrap("foo--bar", &options), vec!["foo--bar"]); + } + + #[test] + fn hyphens_alphanumeric() { + assert_eq!(wrap("Na2-CH4", 5), vec!["Na2-", "CH4"]); + } + + #[test] + fn hyphens_non_alphanumeric() { + let options = Options::new(5).break_words(false); + assert_eq!(wrap("foo(-)bar", &options), vec!["foo(-)bar"]); + } + + #[test] + fn multiple_splits() { + assert_eq!(wrap("foo-bar-baz", 9), vec!["foo-bar-", "baz"]); + } + + #[test] + fn forced_split() { + let options = Options::new(5).break_words(false); + assert_eq!(wrap("foobar-baz", &options), vec!["foobar-", "baz"]); + } + + #[test] + fn multiple_unbroken_words_issue_193() { + let options = Options::new(3).break_words(false); + assert_eq!( + wrap("small large tiny", &options), + vec!["small", "large", "tiny"] + ); + assert_eq!( + wrap("small large tiny", &options), + vec!["small", "large", "tiny"] + ); + } + + #[test] + fn very_narrow_lines_issue_193() { + let options = Options::new(1).break_words(false); + assert_eq!(wrap("fooo x y", &options), vec!["fooo", "x", "y"]); + assert_eq!(wrap("fooo x y", &options), vec!["fooo", "x", "y"]); + } + + #[test] + fn simple_hyphens() { + let options = Options::new(8).word_splitter(WordSplitter::HyphenSplitter); + assert_eq!(wrap("foo bar-baz", &options), vec!["foo bar-", "baz"]); + } + + #[test] + fn no_hyphenation() { + let options = Options::new(8).word_splitter(WordSplitter::NoHyphenation); + assert_eq!(wrap("foo bar-baz", &options), vec!["foo", "bar-baz"]); + } + + #[test] + #[cfg(feature = "hyphenation")] + fn auto_hyphenation_double_hyphenation() { + let dictionary = Standard::from_embedded(Language::EnglishUS).unwrap(); + let options = Options::new(10); + assert_eq!( + wrap("Internationalization", &options), + vec!["Internatio", "nalization"] + ); + + let options = Options::new(10).word_splitter(WordSplitter::Hyphenation(dictionary)); + assert_eq!( + wrap("Internationalization", &options), + vec!["Interna-", "tionaliza-", "tion"] + ); + } + + #[test] + #[cfg(feature = "hyphenation")] + fn auto_hyphenation_issue_158() { + let dictionary = Standard::from_embedded(Language::EnglishUS).unwrap(); + let options = Options::new(10); + assert_eq!( + wrap("participation is the key to success", &options), + vec!["participat", "ion is", "the key to", "success"] + ); + + let options = Options::new(10).word_splitter(WordSplitter::Hyphenation(dictionary)); + assert_eq!( + wrap("participation is the key to success", &options), + vec!["partici-", "pation is", "the key to", "success"] + ); + } + + #[test] + #[cfg(feature = "hyphenation")] + fn split_len_hyphenation() { + // Test that hyphenation takes the width of the whitespace + // into account. + let dictionary = Standard::from_embedded(Language::EnglishUS).unwrap(); + let options = Options::new(15).word_splitter(WordSplitter::Hyphenation(dictionary)); + assert_eq!( + wrap("garbage collection", &options), + vec!["garbage col-", "lection"] + ); + } + + #[test] + #[cfg(feature = "hyphenation")] + fn borrowed_lines() { + // Lines that end with an extra hyphen are owned, the final + // line is borrowed. + use std::borrow::Cow::{Borrowed, Owned}; + let dictionary = Standard::from_embedded(Language::EnglishUS).unwrap(); + let options = Options::new(10).word_splitter(WordSplitter::Hyphenation(dictionary)); + let lines = wrap("Internationalization", &options); + assert_eq!(lines, vec!["Interna-", "tionaliza-", "tion"]); + if let Borrowed(s) = lines[0] { + assert!(false, "should not have been borrowed: {:?}", s); + } + if let Borrowed(s) = lines[1] { + assert!(false, "should not have been borrowed: {:?}", s); + } + if let Owned(ref s) = lines[2] { + assert!(false, "should not have been owned: {:?}", s); + } + } + + #[test] + #[cfg(feature = "hyphenation")] + fn auto_hyphenation_with_hyphen() { + let dictionary = Standard::from_embedded(Language::EnglishUS).unwrap(); + let options = Options::new(8).break_words(false); + assert_eq!( + wrap("over-caffinated", &options), + vec!["over-", "caffinated"] + ); + + let options = options.word_splitter(WordSplitter::Hyphenation(dictionary)); + assert_eq!( + wrap("over-caffinated", &options), + vec!["over-", "caffi-", "nated"] + ); + } + + #[test] + fn break_words() { + assert_eq!(wrap("foobarbaz", 3), vec!["foo", "bar", "baz"]); + } + + #[test] + fn break_words_wide_characters() { + // Even the poor man's version of `ch_width` counts these + // characters as wide. + let options = Options::new(5).word_separator(WordSeparator::AsciiSpace); + assert_eq!(wrap("Ｈｅｌｌｏ", options), vec!["Ｈｅ", "ｌｌ", "ｏ"]); + } + + #[test] + fn break_words_zero_width() { + assert_eq!(wrap("foobar", 0), vec!["f", "o", "o", "b", "a", "r"]); + } + + #[test] + fn break_long_first_word() { + assert_eq!(wrap("testx y", 4), vec!["test", "x y"]); + } + + #[test] + fn wrap_preserves_line_breaks_trims_whitespace() { + assert_eq!(wrap(" ", 80), vec![""]); + assert_eq!(wrap(" \n ", 80), vec!["", ""]); + assert_eq!(wrap(" \n \n \n ", 80), vec!["", "", "", ""]); + } + + #[test] + fn wrap_colored_text() { + // The words are much longer than 6 bytes, but they remain + // intact after filling the text. + let green_hello = "\u{1b}[0m\u{1b}[32mHello\u{1b}[0m"; + let blue_world = "\u{1b}[0m\u{1b}[34mWorld!\u{1b}[0m"; + assert_eq!( + wrap(&format!("{} {}", green_hello, blue_world), 6), + vec![green_hello, blue_world], + ); + } +} diff --git a/third_party/rust/textwrap/src/wrap_algorithms.rs b/third_party/rust/textwrap/src/wrap_algorithms.rs new file mode 100644 index 0000000000..7737e08f99 --- /dev/null +++ b/third_party/rust/textwrap/src/wrap_algorithms.rs @@ -0,0 +1,413 @@ +//! Word wrapping algorithms. +//! +//! After a text has been broken into words (or [`Fragment`]s), one +//! now has to decide how to break the fragments into lines. The +//! simplest algorithm for this is implemented by +//! [`wrap_first_fit()`]: it uses no look-ahead and simply adds +//! fragments to the line as long as they fit. However, this can lead +//! to poor line breaks if a large fragment almost-but-not-quite fits +//! on a line. When that happens, the fragment is moved to the next +//! line and it will leave behind a large gap. +//! +//! A more advanced algorithm, implemented by [`wrap_optimal_fit()`], +//! will take this into account. The optimal-fit algorithm considers +//! all possible line breaks and will attempt to minimize the gaps +//! left behind by overly short lines. +//! +//! While both algorithms run in linear time, the first-fit algorithm +//! is about 4 times faster than the optimal-fit algorithm. + +#[cfg(feature = "smawk")] +mod optimal_fit; +#[cfg(feature = "smawk")] +pub use optimal_fit::{wrap_optimal_fit, OverflowError, Penalties}; + +use crate::core::{Fragment, Word}; + +/// Describes how to wrap words into lines. +/// +/// The simplest approach is to wrap words one word at a time and +/// accept the first way of wrapping which fit +/// ([`WrapAlgorithm::FirstFit`]). If the `smawk` Cargo feature is +/// enabled, a more complex algorithm is available which will look at +/// an entire paragraph at a time in order to find optimal line breaks +/// ([`WrapAlgorithm::OptimalFit`]). +#[derive(Clone, Copy)] +pub enum WrapAlgorithm { + /// Wrap words using a fast and simple algorithm. + /// + /// This algorithm uses no look-ahead when finding line breaks. + /// Implemented by [`wrap_first_fit()`], please see that function + /// for details and examples. + FirstFit, + + /// Wrap words using an advanced algorithm with look-ahead. + /// + /// This wrapping algorithm considers the entire paragraph to find + /// optimal line breaks. When wrapping text, "penalties" are + /// assigned to line breaks based on the gaps left at the end of + /// lines. See [`Penalties`] for details. + /// + /// The underlying wrapping algorithm is implemented by + /// [`wrap_optimal_fit()`], please see that function for examples. + /// + /// **Note:** Only available when the `smawk` Cargo feature is + /// enabled. + #[cfg(feature = "smawk")] + OptimalFit(Penalties), + + /// Custom wrapping function. + /// + /// Use this if you want to implement your own wrapping algorithm. + /// The function can freely decide how to turn a slice of + /// [`Word`]s into lines. + /// + /// # Example + /// + /// ``` + /// use textwrap::core::Word; + /// use textwrap::{wrap, Options, WrapAlgorithm}; + /// + /// fn stair<'a, 'b>(words: &'b [Word<'a>], _: &'b [usize]) -> Vec<&'b [Word<'a>]> { + /// let mut lines = Vec::new(); + /// let mut step = 1; + /// let mut start_idx = 0; + /// while start_idx + step <= words.len() { + /// lines.push(&words[start_idx .. start_idx+step]); + /// start_idx += step; + /// step += 1; + /// } + /// lines + /// } + /// + /// let options = Options::new(10).wrap_algorithm(WrapAlgorithm::Custom(stair)); + /// assert_eq!(wrap("First, second, third, fourth, fifth, sixth", options), + /// vec!["First,", + /// "second, third,", + /// "fourth, fifth, sixth"]); + /// ``` + Custom(for<'a, 'b> fn(words: &'b [Word<'a>], line_widths: &'b [usize]) -> Vec<&'b [Word<'a>]>), +} + +impl PartialEq for WrapAlgorithm { + /// Compare two wrap algorithms. + /// + /// ``` + /// use textwrap::WrapAlgorithm; + /// + /// assert_eq!(WrapAlgorithm::FirstFit, WrapAlgorithm::FirstFit); + /// #[cfg(feature = "smawk")] { + /// assert_eq!(WrapAlgorithm::new_optimal_fit(), WrapAlgorithm::new_optimal_fit()); + /// } + /// ``` + /// + /// Note that `WrapAlgorithm::Custom` values never compare equal: + /// + /// ``` + /// use textwrap::WrapAlgorithm; + /// + /// assert_ne!(WrapAlgorithm::Custom(|words, line_widths| vec![words]), + /// WrapAlgorithm::Custom(|words, line_widths| vec![words])); + /// ``` + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (WrapAlgorithm::FirstFit, WrapAlgorithm::FirstFit) => true, + #[cfg(feature = "smawk")] + (WrapAlgorithm::OptimalFit(a), WrapAlgorithm::OptimalFit(b)) => a == b, + (_, _) => false, + } + } +} + +impl std::fmt::Debug for WrapAlgorithm { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + WrapAlgorithm::FirstFit => f.write_str("FirstFit"), + #[cfg(feature = "smawk")] + WrapAlgorithm::OptimalFit(penalties) => write!(f, "OptimalFit({:?})", penalties), + WrapAlgorithm::Custom(_) => f.write_str("Custom(...)"), + } + } +} + +impl WrapAlgorithm { + /// Create new wrap algorithm. + /// + /// The best wrapping algorithm is used by default, i.e., + /// [`WrapAlgorithm::OptimalFit`] if available, otherwise + /// [`WrapAlgorithm::FirstFit`]. + pub const fn new() -> Self { + #[cfg(not(feature = "smawk"))] + { + WrapAlgorithm::FirstFit + } + + #[cfg(feature = "smawk")] + { + WrapAlgorithm::new_optimal_fit() + } + } + + /// New [`WrapAlgorithm::OptimalFit`] with default penalties. This + /// works well for monospace text. + /// + /// **Note:** Only available when the `smawk` Cargo feature is + /// enabled. + #[cfg(feature = "smawk")] + pub const fn new_optimal_fit() -> Self { + WrapAlgorithm::OptimalFit(Penalties::new()) + } + + /// Wrap words according to line widths. + /// + /// The `line_widths` slice gives the target line width for each + /// line (the last slice element is repeated as necessary). This + /// can be used to implement hanging indentation. + #[inline] + pub fn wrap<'a, 'b>( + &self, + words: &'b [Word<'a>], + line_widths: &'b [usize], + ) -> Vec<&'b [Word<'a>]> { + // Every integer up to 2u64.pow(f64::MANTISSA_DIGITS) = 2**53 + // = 9_007_199_254_740_992 can be represented without loss by + // a f64. Larger line widths will be rounded to the nearest + // representable number. + let f64_line_widths = line_widths.iter().map(|w| *w as f64).collect::>(); + + match self { + WrapAlgorithm::FirstFit => wrap_first_fit(words, &f64_line_widths), + + #[cfg(feature = "smawk")] + WrapAlgorithm::OptimalFit(penalties) => { + // The computation cannot overflow when the line + // widths are restricted to usize. + wrap_optimal_fit(words, &f64_line_widths, penalties).unwrap() + } + + WrapAlgorithm::Custom(func) => func(words, line_widths), + } + } +} + +impl Default for WrapAlgorithm { + fn default() -> Self { + WrapAlgorithm::new() + } +} + +/// Wrap abstract fragments into lines with a first-fit algorithm. +/// +/// The `line_widths` slice gives the target line width for each line +/// (the last slice element is repeated as necessary). This can be +/// used to implement hanging indentation. +/// +/// The fragments must already have been split into the desired +/// widths, this function will not (and cannot) attempt to split them +/// further when arranging them into lines. +/// +/// # First-Fit Algorithm +/// +/// This implements a simple “greedy” algorithm: accumulate fragments +/// one by one and when a fragment no longer fits, start a new line. +/// There is no look-ahead, we simply take first fit of the fragments +/// we find. +/// +/// While fast and predictable, this algorithm can produce poor line +/// breaks when a long fragment is moved to a new line, leaving behind +/// a large gap: +/// +/// ``` +/// use textwrap::core::Word; +/// use textwrap::wrap_algorithms::wrap_first_fit; +/// use textwrap::WordSeparator; +/// +/// // Helper to convert wrapped lines to a Vec. +/// fn lines_to_strings(lines: Vec<&[Word<'_>]>) -> Vec { +/// lines.iter().map(|line| { +/// line.iter().map(|word| &**word).collect::>().join(" ") +/// }).collect::>() +/// } +/// +/// let text = "These few words will unfortunately not wrap nicely."; +/// let words = WordSeparator::AsciiSpace.find_words(text).collect::>(); +/// assert_eq!(lines_to_strings(wrap_first_fit(&words, &[15.0])), +/// vec!["These few words", +/// "will", // <-- short line +/// "unfortunately", +/// "not wrap", +/// "nicely."]); +/// +/// // We can avoid the short line if we look ahead: +/// #[cfg(feature = "smawk")] +/// use textwrap::wrap_algorithms::{wrap_optimal_fit, Penalties}; +/// #[cfg(feature = "smawk")] +/// assert_eq!(lines_to_strings(wrap_optimal_fit(&words, &[15.0], &Penalties::new()).unwrap()), +/// vec!["These few", +/// "words will", +/// "unfortunately", +/// "not wrap", +/// "nicely."]); +/// ``` +/// +/// The [`wrap_optimal_fit()`] function was used above to get better +/// line breaks. It uses an advanced algorithm which tries to avoid +/// short lines. This function is about 4 times faster than +/// [`wrap_optimal_fit()`]. +/// +/// # Examples +/// +/// Imagine you're building a house site and you have a number of +/// tasks you need to execute. Things like pour foundation, complete +/// framing, install plumbing, electric cabling, install insulation. +/// +/// The construction workers can only work during daytime, so they +/// need to pack up everything at night. Because they need to secure +/// their tools and move machines back to the garage, this process +/// takes much more time than the time it would take them to simply +/// switch to another task. +/// +/// You would like to make a list of tasks to execute every day based +/// on your estimates. You can model this with a program like this: +/// +/// ``` +/// use textwrap::core::{Fragment, Word}; +/// use textwrap::wrap_algorithms::wrap_first_fit; +/// +/// #[derive(Debug)] +/// struct Task<'a> { +/// name: &'a str, +/// hours: f64, // Time needed to complete task. +/// sweep: f64, // Time needed for a quick sweep after task during the day. +/// cleanup: f64, // Time needed for full cleanup if day ends with this task. +/// } +/// +/// impl Fragment for Task<'_> { +/// fn width(&self) -> f64 { self.hours } +/// fn whitespace_width(&self) -> f64 { self.sweep } +/// fn penalty_width(&self) -> f64 { self.cleanup } +/// } +/// +/// // The morning tasks +/// let tasks = vec![ +/// Task { name: "Foundation", hours: 4.0, sweep: 2.0, cleanup: 3.0 }, +/// Task { name: "Framing", hours: 3.0, sweep: 1.0, cleanup: 2.0 }, +/// Task { name: "Plumbing", hours: 2.0, sweep: 2.0, cleanup: 2.0 }, +/// Task { name: "Electrical", hours: 2.0, sweep: 1.0, cleanup: 2.0 }, +/// Task { name: "Insulation", hours: 2.0, sweep: 1.0, cleanup: 2.0 }, +/// Task { name: "Drywall", hours: 3.0, sweep: 1.0, cleanup: 2.0 }, +/// Task { name: "Floors", hours: 3.0, sweep: 1.0, cleanup: 2.0 }, +/// Task { name: "Countertops", hours: 1.0, sweep: 1.0, cleanup: 2.0 }, +/// Task { name: "Bathrooms", hours: 2.0, sweep: 1.0, cleanup: 2.0 }, +/// ]; +/// +/// // Fill tasks into days, taking `day_length` into account. The +/// // output shows the hours worked per day along with the names of +/// // the tasks for that day. +/// fn assign_days<'a>(tasks: &[Task<'a>], day_length: f64) -> Vec<(f64, Vec<&'a str>)> { +/// let mut days = Vec::new(); +/// // Assign tasks to days. The assignment is a vector of slices, +/// // with a slice per day. +/// let assigned_days: Vec<&[Task<'a>]> = wrap_first_fit(&tasks, &[day_length]); +/// for day in assigned_days.iter() { +/// let last = day.last().unwrap(); +/// let work_hours: f64 = day.iter().map(|t| t.hours + t.sweep).sum(); +/// let names = day.iter().map(|t| t.name).collect::>(); +/// days.push((work_hours - last.sweep + last.cleanup, names)); +/// } +/// days +/// } +/// +/// // With a single crew working 8 hours a day: +/// assert_eq!( +/// assign_days(&tasks, 8.0), +/// [ +/// (7.0, vec!["Foundation"]), +/// (8.0, vec!["Framing", "Plumbing"]), +/// (7.0, vec!["Electrical", "Insulation"]), +/// (5.0, vec!["Drywall"]), +/// (7.0, vec!["Floors", "Countertops"]), +/// (4.0, vec!["Bathrooms"]), +/// ] +/// ); +/// +/// // With two crews working in shifts, 16 hours a day: +/// assert_eq!( +/// assign_days(&tasks, 16.0), +/// [ +/// (14.0, vec!["Foundation", "Framing", "Plumbing"]), +/// (15.0, vec!["Electrical", "Insulation", "Drywall", "Floors"]), +/// (6.0, vec!["Countertops", "Bathrooms"]), +/// ] +/// ); +/// ``` +/// +/// Apologies to anyone who actually knows how to build a house and +/// knows how long each step takes :-) +pub fn wrap_first_fit<'a, T: Fragment>( + fragments: &'a [T], + line_widths: &[f64], +) -> Vec<&'a [T]> { + // The final line width is used for all remaining lines. + let default_line_width = line_widths.last().copied().unwrap_or(0.0); + let mut lines = Vec::new(); + let mut start = 0; + let mut width = 0.0; + + for (idx, fragment) in fragments.iter().enumerate() { + let line_width = line_widths + .get(lines.len()) + .copied() + .unwrap_or(default_line_width); + if width + fragment.width() + fragment.penalty_width() > line_width && idx > start { + lines.push(&fragments[start..idx]); + start = idx; + width = 0.0; + } + width += fragment.width() + fragment.whitespace_width(); + } + lines.push(&fragments[start..]); + lines +} + +#[cfg(test)] +mod tests { + use super::*; + + #[derive(Debug, PartialEq)] + struct Word(f64); + + #[rustfmt::skip] + impl Fragment for Word { + fn width(&self) -> f64 { self.0 } + fn whitespace_width(&self) -> f64 { 1.0 } + fn penalty_width(&self) -> f64 { 0.0 } + } + + #[test] + fn wrap_string_longer_than_f64() { + let words = vec![ + Word(1e307), + Word(2e307), + Word(3e307), + Word(4e307), + Word(5e307), + Word(6e307), + ]; + // Wrap at just under f64::MAX (~19e307). The tiny + // whitespace_widths disappear because of loss of precision. + assert_eq!( + wrap_first_fit(&words, &[15e307]), + &[ + vec![ + Word(1e307), + Word(2e307), + Word(3e307), + Word(4e307), + Word(5e307) + ], + vec![Word(6e307)] + ] + ); + } +} diff --git a/third_party/rust/textwrap/src/wrap_algorithms/optimal_fit.rs b/third_party/rust/textwrap/src/wrap_algorithms/optimal_fit.rs new file mode 100644 index 0000000000..bdc0334539 --- /dev/null +++ b/third_party/rust/textwrap/src/wrap_algorithms/optimal_fit.rs @@ -0,0 +1,433 @@ +use std::cell::RefCell; + +use crate::core::Fragment; + +/// Penalties for +/// [`WrapAlgorithm::OptimalFit`](crate::WrapAlgorithm::OptimalFit) +/// and [`wrap_optimal_fit`]. +/// +/// This wrapping algorithm in [`wrap_optimal_fit`] considers the +/// entire paragraph to find optimal line breaks. When wrapping text, +/// "penalties" are assigned to line breaks based on the gaps left at +/// the end of lines. The penalties are given by this struct, with +/// [`Penalties::default`] assigning penalties that work well for +/// monospace text. +/// +/// If you are wrapping proportional text, you are advised to assign +/// your own penalties according to your font size. See the individual +/// penalties below for details. +/// +/// **Note:** Only available when the `smawk` Cargo feature is +/// enabled. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Penalties { + /// Per-line penalty. This is added for every line, which makes it + /// expensive to output more lines than the minimum required. + pub nline_penalty: usize, + + /// Per-character cost for lines that overflow the target line width. + /// + /// With a default value of 50², every single character costs as + /// much as leaving a gap of 50 characters behind. This is because + /// we assign as cost of `gap * gap` to a short line. When + /// wrapping monospace text, we can overflow the line by 1 + /// character in extreme cases: + /// + /// ``` + /// use textwrap::core::Word; + /// use textwrap::wrap_algorithms::{wrap_optimal_fit, Penalties}; + /// + /// let short = "foo "; + /// let long = "x".repeat(50); + /// let length = (short.len() + long.len()) as f64; + /// let fragments = vec![Word::from(short), Word::from(&long)]; + /// let penalties = Penalties::new(); + /// + /// // Perfect fit, both words are on a single line with no overflow. + /// let wrapped = wrap_optimal_fit(&fragments, &[length], &penalties).unwrap(); + /// assert_eq!(wrapped, vec![&[Word::from(short), Word::from(&long)]]); + /// + /// // The words no longer fit, yet we get a single line back. While + /// // the cost of overflow (`1 * 2500`) is the same as the cost of the + /// // gap (`50 * 50 = 2500`), the tie is broken by `nline_penalty` + /// // which makes it cheaper to overflow than to use two lines. + /// let wrapped = wrap_optimal_fit(&fragments, &[length - 1.0], &penalties).unwrap(); + /// assert_eq!(wrapped, vec![&[Word::from(short), Word::from(&long)]]); + /// + /// // The cost of overflow would be 2 * 2500, whereas the cost of + /// // the gap is only `49 * 49 + nline_penalty = 2401 + 1000 = + /// // 3401`. We therefore get two lines. + /// let wrapped = wrap_optimal_fit(&fragments, &[length - 2.0], &penalties).unwrap(); + /// assert_eq!(wrapped, vec![&[Word::from(short)], + /// &[Word::from(&long)]]); + /// ``` + /// + /// This only happens if the overflowing word is 50 characters + /// long _and_ if the word overflows the line by exactly one + /// character. If it overflows by more than one character, the + /// overflow penalty will quickly outgrow the cost of the gap, as + /// seen above. + pub overflow_penalty: usize, + + /// When should the a single word on the last line be considered + /// "too short"? + /// + /// If the last line of the text consist of a single word and if + /// this word is shorter than `1 / short_last_line_fraction` of + /// the line width, then the final line will be considered "short" + /// and `short_last_line_penalty` is added as an extra penalty. + /// + /// The effect of this is to avoid a final line consisting of a + /// single small word. For example, with a + /// `short_last_line_penalty` of 25 (the default), a gap of up to + /// 5 columns will be seen as more desirable than having a final + /// short line. + /// + /// ## Examples + /// + /// ``` + /// use textwrap::{wrap, wrap_algorithms, Options, WrapAlgorithm}; + /// + /// let text = "This is a demo of the short last line penalty."; + /// + /// // The first-fit algorithm leaves a single short word on the last line: + /// assert_eq!(wrap(text, Options::new(37).wrap_algorithm(WrapAlgorithm::FirstFit)), + /// vec!["This is a demo of the short last line", + /// "penalty."]); + /// + /// #[cfg(feature = "smawk")] { + /// let mut penalties = wrap_algorithms::Penalties::new(); + /// + /// // Since "penalty." is shorter than 25% of the line width, the + /// // optimal-fit algorithm adds a penalty of 25. This is enough + /// // to move "line " down: + /// assert_eq!(wrap(text, Options::new(37).wrap_algorithm(WrapAlgorithm::OptimalFit(penalties))), + /// vec!["This is a demo of the short last", + /// "line penalty."]); + /// + /// // We can change the meaning of "short" lines. Here, only words + /// // shorter than 1/10th of the line width will be considered short: + /// penalties.short_last_line_fraction = 10; + /// assert_eq!(wrap(text, Options::new(37).wrap_algorithm(WrapAlgorithm::OptimalFit(penalties))), + /// vec!["This is a demo of the short last line", + /// "penalty."]); + /// + /// // If desired, the penalty can also be disabled: + /// penalties.short_last_line_fraction = 4; + /// penalties.short_last_line_penalty = 0; + /// assert_eq!(wrap(text, Options::new(37).wrap_algorithm(WrapAlgorithm::OptimalFit(penalties))), + /// vec!["This is a demo of the short last line", + /// "penalty."]); + /// } + /// ``` + pub short_last_line_fraction: usize, + + /// Penalty for a last line with a single short word. + /// + /// Set this to zero if you do not want to penalize short last lines. + pub short_last_line_penalty: usize, + + /// Penalty for lines ending with a hyphen. + pub hyphen_penalty: usize, +} + +impl Penalties { + /// Default penalties for monospace text. + /// + /// The penalties here work well for monospace text. This is + /// because they expect the gaps at the end of lines to be roughly + /// in the range `0..100`. If the gaps are larger, the + /// `overflow_penalty` and `hyphen_penalty` become insignificant. + pub const fn new() -> Self { + Penalties { + nline_penalty: 1000, + overflow_penalty: 50 * 50, + short_last_line_fraction: 4, + short_last_line_penalty: 25, + hyphen_penalty: 25, + } + } +} + +impl Default for Penalties { + fn default() -> Self { + Self::new() + } +} + +/// Cache for line numbers. This is necessary to avoid a O(n**2) +/// behavior when computing line numbers in [`wrap_optimal_fit`]. +struct LineNumbers { + line_numbers: RefCell>, +} + +impl LineNumbers { + fn new(size: usize) -> Self { + let mut line_numbers = Vec::with_capacity(size); + line_numbers.push(0); + LineNumbers { + line_numbers: RefCell::new(line_numbers), + } + } + + fn get(&self, i: usize, minima: &[(usize, T)]) -> usize { + while self.line_numbers.borrow_mut().len() < i + 1 { + let pos = self.line_numbers.borrow().len(); + let line_number = 1 + self.get(minima[pos].0, minima); + self.line_numbers.borrow_mut().push(line_number); + } + + self.line_numbers.borrow()[i] + } +} + +/// Overflow error during the [`wrap_optimal_fit`] computation. +#[derive(Debug, PartialEq, Eq)] +pub struct OverflowError; + +impl std::fmt::Display for OverflowError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "wrap_optimal_fit cost computation overflowed") + } +} + +impl std::error::Error for OverflowError {} + +/// Wrap abstract fragments into lines with an optimal-fit algorithm. +/// +/// The `line_widths` slice gives the target line width for each line +/// (the last slice element is repeated as necessary). This can be +/// used to implement hanging indentation. +/// +/// The fragments must already have been split into the desired +/// widths, this function will not (and cannot) attempt to split them +/// further when arranging them into lines. +/// +/// # Optimal-Fit Algorithm +/// +/// The algorithm considers all possible break points and picks the +/// breaks which minimizes the gaps at the end of each line. More +/// precisely, the algorithm assigns a cost or penalty to each break +/// point, determined by `cost = gap * gap` where `gap = target_width - +/// line_width`. Shorter lines are thus penalized more heavily since +/// they leave behind a larger gap. +/// +/// We can illustrate this with the text “To be, or not to be: that is +/// the question”. We will be wrapping it in a narrow column with room +/// for only 10 characters. The [greedy +/// algorithm](super::wrap_first_fit) will produce these lines, each +/// annotated with the corresponding penalty: +/// +/// ```text +/// "To be, or" 1² = 1 +/// "not to be:" 0² = 0 +/// "that is" 3² = 9 +/// "the" 7² = 49 +/// "question" 2² = 4 +/// ``` +/// +/// We see that line four with “the” leaves a gap of 7 columns, which +/// gives it a penalty of 49. The sum of the penalties is 63. +/// +/// There are 10 words, which means that there are `2_u32.pow(9)` or +/// 512 different ways to typeset it. We can compute +/// the sum of the penalties for each possible line break and search +/// for the one with the lowest sum: +/// +/// ```text +/// "To be," 4² = 16 +/// "or not to" 1² = 1 +/// "be: that" 2² = 4 +/// "is the" 4² = 16 +/// "question" 2² = 4 +/// ``` +/// +/// The sum of the penalties is 41, which is better than what the +/// greedy algorithm produced. +/// +/// Searching through all possible combinations would normally be +/// prohibitively slow. However, it turns out that the problem can be +/// formulated as the task of finding column minima in a cost matrix. +/// This matrix has a special form (totally monotone) which lets us +/// use a [linear-time algorithm called +/// SMAWK](https://lib.rs/crates/smawk) to find the optimal break +/// points. +/// +/// This means that the time complexity remains O(_n_) where _n_ is +/// the number of words. Compared to +/// [`wrap_first_fit()`](super::wrap_first_fit), this function is +/// about 4 times slower. +/// +/// The optimization of per-line costs over the entire paragraph is +/// inspired by the line breaking algorithm used in TeX, as described +/// in the 1981 article [_Breaking Paragraphs into +/// Lines_](http://www.eprg.org/G53DOC/pdfs/knuth-plass-breaking.pdf) +/// by Knuth and Plass. The implementation here is based on [Python +/// code by David +/// Eppstein](https://github.com/jfinkels/PADS/blob/master/pads/wrap.py). +/// +/// # Errors +/// +/// In case of an overflow during the cost computation, an `Err` is +/// returned. Overflows happens when fragments or lines have infinite +/// widths (`f64::INFINITY`) or if the widths are so large that the +/// gaps at the end of lines have sizes larger than `f64::MAX.sqrt()` +/// (approximately 1e154): +/// +/// ``` +/// use textwrap::core::Fragment; +/// use textwrap::wrap_algorithms::{wrap_optimal_fit, OverflowError, Penalties}; +/// +/// #[derive(Debug, PartialEq)] +/// struct Word(f64); +/// +/// impl Fragment for Word { +/// fn width(&self) -> f64 { self.0 } +/// fn whitespace_width(&self) -> f64 { 1.0 } +/// fn penalty_width(&self) -> f64 { 0.0 } +/// } +/// +/// // Wrapping overflows because 1e155 * 1e155 = 1e310, which is +/// // larger than f64::MAX: +/// assert_eq!(wrap_optimal_fit(&[Word(0.0), Word(0.0)], &[1e155], &Penalties::default()), +/// Err(OverflowError)); +/// ``` +/// +/// When using fragment widths and line widths which fit inside an +/// `u64`, overflows cannot happen. This means that fragments derived +/// from a `&str` cannot cause overflows. +/// +/// **Note:** Only available when the `smawk` Cargo feature is +/// enabled. +pub fn wrap_optimal_fit<'a, 'b, T: Fragment>( + fragments: &'a [T], + line_widths: &'b [f64], + penalties: &'b Penalties, +) -> Result, OverflowError> { + // The final line width is used for all remaining lines. + let default_line_width = line_widths.last().copied().unwrap_or(0.0); + let mut widths = Vec::with_capacity(fragments.len() + 1); + let mut width = 0.0; + widths.push(width); + for fragment in fragments { + width += fragment.width() + fragment.whitespace_width(); + widths.push(width); + } + + let line_numbers = LineNumbers::new(fragments.len()); + + let minima = smawk::online_column_minima(0.0, widths.len(), |minima, i, j| { + // Line number for fragment `i`. + let line_number = line_numbers.get(i, minima); + let line_width = line_widths + .get(line_number) + .copied() + .unwrap_or(default_line_width); + let target_width = line_width.max(1.0); + + // Compute the width of a line spanning fragments[i..j] in + // constant time. We need to adjust widths[j] by subtracting + // the whitespace of fragment[j-1] and then add the penalty. + let line_width = widths[j] - widths[i] - fragments[j - 1].whitespace_width() + + fragments[j - 1].penalty_width(); + + // We compute cost of the line containing fragments[i..j]. We + // start with values[i].1, which is the optimal cost for + // breaking before fragments[i]. + // + // First, every extra line cost NLINE_PENALTY. + let mut cost = minima[i].1 + penalties.nline_penalty as f64; + + // Next, we add a penalty depending on the line length. + if line_width > target_width { + // Lines that overflow get a hefty penalty. + let overflow = line_width - target_width; + cost += overflow * penalties.overflow_penalty as f64; + } else if j < fragments.len() { + // Other lines (except for the last line) get a milder + // penalty which depend on the size of the gap. + let gap = target_width - line_width; + cost += gap * gap; + } else if i + 1 == j + && line_width < target_width / penalties.short_last_line_fraction as f64 + { + // The last line can have any size gap, but we do add a + // penalty if the line is very short (typically because it + // contains just a single word). + cost += penalties.short_last_line_penalty as f64; + } + + // Finally, we discourage hyphens. + if fragments[j - 1].penalty_width() > 0.0 { + // TODO: this should use a penalty value from the fragment + // instead. + cost += penalties.hyphen_penalty as f64; + } + + cost + }); + + for (_, cost) in &minima { + if cost.is_infinite() { + return Err(OverflowError); + } + } + + let mut lines = Vec::with_capacity(line_numbers.get(fragments.len(), &minima)); + let mut pos = fragments.len(); + loop { + let prev = minima[pos].0; + lines.push(&fragments[prev..pos]); + pos = prev; + if pos == 0 { + break; + } + } + + lines.reverse(); + Ok(lines) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[derive(Debug, PartialEq)] + struct Word(f64); + + #[rustfmt::skip] + impl Fragment for Word { + fn width(&self) -> f64 { self.0 } + fn whitespace_width(&self) -> f64 { 1.0 } + fn penalty_width(&self) -> f64 { 0.0 } + } + + #[test] + fn wrap_fragments_with_infinite_widths() { + let words = vec![Word(f64::INFINITY)]; + assert_eq!( + wrap_optimal_fit(&words, &[0.0], &Penalties::default()), + Err(OverflowError) + ); + } + + #[test] + fn wrap_fragments_with_huge_widths() { + let words = vec![Word(1e200), Word(1e250), Word(1e300)]; + assert_eq!( + wrap_optimal_fit(&words, &[1e300], &Penalties::default()), + Err(OverflowError) + ); + } + + #[test] + fn wrap_fragments_with_large_widths() { + // The gaps will be of the sizes between 1e25 and 1e75. This + // makes the `gap * gap` cost fit comfortably in a f64. + let words = vec![Word(1e25), Word(1e50), Word(1e75)]; + assert_eq!( + wrap_optimal_fit(&words, &[1e100], &Penalties::default()), + Ok(vec![&vec![Word(1e25), Word(1e50), Word(1e75)][..]]) + ); + } +} diff --git a/third_party/rust/textwrap/tests/indent.rs b/third_party/rust/textwrap/tests/indent.rs new file mode 100644 index 0000000000..9dd5ad2642 --- /dev/null +++ b/third_party/rust/textwrap/tests/indent.rs @@ -0,0 +1,88 @@ +/// tests cases ported over from python standard library +use textwrap::{dedent, indent}; + +const ROUNDTRIP_CASES: [&str; 3] = [ + // basic test case + "Hi.\nThis is a test.\nTesting.", + // include a blank line + "Hi.\nThis is a test.\n\nTesting.", + // include leading and trailing blank lines + "\nHi.\nThis is a test.\nTesting.\n", +]; + +const WINDOWS_CASES: [&str; 2] = [ + // use windows line endings + "Hi.\r\nThis is a test.\r\nTesting.", + // pathological case + "Hi.\r\nThis is a test.\n\r\nTesting.\r\n\n", +]; + +#[test] +fn test_indent_nomargin_default() { + // indent should do nothing if 'prefix' is empty. + for text in ROUNDTRIP_CASES.iter() { + assert_eq!(&indent(text, ""), text); + } + for text in WINDOWS_CASES.iter() { + assert_eq!(&indent(text, ""), text); + } +} + +#[test] +fn test_roundtrip_spaces() { + // A whitespace prefix should roundtrip with dedent + for text in ROUNDTRIP_CASES.iter() { + assert_eq!(&dedent(&indent(text, " ")), text); + } +} + +#[test] +fn test_roundtrip_tabs() { + // A whitespace prefix should roundtrip with dedent + for text in ROUNDTRIP_CASES.iter() { + assert_eq!(&dedent(&indent(text, "\t\t")), text); + } +} + +#[test] +fn test_roundtrip_mixed() { + // A whitespace prefix should roundtrip with dedent + for text in ROUNDTRIP_CASES.iter() { + assert_eq!(&dedent(&indent(text, " \t \t ")), text); + } +} + +#[test] +fn test_indent_default() { + // Test default indenting of lines that are not whitespace only + let prefix = " "; + let expected = [ + // Basic test case + " Hi.\n This is a test.\n Testing.", + // Include a blank line + " Hi.\n This is a test.\n\n Testing.", + // Include leading and trailing blank lines + "\n Hi.\n This is a test.\n Testing.\n", + ]; + for (text, expect) in ROUNDTRIP_CASES.iter().zip(expected.iter()) { + assert_eq!(&indent(text, prefix), expect) + } + let expected = [ + // Use Windows line endings + " Hi.\r\n This is a test.\r\n Testing.", + // Pathological case + " Hi.\r\n This is a test.\n\r\n Testing.\r\n\n", + ]; + for (text, expect) in WINDOWS_CASES.iter().zip(expected.iter()) { + assert_eq!(&indent(text, prefix), expect) + } +} + +#[test] +fn indented_text_should_have_the_same_number_of_lines_as_the_original_text() { + let texts = ["foo\nbar", "foo\nbar\n", "foo\nbar\nbaz"]; + for original in texts.iter() { + let indented = indent(original, ""); + assert_eq!(&indented, original); + } +} diff --git a/third_party/rust/textwrap/tests/version-numbers.rs b/third_party/rust/textwrap/tests/version-numbers.rs new file mode 100644 index 0000000000..3f429b187a --- /dev/null +++ b/third_party/rust/textwrap/tests/version-numbers.rs @@ -0,0 +1,22 @@ +#[test] +fn test_readme_deps() { + version_sync::assert_markdown_deps_updated!("README.md"); +} + +#[test] +fn test_changelog() { + version_sync::assert_contains_regex!( + "CHANGELOG.md", + r"^## Version {version} \(20\d\d-\d\d-\d\d\)" + ); +} + +#[test] +fn test_html_root_url() { + version_sync::assert_html_root_url_updated!("src/lib.rs"); +} + +#[test] +fn test_dependency_graph() { + version_sync::assert_contains_regex!("src/lib.rs", "master/images/textwrap-{version}.svg"); +} diff --git a/third_party/rust/unicode-linebreak/.cargo-checksum.json b/third_party/rust/unicode-linebreak/.cargo-checksum.json new file mode 100644 index 0000000000..2b4a10002a --- /dev/null +++ b/third_party/rust/unicode-linebreak/.cargo-checksum.json @@ -0,0 +1 @@ +{"files":{"Cargo.toml":"7f8e0b5d66e9c5621c5fb57d4b16b801a24061e37ee337e06f8a004e6895a8dc","LICENSE":"c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4","src/lib.rs":"ef4f143f99e9ef4f17dab02e1b80ed3ac484ae58e0bb64164920720e0ca9425f","src/shared.rs":"e0c98ea10f78491f567e2a18bcb41b3f0ae01ce587d8b3a16ce0dc1097919109","src/tables.rs":"1821b437dfb31164ce8180af3937ca42270f1edf963a2d2e41cbaaf999553c94"},"package":"3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f"} \ No newline at end of file diff --git a/third_party/rust/unicode-linebreak/Cargo.toml b/third_party/rust/unicode-linebreak/Cargo.toml new file mode 100644 index 0000000000..857c779d67 --- /dev/null +++ b/third_party/rust/unicode-linebreak/Cargo.toml @@ -0,0 +1,32 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2021" +rust-version = "1.56" +name = "unicode-linebreak" +version = "0.1.5" +authors = ["Axel Forsman "] +include = [ + "src/**/*", + "LICENSE", +] +description = "Implementation of the Unicode Line Breaking Algorithm" +homepage = "https://github.com/axelf4/unicode-linebreak" +readme = "README.md" +keywords = [ + "unicode", + "text", + "layout", +] +categories = ["internationalization"] +license = "Apache-2.0" +repository = "https://github.com/axelf4/unicode-linebreak" diff --git a/third_party/rust/unicode-linebreak/LICENSE b/third_party/rust/unicode-linebreak/LICENSE new file mode 100644 index 0000000000..261eeb9e9f --- /dev/null +++ b/third_party/rust/unicode-linebreak/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/third_party/rust/unicode-linebreak/src/lib.rs b/third_party/rust/unicode-linebreak/src/lib.rs new file mode 100644 index 0000000000..ca473d8494 --- /dev/null +++ b/third_party/rust/unicode-linebreak/src/lib.rs @@ -0,0 +1,160 @@ +//! Implementation of the Line Breaking Algorithm described in [Unicode Standard Annex #14][UAX14]. +//! +//! Given an input text, locates "line break opportunities", or positions appropriate for wrapping +//! lines when displaying text. +//! +//! # Example +//! +//! ``` +//! use unicode_linebreak::{linebreaks, BreakOpportunity::{Mandatory, Allowed}}; +//! +//! let text = "a b \nc"; +//! assert!(linebreaks(text).eq([ +//! (2, Allowed), // May break after first space +//! (5, Mandatory), // Must break after line feed +//! (6, Mandatory) // Must break at end of text, so that there always is at least one LB +//! ])); +//! ``` +//! +//! [UAX14]: https://www.unicode.org/reports/tr14/ + +#![no_std] +#![deny(missing_docs, missing_debug_implementations)] + +use core::iter::once; + +/// The [Unicode version](https://www.unicode.org/versions/) conformed to. +pub const UNICODE_VERSION: (u8, u8, u8) = (15, 0, 0); + +include!("shared.rs"); +include!("tables.rs"); + +/// Returns the line break property of the specified code point. +/// +/// # Examples +/// +/// ``` +/// use unicode_linebreak::{BreakClass, break_property}; +/// assert_eq!(break_property(0x2CF3), BreakClass::Alphabetic); +/// ``` +#[inline(always)] +pub fn break_property(codepoint: u32) -> BreakClass { + const BMP_INDEX_LENGTH: u32 = BMP_LIMIT >> BMP_SHIFT; + const OMITTED_BMP_INDEX_1_LENGTH: u32 = BMP_LIMIT >> SHIFT_1; + + let data_pos = if codepoint < BMP_LIMIT { + let i = codepoint >> BMP_SHIFT; + BREAK_PROP_TRIE_INDEX[i as usize] + (codepoint & (BMP_DATA_BLOCK_LENGTH - 1)) as u16 + } else if codepoint < BREAK_PROP_TRIE_HIGH_START { + let i1 = codepoint >> SHIFT_1; + let i2 = BREAK_PROP_TRIE_INDEX + [(i1 + BMP_INDEX_LENGTH - OMITTED_BMP_INDEX_1_LENGTH) as usize] + + ((codepoint >> SHIFT_2) & (INDEX_2_BLOCK_LENGTH - 1)) as u16; + let i3_block = BREAK_PROP_TRIE_INDEX[i2 as usize]; + let i3_pos = ((codepoint >> SHIFT_3) & (INDEX_3_BLOCK_LENGTH - 1)) as u16; + + debug_assert!(i3_block & 0x8000 == 0, "18-bit indices are unexpected"); + let data_block = BREAK_PROP_TRIE_INDEX[(i3_block + i3_pos) as usize]; + data_block + (codepoint & (SMALL_DATA_BLOCK_LENGTH - 1)) as u16 + } else { + return XX; + }; + BREAK_PROP_TRIE_DATA[data_pos as usize] +} + +/// Break opportunity type. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +pub enum BreakOpportunity { + /// A line must break at this spot. + Mandatory, + /// A line is allowed to end at this spot. + Allowed, +} + +/// Returns an iterator over line break opportunities in the specified string. +/// +/// Break opportunities are given as tuples of the byte index of the character succeeding the break +/// and the type. +/// +/// Uses the default Line Breaking Algorithm with the tailoring that Complex-Context Dependent +/// (SA) characters get resolved to Ordinary Alphabetic and Symbol Characters (AL) regardless of +/// General_Category. +/// +/// # Examples +/// +/// ``` +/// use unicode_linebreak::{linebreaks, BreakOpportunity::{Mandatory, Allowed}}; +/// assert!(linebreaks("Hello world!").eq(vec![(6, Allowed), (12, Mandatory)])); +/// ``` +pub fn linebreaks(s: &str) -> impl Iterator + Clone + '_ { + use BreakOpportunity::{Allowed, Mandatory}; + + s.char_indices() + .map(|(i, c)| (i, break_property(c as u32) as u8)) + .chain(once((s.len(), eot))) + .scan((sot, false), |state, (i, cls)| { + // ZWJ is handled outside the table to reduce its size + let val = PAIR_TABLE[state.0 as usize][cls as usize]; + let is_mandatory = val & MANDATORY_BREAK_BIT != 0; + let is_break = val & ALLOWED_BREAK_BIT != 0 && (!state.1 || is_mandatory); + *state = ( + val & !(ALLOWED_BREAK_BIT | MANDATORY_BREAK_BIT), + cls == BreakClass::ZeroWidthJoiner as u8, + ); + + Some((i, is_break, is_mandatory)) + }) + .filter_map(|(i, is_break, is_mandatory)| { + if is_break { + Some((i, if is_mandatory { Mandatory } else { Allowed })) + } else { + None + } + }) +} + +/// Divides the string at the last index where further breaks do not depend on prior context. +/// +/// The trivial index at `eot` is excluded. +/// +/// A common optimization is to determine only the nearest line break opportunity before the first +/// character that would cause the line to become overfull, requiring backward traversal, of which +/// there are two approaches: +/// +/// * Cache breaks from forward traversals +/// * Step backward and with `split_at_safe` find a pos to safely search forward from, repeatedly +/// +/// # Examples +/// +/// ``` +/// use unicode_linebreak::{linebreaks, split_at_safe}; +/// let s = "Not allowed to break within em dashes: — —"; +/// let (prev, safe) = split_at_safe(s); +/// let n = prev.len(); +/// assert!(linebreaks(safe).eq(linebreaks(s).filter_map(|(i, x)| i.checked_sub(n).map(|i| (i, x))))); +/// ``` +pub fn split_at_safe(s: &str) -> (&str, &str) { + let mut chars = s.char_indices().rev().scan(None, |state, (i, c)| { + let cls = break_property(c as u32); + let is_safe_pair = state + .replace(cls) + .map_or(false, |prev| is_safe_pair(cls, prev)); // Reversed since iterating backwards + Some((i, is_safe_pair)) + }); + chars.find(|&(_, is_safe_pair)| is_safe_pair); + // Include preceding char for `linebreaks` to pick up break before match (disallowed after sot) + s.split_at(chars.next().map_or(0, |(i, _)| i)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn it_works() { + assert_eq!(break_property(0xA), BreakClass::LineFeed); + assert_eq!(break_property(0xDB80), BreakClass::Surrogate); + assert_eq!(break_property(0xe01ef), BreakClass::CombiningMark); + assert_eq!(break_property(0x10ffff), BreakClass::Unknown); + } +} diff --git a/third_party/rust/unicode-linebreak/src/shared.rs b/third_party/rust/unicode-linebreak/src/shared.rs new file mode 100644 index 0000000000..c73819f069 --- /dev/null +++ b/third_party/rust/unicode-linebreak/src/shared.rs @@ -0,0 +1,134 @@ +/// Unicode line breaking class. +#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)] +#[repr(u8)] +pub enum BreakClass { + // Non-tailorable + /// Cause a line break (after) + Mandatory, + /// Cause a line break (after), except between CR and LF + CarriageReturn, + /// Cause a line break (after) + LineFeed, + /// Prohibit a line break between the character and the preceding character + CombiningMark, + /// Cause a line break (after) + NextLine, + /// Do not occur in well-formed text + Surrogate, + /// Prohibit line breaks before and after + WordJoiner, + /// Provide a break opportunity + ZeroWidthSpace, + /// Prohibit line breaks before and after + NonBreakingGlue, + /// Enable indirect line breaks + Space, + /// Prohibit line breaks within joiner sequences + ZeroWidthJoiner, + // Break opportunities + /// Provide a line break opportunity before and after the character + BeforeAndAfter, + /// Generally provide a line break opportunity after the character + After, + /// Generally provide a line break opportunity before the character + Before, + /// Provide a line break opportunity after the character, except in numeric context + Hyphen, + /// Provide a line break opportunity contingent on additional information + Contingent, + // Characters prohibiting certain breaks + /// Prohibit line breaks before + ClosePunctuation, + /// Prohibit line breaks before + CloseParenthesis, + /// Prohibit line breaks before + Exclamation, + /// Allow only indirect line breaks between pairs + Inseparable, + /// Allow only indirect line breaks before + NonStarter, + /// Prohibit line breaks after + OpenPunctuation, + /// Act like they are both opening and closing + Quotation, + // Numeric context + /// Prevent breaks after any and before numeric + InfixSeparator, + /// Form numeric expressions for line breaking purposes + Numeric, + /// Do not break following a numeric expression + Postfix, + /// Do not break in front of a numeric expression + Prefix, + /// Prevent a break before, and allow a break after + Symbol, + // Other characters + /// Act like AL when the resolved EAW is N; otherwise, act as ID + Ambiguous, + /// Are alphabetic characters or symbols that are used with alphabetic characters + Alphabetic, + /// Treat as NS or ID for strict or normal breaking. + ConditionalJapaneseStarter, + /// Do not break from following Emoji Modifier + EmojiBase, + /// Do not break from preceding Emoji Base + EmojiModifier, + /// Form Korean syllable blocks + HangulLvSyllable, + /// Form Korean syllable blocks + HangulLvtSyllable, + /// Do not break around a following hyphen; otherwise act as Alphabetic + HebrewLetter, + /// Break before or after, except in some numeric context + Ideographic, + /// Form Korean syllable blocks + HangulLJamo, + /// Form Korean syllable blocks + HangulVJamo, + /// Form Korean syllable blocks + HangulTJamo, + /// Keep pairs together. For pairs, break before and after other classes + RegionalIndicator, + /// Provide a line break opportunity contingent on additional, language-specific context analysis + ComplexContext, + /// Have as yet unknown line breaking behavior or unassigned code positions + Unknown, +} + +use BreakClass::{ + After as BA, Alphabetic as AL, Ambiguous as AI, Before as BB, BeforeAndAfter as B2, + CarriageReturn as CR, CloseParenthesis as CP, ClosePunctuation as CL, CombiningMark as CM, + ComplexContext as SA, ConditionalJapaneseStarter as CJ, Contingent as CB, EmojiBase as EB, + EmojiModifier as EM, Exclamation as EX, HangulLJamo as JL, HangulLvSyllable as H2, + HangulLvtSyllable as H3, HangulTJamo as JT, HangulVJamo as JV, HebrewLetter as HL, + Hyphen as HY, Ideographic as ID, InfixSeparator as IS, Inseparable as IN, LineFeed as LF, + Mandatory as BK, NextLine as NL, NonBreakingGlue as GL, NonStarter as NS, Numeric as NU, + OpenPunctuation as OP, Postfix as PO, Prefix as PR, Quotation as QU, RegionalIndicator as RI, + Space as SP, Surrogate as SG, Symbol as SY, Unknown as XX, WordJoiner as WJ, + ZeroWidthJoiner as ZWJ, ZeroWidthSpace as ZW, +}; + +/// Ceiling for code points in the Basic Multilingual Place (BMP). +const BMP_LIMIT: u32 = 0x10000; + +/// Shift size for getting index-3 table offset. +const SHIFT_3: u32 = 4; +/// Shift size for getting index-2 table offset. +const SHIFT_2: u32 = 5 + SHIFT_3; +/// Shift size for getting index-1 table offset. +const SHIFT_1: u32 = 5 + SHIFT_2; +/// Shift size for getting BMP block start. +const BMP_SHIFT: u32 = 6; + +const INDEX_2_BLOCK_LENGTH: u32 = 1 << (SHIFT_1 - SHIFT_2); +const INDEX_3_BLOCK_LENGTH: u32 = 1 << (SHIFT_2 - SHIFT_3); +const SMALL_DATA_BLOCK_LENGTH: u32 = 1 << SHIFT_3; +const BMP_DATA_BLOCK_LENGTH: u32 = 1 << BMP_SHIFT; + +const ALLOWED_BREAK_BIT: u8 = 0x80; +const MANDATORY_BREAK_BIT: u8 = 0x40; + +#[allow(non_upper_case_globals)] +const eot: u8 = 43; +#[allow(non_upper_case_globals)] +const sot: u8 = 44; diff --git a/third_party/rust/unicode-linebreak/src/tables.rs b/third_party/rust/unicode-linebreak/src/tables.rs new file mode 100644 index 0000000000..1a5d16b5ce --- /dev/null +++ b/third_party/rust/unicode-linebreak/src/tables.rs @@ -0,0 +1,10 @@ +const BREAK_PROP_TRIE_HIGH_START: u32 = 918016; +static BREAK_PROP_TRIE_INDEX: [u16; 2844] = [0, 64, 127, 191, 247, 247, 247, 247, 247, 247, 247, 304, 368, 417, 481, 247, 247, 247, 542, 247, 558, 607, 662, 726, 790, 843, 247, 892, 950, 1003, 1029, 1093, 1157, 1221, 1270, 1324, 1384, 1446, 1509, 1571, 1634, 1696, 1759, 1821, 1885, 1947, 2009, 2071, 2135, 2197, 2260, 2322, 2386, 2448, 2512, 2576, 2639, 2703, 2766, 2830, 2894, 2958, 3016, 3080, 3144, 3208, 3256, 3314, 3378, 3410, 3442, 3482, 247, 3546, 3601, 3663, 3710, 3747, 3782, 3814, 3878, 247, 247, 247, 247, 247, 247, 247, 247, 247, 3942, 3974, 4038, 4102, 3144, 4166, 4230, 4262, 4326, 4374, 4438, 4502, 4566, 4620, 4661, 4694, 4758, 4807, 4871, 4930, 4994, 5052, 5112, 5176, 5240, 5301, 247, 247, 247, 5365, 247, 247, 247, 247, 5429, 5487, 553, 5551, 5615, 5677, 5741, 5803, 5867, 5911, 5969, 6015, 6079, 6141, 6203, 6267, 6323, 247, 247, 6366, 6418, 6482, 6514, 6515, 6514, 6566, 6630, 6690, 6754, 6818, 6882, 6943, 7004, 7045, 7099, 7158, 247, 247, 247, 247, 247, 247, 7219, 7259, 247, 247, 247, 247, 247, 7321, 7375, 247, 247, 247, 247, 7398, 7462, 7510, 7574, 7606, 7670, 7734, 7798, 7825, 7889, 7889, 7889, 7931, 7995, 8059, 8120, 8181, 8245, 7889, 7809, 8294, 8262, 8358, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 247, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 8401, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 8452, 8509, 247, 247, 247, 247, 8573, 8637, 8699, 8731, 247, 247, 247, 8795, 8857, 8921, 8985, 9043, 9107, 9164, 9228, 9291, 9355, 9419, 3144, 9480, 9543, 9591, 247, 9639, 9703, 9711, 9719, 9727, 9707, 9715, 9723, 9703, 9711, 9719, 9727, 9707, 9715, 9723, 9703, 9711, 9719, 9727, 9707, 9715, 9723, 9703, 9711, 9719, 9727, 9707, 9715, 9723, 9703, 9711, 9719, 9727, 9707, 9715, 9723, 9703, 9711, 9719, 9727, 9707, 9715, 9723, 9703, 9711, 9719, 9727, 9707, 9715, 9723, 9703, 9711, 9719, 9727, 9707, 9715, 9723, 9703, 9711, 9719, 9727, 9707, 9715, 9723, 9703, 9711, 9719, 9727, 9707, 9715, 9723, 9703, 9711, 9719, 9727, 9707, 9715, 9723, 9703, 9711, 9719, 9727, 9707, 9715, 9723, 9703, 9711, 9719, 9727, 9707, 9715, 9723, 9703, 9711, 9719, 9727, 9707, 9715, 9723, 9703, 9711, 9719, 9727, 9707, 9715, 9723, 9703, 9711, 9719, 9727, 9707, 9715, 9723, 9703, 9711, 9719, 9727, 9707, 9715, 9723, 9703, 9711, 9719, 9727, 9707, 9715, 9723, 9703, 9711, 9719, 9727, 9707, 9715, 9723, 9703, 9711, 9719, 9727, 9707, 9715, 9723, 9703, 9711, 9719, 9727, 9707, 9715, 9723, 9703, 9711, 9719, 9727, 9707, 9715, 9723, 9703, 9711, 9719, 9727, 9707, 9715, 9723, 9703, 9711, 9719, 9727, 9707, 9715, 9723, 9703, 9711, 9719, 9727, 9707, 9715, 9779, 9836, 9900, 9900, 9900, 9900, 9900, 9900, 9900, 9900, 9900, 9900, 9900, 9900, 9900, 9900, 9900, 9900, 9900, 9900, 9900, 9900, 9900, 9900, 9900, 9900, 9900, 9900, 9900, 9900, 9900, 9900, 9900, 9900, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 10028, 10092, 247, 10153, 247, 247, 247, 247, 10172, 247, 10236, 10292, 10356, 10416, 247, 10470, 10534, 10596, 10645, 10708, 2656, 2686, 2715, 2746, 2778, 2778, 2778, 2779, 2778, 2778, 2778, 2779, 2811, 2811, 2811, 2811, 2811, 2811, 2811, 2811, 2811, 2811, 2811, 2811, 2811, 2811, 2811, 2811, 2811, 2811, 2811, 2811, 2811, 2811, 2811, 2811, 2811, 2811, 2811, 2811, 2811, 2811, 2811, 2811, 2811, 2811, 2811, 2811, 2811, 2811, 2811, 2811, 2843, 503, 247, 508, 10772, 616, 616, 9964, 9964, 247, 247, 247, 247, 247, 247, 247, 1253, 10788, 247, 247, 2531, 247, 247, 247, 247, 500, 2522, 1837, 9964, 9964, 247, 247, 10795, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 247, 2522, 247, 247, 247, 1837, 551, 2055, 247, 247, 7593, 247, 1253, 247, 247, 10811, 247, 10827, 247, 247, 9631, 10842, 9964, 9964, 247, 247, 247, 247, 247, 247, 247, 247, 247, 616, 2235, 247, 247, 9631, 247, 2055, 247, 247, 1995, 247, 247, 247, 10844, 504, 504, 10859, 513, 10873, 9964, 9964, 9964, 9964, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 4421, 247, 4422, 1995, 9964, 509, 247, 247, 10889, 9964, 9964, 9964, 9964, 10905, 247, 247, 10915, 247, 10930, 247, 247, 247, 500, 783, 9964, 9964, 9964, 247, 10943, 247, 10954, 247, 1254, 9964, 9964, 9964, 9964, 247, 247, 247, 9627, 247, 630, 247, 247, 10970, 1769, 247, 10986, 4022, 11002, 247, 247, 247, 247, 9964, 9964, 247, 247, 11018, 11034, 247, 247, 247, 11050, 247, 624, 247, 1261, 247, 11066, 781, 9964, 9964, 9964, 9964, 9964, 247, 247, 247, 247, 4022, 9964, 9964, 9964, 247, 247, 247, 6454, 247, 247, 247, 4028, 247, 247, 4052, 2235, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 247, 500, 247, 247, 11082, 6455, 9964, 9964, 9964, 4858, 247, 247, 1995, 247, 362, 11098, 9964, 247, 11114, 9964, 9964, 247, 2055, 9964, 247, 4421, 549, 247, 247, 360, 11130, 630, 2920, 11146, 549, 247, 247, 11161, 11175, 247, 4022, 2235, 549, 247, 361, 11191, 11207, 247, 247, 11223, 549, 247, 247, 365, 11239, 11255, 494, 6452, 247, 513, 356, 11271, 11286, 9964, 9964, 9964, 11302, 501, 11317, 247, 247, 353, 4811, 2235, 11333, 629, 506, 11348, 1947, 11364, 11378, 4817, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 247, 247, 247, 363, 11394, 11410, 6455, 9964, 247, 247, 247, 368, 11426, 2235, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 247, 247, 353, 11442, 11457, 11466, 9964, 9964, 247, 247, 247, 368, 11482, 2235, 11498, 9964, 247, 247, 357, 11514, 2235, 9964, 9964, 9964, 3144, 2817, 2686, 11530, 9476, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 247, 247, 356, 1067, 9964, 9964, 9964, 9964, 9964, 9964, 247, 247, 247, 247, 940, 10032, 11546, 11558, 247, 11574, 11588, 2235, 9964, 9964, 9964, 9964, 622, 247, 247, 11604, 11619, 9964, 8688, 247, 247, 11635, 11651, 11667, 247, 247, 358, 11683, 11698, 247, 247, 247, 247, 4022, 11714, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 506, 247, 353, 3033, 11730, 940, 2522, 11746, 247, 3005, 3032, 4815, 9964, 9964, 9964, 9964, 1801, 247, 247, 11761, 11776, 2235, 11792, 247, 4674, 11808, 2235, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 247, 11824, 11840, 493, 247, 11852, 11866, 2235, 9964, 9964, 9964, 9964, 9964, 1837, 247, 11882, 11897, 11911, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 3798, 9964, 9964, 9964, 9964, 9964, 9964, 247, 247, 247, 247, 247, 247, 500, 11926, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 6453, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 247, 247, 247, 247, 247, 247, 6454, 247, 247, 247, 247, 247, 11942, 247, 247, 11956, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 11966, 247, 247, 247, 247, 247, 247, 247, 247, 11982, 11998, 4816, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 3955, 247, 247, 247, 247, 4421, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 247, 247, 247, 4022, 247, 500, 12014, 247, 247, 247, 247, 500, 2235, 247, 616, 12030, 247, 247, 247, 12046, 12058, 12074, 513, 1256, 247, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 247, 247, 247, 247, 247, 12085, 9964, 9964, 9964, 9964, 9964, 9964, 247, 247, 247, 247, 2056, 367, 368, 368, 12101, 549, 9964, 9964, 9964, 9964, 12117, 4820, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7869, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 4422, 9964, 9964, 7868, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 1671, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7950, 12131, 9964, 12147, 12159, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7865, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 247, 247, 247, 247, 247, 247, 1253, 2522, 4022, 12175, 4818, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 368, 368, 1000, 368, 4815, 247, 247, 247, 247, 247, 247, 247, 6453, 9964, 9964, 9964, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 4422, 247, 247, 623, 247, 247, 247, 12191, 368, 12204, 247, 12216, 247, 247, 247, 1253, 9964, 247, 247, 247, 247, 12230, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 247, 6453, 247, 6453, 247, 247, 247, 247, 247, 4421, 247, 4022, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 247, 247, 247, 247, 247, 510, 247, 247, 247, 502, 12244, 12258, 511, 247, 247, 247, 3549, 1670, 247, 3600, 12271, 493, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 624, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 12281, 12295, 12295, 12295, 368, 368, 368, 11672, 368, 368, 452, 12311, 12323, 4860, 678, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 247, 500, 12335, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 3033, 12351, 12365, 247, 247, 247, 616, 9964, 4856, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 247, 247, 2522, 12381, 9307, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 247, 12397, 9964, 247, 247, 356, 12413, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 247, 356, 2235, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 12429, 500, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 247, 625, 4815, 9964, 9964, 247, 247, 247, 247, 12445, 12461, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 494, 247, 247, 10340, 12477, 9964, 9964, 9964, 9964, 494, 247, 247, 616, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 511, 247, 12492, 12505, 12519, 12535, 12549, 12557, 505, 2055, 12572, 2055, 9964, 9964, 9964, 6455, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 12588, 6514, 6564, 6514, 6514, 6514, 12604, 6514, 6514, 6514, 12588, 7889, 7889, 7889, 12617, 12623, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 12639, 12645, 7889, 12652, 12666, 7889, 7889, 12679, 7889, 7889, 7889, 7889, 12695, 12710, 12720, 12727, 12742, 12756, 12772, 12786, 7889, 7889, 7889, 7889, 6936, 11875, 12802, 6416, 6933, 7889, 7889, 12814, 7889, 12830, 7889, 7889, 7889, 12842, 7889, 12854, 7889, 7889, 7889, 7889, 12865, 247, 247, 12881, 7889, 7889, 12641, 12897, 12903, 7889, 7889, 7889, 247, 247, 247, 247, 247, 247, 247, 12919, 247, 247, 247, 247, 247, 12802, 7889, 7889, 6402, 247, 247, 247, 6935, 6933, 247, 247, 6935, 247, 6400, 7889, 7889, 7889, 7889, 7889, 12935, 12718, 12751, 12950, 7889, 7889, 7889, 12750, 7889, 7889, 7889, 12965, 12713, 12980, 7889, 7889, 247, 247, 247, 247, 247, 12919, 7889, 7889, 7889, 7889, 7889, 7889, 12898, 7889, 7889, 12702, 247, 247, 247, 247, 247, 247, 247, 247, 247, 512, 247, 247, 1253, 9964, 9964, 2235, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7889, 7863, 2093, 9964, 368, 368, 368, 368, 368, 368, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 9964, 368, 368, 368, 368, 368, 368, 368, 368, 368, 368, 368, 368, 368, 368, 368, 9964, 1077, 1109, 1141, 1173, 1205, 1237, 1269, 1295, 1327, 1359, 1391, 1423, 1455, 1487, 1519, 1546, 1578, 1585, 1617, 896, 896, 896, 896, 1638, 1578, 1670, 1699, 896, 896, 896, 896, 896, 1731, 1760, 896, 896, 896, 896, 896, 896, 896, 896, 896, 896, 896, 896, 896, 896, 896, 896, 1578, 1792, 896, 1820, 202, 202, 202, 202, 202, 202, 202, 202, 1852, 202, 1884, 1903, 896, 896, 896, 896, 896, 896, 896, 896, 896, 896, 896, 896, 896, 896, 896, 896, 1920, 1952, 1975, 896, 896, 896, 896, 2007, 896, 896, 896, 896, 896, 896, 896, 2023, 2055, 2087, 2119, 2141, 1578, 2173, 896, 2189, 2221, 2244, 2263, 2279, 2311, 896, 2336, 2368, 2400, 2432, 2464, 2496, 2528, 2560, 202, 2592, 202, 202, 202, 202, 202, 202, 202, 202, 202, 202, 202, 202, 202, 202, 202, 202, 202, 202, 202, 202, 202, 202, 202, 202, 202, 202, 202, 202, 202, 202, 202, 202, 2592, 896, 896, 896, 896, 896, 896, 896, 896, 896, 896, 896, 896, 896, 896, 896, 896, 896, 896, 896, 896, 896, 896, 896, 896, 896, 896, 896, 896, 896, 896, 896, 896, 2624]; +static BREAK_PROP_TRIE_DATA: [BreakClass; 12996] = [ +CM,CM,CM,CM,CM,CM,CM,CM,CM,BA,LF,BK,BK,CR,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,SP,EX,QU,AL,PR,PO,AL,QU,OP,CP,AL,PR,IS,HY,IS,SY,NU,NU,NU,NU,NU,NU,NU,NU,NU,NU,IS,IS,AL,AL,AL,EX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,OP,PR,CP,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,OP,BA,CL,AL,CM,CM,CM,CM,CM,NL,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,GL,OP,PO,PR,PR,PR,AL,AI,AI,AL,AI,QU,AL,BA,AL,AL,PO,PR,AI,AI,BB,AL,AI,AI,AI,AI,AI,QU,AI,AI,AI,OP,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AI,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AI,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AI,BB,AI,AI,AI,BB,AI,AL,AL,AI,AL,AL,AL,AL,AL,AL,AL,AI,AI,AI,AI,AL,AI,AL,BB,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,GL,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,GL,GL,GL,GL,GL,GL,GL,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,AL,AL,AL,AL,AL,AL,AL,AL,XX,XX,AL,AL,AL,AL,IS,AL,XX,XX,XX,XX,AL,AL,AL,AL,AL,AL,AL,XX,AL,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,CM,CM,CM,CM,CM,CM,CM,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,IS,BA,XX,XX,AL,AL,PR,XX,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,BA,CM,AL,CM,CM,AL,CM,CM,EX,CM,XX,XX,XX,XX,XX,XX,XX,XX,HL,HL,HL,HL,HL,HL,HL,HL,HL,HL,HL,HL,HL,HL,HL,HL,HL,HL,HL,HL,HL,HL,HL,HL,HL,HL,HL,XX,XX,XX,XX,HL,HL,HL,HL,AL,AL,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,PO,PO,PO,IS,IS,AL,AL,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,EX,CM,EX,EX,EX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,NU,NU,NU,NU,NU,NU,NU,NU,NU,NU,PO,NU,NU,AL,AL,AL,CM,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,EX,AL,CM,CM,CM,CM,CM,CM,CM,AL,AL,CM,CM,CM,CM,CM,CM,AL,AL,CM,CM,AL,CM,CM,CM,CM,AL,AL,NU,NU,NU,NU,NU,NU,NU,NU,NU,NU,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,AL,AL,CM,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,XX,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,AL,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,NU,NU,NU,NU,NU,NU,NU,NU,NU,NU,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,CM,CM,CM,CM,CM,CM,CM,CM,CM,AL,AL,AL,AL,IS,EX,AL,XX,XX,CM,PR,PR,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,CM,CM,CM,CM,AL,CM,CM,CM,CM,CM,CM,CM,CM,CM,AL,CM,CM,CM,AL,CM,CM,CM,CM,CM,XX,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,CM,CM,CM,XX,XX,AL,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,XX,XX,XX,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,AL,AL,XX,XX,XX,XX,XX,XX,CM,CM,CM,CM,CM,CM,CM,CM,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,AL,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,CM,CM,CM,AL,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,AL,CM,CM,CM,CM,CM,CM,CM,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,CM,CM,BA,BA,NU,NU,NU,NU,NU,NU,NU,NU,NU,NU,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,CM,CM,CM,XX,AL,AL,AL,AL,AL,AL,AL,AL,XX,XX,AL,AL,XX,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,AL,AL,AL,AL,AL,AL,AL,XX,AL,XX,XX,XX,AL,AL,AL,AL,XX,XX,CM,AL,CM,CM,CM,CM,CM,XX,XX,CM,CM,XX,XX,CM,CM,CM,AL,XX,XX,XX,XX,XX,XX,XX,XX,CM,XX,XX,XX,XX,AL,AL,XX,AL,AL,AL,CM,CM,XX,XX,NU,NU,NU,NU,NU,NU,NU,NU,NU,NU,AL,AL,PO,PO,AL,AL,AL,AL,AL,PO,AL,PR,AL,AL,CM,XX,CM,CM,CM,XX,AL,AL,AL,AL,AL,AL,XX,XX,XX,XX,AL,AL,XX,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,AL,AL,AL,AL,AL,AL,AL,XX,AL,AL,XX,AL,AL,XX,AL,AL,XX,XX,CM,XX,CM,CM,CM,XX,XX,XX,XX,CM,CM,XX,XX,CM,CM,CM,XX,XX,XX,CM,XX,XX,XX,XX,XX,XX,XX,AL,AL,AL,AL,XX,AL,XX,XX,XX,XX,XX,XX,XX,NU,NU,NU,NU,NU,NU,NU,NU,NU,NU,CM,CM,AL,AL,AL,CM,AL,XX,XX,XX,XX,XX,XX,XX,XX,XX,CM,CM,CM,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,AL,AL,AL,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,AL,AL,AL,AL,AL,AL,AL,XX,AL,AL,XX,AL,AL,AL,AL,AL,XX,XX,CM,AL,CM,CM,CM,CM,CM,CM,XX,CM,CM,CM,XX,CM,CM,CM,XX,XX,AL,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,AL,AL,CM,CM,XX,XX,NU,NU,NU,NU,NU,NU,NU,NU,NU,NU,AL,PR,XX,XX,XX,XX,XX,XX,XX,AL,CM,CM,CM,CM,CM,CM,XX,CM,CM,CM,XX,AL,AL,AL,AL,AL,AL,AL,AL,XX,XX,AL,AL,XX,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,AL,AL,AL,AL,AL,AL,AL,XX,AL,AL,XX,AL,AL,AL,AL,AL,XX,XX,CM,AL,CM,CM,CM,CM,CM,XX,XX,CM,CM,XX,XX,CM,CM,CM,XX,XX,XX,XX,XX,XX,XX,CM,CM,CM,XX,XX,XX,XX,AL,AL,XX,AL,AL,AL,CM,CM,XX,XX,NU,NU,NU,NU,NU,NU,NU,NU,NU,NU,AL,AL,AL,AL,AL,AL,AL,AL,XX,XX,XX,XX,XX,XX,XX,XX,CM,AL,XX,AL,AL,AL,AL,AL,AL,XX,XX,XX,AL,AL,AL,XX,AL,AL,AL,AL,XX,XX,XX,AL,AL,XX,AL,XX,AL,AL,XX,XX,XX,AL,AL,XX,XX,XX,AL,AL,AL,XX,XX,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,XX,XX,XX,CM,CM,CM,XX,XX,XX,CM,CM,CM,XX,CM,CM,CM,CM,XX,XX,AL,XX,XX,XX,XX,XX,XX,CM,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,NU,NU,NU,NU,NU,NU,NU,NU,NU,NU,AL,AL,AL,AL,AL,AL,AL,AL,AL,PR,AL,XX,XX,XX,XX,XX,CM,CM,CM,CM,CM,AL,AL,AL,AL,AL,AL,AL,AL,XX,AL,AL,AL,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,XX,CM,AL,CM,CM,CM,CM,CM,XX,CM,CM,CM,XX,CM,CM,CM,CM,XX,XX,XX,XX,XX,XX,XX,CM,CM,XX,AL,AL,AL,XX,XX,AL,XX,XX,AL,AL,CM,CM,XX,XX,NU,NU,NU,NU,NU,NU,NU,NU,NU,NU,XX,XX,XX,XX,XX,XX,XX,BB,AL,AL,AL,AL,AL,AL,AL,AL,CM,CM,CM,BB,AL,AL,AL,AL,AL,AL,AL,AL,XX,AL,AL,AL,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,AL,AL,AL,AL,AL,XX,XX,CM,AL,CM,CM,CM,CM,CM,XX,CM,CM,CM,XX,CM,CM,CM,CM,XX,XX,XX,XX,XX,XX,XX,CM,CM,XX,XX,XX,XX,XX,XX,AL,AL,XX,AL,AL,CM,CM,XX,XX,NU,NU,NU,NU,NU,NU,NU,NU,NU,NU,XX,AL,AL,CM,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,CM,CM,CM,CM,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,AL,AL,AL,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,CM,CM,AL,CM,CM,CM,CM,CM,XX,CM,CM,CM,XX,CM,CM,CM,CM,AL,AL,XX,XX,XX,XX,AL,AL,AL,CM,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,CM,CM,XX,XX,NU,NU,NU,NU,NU,NU,NU,NU,NU,NU,AL,AL,AL,AL,AL,AL,AL,AL,AL,PO,AL,AL,AL,AL,AL,AL,XX,CM,CM,CM,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,XX,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,AL,XX,XX,AL,AL,AL,AL,AL,AL,AL,XX,XX,XX,CM,XX,XX,XX,XX,CM,CM,CM,CM,CM,CM,XX,CM,XX,CM,CM,CM,CM,CM,CM,CM,CM,XX,XX,XX,XX,XX,XX,NU,NU,NU,NU,NU,NU,NU,NU,NU,NU,XX,XX,CM,CM,AL,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,XX,XX,XX,XX,PR,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,AL,NU,NU,NU,NU,NU,NU,NU,NU,NU,NU,BA,BA,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,SA,SA,XX,SA,XX,SA,SA,SA,SA,SA,XX,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,XX,SA,XX,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,XX,XX,SA,SA,SA,SA,SA,XX,SA,XX,SA,SA,SA,SA,SA,SA,SA,XX,NU,NU,NU,NU,NU,NU,NU,NU,NU,NU,XX,XX,SA,SA,SA,SA,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,AL,BB,BB,BB,BB,AL,BB,BB,GL,BB,BB,BA,GL,EX,EX,EX,EX,EX,GL,AL,EX,AL,AL,AL,CM,CM,AL,AL,AL,AL,AL,AL,NU,NU,NU,NU,NU,NU,NU,NU,NU,NU,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,BA,CM,AL,CM,AL,CM,OP,CL,OP,CL,CM,CM,AL,AL,AL,AL,AL,AL,AL,AL,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,XX,XX,XX,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,BA,CM,CM,AL,AL,AL,AL,AL,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,XX,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,XX,BA,BA,AL,AL,AL,AL,AL,AL,CM,AL,AL,AL,AL,AL,AL,XX,AL,AL,BB,BB,BA,BB,AL,AL,AL,AL,AL,GL,GL,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,NU,NU,NU,NU,NU,NU,NU,NU,NU,NU,BA,BA,AL,AL,AL,AL,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,NU,NU,NU,NU,NU,NU,NU,NU,NU,NU,SA,SA,SA,SA,SA,SA,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,AL,XX,XX,XX,XX,XX,AL,XX,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JV,JV,JV,JV,JV,JV,JV,JV,JV,JV,JV,JV,JV,JV,JV,JV,JV,JV,JV,JV,JV,JV,JV,JV,JV,JV,JV,JV,JV,JV,JV,JV,JV,JV,JV,JV,JV,JV,JV,JV,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,AL,AL,AL,AL,XX,XX,AL,AL,AL,AL,AL,AL,AL,XX,AL,XX,AL,AL,AL,AL,XX,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,AL,AL,AL,AL,XX,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,AL,AL,AL,AL,XX,XX,AL,AL,AL,AL,AL,AL,AL,XX,AL,AL,AL,AL,XX,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,AL,AL,AL,AL,XX,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,XX,CM,CM,CM,AL,BA,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,XX,XX,XX,XX,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,XX,AL,AL,AL,AL,AL,AL,XX,XX,BA,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,BA,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,OP,CL,XX,XX,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,BA,BA,BA,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,XX,XX,XX,XX,XX,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,CM,CM,CM,CM,XX,XX,XX,XX,XX,XX,XX,XX,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,CM,CM,CM,BA,BA,XX,XX,XX,XX,XX,XX,XX,XX,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,CM,CM,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,AL,AL,AL,XX,CM,CM,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,BA,BA,NS,SA,BA,AL,BA,PR,SA,SA,XX,XX,NU,NU,NU,NU,NU,NU,NU,NU,NU,NU,XX,XX,XX,XX,XX,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,XX,XX,XX,XX,XX,AL,AL,EX,EX,BA,BA,BB,AL,EX,EX,AL,CM,CM,CM,GL,CM,NU,NU,NU,NU,NU,NU,NU,NU,NU,NU,XX,XX,XX,XX,XX,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,XX,XX,XX,XX,XX,XX,AL,AL,AL,AL,AL,CM,CM,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,CM,AL,XX,XX,XX,XX,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,XX,XX,XX,XX,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,XX,XX,XX,XX,AL,XX,XX,XX,EX,EX,NU,NU,NU,NU,NU,NU,NU,NU,NU,NU,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,XX,XX,SA,SA,SA,SA,SA,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,XX,XX,XX,XX,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,XX,XX,XX,XX,XX,XX,NU,NU,NU,NU,NU,NU,NU,NU,NU,NU,SA,XX,XX,XX,SA,SA,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,CM,CM,CM,CM,CM,XX,XX,AL,AL,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,XX,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,XX,XX,CM,NU,NU,NU,NU,NU,NU,NU,NU,NU,NU,XX,XX,XX,XX,XX,XX,NU,NU,NU,NU,NU,NU,NU,NU,NU,NU,XX,XX,XX,XX,XX,XX,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,XX,XX,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,CM,CM,CM,CM,CM,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,AL,AL,AL,AL,AL,AL,AL,AL,XX,XX,XX,NU,NU,NU,NU,NU,NU,NU,NU,NU,NU,BA,BA,AL,BA,BA,BA,BA,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,CM,CM,CM,CM,CM,CM,CM,CM,CM,AL,AL,AL,AL,AL,AL,AL,AL,AL,BA,BA,XX,CM,CM,CM,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,AL,AL,NU,NU,NU,NU,NU,NU,NU,NU,NU,NU,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,XX,XX,XX,XX,XX,XX,XX,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,XX,XX,XX,BA,BA,BA,BA,BA,NU,NU,NU,NU,NU,NU,NU,NU,NU,NU,XX,XX,XX,AL,AL,AL,NU,NU,NU,NU,NU,NU,NU,NU,NU,NU,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,BA,BA,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,XX,XX,XX,XX,XX,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,XX,AL,AL,AL,AL,AL,AL,AL,AL,XX,XX,XX,XX,XX,XX,XX,XX,CM,CM,CM,AL,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,AL,AL,AL,AL,CM,AL,AL,AL,AL,AL,AL,CM,AL,AL,CM,CM,CM,AL,XX,XX,XX,XX,XX,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,GL,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,GL,CM,CM,CM,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,XX,AL,AL,AL,AL,AL,AL,XX,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,XX,AL,AL,AL,AL,AL,AL,XX,XX,AL,AL,AL,AL,AL,AL,AL,AL,XX,AL,XX,AL,XX,AL,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,XX,AL,AL,AL,AL,AL,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,XX,AL,AL,AL,AL,AL,AL,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,XX,AL,AL,AL,XX,AL,AL,AL,AL,AL,AL,AL,BB,AL,XX,BA,BA,BA,BA,BA,BA,BA,GL,BA,BA,BA,ZW,CM,ZWJ,CM,CM,BA,GL,BA,BA,B2,AI,AI,AL,QU,QU,OP,QU,QU,QU,OP,QU,AI,AI,AL,AL,IN,IN,IN,BA,BK,BK,CM,CM,CM,CM,CM,GL,PO,PO,PO,PO,PO,PO,PO,PO,AL,QU,QU,AI,NS,NS,AL,AL,AL,AL,IS,OP,CL,NS,NS,NS,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,BA,PO,BA,BA,BA,BA,AL,BA,BA,BA,WJ,AL,AL,AL,AL,XX,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,AL,AL,XX,XX,AI,AL,AL,AL,AL,AL,AL,AL,AL,OP,CL,AI,AL,AI,AI,AI,AI,AL,AL,AL,AL,AL,AL,AL,AL,OP,CL,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,XX,XX,PR,PR,PR,PR,PR,PR,PR,PO,PR,PR,PR,PR,PR,PR,PR,PR,PR,PR,PR,PR,PR,PR,PO,PR,PR,PR,PR,PO,PR,PR,PO,PR,PR,PR,PR,PR,PR,PR,PR,PR,PR,PR,PR,PR,PR,PR,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,AL,AL,AL,PO,AL,AI,AL,AL,AL,PO,AL,AL,AL,AL,AL,AL,AL,AL,AL,AI,AL,AL,PR,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AI,AI,AL,AL,AL,AL,AL,AL,AL,AL,AI,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AI,AI,AL,AL,AL,AL,AL,AI,AL,AL,AI,AL,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AL,AL,AL,AL,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AL,AL,AL,AL,AL,AL,AL,AL,AL,AI,AL,AL,XX,XX,XX,XX,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AI,AL,AI,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AI,AL,AI,AI,AL,AL,AL,AI,AI,AL,AL,AI,AL,AL,AL,AI,AL,AI,PR,PR,AL,AI,AL,AL,AL,AL,AI,AL,AL,AI,AI,AI,AI,AL,AL,AI,AL,AI,AL,AI,AI,AI,AI,AI,AI,AL,AI,AL,AL,AL,AL,AL,AI,AI,AI,AI,AL,AL,AL,AL,AI,AI,AL,AL,AL,AL,AL,AL,AL,AL,AI,AL,AL,AL,AI,AL,AL,AL,AL,AL,AI,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AI,AI,AL,AL,AI,AI,AI,AI,AL,AL,AI,AI,AL,AL,AI,AI,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AI,AI,AL,AL,AI,AI,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AI,AL,AL,AL,AI,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AI,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AI,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,IN,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,OP,CL,OP,CL,AL,AL,AL,AL,AL,AL,AI,AL,AL,AL,AL,AL,AL,AL,ID,ID,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,OP,CL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,ID,ID,ID,ID,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AL,AL,AL,AL,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AL,AL,AI,AI,AI,AI,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AI,AI,AL,AI,AI,AI,AI,AI,AI,AI,AL,AL,AL,AL,AL,AL,AL,AL,AI,AI,AL,AL,AI,AI,AL,AL,AL,AL,AI,AI,AL,AL,AL,AL,AI,AI,AI,AL,AL,AI,AL,AL,AI,AI,AI,AI,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AI,AI,AI,AI,AL,AL,AL,AL,AL,AL,AL,AL,AL,AI,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,ID,ID,ID,ID,AL,AI,AI,AL,AL,AI,AL,AL,AL,AL,AI,AI,AL,AL,AL,AL,ID,ID,AI,AI,ID,AL,ID,ID,ID,EB,ID,ID,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,ID,ID,ID,AL,AL,AL,AL,AI,AL,AI,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AI,AI,AL,AI,AI,AI,AL,AI,ID,AI,AI,AL,AI,AI,AL,AI,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,ID,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AI,AI,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,ID,ID,ID,ID,ID,ID,ID,ID,ID,AI,AI,AI,AI,ID,AL,ID,ID,ID,AI,ID,ID,AI,AI,AI,ID,ID,AI,AI,ID,AI,AI,ID,ID,ID,AL,AI,AL,AL,AL,AL,AI,AI,ID,AI,AI,AI,AI,AI,AI,ID,ID,ID,ID,ID,AI,ID,ID,EB,ID,AI,AI,ID,ID,ID,ID,ID,AL,AL,AL,ID,ID,EB,EB,EB,EB,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AI,AL,AL,AL,QU,QU,QU,QU,QU,QU,AL,EX,EX,ID,AL,AL,AL,OP,CL,OP,CL,OP,CL,OP,CL,OP,CL,OP,CL,OP,CL,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,OP,CL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,OP,CL,OP,CL,OP,CL,OP,CL,OP,CL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,OP,CL,OP,CL,OP,CL,OP,CL,OP,CL,OP,CL,OP,CL,OP,CL,OP,CL,OP,CL,OP,CL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,OP,CL,OP,CL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,OP,CL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AI,AI,AI,AI,AI,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,CM,CM,CM,AL,AL,XX,XX,XX,XX,XX,EX,BA,BA,BA,AL,EX,BA,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,AL,XX,XX,XX,XX,XX,AL,XX,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,XX,XX,XX,XX,XX,XX,AL,BA,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,CM,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,XX,XX,XX,XX,XX,XX,XX,XX,AL,AL,AL,AL,AL,AL,AL,XX,AL,AL,AL,AL,AL,AL,AL,XX,AL,AL,AL,AL,AL,AL,AL,XX,AL,AL,AL,AL,AL,AL,AL,XX,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,QU,QU,QU,QU,QU,QU,QU,QU,QU,QU,QU,QU,QU,QU,BA,BA,BA,BA,BA,BA,BA,BA,AL,BA,OP,BA,AL,AL,QU,QU,AL,AL,QU,QU,OP,CL,OP,CL,OP,CL,OP,CL,BA,BA,BA,BA,EX,AL,BA,BA,AL,BA,BA,AL,AL,AL,AL,AL,B2,B2,BA,BA,BA,AL,BA,BA,OP,BA,BA,BA,BA,BA,BA,BA,BA,AL,BA,AL,BA,BA,AL,AL,AL,EX,EX,OP,CL,OP,CL,OP,CL,OP,CL,BA,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,XX,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,XX,XX,XX,XX,BA,CL,CL,ID,ID,NS,ID,ID,OP,CL,OP,CL,OP,CL,OP,CL,OP,CL,ID,ID,OP,CL,OP,CL,OP,CL,OP,CL,NS,OP,CL,CL,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,CM,CM,CM,CM,CM,CM,ID,ID,ID,ID,ID,CM,ID,ID,ID,ID,ID,NS,NS,ID,ID,ID,XX,CJ,ID,CJ,ID,CJ,ID,CJ,ID,CJ,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,CJ,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,CJ,ID,CJ,ID,CJ,ID,ID,ID,ID,ID,ID,CJ,ID,ID,ID,ID,ID,ID,CJ,CJ,XX,XX,CM,CM,NS,NS,NS,NS,ID,NS,CJ,ID,CJ,ID,CJ,ID,CJ,ID,CJ,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,CJ,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,CJ,ID,CJ,ID,CJ,ID,ID,ID,ID,ID,ID,CJ,ID,ID,ID,ID,ID,ID,CJ,CJ,ID,ID,ID,ID,NS,CJ,NS,NS,ID,XX,XX,XX,XX,XX,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,XX,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,CJ,CJ,CJ,CJ,CJ,CJ,CJ,CJ,CJ,CJ,CJ,CJ,CJ,CJ,CJ,CJ,ID,ID,ID,ID,ID,ID,ID,ID,AI,AI,AI,AI,AI,AI,AI,AI,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,NS,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,XX,XX,XX,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,XX,XX,XX,XX,XX,XX,XX,XX,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,BA,BA,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,BA,EX,BA,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,NU,NU,NU,NU,NU,NU,NU,NU,NU,NU,AL,AL,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,CM,CM,CM,CM,AL,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,CM,CM,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,CM,CM,AL,BA,BA,BA,BA,BA,XX,XX,XX,XX,XX,XX,XX,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,XX,XX,XX,XX,AL,AL,XX,AL,XX,AL,AL,AL,AL,AL,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,CM,AL,AL,AL,CM,AL,AL,AL,AL,CM,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,CM,CM,CM,CM,CM,AL,AL,AL,AL,CM,XX,XX,XX,AL,AL,AL,AL,AL,AL,AL,AL,PO,AL,XX,XX,XX,XX,XX,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,BB,BB,EX,EX,XX,XX,XX,XX,XX,XX,XX,XX,CM,CM,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,XX,XX,XX,XX,XX,XX,XX,XX,BA,BA,NU,NU,NU,NU,NU,NU,NU,NU,NU,NU,XX,XX,XX,XX,XX,XX,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,BB,AL,AL,CM,NU,NU,NU,NU,NU,NU,NU,NU,NU,NU,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,CM,CM,CM,CM,CM,CM,CM,CM,BA,BA,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,AL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,JL,XX,XX,XX,CM,CM,CM,CM,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,AL,AL,AL,AL,AL,AL,BA,BA,BA,AL,AL,AL,AL,XX,AL,NU,NU,NU,NU,NU,NU,NU,NU,NU,NU,XX,XX,XX,XX,AL,AL,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,NU,NU,NU,NU,NU,NU,NU,NU,NU,NU,SA,SA,SA,SA,SA,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,XX,XX,XX,XX,XX,XX,XX,XX,XX,AL,AL,AL,CM,AL,AL,AL,AL,AL,AL,AL,AL,CM,CM,XX,XX,NU,NU,NU,NU,NU,NU,NU,NU,NU,NU,XX,XX,AL,BA,BA,BA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,SA,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,SA,SA,SA,SA,SA,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,CM,CM,CM,CM,CM,BA,BA,AL,AL,AL,CM,CM,XX,XX,XX,XX,XX,XX,XX,XX,XX,AL,AL,AL,AL,AL,AL,XX,XX,AL,AL,AL,AL,AL,AL,XX,XX,AL,AL,AL,AL,AL,AL,XX,XX,XX,XX,XX,XX,XX,XX,XX,AL,AL,AL,AL,AL,AL,AL,XX,AL,AL,AL,AL,AL,AL,AL,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,XX,XX,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,CM,CM,CM,CM,CM,CM,CM,CM,BA,CM,CM,XX,XX,NU,NU,NU,NU,NU,NU,NU,NU,NU,NU,XX,XX,XX,XX,XX,XX,H2,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H2,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H2,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H2,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,H3,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,JV,JV,JV,JV,JV,JV,JV,JV,JV,JV,JV,JV,JV,JV,JV,JV,XX,XX,XX,XX,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,JT,XX,XX,XX,XX,SG,SG,SG,SG,SG,SG,SG,SG,SG,SG,SG,SG,SG,SG,SG,SG,SG,SG,SG,SG,SG,SG,SG,SG,SG,SG,SG,SG,SG,SG,SG,SG,SG,SG,SG,SG,SG,SG,SG,SG,SG,SG,SG,SG,SG,SG,SG,SG,SG,SG,SG,SG,SG,SG,SG,SG,SG,SG,SG,SG,SG,SG,SG,SG,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,AL,AL,AL,AL,AL,AL,AL,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,AL,AL,AL,AL,AL,XX,XX,XX,XX,XX,HL,CM,HL,HL,HL,HL,HL,HL,HL,HL,HL,HL,AL,HL,HL,HL,HL,HL,HL,HL,HL,HL,HL,HL,HL,HL,XX,HL,HL,HL,HL,HL,XX,HL,XX,HL,HL,XX,HL,HL,XX,HL,HL,HL,HL,HL,HL,HL,HL,HL,HL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,CL,OP,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,XX,XX,XX,XX,XX,XX,AL,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,PO,AL,AL,AL,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,IS,CL,CL,IS,IS,EX,EX,OP,CL,IN,XX,XX,XX,XX,XX,XX,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,ID,ID,ID,ID,ID,OP,CL,OP,CL,OP,CL,OP,CL,OP,CL,OP,CL,ID,ID,OP,CL,ID,ID,ID,ID,ID,ID,ID,CL,ID,CL,XX,NS,NS,EX,EX,ID,OP,CL,OP,CL,OP,CL,ID,ID,ID,ID,ID,ID,ID,ID,XX,ID,PR,PO,ID,XX,XX,XX,XX,AL,AL,AL,AL,AL,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,XX,WJ,XX,EX,ID,ID,PR,PO,ID,ID,OP,CL,ID,ID,CL,ID,CL,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,NS,NS,ID,ID,ID,EX,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,OP,ID,CL,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,OP,ID,CL,ID,OP,CL,CL,OP,CL,CL,NS,ID,CJ,CJ,CJ,CJ,CJ,CJ,CJ,CJ,CJ,CJ,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,NS,NS,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,XX,XX,ID,ID,ID,ID,ID,ID,XX,XX,ID,ID,ID,ID,ID,ID,XX,XX,ID,ID,ID,ID,ID,ID,XX,XX,ID,ID,ID,XX,XX,XX,PO,PR,ID,ID,ID,PR,PR,XX,AL,AL,AL,AL,AL,AL,AL,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,CM,CM,CM,CB,AI,XX,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,AL,AL,XX,AL,BA,BA,BA,XX,XX,XX,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,CM,XX,XX,AL,AL,AL,AL,AL,AL,CM,CM,CM,CM,CM,XX,XX,XX,XX,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,BA,AL,AL,AL,AL,AL,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,AL,AL,AL,XX,AL,AL,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,AL,AL,AL,AL,AL,AL,AL,XX,AL,AL,XX,XX,XX,AL,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,XX,XX,XX,XX,AL,AL,AL,AL,AL,AL,XX,XX,AL,XX,AL,AL,AL,AL,AL,AL,XX,AL,AL,XX,XX,XX,AL,XX,XX,AL,AL,AL,AL,AL,AL,XX,BA,AL,AL,AL,AL,AL,AL,AL,AL,XX,AL,AL,XX,XX,XX,XX,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,XX,XX,BA,AL,CM,CM,CM,XX,CM,CM,XX,XX,XX,XX,XX,CM,CM,CM,CM,AL,AL,AL,AL,AL,AL,XX,XX,CM,CM,CM,XX,XX,XX,XX,CM,BA,BA,BA,BA,BA,BA,BA,BA,AL,XX,XX,XX,XX,XX,XX,XX,AL,AL,AL,AL,AL,CM,CM,XX,XX,XX,XX,AL,AL,AL,AL,AL,BA,BA,BA,BA,BA,BA,IN,XX,XX,XX,XX,XX,XX,XX,XX,XX,AL,AL,AL,AL,AL,AL,XX,XX,XX,BA,BA,BA,BA,BA,BA,BA,AL,AL,XX,XX,XX,XX,XX,XX,XX,AL,AL,AL,AL,XX,XX,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,CM,CM,BA,XX,XX,CM,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,XX,XX,XX,XX,XX,AL,AL,CM,CM,CM,CM,AL,AL,AL,AL,XX,XX,XX,XX,XX,XX,CM,CM,CM,CM,CM,CM,CM,BA,BA,AL,AL,AL,AL,AL,XX,XX,CM,AL,AL,CM,CM,AL,XX,XX,XX,XX,XX,XX,XX,XX,XX,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,AL,AL,AL,BA,BA,CM,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,AL,XX,XX,CM,CM,CM,CM,CM,XX,NU,NU,NU,NU,NU,NU,NU,NU,NU,NU,BA,BA,BA,BA,AL,CM,CM,AL,XX,XX,XX,XX,XX,XX,XX,XX,AL,AL,AL,CM,AL,BB,AL,XX,XX,XX,XX,XX,XX,XX,XX,XX,CM,AL,AL,AL,AL,BA,BA,AL,BA,CM,CM,CM,CM,AL,CM,CM,NU,NU,NU,NU,NU,NU,NU,NU,NU,NU,AL,BB,AL,BA,BA,BA,CM,CM,CM,CM,CM,CM,CM,CM,BA,BA,AL,BA,BA,AL,CM,AL,CM,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,AL,AL,AL,AL,AL,AL,AL,XX,AL,XX,AL,AL,AL,AL,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,BA,XX,XX,XX,XX,XX,XX,CM,CM,CM,CM,XX,AL,AL,AL,AL,AL,AL,AL,AL,XX,XX,AL,XX,AL,AL,XX,AL,AL,AL,AL,AL,XX,CM,CM,AL,CM,CM,AL,XX,XX,XX,XX,XX,XX,CM,XX,XX,XX,XX,XX,AL,AL,AL,CM,CM,XX,XX,CM,CM,CM,CM,CM,CM,CM,XX,XX,XX,CM,CM,CM,CM,CM,CM,CM,AL,AL,AL,AL,BA,BA,BA,BA,AL,NU,NU,NU,NU,NU,NU,NU,NU,NU,NU,BA,BA,XX,AL,CM,AL,CM,CM,CM,CM,AL,AL,AL,AL,XX,XX,XX,XX,XX,XX,XX,XX,CM,CM,CM,CM,CM,CM,XX,XX,CM,CM,CM,CM,CM,CM,CM,CM,BB,BA,BA,EX,EX,AL,AL,AL,BA,BA,BA,BA,BA,BA,BA,BA,AL,AL,AL,AL,CM,CM,XX,XX,CM,BA,BA,AL,AL,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,BB,BB,BB,BB,BB,BB,BB,BB,BB,BB,BB,BB,BB,XX,XX,XX,CM,CM,CM,CM,CM,CM,CM,CM,AL,AL,XX,XX,XX,XX,XX,XX,NU,NU,NU,NU,NU,NU,NU,NU,NU,NU,SA,SA,BA,BA,BA,SA,AL,AL,AL,AL,AL,AL,AL,XX,XX,AL,XX,XX,AL,AL,AL,AL,XX,AL,AL,XX,AL,AL,AL,AL,AL,AL,AL,AL,CM,CM,CM,CM,CM,CM,XX,CM,CM,XX,XX,CM,CM,CM,CM,AL,CM,CM,BA,BA,BA,XX,XX,XX,XX,XX,XX,XX,XX,XX,AL,CM,CM,CM,CM,CM,CM,CM,XX,XX,CM,CM,CM,CM,CM,CM,AL,BB,AL,CM,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,AL,AL,AL,CM,CM,CM,CM,CM,CM,CM,AL,CM,CM,CM,CM,BB,AL,BA,BA,BA,BA,BB,AL,CM,XX,XX,XX,XX,XX,XX,XX,XX,AL,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,AL,AL,AL,AL,CM,CM,CM,CM,CM,CM,CM,CM,CM,CM,BA,BA,BA,AL,BB,BB,BA,BA,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,BB,BB,BB,BB,BB,BB,BB,BB,BB,BB,XX,XX,XX,XX,XX,XX,AL,BA,BA,BA,BA,BA,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,BB,EX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,CM,CM,CM,CM,CM,CM,XX,XX,XX,CM,XX,CM,CM,XX,CM,CM,CM,CM,CM,CM,AL,CM,XX,XX,XX,XX,XX,XX,XX,XX,AL,AL,AL,AL,AL,AL,XX,AL,AL,XX,AL,AL,AL,AL,AL,AL,CM,CM,XX,CM,CM,CM,CM,CM,AL,XX,XX,XX,XX,XX,XX,XX,AL,AL,AL,CM,CM,CM,CM,AL,AL,XX,XX,XX,XX,XX,XX,XX,CM,CM,AL,CM,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,CM,CM,CM,CM,CM,CM,CM,XX,XX,XX,CM,CM,CM,BA,BA,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,PO,PO,PO,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,BA,BA,BA,BA,BA,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,AL,AL,AL,AL,AL,AL,AL,AL,OP,OP,OP,CL,CL,CL,AL,AL,CL,AL,AL,AL,OP,CL,OP,CL,AL,AL,AL,AL,AL,AL,AL,AL,AL,OP,CL,CL,AL,AL,AL,AL,GL,GL,GL,GL,GL,GL,GL,OP,CL,GL,GL,GL,OP,CL,OP,CL,CM,AL,AL,AL,AL,AL,AL,CM,CM,CM,CM,CM,CM,CM,CM,CM,NU,NU,NU,NU,NU,NU,NU,NU,NU,NU,XX,XX,XX,XX,BA,BA,CM,CM,CM,CM,CM,BA,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,CM,CM,CM,CM,CM,CM,CM,BA,BA,BA,AL,AL,AL,AL,AL,AL,BA,AL,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,NU,NU,NU,NU,NU,NU,NU,NU,NU,NU,XX,AL,AL,AL,AL,AL,AL,AL,BA,BA,AL,AL,XX,XX,XX,XX,XX,CM,CM,CM,CM,CM,CM,CM,CM,XX,XX,XX,XX,XX,XX,XX,CM,NS,NS,NS,NS,GL,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,CJ,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,CJ,CJ,CJ,XX,XX,CJ,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,CJ,CJ,CJ,CJ,XX,XX,XX,XX,XX,XX,XX,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,XX,AL,CM,CM,BA,AL,AL,AL,AL,AL,CM,CM,CM,CM,CM,AL,AL,AL,CM,CM,CM,AL,AL,CM,CM,CM,CM,CM,CM,CM,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,CM,CM,CM,CM,AL,AL,CM,CM,CM,AL,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,AL,XX,XX,AL,AL,XX,XX,AL,AL,AL,AL,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,AL,XX,AL,AL,AL,AL,AL,XX,AL,XX,XX,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,XX,XX,NU,NU,NU,NU,NU,NU,NU,NU,NU,NU,NU,NU,NU,NU,NU,NU,AL,AL,AL,AL,AL,CM,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,CM,AL,AL,BA,BA,BA,BA,AL,XX,XX,XX,XX,XX,AL,AL,AL,AL,AL,AL,XX,XX,XX,XX,XX,CM,CM,CM,CM,CM,CM,CM,CM,CM,XX,XX,CM,CM,CM,CM,CM,XX,CM,CM,XX,CM,CM,CM,CM,CM,XX,XX,XX,XX,XX,CM,CM,CM,CM,CM,CM,CM,AL,AL,AL,AL,AL,AL,AL,XX,XX,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,CM,XX,NU,NU,NU,NU,NU,NU,NU,NU,NU,NU,XX,XX,XX,XX,XX,PR,AL,AL,AL,AL,AL,AL,AL,XX,AL,AL,AL,AL,XX,AL,AL,XX,AL,AL,AL,AL,CM,CM,CM,CM,CM,CM,CM,AL,XX,XX,XX,XX,NU,NU,NU,NU,NU,NU,NU,NU,NU,NU,XX,XX,XX,XX,OP,OP,PO,AL,AL,AL,AL,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,XX,AL,AL,XX,AL,XX,XX,AL,XX,AL,AL,AL,AL,AL,AL,AL,XX,AL,AL,AL,AL,XX,AL,XX,AL,XX,XX,XX,XX,AL,XX,XX,XX,XX,AL,XX,AL,XX,AL,XX,AL,AL,AL,XX,AL,AL,XX,AL,XX,XX,AL,XX,AL,XX,AL,XX,AL,XX,AL,AL,XX,AL,XX,XX,AL,AL,AL,AL,XX,AL,AL,AL,AL,XX,AL,AL,AL,AL,XX,AL,XX,AL,AL,AL,XX,AL,AL,AL,AL,AL,XX,AL,AL,AL,AL,AL,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,ID,ID,ID,AI,AI,AI,AI,AI,AI,AI,AI,AI,AI,AL,AL,AL,ID,ID,ID,ID,ID,ID,RI,RI,RI,RI,RI,RI,RI,RI,RI,RI,RI,RI,RI,RI,RI,RI,ID,ID,ID,ID,ID,EB,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,AL,AL,ID,ID,ID,ID,ID,AL,ID,ID,ID,EB,EB,EB,ID,ID,EB,ID,ID,EB,EB,EB,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,EM,EM,EM,EM,EM,ID,ID,EB,EB,ID,ID,EB,EB,EB,EB,EB,EB,EB,EB,EB,EB,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,EB,EB,EB,EB,EB,EB,EB,EB,EB,EB,ID,ID,ID,EB,ID,ID,ID,EB,EB,EB,ID,EB,EB,EB,ID,ID,ID,ID,ID,ID,ID,EB,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,AL,ID,AL,ID,AL,ID,ID,ID,ID,ID,EB,ID,ID,ID,ID,AL,AL,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,AL,AL,AL,AL,AL,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,EB,EB,ID,ID,ID,ID,EB,ID,ID,ID,ID,ID,EB,ID,ID,ID,ID,EB,EB,ID,ID,ID,ID,ID,ID,ID,ID,ID,AL,AL,AL,AL,AL,AL,AL,AL,ID,ID,ID,ID,AL,AL,AL,AL,AL,AL,ID,ID,ID,ID,ID,ID,EB,EB,EB,ID,ID,ID,EB,EB,EB,EB,EB,AL,AL,AL,AL,AL,AL,QU,QU,QU,NS,NS,NS,AL,AL,AL,AL,ID,ID,ID,ID,EB,EB,EB,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,EB,ID,ID,ID,AL,AL,AL,AL,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,ID,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,AL,EB,ID,ID,EB,EB,EB,EB,EB,EB,EB,EB,EB,EB,ID,ID,EB,EB,EB,ID,ID,ID,ID,ID,EB,EB,ID,EB,EB,ID,EB,ID,ID,ID,ID,EB,EB,EB,EB,EB,EB,EB,EB,EB,EB,EB,EB,EB,ID,ID,]; + +static PAIR_TABLE: [[u8; 44]; 53] = [[192,193,194,221,196,221,198,199,200,201,221,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,221,221,212,223,224,225,226,227,228,229,230,231,232,221,221,235,],[192,193,2,221,196,221,198,199,200,201,221,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,221,221,212,223,224,225,226,227,228,229,230,231,232,221,221,235,],[192,193,194,221,196,221,198,199,200,201,221,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,221,221,212,223,224,225,226,227,228,229,230,231,232,221,221,235,],[0,1,2,3,4,29,6,7,8,9,3,139,12,141,14,143,16,17,18,19,20,21,22,23,24,25,26,27,29,29,20,159,160,161,162,35,164,165,166,167,168,29,29,235,],[192,193,194,221,196,221,198,199,200,201,221,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,221,221,212,223,224,225,226,227,228,229,230,231,232,221,221,235,],[0,1,2,29,4,29,6,7,8,9,29,139,12,141,14,143,16,17,18,19,20,21,22,23,24,25,26,27,29,29,20,159,160,161,162,35,164,165,166,167,168,29,29,235,],[0,1,2,6,4,29,6,7,8,9,6,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,29,29,20,31,32,33,34,35,36,37,38,39,40,29,29,235,],[0,1,2,157,4,157,134,7,136,45,157,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,157,157,148,159,160,161,162,163,164,165,166,167,168,157,157,235,],[0,1,2,8,4,29,6,7,8,9,8,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,29,29,20,31,32,33,34,35,36,37,38,39,40,29,29,235,],[0,1,2,157,4,157,6,7,136,9,157,139,140,141,142,143,16,17,18,147,148,149,150,23,152,153,154,27,157,157,148,159,160,161,162,163,164,165,166,167,168,157,157,235,],[0,1,2,10,4,29,6,7,8,9,10,139,12,141,14,143,16,17,18,19,20,21,22,23,24,25,26,27,29,29,20,159,160,161,162,35,164,165,166,167,168,29,29,235,],[0,1,2,11,4,157,6,7,8,50,11,11,12,141,14,143,16,17,18,19,20,149,22,23,152,153,154,27,157,157,20,159,160,161,162,163,164,165,166,167,168,157,157,235,],[0,1,2,12,4,157,6,7,136,9,12,139,12,141,14,143,16,17,18,19,20,149,22,23,152,153,154,27,157,157,20,159,160,161,162,163,164,165,166,167,168,157,157,235,],[0,1,2,13,4,29,6,7,8,9,13,11,12,13,14,143,16,17,18,19,20,21,22,23,24,25,26,27,29,29,20,31,32,33,34,35,36,37,38,39,40,29,29,235,],[0,1,2,14,4,157,6,7,136,9,14,139,12,141,14,143,16,17,18,19,20,149,22,23,24,153,154,27,157,157,20,159,160,161,162,163,164,165,166,167,168,157,157,235,],[0,1,2,15,4,157,6,7,8,9,15,139,140,141,142,143,16,17,18,147,148,149,22,23,152,153,154,27,157,157,148,159,160,161,162,163,164,165,166,167,168,157,157,235,],[0,1,2,16,4,157,6,7,8,48,16,139,12,141,14,143,16,17,18,19,20,149,22,23,152,25,26,27,157,157,20,159,160,161,162,163,164,165,166,167,168,157,157,235,],[0,1,2,17,4,29,6,7,8,49,17,139,12,141,14,143,16,17,18,19,20,149,22,23,24,25,26,27,29,29,20,159,160,161,162,35,164,165,166,167,168,29,29,235,],[0,1,2,18,4,157,6,7,8,9,18,139,12,141,14,143,16,17,18,19,20,149,22,23,152,153,154,27,157,157,20,159,160,161,162,163,164,165,166,167,168,157,157,235,],[0,1,2,19,4,157,6,7,8,9,19,139,12,141,14,143,16,17,18,19,20,149,22,23,152,153,154,27,157,157,20,159,160,161,162,163,164,165,166,167,168,157,157,235,],[0,1,2,20,4,157,6,7,8,9,20,139,12,141,14,143,16,17,18,19,20,149,22,23,152,153,154,27,157,157,20,159,160,161,162,163,164,165,166,167,168,157,157,235,],[0,1,2,21,4,29,6,7,8,46,21,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,29,29,20,31,32,33,34,35,36,37,38,39,40,29,29,235,],[0,1,2,22,4,29,6,7,8,47,22,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,29,29,20,31,32,33,34,35,36,37,38,39,40,29,29,235,],[0,1,2,23,4,29,6,7,8,9,23,139,12,141,14,143,16,17,18,19,20,149,22,23,24,153,154,27,29,29,20,159,160,161,162,35,164,165,166,167,168,29,29,235,],[0,1,2,24,4,29,6,7,8,9,24,139,12,141,14,143,16,17,18,19,20,21,22,23,24,25,26,27,29,29,20,159,160,161,162,35,164,165,166,167,168,29,29,235,],[0,1,2,25,4,29,6,7,8,9,25,139,12,141,14,143,16,17,18,19,20,21,22,23,24,153,154,27,29,29,20,159,160,161,162,35,164,165,166,167,168,29,29,235,],[0,1,2,26,4,29,6,7,8,9,26,139,12,141,14,143,16,17,18,19,20,21,22,23,24,153,154,27,29,29,20,31,32,33,34,35,36,37,38,39,168,29,29,235,],[0,1,2,27,4,157,6,7,8,9,27,139,12,141,14,143,16,17,18,19,20,149,22,23,24,153,154,27,157,157,20,159,160,161,162,35,164,165,166,167,168,157,157,235,],[0,1,2,29,4,29,6,7,8,9,29,139,12,141,14,143,16,17,18,19,20,21,22,23,24,25,26,27,29,29,20,159,160,161,162,35,164,165,166,167,168,29,29,235,],[0,1,2,29,4,29,6,7,8,9,29,139,12,141,14,143,16,17,18,19,20,21,22,23,24,25,26,27,29,29,20,159,160,161,162,35,164,165,166,167,168,29,29,235,],[0,1,2,20,4,157,6,7,8,9,20,139,12,141,14,143,16,17,18,19,20,149,22,23,152,153,154,27,157,157,20,159,160,161,162,163,164,165,166,167,168,157,157,235,],[0,1,2,31,4,157,6,7,8,9,31,139,12,141,14,143,16,17,18,19,20,149,22,23,152,25,154,27,157,157,20,159,32,161,162,163,164,165,166,167,168,157,157,235,],[0,1,2,32,4,157,6,7,8,9,32,139,12,141,14,143,16,17,18,19,20,149,22,23,152,25,154,27,157,157,20,159,160,161,162,163,164,165,166,167,168,157,157,235,],[0,1,2,33,4,157,6,7,8,9,33,139,12,141,14,143,16,17,18,19,20,149,22,23,152,25,154,27,157,157,20,159,160,161,162,163,164,165,38,39,168,157,157,235,],[0,1,2,34,4,157,6,7,8,9,34,139,12,141,14,143,16,17,18,19,20,149,22,23,152,25,154,27,157,157,20,159,160,161,162,163,164,165,166,39,168,157,157,235,],[0,1,2,35,4,29,6,7,8,9,35,139,51,141,51,143,16,17,18,19,20,21,22,23,24,25,26,27,29,29,20,159,160,161,162,35,164,165,166,167,168,29,29,235,],[0,1,2,36,4,157,6,7,8,9,36,139,12,141,14,143,16,17,18,19,20,149,22,23,152,25,154,27,157,157,20,159,160,161,162,163,164,165,166,167,168,157,157,235,],[0,1,2,37,4,157,6,7,8,9,37,139,12,141,14,143,16,17,18,19,20,149,22,23,152,25,154,27,157,157,20,159,160,33,34,163,164,37,38,167,168,157,157,235,],[0,1,2,38,4,157,6,7,8,9,38,139,12,141,14,143,16,17,18,19,20,149,22,23,152,25,154,27,157,157,20,159,160,161,162,163,164,165,38,39,168,157,157,235,],[0,1,2,39,4,157,6,7,8,9,39,139,12,141,14,143,16,17,18,19,20,149,22,23,152,25,154,27,157,157,20,159,160,161,162,163,164,165,166,39,168,157,157,235,],[0,1,2,40,4,157,6,7,8,9,40,139,12,141,14,143,16,17,18,19,20,149,22,23,152,153,154,27,157,157,20,159,160,161,162,163,164,165,166,167,52,157,157,235,],[0,1,2,29,4,29,6,7,8,9,29,139,12,141,14,143,16,17,18,19,20,21,22,23,24,25,26,27,29,29,20,159,160,161,162,35,164,165,166,167,168,29,29,235,],[0,1,2,29,4,29,6,7,8,9,29,139,12,141,14,143,16,17,18,19,20,21,22,23,24,25,26,27,29,29,20,159,160,161,162,35,164,165,166,167,168,29,29,235,],[0,1,2,157,4,157,6,7,136,9,157,139,12,141,14,143,16,17,18,19,20,149,22,23,152,153,154,27,157,157,20,159,160,161,162,163,164,165,166,167,168,157,157,235,],[0,1,2,29,4,29,6,7,8,9,29,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,29,29,20,31,32,33,34,35,36,37,38,39,40,29,29,43,],[0,1,2,157,4,157,134,7,136,45,157,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,157,157,148,159,160,161,162,163,164,165,166,167,168,157,157,235,],[0,1,2,29,4,29,6,7,8,46,29,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,29,29,20,31,32,33,34,35,36,37,38,39,40,29,29,235,],[0,1,2,157,4,157,6,7,136,47,157,139,140,141,142,143,16,17,18,147,148,21,150,23,152,153,154,27,157,157,148,159,160,161,162,163,164,165,166,167,168,157,157,235,],[0,1,2,157,4,157,6,7,136,48,157,139,140,141,142,143,16,17,18,147,20,149,150,23,152,153,154,27,157,157,20,159,160,161,162,163,164,165,166,167,168,157,157,235,],[0,1,2,157,4,157,6,7,136,49,157,139,140,141,142,143,16,17,18,147,20,149,150,23,152,153,154,27,157,157,20,159,160,161,162,163,164,165,166,167,168,157,157,235,],[0,1,2,157,4,157,6,7,136,50,157,11,140,141,142,143,16,17,18,147,148,149,150,23,152,153,154,27,157,157,148,159,160,161,162,163,164,165,166,167,168,157,157,235,],[0,1,2,51,4,29,6,7,8,9,51,11,12,13,14,143,16,17,18,19,20,21,22,23,24,25,26,27,29,29,20,31,32,33,34,35,36,37,38,39,40,29,29,235,],[0,1,2,52,4,157,6,7,8,9,52,139,12,141,14,143,16,17,18,19,20,149,22,23,152,153,154,27,157,157,20,159,160,161,162,163,164,165,166,167,168,157,157,235,],]; + + fn is_safe_pair(a: BreakClass, b: BreakClass) -> bool { + !matches!((a, b), (CM, CM)|(SP, CM)|(ZWJ, CM)|(BA, CM)|(HY, CM)|(RI, CM)|(CM, SG)|(SP, SG)|(ZWJ, SG)|(BA, SG)|(HY, SG)|(SP, WJ)|(CM, GL)|(SP, GL)|(ZWJ, GL)|(BA, GL)|(HY, GL)|(CM, SP)|(SP, SP)|(ZWJ, SP)|(CM, ZWJ)|(SP, ZWJ)|(ZWJ, ZWJ)|(BA, ZWJ)|(HY, ZWJ)|(RI, ZWJ)|(CM, B2)|(SP, B2)|(ZWJ, B2)|(BA, B2)|(HY, B2)|(CM, BA)|(SP, BA)|(ZWJ, BA)|(CM, BB)|(SP, BB)|(ZWJ, BB)|(BA, BB)|(HY, BB)|(CM, HY)|(SP, HY)|(ZWJ, HY)|(CM, CB)|(SP, CB)|(ZWJ, CB)|(SP, CL)|(SP, CP)|(SP, EX)|(CM, IN)|(SP, IN)|(ZWJ, IN)|(CM, NS)|(SP, NS)|(ZWJ, NS)|(CM, OP)|(SP, OP)|(ZWJ, OP)|(BA, OP)|(HY, OP)|(SP, QU)|(SP, IS)|(CM, NU)|(SP, NU)|(ZWJ, NU)|(BA, NU)|(CM, PO)|(SP, PO)|(ZWJ, PO)|(BA, PO)|(HY, PO)|(CM, PR)|(SP, PR)|(ZWJ, PR)|(BA, PR)|(HY, PR)|(SP, SY)|(CM, AI)|(SP, AI)|(ZWJ, AI)|(BA, AI)|(HY, AI)|(CM, AL)|(SP, AL)|(ZWJ, AL)|(BA, AL)|(HY, AL)|(CM, CJ)|(SP, CJ)|(ZWJ, CJ)|(CM, EB)|(SP, EB)|(ZWJ, EB)|(BA, EB)|(HY, EB)|(CM, EM)|(SP, EM)|(ZWJ, EM)|(BA, EM)|(HY, EM)|(CM, H2)|(SP, H2)|(ZWJ, H2)|(BA, H2)|(HY, H2)|(CM, H3)|(SP, H3)|(ZWJ, H3)|(BA, H3)|(HY, H3)|(CM, HL)|(SP, HL)|(ZWJ, HL)|(BA, HL)|(HY, HL)|(CM, ID)|(SP, ID)|(ZWJ, ID)|(BA, ID)|(HY, ID)|(CM, JL)|(SP, JL)|(ZWJ, JL)|(BA, JL)|(HY, JL)|(CM, JV)|(SP, JV)|(ZWJ, JV)|(BA, JV)|(HY, JV)|(CM, JT)|(SP, JT)|(ZWJ, JT)|(BA, JT)|(HY, JT)|(CM, RI)|(SP, RI)|(ZWJ, RI)|(BA, RI)|(HY, RI)|(RI, RI)|(CM, SA)|(SP, SA)|(ZWJ, SA)|(BA, SA)|(HY, SA)|(CM, XX)|(SP, XX)|(ZWJ, XX)|(BA, XX)|(HY, XX)) + } diff --git a/third_party/rust/uniffi-example-arithmetic/.cargo-checksum.json b/third_party/rust/uniffi-example-arithmetic/.cargo-checksum.json index 24e445a094..33a5447821 100644 --- a/third_party/rust/uniffi-example-arithmetic/.cargo-checksum.json +++ b/third_party/rust/uniffi-example-arithmetic/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"Cargo.toml":"056d32277bd81040868270ae29d54b3d59ba08833cc307e8aee14a61b914d658","build.rs":"7d98b49c1d9c868c4199f0417eaa017ab459cdd536e9a29851d5f707941f9ead","src/arithmetic.udl":"8554c6907ece627645f6b896f71430e5412bf19b0ac6becf63eb9a69868d0f7a","src/lib.rs":"c454193443e92d49f997c760f4131192fb66bf213bbac1710c1ebde19e765e5d","tests/bindings/test_arithmetic.kts":"e0e9347755db4e18f70b1b74c2d5a4aa328373015090ed959b46d65c2a205d92","tests/bindings/test_arithmetic.py":"3e41d69e21e96a6830197c760f3b7bddd754edc0c5515b7bd33b79cccb10f941","tests/bindings/test_arithmetic.rb":"ea0fdce0a4c7b557b427db77521da05240cd6e87d60a128ac2307fab3bbbc76d","tests/bindings/test_arithmetic.swift":"455b87d95fc690af9c35f9e43676e9c855dedddd2fc1c9e1cbaa6a02835c2d4c","tests/test_generated_bindings.rs":"26b92d6b3e648f6fadd4182cbdba4f412b73da48a789785fd98cd486b29abf05","uniffi.toml":"a2d4f46fa51dc1be1e8bcdf67ec13223637fc1b6c6437455cf53c2dae065fb45"},"package":null} \ No newline at end of file +{"files":{"Cargo.toml":"359cffb76e0eac82aeec6a667f7670fa4b88346c2dd7c17febe71731fd6df58b","build.rs":"7d98b49c1d9c868c4199f0417eaa017ab459cdd536e9a29851d5f707941f9ead","src/arithmetic.udl":"8554c6907ece627645f6b896f71430e5412bf19b0ac6becf63eb9a69868d0f7a","src/lib.rs":"c454193443e92d49f997c760f4131192fb66bf213bbac1710c1ebde19e765e5d","tests/bindings/test_arithmetic.kts":"e0e9347755db4e18f70b1b74c2d5a4aa328373015090ed959b46d65c2a205d92","tests/bindings/test_arithmetic.py":"3e41d69e21e96a6830197c760f3b7bddd754edc0c5515b7bd33b79cccb10f941","tests/bindings/test_arithmetic.rb":"ea0fdce0a4c7b557b427db77521da05240cd6e87d60a128ac2307fab3bbbc76d","tests/bindings/test_arithmetic.swift":"455b87d95fc690af9c35f9e43676e9c855dedddd2fc1c9e1cbaa6a02835c2d4c","tests/test_generated_bindings.rs":"26b92d6b3e648f6fadd4182cbdba4f412b73da48a789785fd98cd486b29abf05","uniffi.toml":"a2d4f46fa51dc1be1e8bcdf67ec13223637fc1b6c6437455cf53c2dae065fb45"},"package":null} \ No newline at end of file diff --git a/third_party/rust/uniffi-example-arithmetic/Cargo.toml b/third_party/rust/uniffi-example-arithmetic/Cargo.toml index 4efddceb0a..52f08bb9f6 100644 --- a/third_party/rust/uniffi-example-arithmetic/Cargo.toml +++ b/third_party/rust/uniffi-example-arithmetic/Cargo.toml @@ -28,15 +28,15 @@ crate-type = [ thiserror = "1.0" [dependencies.uniffi] -version = "0.25" +version = "0.27" path = "../../uniffi" [dev-dependencies.uniffi] -version = "0.25" +version = "0.27" path = "../../uniffi" features = ["bindgen-tests"] [build-dependencies.uniffi] -version = "0.25" +version = "0.27" path = "../../uniffi" features = ["build"] diff --git a/third_party/rust/uniffi-example-geometry/.cargo-checksum.json b/third_party/rust/uniffi-example-geometry/.cargo-checksum.json index 43ff278460..52b8ef33db 100644 --- a/third_party/rust/uniffi-example-geometry/.cargo-checksum.json +++ b/third_party/rust/uniffi-example-geometry/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"Cargo.toml":"caa1fbdbfdf87670cf571dda2e693e8bc9ba38a0a8fbf803014664e84cde3213","build.rs":"fc5da645c8862e15f3b6879db179a1e5eec6161dc1cfbf95a4db9daf107a133f","src/geometry.udl":"7da7a6ec080c7117ec3c25206e23f9ed436e60b1a26fba34f991547680443550","src/lib.rs":"f9d004c97efb1a719368169f0aab181f27439eda3520c1afaca2420433226682","tests/bindings/test_geometry.kts":"e537185e3c699df1c0468525700e8a38f9a504b2a663c38442146b951e38e9a7","tests/bindings/test_geometry.py":"3ea483b8a4fbe13aefa6641177ae149f75f734bc32bf0da533b97c1abf3dc317","tests/bindings/test_geometry.rb":"17c2fe8a7b477419a6646983dd88f1b07a0304b58a568c03e9bfa640d5b2df5c","tests/bindings/test_geometry.swift":"a61fec6bfe16020809e20e4da372748c24366767138c5672a0bfff85c4b62d78","tests/test_generated_bindings.rs":"ff8fc093ccb6ee3ee2235c09276c7bb87234ad143667429cb721e46379577f3d"},"package":null} \ No newline at end of file +{"files":{"Cargo.toml":"2d2c1ec66f7e2aaa4553b5338e68a7b64d68b84152547e1ccc4eeb019ca97103","build.rs":"fc5da645c8862e15f3b6879db179a1e5eec6161dc1cfbf95a4db9daf107a133f","src/geometry.udl":"7da7a6ec080c7117ec3c25206e23f9ed436e60b1a26fba34f991547680443550","src/lib.rs":"f9d004c97efb1a719368169f0aab181f27439eda3520c1afaca2420433226682","tests/bindings/test_geometry.kts":"e537185e3c699df1c0468525700e8a38f9a504b2a663c38442146b951e38e9a7","tests/bindings/test_geometry.py":"600e74ba0eba4e35d824c8f6d5bd5a2120a470017e8465c32d1e954a1939d323","tests/bindings/test_geometry.rb":"651de70af595f8b52ef030a03356939e8c1d0b40e44b4155b45d565d1d1dcbed","tests/bindings/test_geometry.swift":"a61fec6bfe16020809e20e4da372748c24366767138c5672a0bfff85c4b62d78","tests/test_generated_bindings.rs":"ff8fc093ccb6ee3ee2235c09276c7bb87234ad143667429cb721e46379577f3d"},"package":null} \ No newline at end of file diff --git a/third_party/rust/uniffi-example-geometry/Cargo.toml b/third_party/rust/uniffi-example-geometry/Cargo.toml index b9fe377732..b9c8a0beb6 100644 --- a/third_party/rust/uniffi-example-geometry/Cargo.toml +++ b/third_party/rust/uniffi-example-geometry/Cargo.toml @@ -25,15 +25,15 @@ crate-type = [ ] [dependencies.uniffi] -version = "0.25" +version = "0.27" path = "../../uniffi" [dev-dependencies.uniffi] -version = "0.25" +version = "0.27" path = "../../uniffi" features = ["bindgen-tests"] [build-dependencies.uniffi] -version = "0.25" +version = "0.27" path = "../../uniffi" features = ["build"] diff --git a/third_party/rust/uniffi-example-geometry/tests/bindings/test_geometry.py b/third_party/rust/uniffi-example-geometry/tests/bindings/test_geometry.py index a40e2af6ec..fd6772be24 100644 --- a/third_party/rust/uniffi-example-geometry/tests/bindings/test_geometry.py +++ b/third_party/rust/uniffi-example-geometry/tests/bindings/test_geometry.py @@ -1,10 +1,10 @@ from geometry import * -ln1 = Line(Point(0,0), Point(1,2)) -ln2 = Line(Point(1,1), Point(2,2)) +ln1 = Line(start=Point(coord_x=0, coord_y=0), end=Point(coord_x=1, coord_y=2)) +ln2 = Line(start=Point(coord_x=1, coord_y=1), end=Point(coord_x=2, coord_y=2)) assert gradient(ln1) == 2 assert gradient(ln2) == 1 -assert intersection(ln1, ln2) == Point(0, 0) +assert intersection(ln1, ln2) == Point(coord_x=0, coord_y=0) assert intersection(ln1, ln1) is None diff --git a/third_party/rust/uniffi-example-geometry/tests/bindings/test_geometry.rb b/third_party/rust/uniffi-example-geometry/tests/bindings/test_geometry.rb index 8b1280d823..90fdff684e 100644 --- a/third_party/rust/uniffi-example-geometry/tests/bindings/test_geometry.rb +++ b/third_party/rust/uniffi-example-geometry/tests/bindings/test_geometry.rb @@ -6,11 +6,11 @@ require 'geometry' include Test::Unit::Assertions include Geometry -ln1 = Line.new(Point.new(0.0, 0.0), Point.new(1.0, 2.0)) -ln2 = Line.new(Point.new(1.0, 1.0), Point.new(2.0, 2.0)) +ln1 = Line.new(start: Point.new(coord_x: 0.0, coord_y: 0.0), _end: Point.new(coord_x: 1.0, coord_y: 2.0)) +ln2 = Line.new(start: Point.new(coord_x: 1.0, coord_y: 1.0), _end: Point.new(coord_x: 2.0, coord_y: 2.0)) assert_equal Geometry.gradient(ln1), 2 assert_equal Geometry.gradient(ln2), 1 -assert_equal Geometry.intersection(ln1, ln2), Point.new(0, 0) +assert_equal Geometry.intersection(ln1, ln2), Point.new(coord_x: 0, coord_y: 0) assert Geometry.intersection(ln1, ln1).nil? diff --git a/third_party/rust/uniffi-example-rondpoint/.cargo-checksum.json b/third_party/rust/uniffi-example-rondpoint/.cargo-checksum.json index 87e0d96e40..2f1f261a13 100644 --- a/third_party/rust/uniffi-example-rondpoint/.cargo-checksum.json +++ b/third_party/rust/uniffi-example-rondpoint/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"Cargo.toml":"e945bedc4897a588199714e71d6210a77d051925af39b345e605fc58fb782b50","build.rs":"ba88cce38ecd3321a7a93623755e3339e255360a7f946d3779ded804662c081a","src/lib.rs":"740d70ab5ca22eefcc291a56a9e4ed84e9669f4cfe3890e7d79bc56ae4b991a3","src/rondpoint.udl":"c903cb8c95b3ec1b103350857c3c3bc428bfd90c86a6c48089db9e0fc6e41eb5","tests/bindings/test_rondpoint.kts":"4aac8353278807f4add95c81f4c6c61187204b9767f882fd64872ed8ac1f6451","tests/bindings/test_rondpoint.py":"d618274170af767f8a5614a2565ea698b26ea3e1a222d5c110e7b2d00763e73b","tests/bindings/test_rondpoint.rb":"9cc49df311823d6caedbe7b05ff8c4da6329063c2ce16810192aaaa7edcdf5f5","tests/bindings/test_rondpoint.swift":"fa806e7e09c22ed44496658f6e0781765447bbdd250d7adf4b1152248ed70e69","tests/test_generated_bindings.rs":"5464f89e91c458f164b83a454c6df67a2953873e8a785a4720a2253d843f88e5"},"package":null} \ No newline at end of file +{"files":{"Cargo.toml":"c0706abf631a178bfbc2e53b0b714d4a7b73059c46e1326efa4f52d235dc05f5","build.rs":"ba88cce38ecd3321a7a93623755e3339e255360a7f946d3779ded804662c081a","src/lib.rs":"740d70ab5ca22eefcc291a56a9e4ed84e9669f4cfe3890e7d79bc56ae4b991a3","src/rondpoint.udl":"c903cb8c95b3ec1b103350857c3c3bc428bfd90c86a6c48089db9e0fc6e41eb5","tests/bindings/test_rondpoint.kts":"4aac8353278807f4add95c81f4c6c61187204b9767f882fd64872ed8ac1f6451","tests/bindings/test_rondpoint.py":"af25a56c35da9a934fb9f098c25f57329c53d461be378e4c5089b12a45efa28b","tests/bindings/test_rondpoint.rb":"d4b4523084534266ea7ef3161021b9903cb8d7a75cf4624c59055af9f02d22f9","tests/bindings/test_rondpoint.swift":"fa806e7e09c22ed44496658f6e0781765447bbdd250d7adf4b1152248ed70e69","tests/test_generated_bindings.rs":"5464f89e91c458f164b83a454c6df67a2953873e8a785a4720a2253d843f88e5"},"package":null} \ No newline at end of file diff --git a/third_party/rust/uniffi-example-rondpoint/Cargo.toml b/third_party/rust/uniffi-example-rondpoint/Cargo.toml index 64232cf2d2..44d9628df4 100644 --- a/third_party/rust/uniffi-example-rondpoint/Cargo.toml +++ b/third_party/rust/uniffi-example-rondpoint/Cargo.toml @@ -25,15 +25,15 @@ crate-type = [ ] [dependencies.uniffi] -version = "0.25" +version = "0.27" path = "../../uniffi" [dev-dependencies.uniffi] -version = "0.25" +version = "0.27" path = "../../uniffi" features = ["bindgen-tests"] [build-dependencies.uniffi] -version = "0.25" +version = "0.27" path = "../../uniffi" features = ["build"] diff --git a/third_party/rust/uniffi-example-rondpoint/tests/bindings/test_rondpoint.py b/third_party/rust/uniffi-example-rondpoint/tests/bindings/test_rondpoint.py index ecfcc1e527..0b47c0fa5a 100644 --- a/third_party/rust/uniffi-example-rondpoint/tests/bindings/test_rondpoint.py +++ b/third_party/rust/uniffi-example-rondpoint/tests/bindings/test_rondpoint.py @@ -2,7 +2,7 @@ import sys import ctypes from rondpoint import * -dico = Dictionnaire(Enumeration.DEUX, True, 0, 123456789) +dico = Dictionnaire(un=Enumeration.DEUX, deux=True, petit_nombre=0, gros_nombre=123456789) copyDico = copie_dictionnaire(dico) assert dico == copyDico diff --git a/third_party/rust/uniffi-example-rondpoint/tests/bindings/test_rondpoint.rb b/third_party/rust/uniffi-example-rondpoint/tests/bindings/test_rondpoint.rb index 0121f6e0f9..faa4062019 100644 --- a/third_party/rust/uniffi-example-rondpoint/tests/bindings/test_rondpoint.rb +++ b/third_party/rust/uniffi-example-rondpoint/tests/bindings/test_rondpoint.rb @@ -6,7 +6,12 @@ require 'rondpoint' include Test::Unit::Assertions include Rondpoint -dico = Dictionnaire.new Enumeration::DEUX, true, 0, 123_456_789 +dico = Dictionnaire.new( + un: Enumeration::DEUX, + deux: true, + petit_nombre: 0, + gros_nombre: 123_456_789 +) assert_equal dico, Rondpoint.copie_dictionnaire(dico) diff --git a/third_party/rust/uniffi-example-sprites/.cargo-checksum.json b/third_party/rust/uniffi-example-sprites/.cargo-checksum.json index 3542d7cfd7..8a890fb65c 100644 --- a/third_party/rust/uniffi-example-sprites/.cargo-checksum.json +++ b/third_party/rust/uniffi-example-sprites/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"Cargo.toml":"e2d06b68b0865f390496bffce512d19d5fc473b64ff23c705daabb4169b8301c","build.rs":"7e9d92d7c8fc17b359a29117b137ffc4d32f6c10b450d03e30a396339d8c9099","src/lib.rs":"d7984c0c10011b1bd939bce71dae7437ebb9090583b5d1b1cc133c2e5f60ab66","src/sprites.udl":"bfd35f04ba0549301189dfb8fc45b0f39bad00956c324f33be0e845fb7ff78aa","tests/bindings/test_sprites.kts":"06ed115325f37ce59ed6f33e2d651cd2aa352fddcc644580f62a6da6ca075844","tests/bindings/test_sprites.py":"2e6ce838cfb387586257703c3500062438e840dd7ae57d185cdc244dc0745b8f","tests/bindings/test_sprites.rb":"6289a1833c7c8f4583ee4f0488d680de2ee46cfb203095a9b66d7234e2f07d53","tests/bindings/test_sprites.swift":"b2c0a6f4d5edfd7de7c2ba77b838865ffda153a6f364f273456175192d3e6e00","tests/test_generated_bindings.rs":"9a22d693c97fc6d90031cc60f61ece1d9279165ad6a92c9fe937448e126e8de6"},"package":null} \ No newline at end of file +{"files":{"Cargo.toml":"94385b28fdc7e33b7019fada486124c027d29b3d2b11b57e8d488658386f7974","build.rs":"7e9d92d7c8fc17b359a29117b137ffc4d32f6c10b450d03e30a396339d8c9099","src/lib.rs":"d7984c0c10011b1bd939bce71dae7437ebb9090583b5d1b1cc133c2e5f60ab66","src/sprites.udl":"bfd35f04ba0549301189dfb8fc45b0f39bad00956c324f33be0e845fb7ff78aa","tests/bindings/test_sprites.kts":"06ed115325f37ce59ed6f33e2d651cd2aa352fddcc644580f62a6da6ca075844","tests/bindings/test_sprites.py":"f976f2be7ab8b88e8b84def760a849c0d98f4c7b481f054f92b566325d78d52d","tests/bindings/test_sprites.rb":"009d545bb7167b7218211430cfaeeb143cc30617eedcf3e51baafe752ad43241","tests/bindings/test_sprites.swift":"b2c0a6f4d5edfd7de7c2ba77b838865ffda153a6f364f273456175192d3e6e00","tests/test_generated_bindings.rs":"9a22d693c97fc6d90031cc60f61ece1d9279165ad6a92c9fe937448e126e8de6"},"package":null} \ No newline at end of file diff --git a/third_party/rust/uniffi-example-sprites/Cargo.toml b/third_party/rust/uniffi-example-sprites/Cargo.toml index 6caed17ab1..c6dc9d1f4b 100644 --- a/third_party/rust/uniffi-example-sprites/Cargo.toml +++ b/third_party/rust/uniffi-example-sprites/Cargo.toml @@ -25,15 +25,15 @@ crate-type = [ ] [dependencies.uniffi] -version = "0.25" +version = "0.27" path = "../../uniffi" [dev-dependencies.uniffi] -version = "0.25" +version = "0.27" path = "../../uniffi" features = ["bindgen-tests"] [build-dependencies.uniffi] -version = "0.25" +version = "0.27" path = "../../uniffi" features = ["build"] diff --git a/third_party/rust/uniffi-example-sprites/tests/bindings/test_sprites.py b/third_party/rust/uniffi-example-sprites/tests/bindings/test_sprites.py index 5142c2fc42..d04742e076 100644 --- a/third_party/rust/uniffi-example-sprites/tests/bindings/test_sprites.py +++ b/third_party/rust/uniffi-example-sprites/tests/bindings/test_sprites.py @@ -1,17 +1,17 @@ from sprites import * sempty = Sprite(None) -assert sempty.get_position() == Point(0, 0) +assert sempty.get_position() == Point(x=0, y=0) -s = Sprite(Point(0, 1)) -assert s.get_position() == Point(0, 1) +s = Sprite(Point(x=0, y=1)) +assert s.get_position() == Point(x=0, y=1) -s.move_to(Point(1, 2)) -assert s.get_position() == Point(1, 2) +s.move_to(Point(x=1, y=2)) +assert s.get_position() == Point(x=1, y=2) -s.move_by(Vector(-4, 2)) -assert s.get_position() == Point(-3, 4) +s.move_by(Vector(dx=-4, dy=2)) +assert s.get_position() == Point(x=-3, y=4) -srel = Sprite.new_relative_to(Point(0, 1), Vector(1, 1.5)) -assert srel.get_position() == Point(1, 2.5) +srel = Sprite.new_relative_to(Point(x=0, y=1), Vector(dx=1, dy=1.5)) +assert srel.get_position() == Point(x=1, y=2.5) diff --git a/third_party/rust/uniffi-example-sprites/tests/bindings/test_sprites.rb b/third_party/rust/uniffi-example-sprites/tests/bindings/test_sprites.rb index 9d79b57026..fa73043979 100644 --- a/third_party/rust/uniffi-example-sprites/tests/bindings/test_sprites.rb +++ b/third_party/rust/uniffi-example-sprites/tests/bindings/test_sprites.rb @@ -7,16 +7,16 @@ include Test::Unit::Assertions include Sprites sempty = Sprite.new(nil) -assert_equal sempty.get_position, Point.new(0, 0) +assert_equal sempty.get_position, Point.new(x: 0, y: 0) -s = Sprite.new(Point.new(0, 1)) -assert_equal s.get_position, Point.new(0, 1) +s = Sprite.new(Point.new(x: 0, y: 1)) +assert_equal s.get_position, Point.new(x: 0, y: 1) -s.move_to(Point.new(1, 2)) -assert_equal s.get_position, Point.new(1, 2) +s.move_to(Point.new(x: 1, y: 2)) +assert_equal s.get_position, Point.new(x: 1, y: 2) -s.move_by(Vector.new(-4, 2)) -assert_equal s.get_position, Point.new(-3, 4) +s.move_by(Vector.new(dx: -4, dy: 2)) +assert_equal s.get_position, Point.new(x: -3, y: 4) -srel = Sprite.new_relative_to(Point.new(0, 1), Vector.new(1, 1.5)) -assert_equal srel.get_position, Point.new(1, 2.5) +srel = Sprite.new_relative_to(Point.new(x: 0, y: 1), Vector.new(dx: 1, dy: 1.5)) +assert_equal srel.get_position, Point.new(x: 1, y: 2.5) diff --git a/third_party/rust/uniffi-example-todolist/.cargo-checksum.json b/third_party/rust/uniffi-example-todolist/.cargo-checksum.json index a0bf1d826c..6b3a53fc5e 100644 --- a/third_party/rust/uniffi-example-todolist/.cargo-checksum.json +++ b/third_party/rust/uniffi-example-todolist/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"Cargo.toml":"cb308f7bd5299ed45a3bd516390e21e28d0b02a1913e25917e30e9b8ebe084e3","build.rs":"709d073560a66876d2b975f5145a2dc36e1e3420d79328c8b62666526cae5b2d","src/lib.rs":"ccf6851beb2a3e481541dd8a3c3e3d2fbd513a410eef820221942098212a8184","src/todolist.udl":"1f8a24049c2340b9184e95facfc191ecdcb91541729ae7f20b4625d67685f13c","tests/bindings/test_todolist.kts":"f3d29b48e0193563fc4f131d91ea697f758174dcdb80ea554f233949e575bf55","tests/bindings/test_todolist.py":"f7430af9347df0daa954d38bc2203ce400affbb9a53fced4bb67a6796afa0664","tests/bindings/test_todolist.rb":"6524b5271a9cc0e2d78ca9f86ccb6973889926688a0843b4505a4f62d48f6dcb","tests/bindings/test_todolist.swift":"d1911b85fe0c8c0b42e5421b5af5d7359c9a65bba477d23560eb4b0f52e80662","tests/test_generated_bindings.rs":"46ef1fbedaac0c4867812ef2632a641eab36ab0ee12f5757567dd037aeddcbd3"},"package":null} \ No newline at end of file +{"files":{"Cargo.toml":"9b182bf2d363240fe1f7b41c73da4bde96d87562812885e4f3175cb6b92ea56b","build.rs":"709d073560a66876d2b975f5145a2dc36e1e3420d79328c8b62666526cae5b2d","src/lib.rs":"ccf6851beb2a3e481541dd8a3c3e3d2fbd513a410eef820221942098212a8184","src/todolist.udl":"1f8a24049c2340b9184e95facfc191ecdcb91541729ae7f20b4625d67685f13c","tests/bindings/test_todolist.kts":"f3d29b48e0193563fc4f131d91ea697f758174dcdb80ea554f233949e575bf55","tests/bindings/test_todolist.py":"8ea1377fe336adaf4cade0fe073ae5825c3b38c63e46b7f4e92b87c0fe57c036","tests/bindings/test_todolist.rb":"e8840eb81477048718a56c5fa02c0fb2e28cf0a598a20c9393ba2262d6ffb89b","tests/bindings/test_todolist.swift":"d1911b85fe0c8c0b42e5421b5af5d7359c9a65bba477d23560eb4b0f52e80662","tests/test_generated_bindings.rs":"46ef1fbedaac0c4867812ef2632a641eab36ab0ee12f5757567dd037aeddcbd3"},"package":null} \ No newline at end of file diff --git a/third_party/rust/uniffi-example-todolist/Cargo.toml b/third_party/rust/uniffi-example-todolist/Cargo.toml index 9421e726a0..cf4de491cd 100644 --- a/third_party/rust/uniffi-example-todolist/Cargo.toml +++ b/third_party/rust/uniffi-example-todolist/Cargo.toml @@ -29,15 +29,15 @@ once_cell = "1.12" thiserror = "1.0" [dependencies.uniffi] -version = "0.25" +version = "0.27" path = "../../uniffi" [dev-dependencies.uniffi] -version = "0.25" +version = "0.27" path = "../../uniffi" features = ["bindgen-tests"] [build-dependencies.uniffi] -version = "0.25" +version = "0.27" path = "../../uniffi" features = ["build"] diff --git a/third_party/rust/uniffi-example-todolist/tests/bindings/test_todolist.py b/third_party/rust/uniffi-example-todolist/tests/bindings/test_todolist.py index 017e999fb2..7b676c83de 100644 --- a/third_party/rust/uniffi-example-todolist/tests/bindings/test_todolist.py +++ b/third_party/rust/uniffi-example-todolist/tests/bindings/test_todolist.py @@ -2,7 +2,7 @@ from todolist import * todo = TodoList() -entry = TodoEntry("Write bindings for strings in records") +entry = TodoEntry(text="Write bindings for strings in records") todo.add_item("Write python bindings") @@ -20,7 +20,7 @@ assert(todo.get_last_entry().text == "Write bindings for strings in records") todo.add_item("Test Ünicode hàndling without an entry can't believe I didn't test this at first 🤣") assert(todo.get_last() == "Test Ünicode hàndling without an entry can't believe I didn't test this at first 🤣") -entry2 = TodoEntry("Test Ünicode hàndling in an entry can't believe I didn't test this at first 🤣") +entry2 = TodoEntry(text="Test Ünicode hàndling in an entry can't believe I didn't test this at first 🤣") todo.add_entry(entry2) assert(todo.get_last_entry().text == "Test Ünicode hàndling in an entry can't believe I didn't test this at first 🤣") diff --git a/third_party/rust/uniffi-example-todolist/tests/bindings/test_todolist.rb b/third_party/rust/uniffi-example-todolist/tests/bindings/test_todolist.rb index d9e04f92e7..fc1a823f52 100644 --- a/third_party/rust/uniffi-example-todolist/tests/bindings/test_todolist.rb +++ b/third_party/rust/uniffi-example-todolist/tests/bindings/test_todolist.rb @@ -7,7 +7,7 @@ include Test::Unit::Assertions include Todolist todo = TodoList.new -entry = TodoEntry.new 'Write bindings for strings in records' +entry = TodoEntry.new(text: 'Write bindings for strings in records') todo.add_item('Write ruby bindings') @@ -25,7 +25,7 @@ assert_equal todo.get_last_entry.text, 'Write bindings for strings in records' todo.add_item("Test Ünicode hàndling without an entry can't believe I didn't test this at first 🤣") assert_equal todo.get_last, "Test Ünicode hàndling without an entry can't believe I didn't test this at first 🤣" -entry2 = TodoEntry.new("Test Ünicode hàndling in an entry can't believe I didn't test this at first 🤣") +entry2 = TodoEntry.new(text: "Test Ünicode hàndling in an entry can't believe I didn't test this at first 🤣") todo.add_entry(entry2) assert_equal todo.get_last_entry.text, "Test Ünicode hàndling in an entry can't believe I didn't test this at first 🤣" @@ -44,4 +44,4 @@ todo.add_item "Test liveness after being demoted from default" assert todo.get_last == "Test liveness after being demoted from default" todo2.add_item "Test shared state through local vs default reference" -assert Todolist.get_default_list.get_last == "Test shared state through local vs default reference" \ No newline at end of file +assert Todolist.get_default_list.get_last == "Test shared state through local vs default reference" diff --git a/third_party/rust/uniffi/.cargo-checksum.json b/third_party/rust/uniffi/.cargo-checksum.json index 74d539b505..98c4aca54e 100644 --- a/third_party/rust/uniffi/.cargo-checksum.json +++ b/third_party/rust/uniffi/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"Cargo.toml":"5a5cf41b9eb4aac8e312fc9584c0d47585a5a20b13bc7cfb14c9b8813ea596e6","release.toml":"1aa1b131d4cc93b5eba8758a4401c70bc0d7fe5861e2ec147e9259fe7c0da472","src/cli.rs":"0b4791c263d6cf54e4e63dff9a8ead59838d5e7b45fbf5b7f77ab16f602bdb0d","src/lib.rs":"6bc2c11f466fbcd128827a57b5f93a77f716262200f4e5ad2ed8dd75845320fc","tests/ui/proc_macro_arc.rs":"fedc429603753e8ef953642a7295323ccb3f76fd3ae1ab181ad90c5eb88212bb","tests/ui/proc_macro_arc.stderr":"a24af227b907328c9cac6317ec9f43dbc45d7f7c77c603e5d72db7fa050e8b01","tests/ui/version_mismatch.rs":"16ea359e5853517ee0d0704c015ae8c825533109fbefd715130d0f4a51f15898","tests/ui/version_mismatch.stderr":"21dcb836253312ba8e3a0502cce6ff279818aaaadcea9628a41b196e0c8c94b6"},"package":"21345172d31092fd48c47fd56c53d4ae9e41c4b1f559fb8c38c1ab1685fd919f"} \ No newline at end of file +{"files":{"Cargo.toml":"eb974d356d4da93a076434ff428c448f70e036a724bd9a0a7eae6b9ddff2346e","README.md":"37c1af00ec81a9f1bc206ab3578356e5f9ad4077dc46dd1bb623d81d804948b8","release.toml":"1aa1b131d4cc93b5eba8758a4401c70bc0d7fe5861e2ec147e9259fe7c0da472","src/cli.rs":"5c0b9bb93665f2f49f7e90335e65206887e26e96f2a533eb1203be27c9380c84","src/lib.rs":"422503d7cbac1360852287b1810c99663669625b9abf080a5fec22058bb73d8c","tests/ui/proc_macro_arc.rs":"fedc429603753e8ef953642a7295323ccb3f76fd3ae1ab181ad90c5eb88212bb","tests/ui/proc_macro_arc.stderr":"a24af227b907328c9cac6317ec9f43dbc45d7f7c77c603e5d72db7fa050e8b01","tests/ui/version_mismatch.rs":"16ea359e5853517ee0d0704c015ae8c825533109fbefd715130d0f4a51f15898","tests/ui/version_mismatch.stderr":"21dcb836253312ba8e3a0502cce6ff279818aaaadcea9628a41b196e0c8c94b6"},"package":"a5566fae48a5cb017005bf9cd622af5236b2a203a13fb548afde3506d3c68277"} \ No newline at end of file diff --git a/third_party/rust/uniffi/Cargo.toml b/third_party/rust/uniffi/Cargo.toml index 475c8ab9be..374e365502 100644 --- a/third_party/rust/uniffi/Cargo.toml +++ b/third_party/rust/uniffi/Cargo.toml @@ -12,11 +12,12 @@ [package] edition = "2021" name = "uniffi" -version = "0.25.3" +version = "0.27.1" authors = ["Firefox Sync Team "] description = "a multi-language bindings generator for rust" homepage = "https://mozilla.github.io/uniffi-rs" documentation = "https://mozilla.github.io/uniffi-rs" +readme = "README.md" keywords = [ "ffi", "bindgen", @@ -41,18 +42,18 @@ features = [ optional = true [dependencies.uniffi_bindgen] -version = "=0.25.3" +version = "=0.27.1" optional = true [dependencies.uniffi_build] -version = "=0.25.3" +version = "=0.27.1" optional = true [dependencies.uniffi_core] -version = "=0.25.3" +version = "=0.27.1" [dependencies.uniffi_macros] -version = "=0.25.3" +version = "=0.27.1" [dev-dependencies.trybuild] version = "1" diff --git a/third_party/rust/uniffi/README.md b/third_party/rust/uniffi/README.md new file mode 100644 index 0000000000..64ac3486a3 --- /dev/null +++ b/third_party/rust/uniffi/README.md @@ -0,0 +1,81 @@ +# UniFFI - a multi-language bindings generator for Rust + +UniFFI is a toolkit for building cross-platform software components in Rust. + +For the impatient, see [**the UniFFI user guide**](https://mozilla.github.io/uniffi-rs/) +or [**the UniFFI examples**](https://github.com/mozilla/uniffi-rs/tree/main/examples#example-uniffi-components). + +By writing your core business logic in Rust and describing its interface in an "object model", +you can use UniFFI to help you: + +* Compile your Rust code into a shared library for use on different target platforms. +* Generate bindings to load and use the library from different target languages. + +You can describe your object model in an [interface definition file](https://mozilla.github.io/uniffi-rs/udl_file_spec.html) +or [by using proc-macros](https://mozilla.github.io/uniffi-rs/proc_macro/index.html). + +UniFFI is currently used extensively by Mozilla in Firefox mobile and desktop browsers; +written once in Rust, auto-generated bindings allow that functionality to be called +from both Kotlin (for Android apps) and Swift (for iOS apps). +It also has a growing community of users shipping various cool things to many users. + +UniFFI comes with support for **Kotlin**, **Swift**, **Python** and **Ruby** with 3rd party bindings available for **C#** and **Golang**. +Additional foreign language bindings can be developed externally and we welcome contributions to list them here. +See [Third-party foreign language bindings](#third-party-foreign-language-bindings). + +## User Guide + +You can read more about using the tool in [**the UniFFI user guide**](https://mozilla.github.io/uniffi-rs/). + +We consider it ready for production use, but UniFFI is a long way from a 1.0 release with lots of internal work still going on. +We try hard to avoid breaking simple consumers, but more advanced things might break as you upgrade over time. + +### Etymology and Pronunciation + +ˈjuːnɪfaɪ. Pronounced to rhyme with "unify". + +A portmanteau word that also puns with "unify", to signify the joining of one codebase accessed from many languages. + +uni - [Latin ūni-, from ūnus, one] +FFI - [Abbreviation, Foreign Function Interface] + +## Alternative tools + +Other tools we know of which try and solve a similarly shaped problem are: + +* [Diplomat](https://github.com/rust-diplomat/diplomat/) - see our [writeup of + the different approach taken by that tool](docs/diplomat-and-macros.md) +* [Interoptopus](https://github.com/ralfbiedert/interoptopus/) + +(Please open a PR if you think other tools should be listed!) + +## Third-party foreign language bindings + +* [Kotlin Multiplatform support](https://gitlab.com/trixnity/uniffi-kotlin-multiplatform-bindings). The repository contains Kotlin Multiplatform bindings generation for UniFFI, letting you target both JVM and Native. +* [Go bindings](https://github.com/NordSecurity/uniffi-bindgen-go) +* [C# bindings](https://github.com/NordSecurity/uniffi-bindgen-cs) +* [Dart bindings](https://github.com/NiallBunting/uniffi-rs-dart) + +### External resources + +There are a few third-party resources that make it easier to work with UniFFI: + +* [Plugin support for `.udl` files](https://github.com/Lonami/uniffi-dl) for the IDEA platform ([*uniffi-dl* in the JetBrains marketplace](https://plugins.jetbrains.com/plugin/20527-uniffi-dl)). It provides syntax highlighting, code folding, code completion, reference resolution and navigation (among others features) for the [UniFFI Definition Language (UDL)](https://mozilla.github.io/uniffi-rs/). +* [cargo swift](https://github.com/antoniusnaumann/cargo-swift), a cargo plugin to build a Swift Package from Rust code. It provides an init command for setting up a UniFFI crate and a package command for building a Swift package from Rust code - without the need for additional configuration or build scripts. +* [Cargo NDK Gradle Plugin](https://github.com/willir/cargo-ndk-android-gradle) allows you to build Rust code using [`cargo-ndk`](https://github.com/bbqsrc/cargo-ndk), which generally makes Android library builds less painful. +* [`uniffi-starter`](https://github.com/ianthetechie/uniffi-starter) is a minimal project demonstrates a wide range of UniFFI in a complete project in a compact manner. It includes a full Android library build process, an XCFramework generation script, and example Swift package structure. + +(Please open a PR if you think other resources should be listed!) + +## Contributing + +If this tool sounds interesting to you, please help us develop it! You can: + +* View the [contributor guidelines](./docs/contributing.md). +* File or work on [issues](https://github.com/mozilla/uniffi-rs/issues) here in GitHub. +* Join discussions in the [#uniffi:mozilla.org](https://matrix.to/#/#uniffi:mozilla.org) + room on Matrix. + +## Code of Conduct + +This project is governed by Mozilla's [Community Participation Guidelines](./CODE_OF_CONDUCT.md). diff --git a/third_party/rust/uniffi/src/cli.rs b/third_party/rust/uniffi/src/cli.rs index b2d3adc2ae..77d7f219a9 100644 --- a/third_party/rust/uniffi/src/cli.rs +++ b/third_party/rust/uniffi/src/cli.rs @@ -5,6 +5,7 @@ use camino::Utf8PathBuf; use clap::{Parser, Subcommand}; use uniffi_bindgen::bindings::TargetLanguage; +use uniffi_bindgen::BindingGeneratorDefault; // Structs to help our cmdline parsing. Note that docstrings below form part // of the "help" output. @@ -35,7 +36,7 @@ enum Commands { #[clap(long, short)] no_format: bool, - /// Path to the optional uniffi config file. If not provided, uniffi-bindgen will try to guess it from the UDL's file location. + /// Path to optional uniffi config file. This config is merged with the `uniffi.toml` config present in each crate, with its values taking precedence. #[clap(long, short)] config: Option, @@ -95,21 +96,29 @@ pub fn run_main() -> anyhow::Result<()> { if lib_file.is_some() { panic!("--lib-file is not compatible with --library.") } - if config.is_some() { - panic!("--config is not compatible with --library. The config file(s) will be found automatically.") - } let out_dir = out_dir.expect("--out-dir is required when using --library"); if language.is_empty() { panic!("please specify at least one language with --language") } uniffi_bindgen::library_mode::generate_bindings( - &source, crate_name, &language, &out_dir, !no_format, + &source, + crate_name, + &BindingGeneratorDefault { + target_languages: language, + try_format_code: !no_format, + }, + config.as_deref(), + &out_dir, + !no_format, )?; } else { uniffi_bindgen::generate_bindings( &source, config.as_deref(), - language, + BindingGeneratorDefault { + target_languages: language, + try_format_code: !no_format, + }, out_dir.as_deref(), lib_file.as_deref(), crate_name.as_deref(), diff --git a/third_party/rust/uniffi/src/lib.rs b/third_party/rust/uniffi/src/lib.rs index 0625bd9c66..319b3c7836 100644 --- a/third_party/rust/uniffi/src/lib.rs +++ b/third_party/rust/uniffi/src/lib.rs @@ -17,8 +17,11 @@ pub use uniffi_bindgen::bindings::ruby::run_test as ruby_run_test; pub use uniffi_bindgen::bindings::swift::run_test as swift_run_test; #[cfg(feature = "bindgen")] pub use uniffi_bindgen::{ - bindings::TargetLanguage, generate_bindings, generate_component_scaffolding, - generate_component_scaffolding_for_crate, print_repr, + bindings::kotlin::gen_kotlin::KotlinBindingGenerator, + bindings::python::gen_python::PythonBindingGenerator, + bindings::ruby::gen_ruby::RubyBindingGenerator, + bindings::swift::gen_swift::SwiftBindingGenerator, bindings::TargetLanguage, generate_bindings, + generate_component_scaffolding, generate_component_scaffolding_for_crate, print_repr, }; #[cfg(feature = "build")] pub use uniffi_build::{generate_scaffolding, generate_scaffolding_for_crate}; diff --git a/third_party/rust/uniffi_bindgen/.cargo-checksum.json b/third_party/rust/uniffi_bindgen/.cargo-checksum.json index 880d79a96f..40b0b6e3cc 100644 --- a/third_party/rust/uniffi_bindgen/.cargo-checksum.json +++ b/third_party/rust/uniffi_bindgen/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"Cargo.toml":"5285a403f48ecf7a073cbe1839c7afbd0e14998e8315b1ff8ecefb9e7171fc4e","askama.toml":"1a245b7803adca782837e125c49100147d2de0d5a1c949ff95e91af1701f6058","src/backend/config.rs":"4861dbf251dbb10beb1ed7e3eea7d79499a0de1cd9ce9ee8381a0e729c097dea","src/backend/filters.rs":"2da4eaa9af92e449f2fa20d06fc2ab2f758a9a10d3ad6cb8a94975184d40d2ff","src/backend/mod.rs":"2ee9d974cd259f7fb156028b4f4f7601691e94fb5692a6daf0d362df3ecf79a8","src/backend/types.rs":"598df3a861f5d53b2c710848943f6049dd43cb4f37aa81f2c08fd36fc5b2f5d5","src/bindings/kotlin/gen_kotlin/callback_interface.rs":"a6c7796ca66cbaabeef401b939d3c707bba17a77581da36a3a0b46f87630440e","src/bindings/kotlin/gen_kotlin/compounds.rs":"ebd2111a74032b336e0768facfb756a9422da2f9b413ee929b24c1c4315e6a06","src/bindings/kotlin/gen_kotlin/custom.rs":"7e619f7320796ecd8c4ced82904b4bd3c6a0043b55d5829274ab3040cdf9cd7f","src/bindings/kotlin/gen_kotlin/enum_.rs":"6559bb00d8e359126b016e549263c0c9bc1dfc5654ed662c0c2912b47931b1e4","src/bindings/kotlin/gen_kotlin/executor.rs":"58a192123fd2dd4b625f29d95ae6bf5161c2fef7bf50aa8790c3ae0e7a9430d9","src/bindings/kotlin/gen_kotlin/external.rs":"bcd2a44f2559a124aa287944ab59296239033372c6b4a7a3b625b1d41c441de2","src/bindings/kotlin/gen_kotlin/miscellany.rs":"6541987e627c4ff27a17ebe919b2b5cd97cb66ff41187ed636396b4e35ea2414","src/bindings/kotlin/gen_kotlin/mod.rs":"ccae80314058df0b4988d0965ab62b0dc872e8676592e7b55037cd54011e6854","src/bindings/kotlin/gen_kotlin/object.rs":"539ec05386c1e844bef09d4de8374760daa5ba99b009615c04be9c3927feb4c9","src/bindings/kotlin/gen_kotlin/primitives.rs":"2c3020416384a67855ca5086e485c4db6d7dcc3ce51343217b4a914b318ae350","src/bindings/kotlin/gen_kotlin/record.rs":"96fd1a180095a062b4a9b71d4f603b232f0133f46355a3e427c4064701d900f2","src/bindings/kotlin/gen_kotlin/variant.rs":"d111d6888745195fc2c24bdddc57359e771616102a8d182c5c8ad268b0a13460","src/bindings/kotlin/mod.rs":"ef88eb9b5b7d6f920c62a525ea4d4bf2a3b1a9154afaa012cdb2feea597fbf23","src/bindings/kotlin/templates/Async.kt":"064ce385ac0e68719de625e0172908257714b62da296ff14b2c0153b9966212a","src/bindings/kotlin/templates/BooleanHelper.kt":"28e8a5088c8d58c9bfdbc575af8d8725060521fdd7d092684a8044b24ae567c7","src/bindings/kotlin/templates/ByteArrayHelper.kt":"6fbb424556f631beb7f28c4168c568ad840445496a29d5c8f40a9a591b1661b1","src/bindings/kotlin/templates/CallbackInterfaceRuntime.kt":"29fab10a8f6b699471e793e8d53f5bed74803a8c433ff80ccef5f56cf742c54a","src/bindings/kotlin/templates/CallbackInterfaceTemplate.kt":"3b063ea03959c95327b6082c7edc0db0df83ee8b9e8d643b92ca45cf4fda458a","src/bindings/kotlin/templates/CustomTypeTemplate.kt":"be9bdc716731f3935a4d48728e33bfeb4acd514f3719ddbb273adcd6fb4ab31f","src/bindings/kotlin/templates/DurationHelper.kt":"414a98161538a26f3a9b357353270c1f245ad6ceed99496aca7162cf473a92fd","src/bindings/kotlin/templates/EnumTemplate.kt":"865fb1badd1a128390903ab8d9f42f9208c6db0eac5e53b88207282176cfd67f","src/bindings/kotlin/templates/ErrorTemplate.kt":"394c0093c0c86a0f2a14cd5fa60a70ba9970c65448867b6aca86fc25cfe08a4c","src/bindings/kotlin/templates/ExternalTypeTemplate.kt":"40df49116f9ea227c9a64a4f45bb7c2e99275c62e93f75290808e2c930911fba","src/bindings/kotlin/templates/FfiConverterTemplate.kt":"aa22962aaa9f641d48ccf44cb56d9f8a7736cbfaa01e1a1656662cfe5dd5c1d7","src/bindings/kotlin/templates/Float32Helper.kt":"662d95af3b629d143fb4d47cb7e9aa26ed28a5f3846de0341e28b0f9fb08bc25","src/bindings/kotlin/templates/Float64Helper.kt":"a77d099fa7d91e8702c1700e7949ffb6aaba9c6e8041ff48bab34b8e1fc9a0aa","src/bindings/kotlin/templates/ForeignExecutorTemplate.kt":"09c63a67adb8c6cb807108f02d7695d3425401ea0cc51b582cfd469a322fcce0","src/bindings/kotlin/templates/Helpers.kt":"90a11ec576e82265ba0f95fc330053779aada5976477f0d4a6b38619da1282cf","src/bindings/kotlin/templates/Int16Helper.kt":"7f83c4a48e1f3b2a59a3ca6a2662be8bc9baf3a5a748b31223cb3f51721ef249","src/bindings/kotlin/templates/Int32Helper.kt":"e02e4702175554b09fd2dd6ac3089dcd2c395f08ec60e762159566a9c9889450","src/bindings/kotlin/templates/Int64Helper.kt":"7a6fd6ca486852c89399c699935a9dfa1c32b9356d9a965cfde532581f05d9fa","src/bindings/kotlin/templates/Int8Helper.kt":"0554545494b6b9a76ce94f9c1723f8cf4230a13076feb75d620b1c9ca1ac4668","src/bindings/kotlin/templates/MapTemplate.kt":"07d20d8cf58a4bca950ac22dbec5e3471ac6c18c3cca562e45628de827b03ccf","src/bindings/kotlin/templates/NamespaceLibraryTemplate.kt":"22226bb8dde52f12c77b40bbb8f7ceb7dcef313950b8d42b2f5fe44777158bfc","src/bindings/kotlin/templates/ObjectRuntime.kt":"7f38f54a0c889d7534d23afdace6b87b6ced5c024a36b5450078a06e071caad8","src/bindings/kotlin/templates/ObjectTemplate.kt":"3f3baea52b6923446827ea1ee3f5160edfe81e00c11f61ea1f72dbc6b796a6d8","src/bindings/kotlin/templates/OptionalTemplate.kt":"c916c4545d37087ee01a6b6aef966928691d26be539c388fce608e9e3ff4b0e7","src/bindings/kotlin/templates/RecordTemplate.kt":"8d573856de75b55b985594ac4e4a6f08da931dce6b52420654b5bb080eec414f","src/bindings/kotlin/templates/RustBufferTemplate.kt":"002878ce9ce9924231e55853b86768fc3dec2caef6a28098f01c3edd739ca076","src/bindings/kotlin/templates/SequenceTemplate.kt":"477a0f6714af151ca58cfc7c4f2cf0e878d391dd9db4efe964f19d5ad4f544f0","src/bindings/kotlin/templates/StringHelper.kt":"ec0441da90a394616d0ba3492eca50602161fe42062bc4f60e9a23191e71e009","src/bindings/kotlin/templates/TimestampHelper.kt":"353c2890f06ad6dda238e9aebb4bdff7bb838e17e46abf351ed3ff1fbc4e6580","src/bindings/kotlin/templates/TopLevelFunctionTemplate.kt":"888ef82e2637ab104f0821803666c77212b5d5940414f71e899f8f8968ffe572","src/bindings/kotlin/templates/Types.kt":"4d87ef529f666db532fb5355a339fd50be3edd6225a703564455b81ef4d4a16d","src/bindings/kotlin/templates/UInt16Helper.kt":"e84a1f30a5a899ba2c5db614d3f3c74f25bccf6dd99bf68b8830829332d051e9","src/bindings/kotlin/templates/UInt32Helper.kt":"7cdf08cc580046935f27ba07b53685968608a102e0a6be305111037c63d7ddf8","src/bindings/kotlin/templates/UInt64Helper.kt":"fd7baacbf3ab6202ff83edcc66e5f7beb11a10053ba66d0b49547616cc7cbe1f","src/bindings/kotlin/templates/UInt8Helper.kt":"bbf5a6d66c995aea9fe2fa9840c6bfa78b03520a09469b984f0e1d43191e453a","src/bindings/kotlin/templates/macros.kt":"0f64366a9d7523b3d20d7e9d8b04eb064568772dec529a12f878acf5d2246a41","src/bindings/kotlin/templates/wrapper.kt":"d515ca22d12f13b1b99c5daa411ea35d9a288076e4b7208eaf88170e5da7477c","src/bindings/kotlin/test.rs":"0f752ab0afde20194afca07af94da9d1422300032696d5f845cd864fc63c5d51","src/bindings/mod.rs":"949f323d6eb5c018497103dbb9dcffb8f395eb5960694b551a24b4887e853afb","src/bindings/python/gen_python/callback_interface.rs":"5df3e091d3c88ef7645e570f693942161a9b9c6307419c15a2534fbc5da974af","src/bindings/python/gen_python/compounds.rs":"9b7187d35826e1b12dbc2b16a13aec783a51f0952e3e2d24adaefbd0ac005016","src/bindings/python/gen_python/custom.rs":"81501641648eb638f5a338c01a71db0d0e96601c3dda83acdb2d49072b387d42","src/bindings/python/gen_python/enum_.rs":"7c3f8f6a97c1491175c8b93b8f9ab13748e2f8084bb717836b6935d024805439","src/bindings/python/gen_python/error.rs":"161bd2e041e3a63a91899de173eec8450cc10e1e9552d064969aa72a02fdfd5e","src/bindings/python/gen_python/executor.rs":"dbbf2292c79f73dacb317a8645185e42f328a017951662975c0488399c562058","src/bindings/python/gen_python/external.rs":"0325e9a39645eb5454d716d4db76a4a31083ddfafa8e9ca063257292372a0637","src/bindings/python/gen_python/miscellany.rs":"d6f6305dd0af85b7ba87b70cbe6ecba00c83d5082c5bdcaf25962fff853973ea","src/bindings/python/gen_python/mod.rs":"4ac0dc0fd9aaabf2e1f9245d13e0090ee0e9c1235ef203bec9314cac5974e9f0","src/bindings/python/gen_python/object.rs":"a4d4c20a0a52687feff2b9a547a13aa9bda09b3af9ec26508646658a88eec8b3","src/bindings/python/gen_python/primitives.rs":"b830c68e20d8539b8ac5566f1ca0dd262c1b14712a747f79e70004cd8f409ba1","src/bindings/python/gen_python/record.rs":"f8e12ce43d7e0f37f05420a849e7867b7251f9790933609a4cb99050fd063089","src/bindings/python/mod.rs":"eac32ce383460d58d3ccf1d406173465fc8a1db8a24408df67620b7d14dcd0cd","src/bindings/python/templates/Async.py":"4a35a878883a548f3bbed929a9ec74c133e2e9cf08375989503e73ddc2f9c648","src/bindings/python/templates/BooleanHelper.py":"c19e38ae3daa29a831f2394a0a2e74c924711e55ddc85db8ac9b5b8b6da9cd92","src/bindings/python/templates/BytesHelper.py":"e8fb9919acc784fb056bba4ab8d5c04ca7b2275211f8397ef2a391833e3d5e8f","src/bindings/python/templates/CallbackInterfaceRuntime.py":"795d8826d5d2b397a91c531c6b1b76d9425728efdcb90514170c8c4f35053e40","src/bindings/python/templates/CallbackInterfaceTemplate.py":"3f38e7b290fce198d188a63dcfa74486810153816283e683d6e04896c2dbea9a","src/bindings/python/templates/CustomType.py":"12064dde5e1baf4d78e541837c414cdc7ee9e827a284c54a98ac92dfaf3478e8","src/bindings/python/templates/DurationHelper.py":"271c301bc480cd48d5df2aec15789dd360f4d3098a9f360d7f8f33fa0a7fcd0a","src/bindings/python/templates/EnumTemplate.py":"1cbc2206f045c3050be1912df581a5393d6f0a4a79c96d8b49661696d25830e0","src/bindings/python/templates/ErrorTemplate.py":"f9ab6c910024e88ff92a7575d5d00cdafe448b2e85d07a9edea57ae7b6dc5864","src/bindings/python/templates/ExternalTemplate.py":"ed4d65caf2de3fd2c2a3fd2658eb44cf91cd2f0878c017be63afa4394bf56be3","src/bindings/python/templates/Float32Helper.py":"80c0a0619d2c58c100ea8db37125878c8e8cf56c42f77195aa9b4b6b6d5716c8","src/bindings/python/templates/Float64Helper.py":"62c3ed0d646d3383c890d1f8fd7cb8639433971b9ba9261ac43c1391472eb141","src/bindings/python/templates/ForeignExecutorTemplate.py":"6a7903acc65b9dac17524767d94b142d38d25d5f5bb27133ee9fd7ed8fb5f5a9","src/bindings/python/templates/Helpers.py":"3265eeb5917e0090a7dbd50fdbd12e7d1b1832a58b347cd003ecf8a433c9ebcd","src/bindings/python/templates/Int16Helper.py":"ef7fd0035a80aef556bdbfbcf074751d4e25661f4e07f9bf41f48601d171e5aa","src/bindings/python/templates/Int32Helper.py":"af7e0176ed41260089426498946e47565a7d57e98dffaae4562dfe541c3019d1","src/bindings/python/templates/Int64Helper.py":"171908319be9edcfe3b178d1d74f0173df2aae6a4a92895a2079fa476caed7df","src/bindings/python/templates/Int8Helper.py":"d02a4a5452ec1096b1b1953e4d661d699f9f8f0ce5086a6f3577a0119f479666","src/bindings/python/templates/MapTemplate.py":"fd0bd7e396a6288a16ecb3dae087ab725be556789887a4dc0c00ab97d815f3fb","src/bindings/python/templates/NamespaceLibraryTemplate.py":"b78b7161d43a95f5a0b5d7da6cae0d8bf47017c6c77ad210e15be9531413baa9","src/bindings/python/templates/ObjectTemplate.py":"3af0c737d1b482169d62dee1b6c49f1a18ea3f485e9fc323b070f35cf00c242c","src/bindings/python/templates/OptionalTemplate.py":"59df962441fb1c50cb99be014c87ecf69450c1a3b60fa6763bd40e9e948124b1","src/bindings/python/templates/PointerManager.py":"22faf6a2801cf756f3b09415b597f0cd403a3872ac99a7e44e3b7b6217606cd7","src/bindings/python/templates/RecordTemplate.py":"04fcbd662bc9817597366046e09321d2516eb8240e796dd9b6f971c347475429","src/bindings/python/templates/RustBufferHelper.py":"8a8c20d195534e465a173dc778ae98957c50e209ec824af2d2b143f5ba6061f5","src/bindings/python/templates/RustBufferTemplate.py":"f5e247b0f8988f29ce94ab50b5fe7749d0423cda1d37b6145cc8a6c5a9818449","src/bindings/python/templates/SequenceTemplate.py":"047f19074fe08982b59001da2ba7318b331ce431d73503e111330579a3ba065f","src/bindings/python/templates/StringHelper.py":"a3f874ea9330413e854c2ebbeff5507e32a166de6967c5cea63ede1f021e267a","src/bindings/python/templates/TimestampHelper.py":"de099ce51ceaa86519c28bb38e21933ead36ff341f4907695029212bcdfed3ac","src/bindings/python/templates/TopLevelFunctionTemplate.py":"93b6101fae2cafdf1a9325bed07019609ac35bacef2dc31ba4be5c256d827473","src/bindings/python/templates/Types.py":"feb69d895e9279e52479146e1dfe2fec48c547378ef2cc0fb988f6acaf6bcc63","src/bindings/python/templates/UInt16Helper.py":"5fbb30ece1f9a2680b60baf680ec4e2936d64de2ff107018e751ef1c041443ef","src/bindings/python/templates/UInt32Helper.py":"84207c380e38a38bb919d58769384a0f4fa175ebbd04ac451b37ccfe01ff68a1","src/bindings/python/templates/UInt64Helper.py":"4606f381834740319a9f604a418ba149917a6dbd43d3d3d8da50c655893e2c8e","src/bindings/python/templates/UInt8Helper.py":"aecf7cf08b7dc75fe81e8dcac78dfce166b132d5c20b4301d845c479ab9f49ba","src/bindings/python/templates/macros.py":"7d0d08f418edf65ff365bf1fb37e3132aebb720dfecdcef3e3ce50529fe0eb6b","src/bindings/python/templates/wrapper.py":"91e8cbf18e5b7d0b2be31c0e09b230318d19406c316b453dab973341eb2c6add","src/bindings/python/test.rs":"48e93959ce3e34ff0191126416301b170239d3e2665711da786e0b8b7a90a2ab","src/bindings/ruby/gen_ruby/mod.rs":"7f3a94537c331a941e6e010e35563247f11f1fedaf971cb8538e17797cb17efa","src/bindings/ruby/gen_ruby/tests.rs":"7dcb86b08e643c43503f4cac6396833497f6988b004321c0067700ee29ffbf32","src/bindings/ruby/mod.rs":"0fdfab5306dc5c05fbcbfb273340d96ad70c5caf5074834ad6851af1a9a56734","src/bindings/ruby/templates/EnumTemplate.rb":"5480edb347f5829e478d19474691babd72f37616ed846d519b5a61cb1d6cf047","src/bindings/ruby/templates/ErrorTemplate.rb":"301c177e639f0a27f19d4935c5317e672aadecbee2f9bfa778df982320f5148d","src/bindings/ruby/templates/Helpers.rb":"ce7ed4be97dad507b991c69c28dc7bb6427e5e79a4b2fba9dad9dccabc3e090c","src/bindings/ruby/templates/NamespaceLibraryTemplate.rb":"9a24c427b9eba99d9e13181a5559a385b5d1d16beae2b72a402f2645b22a9048","src/bindings/ruby/templates/ObjectTemplate.rb":"0cfd9438e4821cf2164b23d748b3227a8cffbe2fab5b7eb70832228ccb628ee0","src/bindings/ruby/templates/RecordTemplate.rb":"4aeff886928ca972e5dc9b799581b30c66a6f6dce446af3285dd3ed6b422dea9","src/bindings/ruby/templates/RustBufferBuilder.rb":"8da4e425b36dde4f171b238cbe57e02fb55e91a45a82134c1dccc0fc360733c0","src/bindings/ruby/templates/RustBufferStream.rb":"43ad2defc772fd24b68df0736533c26597ba007e89b6a5ba0d31fbe356648151","src/bindings/ruby/templates/RustBufferTemplate.rb":"405a32592cab145175b64e21398f83e6e0f16354552c0395479270e732910e08","src/bindings/ruby/templates/TopLevelFunctionTemplate.rb":"88213e7e25bef664da939c04dd5621f438af735ffcb4d2d0c24a529538630069","src/bindings/ruby/templates/macros.rb":"79d7d0e9af749dadbf242f37c0f86af7c616ea5318da127747def40f6cdb20d1","src/bindings/ruby/templates/wrapper.rb":"f82b41543546f8e5804cd0e1785f4735d9dd95383566d0e5ba1cd4d9e8c0578d","src/bindings/ruby/test.rs":"d19837119725233bd9971ca2dfc3256156071c64e6dfaf07ad2307432c055bbb","src/bindings/swift/gen_swift/callback_interface.rs":"6b51276350f506f96fefd0ae8cb3afdcd514e8a529d9e982afc68cbf68d74578","src/bindings/swift/gen_swift/compounds.rs":"1aba37cf2f438423a4ce476eea6a36f71f1d5daddbb77c88556bc3abde287ca7","src/bindings/swift/gen_swift/custom.rs":"bddb601b4ea8810ecbad01271d5ec0b3958999b09bc9382c83637dfd43451734","src/bindings/swift/gen_swift/enum_.rs":"87be67ec3394616368d9ef8e99b7f234c053b3bee9a7f9e6f2dff37f147c8837","src/bindings/swift/gen_swift/executor.rs":"ab672e2d05acbc2c4a839af22034aa557d5e69f1d9c913158310ea1e93851557","src/bindings/swift/gen_swift/external.rs":"321974136d58e649e60b2a3f70a369dce2d49f474f79579f8e0d66eb63d2d634","src/bindings/swift/gen_swift/miscellany.rs":"7fc2444596d76545ad82ee6c4bed64a29dd4a0438d50bfaafe511f41f6a0e409","src/bindings/swift/gen_swift/mod.rs":"41e4bf2fbe622d0dba85363455e287e00dc48e4043910cef35c30ce170acf52b","src/bindings/swift/gen_swift/object.rs":"2269f65a6b58a24bd08fedb133a38b37663bcf11d0586c50a67028022706a156","src/bindings/swift/gen_swift/primitives.rs":"c8346601008ac6a6d07f08ec7395182c45a4d86c163dc1d6d9c326c49f2acda1","src/bindings/swift/gen_swift/record.rs":"5ad98ab04a5d8178daf0956db819c87d26aae7bf968184e88d512e34c02feb90","src/bindings/swift/mod.rs":"26ba270cb7913661f3cee703038d1ea4a70bff64c3b31351d6bc77e67cdee20d","src/bindings/swift/templates/Async.swift":"84b9be2b5eca2dcfad7eee0cb8d34fec613a4bfdc8a7170b8d11575e457f567b","src/bindings/swift/templates/BooleanHelper.swift":"f02e391bed44ca0e03c66c4e6a1545caaae490fc72c4cf5935e66220082a8c30","src/bindings/swift/templates/BridgingHeaderTemplate.h":"3f468869e77b9293836822b8f2ac348716ab5d487f7b8fef1a87ec30ddafa8d5","src/bindings/swift/templates/CallbackInterfaceRuntime.swift":"2c71ac715ad0bca6f73559748453ba37ca242c90de38f76876989be05d21c49b","src/bindings/swift/templates/CallbackInterfaceTemplate.swift":"790f50b49d5b07dd44e8b215ed1fff02991c1f65aba9a9a9925275a544348813","src/bindings/swift/templates/CustomType.swift":"71520eb38a4be9035dca9e3e0402f386e7eaa79b28568bbc2f20d3fd53b4544d","src/bindings/swift/templates/DataHelper.swift":"df11547a2df57dcca0ff9cddc691bb5fa07d5ffd3d328d1c3b4443078008b111","src/bindings/swift/templates/DurationHelper.swift":"cbc41aaa58bda6c2313ede36a9f656a01a28f9c72aa1624e0e1c1da7b841ffb6","src/bindings/swift/templates/EnumTemplate.swift":"fe205dd28defea8ed6126a45b2a95240a920dfebda8927134a50c3b6d0d7e9d7","src/bindings/swift/templates/ErrorTemplate.swift":"3dddb278763b75b38294c1165522fa91078a951ce05c91fbdfde43b5a097f34f","src/bindings/swift/templates/Float32Helper.swift":"ea32538058c4b3c72b1cd2530ac00d0349fadab5e1bc617d33aae4c87692fc98","src/bindings/swift/templates/Float64Helper.swift":"e27e9424dc6e97b8cacc6ca4c796dd2d16dcfcb877e2f19c45eca03381a41e78","src/bindings/swift/templates/ForeignExecutorTemplate.swift":"205933825e691fec525286d263ea34d592cc462257764ee76325bf98cb3cd240","src/bindings/swift/templates/Helpers.swift":"a88fd909787b855998671e551cdb3284109e2fd2b2e7492b1c93c82aad0e9d35","src/bindings/swift/templates/Int16Helper.swift":"204906911813a3931436057c23032f4c4e39e023df90d641d6c6086aefe2f820","src/bindings/swift/templates/Int32Helper.swift":"0997f059c9c4edd3c41aee0bbad4aa2bda6d791a0d623ad8014d5aa6bdae718d","src/bindings/swift/templates/Int64Helper.swift":"bcf8c2deebb3ee9bce87735adc4bd100981989943b69f6a7fb499a9aec4c25d9","src/bindings/swift/templates/Int8Helper.swift":"ad1ec0fa213724933fa4dc4e2e304e13ac4722b774bfffac44793986b997dd33","src/bindings/swift/templates/MapTemplate.swift":"53971ec388417b02519f8deb8d66361ab4693eae77d116f6051cbea4738054ec","src/bindings/swift/templates/ModuleMapTemplate.modulemap":"99ad1e9bf550a21497296f9248ecd4385dd6d0b5892951d24cf990cdbf3eec2c","src/bindings/swift/templates/ObjectTemplate.swift":"4633572ac6d27a0f82f3b125c2136ad4fa126391c0b64b5db1bde4b04a77f807","src/bindings/swift/templates/OptionalTemplate.swift":"2376487ceadca3975f0e82ddf6ce61af8bbbf5b0592fa9cd977460f148d8c99d","src/bindings/swift/templates/RecordTemplate.swift":"16e0b98354b624a8922d7d384a005fa660a6a388d70381872ebbcd0de9fb78a4","src/bindings/swift/templates/RustBufferTemplate.swift":"f4422fdf0cb5b4db267d461f063dedc319ea1a5a13bae1b82c3f108ba8c658bb","src/bindings/swift/templates/SequenceTemplate.swift":"8425b279197582a94b4cf363ab0463907a68a624e24045720ef7df2bcacf3383","src/bindings/swift/templates/StringHelper.swift":"968b9b9b7fbe06a2ac2143016edaff3552e201652accb8d613b03645f0d24a90","src/bindings/swift/templates/TimestampHelper.swift":"82eece13aa186c8e3745c8ad2f1290639ca9689573018a2bdc5c75afbae58c26","src/bindings/swift/templates/TopLevelFunctionTemplate.swift":"ffe0287861e67dcad2be77f30c00c0a326ab59ffbd37409de3bafc969d49df26","src/bindings/swift/templates/Types.swift":"98c654bfc5d2d4ece965cfe15b00e7151b815e26bfb55abf17e799efffccc2c0","src/bindings/swift/templates/UInt16Helper.swift":"d6fba577324fc0e9e50329c968df99341de418011be126bd29702f8a94d87c02","src/bindings/swift/templates/UInt32Helper.swift":"5e0cf151a0c40098b3d96815ba3de23e15fe52f3c517577e1b0c7e7a4c12428f","src/bindings/swift/templates/UInt64Helper.swift":"17237b38d09ced8d2a8ff1ad9ca86873a19e417280e0e60f33d7063397ea4b7b","src/bindings/swift/templates/UInt8Helper.swift":"c4cb2ee4a78b54ca8d0013383c6b43e9ecd42776e3dc7d6e40086325d41714e5","src/bindings/swift/templates/macros.swift":"438091831b355b0ba6726dab7c17c0687ca58854c7799e946a8012e95a42f4ba","src/bindings/swift/templates/wrapper.swift":"d83a1b8ac3ffc761d4e560adae57d5ad275e0fd1bf3fdc35f3b3b3699317c6e6","src/bindings/swift/test.rs":"c15d19e7f324613e2dbd7995dffba875ed430919a0882f05e8e1f6cc8aea613c","src/interface/callbacks.rs":"34384f1a4e89cd30e2c35beab2bbd8cc0a9e3dd21ed0b390859610fbe209b16a","src/interface/enum_.rs":"3a4f9e77d80128444a8a226ef1e5ad7cd339f8d815aa7012554cf94c2e4edd98","src/interface/ffi.rs":"fa23a2e6fcd89d956523bd0aa5c946138ac629752f0be9085e09f78aade75fac","src/interface/function.rs":"835ee542cb19c67fb95502a790771e54deb712272baed8bdc3f6bfabd9c69d6f","src/interface/mod.rs":"5bd3b5a2dd91382089323a21d4d0419dd5621799add4786a4710df0afebcca69","src/interface/object.rs":"2364c4e7f887d6e4256d065f9ad049ffad19f92fe1c1d76789c33a22376bdd7f","src/interface/record.rs":"931dbaa8cb440debfcf14d51be3e6c517ba2c28655033ad2486e384a9bea25c4","src/interface/universe.rs":"66deaa33394e401e3005670731a0685068302fc01640655335a7cbe9fb4cf48e","src/lib.rs":"64a6239dbaa8196089600e7b0c577b43eefc4a9935893292298bf400fd77a03d","src/library_mode.rs":"89c9fc47db09ccb779fd2842d3c7e07b7863164bb5585eeaf959490ca5555438","src/macro_metadata/ci.rs":"07482b87bf5912277d114b24ad3e560cac94ca0087bd6ee9bee2c328d992bb18","src/macro_metadata/extract.rs":"a106a5b6ad8e5deba474646162b83ec4065796c58c60fc13dfbf1a72ed713833","src/macro_metadata/mod.rs":"bcb5e9a015510e9d74c288da928a4bfb8d80926a8ff85227c0eae8cdb2605519","src/scaffolding/mod.rs":"65749e72181c63edae55d49493aa49a1efece93c0a27deda230f5de4b9ba7d60","src/scaffolding/templates/CallbackInterfaceTemplate.rs":"0581f257c0eb7f0be922ef3ebf7892aafad15e9dc7865e53fc9396375f3188fb","src/scaffolding/templates/Checksums.rs":"ee926e840875c2e48e1d0cc5185c11f7a1ed3bde5264b07540812cb13c1d7481","src/scaffolding/templates/EnumTemplate.rs":"350cdcb4f23be6e6e2b442e0d4ea65549bb35a407bef1ac745a37a2e2b527fae","src/scaffolding/templates/ErrorTemplate.rs":"b93e8bc08d818fbf8976b766635ae192042b78c85f8086d4ee41ae03ab7f3b6f","src/scaffolding/templates/ExternalTypesTemplate.rs":"6cf8a89d9e6f1b6f5af4bc86e49454323fa4981846ec76d6661fcff41c348a8e","src/scaffolding/templates/ObjectTemplate.rs":"df614156bee529fd28905a6ad2270685e51736eade6af856edba5de6e9484141","src/scaffolding/templates/RecordTemplate.rs":"5b0f351739d5770f874fb7da56700c557df1d73ac219f3b18c4212d26fc422e0","src/scaffolding/templates/ReexportUniFFIScaffolding.rs":"aa8a1ffa98b6033707d965f90b5709474ed6bc79486fb47dacae8417fc056cf8","src/scaffolding/templates/TopLevelFunctionTemplate.rs":"96b99b38d074492673797737ddac0683c803a3908e03cc9b999d16f7a76ed178","src/scaffolding/templates/UdlMetadata.rs":"d7c50af1de92ef85630b385a910c7b29875502d622eb90da5541a7012b93d9e2","src/scaffolding/templates/macros.rs":"ea6bacd8dd9116ad739bdafe893d70407050f35e4a7ac8dd2c78b8ef34263e8e","src/scaffolding/templates/scaffolding_template.rs":"c8e18306a73ec5b764f665660fc5c91d498b63b6c3f489e524b2bae50f81f231"},"package":"fd992f2929a053829d5875af1eff2ee3d7a7001cb3b9a46cc7895f2caede6940"} \ No newline at end of file +{"files":{"Cargo.toml":"037dacd80bb367cfc530c5ca19fbfac091f385cf88ad5bd33c2009fde6d06e3e","README.md":"37c1af00ec81a9f1bc206ab3578356e5f9ad4077dc46dd1bb623d81d804948b8","askama.toml":"1a245b7803adca782837e125c49100147d2de0d5a1c949ff95e91af1701f6058","src/backend/config.rs":"4861dbf251dbb10beb1ed7e3eea7d79499a0de1cd9ce9ee8381a0e729c097dea","src/backend/filters.rs":"8a818952896f9c5f438d6705bb28f56e77fbfce6d9757a2d74e1b3a925ed36e1","src/backend/mod.rs":"2ee9d974cd259f7fb156028b4f4f7601691e94fb5692a6daf0d362df3ecf79a8","src/backend/types.rs":"598df3a861f5d53b2c710848943f6049dd43cb4f37aa81f2c08fd36fc5b2f5d5","src/bindings/kotlin/gen_kotlin/callback_interface.rs":"741100c2b4b484583d34408b276394078a24918c47101fbaa6233df9c4da32f2","src/bindings/kotlin/gen_kotlin/compounds.rs":"b40d1ab8c70d7da458ff45d2ce58efb6cc3b24bf560c093cbec7d0854d461dc4","src/bindings/kotlin/gen_kotlin/custom.rs":"7e619f7320796ecd8c4ced82904b4bd3c6a0043b55d5829274ab3040cdf9cd7f","src/bindings/kotlin/gen_kotlin/enum_.rs":"6559bb00d8e359126b016e549263c0c9bc1dfc5654ed662c0c2912b47931b1e4","src/bindings/kotlin/gen_kotlin/external.rs":"38f42be67105b9a2ca5ecefec959e6659af728bedb9d107a51c595fe6ff5d332","src/bindings/kotlin/gen_kotlin/miscellany.rs":"6541987e627c4ff27a17ebe919b2b5cd97cb66ff41187ed636396b4e35ea2414","src/bindings/kotlin/gen_kotlin/mod.rs":"34328d11c59a67159620a21bc660fae149bbf452a817d6c9d3f7657cc5b79134","src/bindings/kotlin/gen_kotlin/object.rs":"1cb8d1f5eaf06ceaadb6d2cedca482fdd1502c24500ba270d8fcac48ef2f1231","src/bindings/kotlin/gen_kotlin/primitives.rs":"249896ec7d18f0f8d1d5dc8dc66ea6f3d0cc7b13344ab6892fb985f339d99b9f","src/bindings/kotlin/gen_kotlin/record.rs":"96fd1a180095a062b4a9b71d4f603b232f0133f46355a3e427c4064701d900f2","src/bindings/kotlin/gen_kotlin/variant.rs":"d111d6888745195fc2c24bdddc57359e771616102a8d182c5c8ad268b0a13460","src/bindings/kotlin/mod.rs":"ef88eb9b5b7d6f920c62a525ea4d4bf2a3b1a9154afaa012cdb2feea597fbf23","src/bindings/kotlin/templates/Async.kt":"2130ef176ffabe85cc737059203f2bda38df1f22f2398aa338c9d3099e6d46e2","src/bindings/kotlin/templates/BooleanHelper.kt":"50d8a5109e2d2676f25a02772079efbaac61776a76e3e84eebd1fb13294842de","src/bindings/kotlin/templates/ByteArrayHelper.kt":"dc4aafffacb1fa8f3b4e15f714c13b8d715eec178c63bdba6260baf612dd80d8","src/bindings/kotlin/templates/CallbackInterfaceImpl.kt":"be4d5a5d3ed4f7b1d4c9822905c5732bdb8593c3dbf8d4aabab62291d7ccec99","src/bindings/kotlin/templates/CallbackInterfaceRuntime.kt":"76689c1bfa8aa7dc6e2c9e77c42212b9f317763fb35cd7704ca470675dd2648d","src/bindings/kotlin/templates/CallbackInterfaceTemplate.kt":"b497250899bfd0c79bd01d77f23454b12b108fa269055d6f3699be74fb93d015","src/bindings/kotlin/templates/CustomTypeTemplate.kt":"d42eba4334c39749037d14ef9e2219a2e515479c18905df3f49534424317c848","src/bindings/kotlin/templates/DurationHelper.kt":"dfb45fe1b47bc04dd8c70cb98531c40606eca554791132ee6bed2846f8ee099c","src/bindings/kotlin/templates/EnumTemplate.kt":"7cefbb1e29d4e89420f6a95275bbab891984a56978cf4852a1e52bcc82afd9e8","src/bindings/kotlin/templates/ErrorTemplate.kt":"8f41de90753a42cfe33ba837997baa2954208b987e70cec13ecb4124faf25aa6","src/bindings/kotlin/templates/ExternalTypeTemplate.kt":"b1df8566d000431bfc3820a2e455426e810cba6d8683e77ab78ab0bb7d003720","src/bindings/kotlin/templates/FfiConverterTemplate.kt":"bc0bdbc99ee2459f50c84abf6c1bb236a6179eaf519f1c5f5b3f72d8184dc662","src/bindings/kotlin/templates/Float32Helper.kt":"789246343d34594fc39072c1a5393b848cecadb353659fc6e9080fc7e760fc21","src/bindings/kotlin/templates/Float64Helper.kt":"b87eac72da313b1d559b1738bba1c771f43bb7566fdbb3a34546dd56beeb5832","src/bindings/kotlin/templates/HandleMap.kt":"feb456ea4dfb2ad07331d49d606faf396737817c6f6712a2d9a9d843daebdc1e","src/bindings/kotlin/templates/Helpers.kt":"e7657732f44e8092d492ef291e3ce11aa803531d82be99645b8513c00dca99ac","src/bindings/kotlin/templates/Int16Helper.kt":"54bb1aefbcae1c3c10e0cdc6a9d45e070e3ca57c9ffa53b2d65e5c59808c9743","src/bindings/kotlin/templates/Int32Helper.kt":"49b3e5274d5c7853d227ec986d0a0c71621a448391b2c9aaf4351b86f310dadf","src/bindings/kotlin/templates/Int64Helper.kt":"264fd99a4109f0b2c40b806fc1a0181f27331346a02d94e398f1e34110e3414b","src/bindings/kotlin/templates/Int8Helper.kt":"a50c315d8474a212914f10f43ca7e75061bb73067e0de686b182891c6c7f1bc3","src/bindings/kotlin/templates/Interface.kt":"c66912069c3f61848bbee3d1886abbfa74895f759853f6b4a3c62cef5976766c","src/bindings/kotlin/templates/MapTemplate.kt":"f7e0360d3be74e543573bd56925bf25c6c22e6203aecc1cf519464704eeeb0ee","src/bindings/kotlin/templates/NamespaceLibraryTemplate.kt":"db4c30cfcb709c5413892cf3cf69391ba36c6160543274e8d1f2bedf9001d058","src/bindings/kotlin/templates/ObjectCleanerHelper.kt":"9ebcfcb3fe7788e93cf8cba30fd7470b363719e9ed25cdde43c95aebe1b90c2a","src/bindings/kotlin/templates/ObjectCleanerHelperAndroid.kt":"049e1e32a23b7923393e3dcabce49532737d44e9dbb331f62984ada67bde3125","src/bindings/kotlin/templates/ObjectCleanerHelperJvm.kt":"b6287f72afdb0ab9af5e56136c28e6a4f5e18a50305bce8923ee061b9406cfc3","src/bindings/kotlin/templates/ObjectTemplate.kt":"6a776feb36b0379c43e0013a26ba85cdef385aa1e59b4c2efa7a794140aa99bb","src/bindings/kotlin/templates/OptionalTemplate.kt":"918f2029e60710f4b048a77830b12b388c917af1a488c9f05f38323c58ee0f9e","src/bindings/kotlin/templates/README.md":"83587ff54a31fa47d2c0849cb5db52d6f079551e1cfb73c76c6dd02a7b164ad9","src/bindings/kotlin/templates/RecordTemplate.kt":"677bb63ae4fae9117e9c77928370a8911ab959c6b884c6af2ba4efc686c52721","src/bindings/kotlin/templates/RustBufferTemplate.kt":"b3b78b2c41cbfff6262d758f9ebe064e76d20841ec4db7705142449f7ffc75a9","src/bindings/kotlin/templates/SequenceTemplate.kt":"c1aa28ca87528c97c62656f850205023c2eb8d218264ef7b1e70207ab4f1b9b6","src/bindings/kotlin/templates/StringHelper.kt":"4e942e36af05dc823d5f28ab336c55ff86bf0f95bbb748399bcaa8ad291c7032","src/bindings/kotlin/templates/TimestampHelper.kt":"70137e78de18796996889e8d648f1e90830aa652a93a0b4639ff9f7ccb967a25","src/bindings/kotlin/templates/TopLevelFunctionTemplate.kt":"68c714cc8c7fa244166c5902a59c90317cdfc402193624cade405e3454f9bf67","src/bindings/kotlin/templates/Types.kt":"c725f7e57eda5b2d52c3c92b24f93d9321591e72c89d16973163b3b8d713b85b","src/bindings/kotlin/templates/UInt16Helper.kt":"ee96270f426933cfcd914894d4c7895544f7e3d4a7c24be78afc2896b46cbcbe","src/bindings/kotlin/templates/UInt32Helper.kt":"b2d7543098277e7b92502a0a6693dc25dd42e360f776b19987a48dd7fc6db7fd","src/bindings/kotlin/templates/UInt64Helper.kt":"fc855eb78a4b50d76fc53509dae8218c48a221db5bf73cf5368d755fb9aae478","src/bindings/kotlin/templates/UInt8Helper.kt":"af22d9e6f99fe9d8d7d5175cb03f7a9f62628c9dd939dbbfb5a4085359e52e0e","src/bindings/kotlin/templates/macros.kt":"0a221962503f6977b129eb3c1e3772e3e9d51cbab6d813c55b0387c24d784184","src/bindings/kotlin/templates/wrapper.kt":"a02028a86c620679602f26714c7feb4a306867cda1cba8240ca6e83d99cebd91","src/bindings/kotlin/test.rs":"28bf88a9e9aa9510adbe78005a2027a62818433f49426172046dc83a3ad41911","src/bindings/mod.rs":"949f323d6eb5c018497103dbb9dcffb8f395eb5960694b551a24b4887e853afb","src/bindings/python/gen_python/callback_interface.rs":"5df3e091d3c88ef7645e570f693942161a9b9c6307419c15a2534fbc5da974af","src/bindings/python/gen_python/compounds.rs":"4a83b02e11ae969ab360ba61df44d91dc790f372b5960b350b0b019a57d19de4","src/bindings/python/gen_python/custom.rs":"81501641648eb638f5a338c01a71db0d0e96601c3dda83acdb2d49072b387d42","src/bindings/python/gen_python/enum_.rs":"7c3f8f6a97c1491175c8b93b8f9ab13748e2f8084bb717836b6935d024805439","src/bindings/python/gen_python/error.rs":"161bd2e041e3a63a91899de173eec8450cc10e1e9552d064969aa72a02fdfd5e","src/bindings/python/gen_python/external.rs":"d7101124c22dd7837e227a7f1b683c57b92229a2cd5b25b06740f2fe3d76bed5","src/bindings/python/gen_python/miscellany.rs":"d6f6305dd0af85b7ba87b70cbe6ecba00c83d5082c5bdcaf25962fff853973ea","src/bindings/python/gen_python/mod.rs":"b8aac9a146551cd660f1cd310f8ef02e3bd4a11540a087551bbb7c7706b99e16","src/bindings/python/gen_python/object.rs":"a4d4c20a0a52687feff2b9a547a13aa9bda09b3af9ec26508646658a88eec8b3","src/bindings/python/gen_python/primitives.rs":"b830c68e20d8539b8ac5566f1ca0dd262c1b14712a747f79e70004cd8f409ba1","src/bindings/python/gen_python/record.rs":"f8e12ce43d7e0f37f05420a849e7867b7251f9790933609a4cb99050fd063089","src/bindings/python/mod.rs":"eac32ce383460d58d3ccf1d406173465fc8a1db8a24408df67620b7d14dcd0cd","src/bindings/python/templates/Async.py":"f1cf32d8e28b5e2fcbad6ccd00d03fd49f4b54eac47adcfe23cbf786d523eee7","src/bindings/python/templates/BooleanHelper.py":"cf7bcd414197258b0cfa54c6ad2aeb81a1a6a4a45af5b6aaf6f8e484bc5af59d","src/bindings/python/templates/BytesHelper.py":"8c39cf1760678316cf2b3903632f2bacae4f8aaa961b37eeb03e06e9f07241d1","src/bindings/python/templates/CallbackInterfaceImpl.py":"7dbb049ffebc3565ffb4605d53843c69f782f3e86472e060e3194be4986d328b","src/bindings/python/templates/CallbackInterfaceRuntime.py":"54dbda8a6ffe284ef2045da290a69d37974fed672eb57309c9fc7ac665969397","src/bindings/python/templates/CallbackInterfaceTemplate.py":"ef235bd7927592eb19a2db422352a435b7466595ef31e4822a16c3caa24cdda6","src/bindings/python/templates/CustomType.py":"4647a60dbe63ead2b23d07cf3a3a4a190a219d81357532364fd4afdf990d6e1b","src/bindings/python/templates/DurationHelper.py":"eb9278b546f79b71525ae61a5b30bfe4a1260fd2268c87c600d157bf9b0e2a44","src/bindings/python/templates/EnumTemplate.py":"49903d969b8b160d8f1a0747c803d5f54a6f000a6781493eacad1f6ca7811d7a","src/bindings/python/templates/ErrorTemplate.py":"d7af297596e5ef894e3fdaeb92bd6446843c987f8283c77973bd10fce537c9c1","src/bindings/python/templates/ExternalTemplate.py":"0cd36fc89f0a587dadfe0cb89c4d45a641822ba07cb9410299bdcd73ad3edb79","src/bindings/python/templates/Float32Helper.py":"4aa522163f121fcb84d2f024774d8dd9321c31f09b9a95da3a3131b6d2756971","src/bindings/python/templates/Float64Helper.py":"e7fa247fd9c3907b818f0d1ba28c2cee897e75fdd07fdacad1b8a2b5c26ba418","src/bindings/python/templates/HandleMap.py":"9dbfdcb4ddde5927fd9b9fb26b5194bc16b1d2280c2259895fd0ea443af4afd6","src/bindings/python/templates/Helpers.py":"09ddd46d6fcc6ee7e9b1c123b0830426c967f94e22ab18b3ee248b873f7d5ebc","src/bindings/python/templates/Int16Helper.py":"613345b35e63e7284caf97de9630747ec9cdadc8dd3f8451d2e878cb762958f5","src/bindings/python/templates/Int32Helper.py":"758b093b66dc0a8d3f0b13b9388d21f47de31b5e948689041c4d43ef98cf2c4f","src/bindings/python/templates/Int64Helper.py":"c7e76441ee14e78e856f8819f73243bc04b33ec16083ae7390e0ec27141855f2","src/bindings/python/templates/Int8Helper.py":"d963a76b218a32ea2b3bb26f265dbbc47e859b7d1bc939b43fd9b93c51a62292","src/bindings/python/templates/MapTemplate.py":"9dc81ebced353d0137ef6fe3187e170e3e72d32a3b5520dbbcc1f95354ebf62d","src/bindings/python/templates/NamespaceLibraryTemplate.py":"e480a80a27ed5e54a3ff9c72d3d6ab13343764da6c413d813c4bd72429139193","src/bindings/python/templates/ObjectTemplate.py":"976aa726baf36b53d1c319b262c34b8b2de2e414cb8d3c645ed3bf006833b9e8","src/bindings/python/templates/OptionalTemplate.py":"2629f3b46ff394df620bbff1699935e6844d9aa017e74ac43c0b38acd05f8d42","src/bindings/python/templates/Protocol.py":"8446fe51d7c9d16d7086694cee8016c6f571dc5c930fd18848fadcf109aa0566","src/bindings/python/templates/RecordTemplate.py":"c99d10cc061af339349bb0c7e8b67223fdcd9064362badc137a2ad0df17c57c0","src/bindings/python/templates/RustBufferHelper.py":"a48e5ed1dcde19993ae50bec9b881afa3bc6dd5f7d8257fd60214f2100224929","src/bindings/python/templates/RustBufferTemplate.py":"017f31fd5075306f5c8c2bd0e3a21ea965c694c0daf2523187ab076fc786e9ca","src/bindings/python/templates/SequenceTemplate.py":"1b262e5f546a1923de6968e0233cc621a5fae16062e9e6ac874c9b62d8f145df","src/bindings/python/templates/StringHelper.py":"b303b7fcbbc0981a28c6a7d0cc5bd90f8e9c8b8d572792e217a324b2bdb95dbd","src/bindings/python/templates/TimestampHelper.py":"b3da14de54822f44ada4459355c842550b944b3cd2a85a4eac0f59e82d646877","src/bindings/python/templates/TopLevelFunctionTemplate.py":"1d9da1b6ca2175b30f3277a46a1749590490e82bee6b990ff35efb04e5f102ef","src/bindings/python/templates/Types.py":"3653e2cf773493c6ddfd13ef298b0c7cb33fabc1dba495fca64b9287aae03042","src/bindings/python/templates/UInt16Helper.py":"8ffe4b69a5d4a2b3c5677ff1d8954efc67ab67713ffe297380e930e0379d493d","src/bindings/python/templates/UInt32Helper.py":"83f9603aceae05f2134c7183313ab0a1a8f64cabd8070ae19557494fe41dd6d2","src/bindings/python/templates/UInt64Helper.py":"97269025377a256e821e57991b07e17af05f4d1c4228e01fe5f243d784cb509d","src/bindings/python/templates/UInt8Helper.py":"4896723ed0ab8f5aef4a58d599e0a0dbd63d373f5740821c21b4b429b6a7afda","src/bindings/python/templates/macros.py":"d766feb4dedd2d0e4cd2052da7a69c0b074b97f880b857ee457faa43975230ed","src/bindings/python/templates/wrapper.py":"ab05168e3d01d1a26e9589cd9855d7776c46c59d699f1402a29dfae6eb9ebfbc","src/bindings/python/test.rs":"69d3ee230820f38d743438c8212e1bfc4e92f948d9e73548a38c093e164b2759","src/bindings/ruby/gen_ruby/mod.rs":"861be105f9001d4ad8f7b8ac4a303a95459ec7de7a0c2fdac14a083c43d5a07c","src/bindings/ruby/gen_ruby/tests.rs":"7dcb86b08e643c43503f4cac6396833497f6988b004321c0067700ee29ffbf32","src/bindings/ruby/mod.rs":"0fdfab5306dc5c05fbcbfb273340d96ad70c5caf5074834ad6851af1a9a56734","src/bindings/ruby/templates/EnumTemplate.rb":"5480edb347f5829e478d19474691babd72f37616ed846d519b5a61cb1d6cf047","src/bindings/ruby/templates/ErrorTemplate.rb":"301c177e639f0a27f19d4935c5317e672aadecbee2f9bfa778df982320f5148d","src/bindings/ruby/templates/Helpers.rb":"ce7ed4be97dad507b991c69c28dc7bb6427e5e79a4b2fba9dad9dccabc3e090c","src/bindings/ruby/templates/NamespaceLibraryTemplate.rb":"9a24c427b9eba99d9e13181a5559a385b5d1d16beae2b72a402f2645b22a9048","src/bindings/ruby/templates/ObjectTemplate.rb":"a1c0cc38865195d61df3540284f4756f1b6406b205d74e3855e7089d763b2791","src/bindings/ruby/templates/RecordTemplate.rb":"343a4b159cf298045747fb48f17552e3bf2c9775fa5b4fa40b424976dc67e33a","src/bindings/ruby/templates/RustBufferBuilder.rb":"a36d9183f3e66cbbb1c3e584b78ab86e01bd6b89a4a5ef9614c5df24dc383acc","src/bindings/ruby/templates/RustBufferStream.rb":"ab4fc736906e320fca56dca280daf40138ba443d957c42fbf5cfbf1c6acf463a","src/bindings/ruby/templates/RustBufferTemplate.rb":"de577fbae811f72e260270656f2c12ad7a4d157c78f97898d0cd4e309d92ad6a","src/bindings/ruby/templates/TopLevelFunctionTemplate.rb":"26c9c2d53853792270795bd822e41968e995375478d246f808f9935af77a7d6a","src/bindings/ruby/templates/macros.rb":"dc60ed79844b04fe828a24aef3550a6b6c30f7c0b66f03608d7c56725287ceed","src/bindings/ruby/templates/wrapper.rb":"f82b41543546f8e5804cd0e1785f4735d9dd95383566d0e5ba1cd4d9e8c0578d","src/bindings/ruby/test.rs":"027d62085498b20977f025117e1fb7c30923a189961d679823f16ca62a575d0d","src/bindings/swift/gen_swift/callback_interface.rs":"1a2b56d16db841574be0762d66b57fbaef0519273d45c47ca687bf656546f201","src/bindings/swift/gen_swift/compounds.rs":"d62206bdab8a2a65b19342933efad54c171f0f8c217b82ee8b41617043662fe5","src/bindings/swift/gen_swift/custom.rs":"bddb601b4ea8810ecbad01271d5ec0b3958999b09bc9382c83637dfd43451734","src/bindings/swift/gen_swift/enum_.rs":"87be67ec3394616368d9ef8e99b7f234c053b3bee9a7f9e6f2dff37f147c8837","src/bindings/swift/gen_swift/external.rs":"a1d34b688679a74b0ddcfcb1147a7064b53883d9df9c0670f950078516492ee7","src/bindings/swift/gen_swift/miscellany.rs":"7fc2444596d76545ad82ee6c4bed64a29dd4a0438d50bfaafe511f41f6a0e409","src/bindings/swift/gen_swift/mod.rs":"e6c12506217d0a5479e946998a24ee984e4ea4c4f19334cbd014f53504300181","src/bindings/swift/gen_swift/object.rs":"45a6d6bb053f3ef397ab8c6feba8d0e126a8d14cd87597d25015f97c6ffc3417","src/bindings/swift/gen_swift/primitives.rs":"26a29ea764988d9e021bbac6505ef45e49ae42426522d6e3822e949b6f0b589c","src/bindings/swift/gen_swift/record.rs":"5ad98ab04a5d8178daf0956db819c87d26aae7bf968184e88d512e34c02feb90","src/bindings/swift/mod.rs":"26ba270cb7913661f3cee703038d1ea4a70bff64c3b31351d6bc77e67cdee20d","src/bindings/swift/templates/Async.swift":"1645ac8dbea8575dec05acf0aeb18e210f76231c36ea0178b183e02a3ff6e18f","src/bindings/swift/templates/BooleanHelper.swift":"f02e391bed44ca0e03c66c4e6a1545caaae490fc72c4cf5935e66220082a8c30","src/bindings/swift/templates/BridgingHeaderTemplate.h":"4e1e91859c4fc6f40db32648645f046fb7e71841f44ae84737ea85bdecff7fa3","src/bindings/swift/templates/CallbackInterfaceImpl.swift":"514a0932c445e4040460da2969e4f21595e17b9b960eb23c6d1526e47dd56c51","src/bindings/swift/templates/CallbackInterfaceRuntime.swift":"a5def6b3b41698a42e6ccf5c85d365fe0abc7eff629d9f49d9d396ee90aad3a0","src/bindings/swift/templates/CallbackInterfaceTemplate.swift":"4dcab3e590f897499782aef3c657b9b838b312d8b49a018bf0f1ebde15ada786","src/bindings/swift/templates/CustomType.swift":"71520eb38a4be9035dca9e3e0402f386e7eaa79b28568bbc2f20d3fd53b4544d","src/bindings/swift/templates/DataHelper.swift":"df11547a2df57dcca0ff9cddc691bb5fa07d5ffd3d328d1c3b4443078008b111","src/bindings/swift/templates/DurationHelper.swift":"cbc41aaa58bda6c2313ede36a9f656a01a28f9c72aa1624e0e1c1da7b841ffb6","src/bindings/swift/templates/EnumTemplate.swift":"4b980f8bfe65266d27d561e88c7d79d87f426b35b4b842ef80c5d56841e2f672","src/bindings/swift/templates/ErrorTemplate.swift":"1233d119320a44dbf6099681595dda9bf5dd2a1474af4380b704bff0563c38ef","src/bindings/swift/templates/Float32Helper.swift":"ea32538058c4b3c72b1cd2530ac00d0349fadab5e1bc617d33aae4c87692fc98","src/bindings/swift/templates/Float64Helper.swift":"e27e9424dc6e97b8cacc6ca4c796dd2d16dcfcb877e2f19c45eca03381a41e78","src/bindings/swift/templates/HandleMap.swift":"acd2b06d678e64a573f7b842c7d08b87140ddb5d7146c0bf3401d99999399ec2","src/bindings/swift/templates/Helpers.swift":"491553eb82cdc5c944451a541d4e4655537cccb961f220783459b57b2311ca84","src/bindings/swift/templates/Int16Helper.swift":"204906911813a3931436057c23032f4c4e39e023df90d641d6c6086aefe2f820","src/bindings/swift/templates/Int32Helper.swift":"0997f059c9c4edd3c41aee0bbad4aa2bda6d791a0d623ad8014d5aa6bdae718d","src/bindings/swift/templates/Int64Helper.swift":"bcf8c2deebb3ee9bce87735adc4bd100981989943b69f6a7fb499a9aec4c25d9","src/bindings/swift/templates/Int8Helper.swift":"ad1ec0fa213724933fa4dc4e2e304e13ac4722b774bfffac44793986b997dd33","src/bindings/swift/templates/MapTemplate.swift":"53971ec388417b02519f8deb8d66361ab4693eae77d116f6051cbea4738054ec","src/bindings/swift/templates/ModuleMapTemplate.modulemap":"99ad1e9bf550a21497296f9248ecd4385dd6d0b5892951d24cf990cdbf3eec2c","src/bindings/swift/templates/ObjectTemplate.swift":"37e57815e60900ae48b953fe01e01535d4ab8076f6160fc93c37dd08fdee47a4","src/bindings/swift/templates/OptionalTemplate.swift":"2376487ceadca3975f0e82ddf6ce61af8bbbf5b0592fa9cd977460f148d8c99d","src/bindings/swift/templates/Protocol.swift":"2614b1378cadf14e7617fedd7367c227ac2a774d528acd3a42e44fd0c4f58528","src/bindings/swift/templates/RecordTemplate.swift":"f9f576b72fda9d1e1db34d1765ec6ec8206103a297329720c1c9a1f58ad085b5","src/bindings/swift/templates/RustBufferTemplate.swift":"89ed33846c0cfb220e823a1002238b16f006f3170d8de0dbbf7775d4f8143c31","src/bindings/swift/templates/SequenceTemplate.swift":"8425b279197582a94b4cf363ab0463907a68a624e24045720ef7df2bcacf3383","src/bindings/swift/templates/StringHelper.swift":"968b9b9b7fbe06a2ac2143016edaff3552e201652accb8d613b03645f0d24a90","src/bindings/swift/templates/TimestampHelper.swift":"82eece13aa186c8e3745c8ad2f1290639ca9689573018a2bdc5c75afbae58c26","src/bindings/swift/templates/TopLevelFunctionTemplate.swift":"7aa473a5b12ad7623f61d6c31f6879f269f51d2c4134dd899ce24c7b31ef35f1","src/bindings/swift/templates/Types.swift":"15e255e35e267f2aca49ed5a4fe16ef79520f4261433fd30c5e6c7f637a4d3f6","src/bindings/swift/templates/UInt16Helper.swift":"d6fba577324fc0e9e50329c968df99341de418011be126bd29702f8a94d87c02","src/bindings/swift/templates/UInt32Helper.swift":"5e0cf151a0c40098b3d96815ba3de23e15fe52f3c517577e1b0c7e7a4c12428f","src/bindings/swift/templates/UInt64Helper.swift":"17237b38d09ced8d2a8ff1ad9ca86873a19e417280e0e60f33d7063397ea4b7b","src/bindings/swift/templates/UInt8Helper.swift":"c4cb2ee4a78b54ca8d0013383c6b43e9ecd42776e3dc7d6e40086325d41714e5","src/bindings/swift/templates/macros.swift":"b30ffd93fe2213e13c3b9910bf2404403b4b231d4cd32c81e0f76c3bb4d151b5","src/bindings/swift/templates/wrapper.swift":"e553af470320391d150e6489eac549064689a37e5db6947914ce5609d0128031","src/bindings/swift/test.rs":"f55ba6c05c250093b26ae91404fd9200951462c1cd99e6b2718f7fb4ebcb7fbb","src/interface/callbacks.rs":"4a019376ec8fbaec495a9e3a1d5cb079af65767b6d85bc9f508f92a1e7f5344f","src/interface/enum_.rs":"7baee60e02cc7f751d7a941e877c10a6afaffea626e79897a0e8b17702f13c15","src/interface/ffi.rs":"11b48aaf22fd9cd9eeded30afe950b26cc1c6d8ec6f9385c9e4cd3bdb2881f43","src/interface/function.rs":"be0f9f268e1947381fa235c5a0cf3c1965fd73121172d31f9c130acf539f2ac0","src/interface/mod.rs":"b97b11295b91691e7e6b7b023bba019729ad02f2204bba460c48acf62c5ee363","src/interface/object.rs":"d37d55edc62f52cf7fac4e3b8be1e46557dcbcfa8eb2e5998a91be2c6c062d92","src/interface/record.rs":"d8ddf873c35beaff45ab522bc4cb809c459a7937fd4061dae8c2db0db4c4edb4","src/interface/universe.rs":"76f368ff2b5326c517025a460405d343618bcc9fc9cfb28346313c8f7a335050","src/lib.rs":"2e3adccd5f0a3dacce6e533edfb5640ddee05e4f87ce8144cba859a14af219f6","src/library_mode.rs":"43ee55e4bb8d27dcec8a164961f22de941603d79d4e10c270ba9e7a751d92a1b","src/macro_metadata/ci.rs":"fa87ef42065c821aa89d3fb7938888c035ce6fc03bb3fce575a878c8e49012fb","src/macro_metadata/extract.rs":"7554c7b19b50d40e90bb503320311c2caa3b1e45e4376a9266ce32c0d48afffb","src/macro_metadata/mod.rs":"bcb5e9a015510e9d74c288da928a4bfb8d80926a8ff85227c0eae8cdb2605519","src/scaffolding/mod.rs":"66c3f2d9e81ded234fdd5b34cbb1da334cc271fa6ff3daa41b9acf9ac02f2194","src/scaffolding/templates/CallbackInterfaceTemplate.rs":"11acee064df46f7b5132401ae49c03c77f296bc04065085d6fd5c4ad6b628718","src/scaffolding/templates/Checksums.rs":"ee926e840875c2e48e1d0cc5185c11f7a1ed3bde5264b07540812cb13c1d7481","src/scaffolding/templates/EnumTemplate.rs":"305b8f0e6ec38300f0ae576a1bf1c576d0088d0df8d0b45818ad25f0216a7ac0","src/scaffolding/templates/ErrorTemplate.rs":"e6ec4e1d4c594d9f14a8dfe0a24103a66c0cc91d2129f0e1644775740f85bea1","src/scaffolding/templates/ExternalTypesTemplate.rs":"4c45cefca1774de3f3b650ce3b9a1b1b8fc10c62e0e48e54ac300748c32959f6","src/scaffolding/templates/ObjectTemplate.rs":"80689b74cbb426e6ce8bce77359b122d747b34b48d9a30aef44f93c9aa726fa7","src/scaffolding/templates/RecordTemplate.rs":"644177d86b52bf39c277b4e60a66f594b3fb0454f6b62837f9041297135c09c9","src/scaffolding/templates/ReexportUniFFIScaffolding.rs":"aa8a1ffa98b6033707d965f90b5709474ed6bc79486fb47dacae8417fc056cf8","src/scaffolding/templates/TopLevelFunctionTemplate.rs":"c11a688cafc2e21c3be105533b34c1f73eab55202936f7c8a97191d7e27f26e0","src/scaffolding/templates/UdlMetadata.rs":"d7c50af1de92ef85630b385a910c7b29875502d622eb90da5541a7012b93d9e2","src/scaffolding/templates/macros.rs":"ea6bacd8dd9116ad739bdafe893d70407050f35e4a7ac8dd2c78b8ef34263e8e","src/scaffolding/templates/scaffolding_template.rs":"c8e18306a73ec5b764f665660fc5c91d498b63b6c3f489e524b2bae50f81f231"},"package":"4a77bb514bcd4bf27c9bd404d7c3f2a6a8131b957eba9c22cfeb7751c4278e09"} \ No newline at end of file diff --git a/third_party/rust/uniffi_bindgen/Cargo.toml b/third_party/rust/uniffi_bindgen/Cargo.toml index 9469a9cf25..f0c7af8665 100644 --- a/third_party/rust/uniffi_bindgen/Cargo.toml +++ b/third_party/rust/uniffi_bindgen/Cargo.toml @@ -12,11 +12,12 @@ [package] edition = "2021" name = "uniffi_bindgen" -version = "0.25.3" +version = "0.27.1" authors = ["Firefox Sync Team "] description = "a multi-language bindings generator for rust (codegen and cli tooling)" homepage = "https://mozilla.github.io/uniffi-rs" documentation = "https://mozilla.github.io/uniffi-rs" +readme = "README.md" keywords = [ "ffi", "bindgen", @@ -54,7 +55,7 @@ version = "2.7.0" version = "0.3" [dependencies.goblin] -version = "0.6" +version = "0.8" [dependencies.heck] version = "0.4" @@ -67,15 +68,19 @@ version = "1.0" [dependencies.serde] version = "1" +features = ["derive"] + +[dependencies.textwrap] +version = "0.16" [dependencies.toml] version = "0.5" [dependencies.uniffi_meta] -version = "=0.25.3" +version = "=0.27.1" [dependencies.uniffi_testing] -version = "=0.25.3" +version = "=0.27.1" [dependencies.uniffi_udl] -version = "=0.25.3" +version = "=0.27.1" diff --git a/third_party/rust/uniffi_bindgen/README.md b/third_party/rust/uniffi_bindgen/README.md new file mode 100644 index 0000000000..64ac3486a3 --- /dev/null +++ b/third_party/rust/uniffi_bindgen/README.md @@ -0,0 +1,81 @@ +# UniFFI - a multi-language bindings generator for Rust + +UniFFI is a toolkit for building cross-platform software components in Rust. + +For the impatient, see [**the UniFFI user guide**](https://mozilla.github.io/uniffi-rs/) +or [**the UniFFI examples**](https://github.com/mozilla/uniffi-rs/tree/main/examples#example-uniffi-components). + +By writing your core business logic in Rust and describing its interface in an "object model", +you can use UniFFI to help you: + +* Compile your Rust code into a shared library for use on different target platforms. +* Generate bindings to load and use the library from different target languages. + +You can describe your object model in an [interface definition file](https://mozilla.github.io/uniffi-rs/udl_file_spec.html) +or [by using proc-macros](https://mozilla.github.io/uniffi-rs/proc_macro/index.html). + +UniFFI is currently used extensively by Mozilla in Firefox mobile and desktop browsers; +written once in Rust, auto-generated bindings allow that functionality to be called +from both Kotlin (for Android apps) and Swift (for iOS apps). +It also has a growing community of users shipping various cool things to many users. + +UniFFI comes with support for **Kotlin**, **Swift**, **Python** and **Ruby** with 3rd party bindings available for **C#** and **Golang**. +Additional foreign language bindings can be developed externally and we welcome contributions to list them here. +See [Third-party foreign language bindings](#third-party-foreign-language-bindings). + +## User Guide + +You can read more about using the tool in [**the UniFFI user guide**](https://mozilla.github.io/uniffi-rs/). + +We consider it ready for production use, but UniFFI is a long way from a 1.0 release with lots of internal work still going on. +We try hard to avoid breaking simple consumers, but more advanced things might break as you upgrade over time. + +### Etymology and Pronunciation + +ˈjuːnɪfaɪ. Pronounced to rhyme with "unify". + +A portmanteau word that also puns with "unify", to signify the joining of one codebase accessed from many languages. + +uni - [Latin ūni-, from ūnus, one] +FFI - [Abbreviation, Foreign Function Interface] + +## Alternative tools + +Other tools we know of which try and solve a similarly shaped problem are: + +* [Diplomat](https://github.com/rust-diplomat/diplomat/) - see our [writeup of + the different approach taken by that tool](docs/diplomat-and-macros.md) +* [Interoptopus](https://github.com/ralfbiedert/interoptopus/) + +(Please open a PR if you think other tools should be listed!) + +## Third-party foreign language bindings + +* [Kotlin Multiplatform support](https://gitlab.com/trixnity/uniffi-kotlin-multiplatform-bindings). The repository contains Kotlin Multiplatform bindings generation for UniFFI, letting you target both JVM and Native. +* [Go bindings](https://github.com/NordSecurity/uniffi-bindgen-go) +* [C# bindings](https://github.com/NordSecurity/uniffi-bindgen-cs) +* [Dart bindings](https://github.com/NiallBunting/uniffi-rs-dart) + +### External resources + +There are a few third-party resources that make it easier to work with UniFFI: + +* [Plugin support for `.udl` files](https://github.com/Lonami/uniffi-dl) for the IDEA platform ([*uniffi-dl* in the JetBrains marketplace](https://plugins.jetbrains.com/plugin/20527-uniffi-dl)). It provides syntax highlighting, code folding, code completion, reference resolution and navigation (among others features) for the [UniFFI Definition Language (UDL)](https://mozilla.github.io/uniffi-rs/). +* [cargo swift](https://github.com/antoniusnaumann/cargo-swift), a cargo plugin to build a Swift Package from Rust code. It provides an init command for setting up a UniFFI crate and a package command for building a Swift package from Rust code - without the need for additional configuration or build scripts. +* [Cargo NDK Gradle Plugin](https://github.com/willir/cargo-ndk-android-gradle) allows you to build Rust code using [`cargo-ndk`](https://github.com/bbqsrc/cargo-ndk), which generally makes Android library builds less painful. +* [`uniffi-starter`](https://github.com/ianthetechie/uniffi-starter) is a minimal project demonstrates a wide range of UniFFI in a complete project in a compact manner. It includes a full Android library build process, an XCFramework generation script, and example Swift package structure. + +(Please open a PR if you think other resources should be listed!) + +## Contributing + +If this tool sounds interesting to you, please help us develop it! You can: + +* View the [contributor guidelines](./docs/contributing.md). +* File or work on [issues](https://github.com/mozilla/uniffi-rs/issues) here in GitHub. +* Join discussions in the [#uniffi:mozilla.org](https://matrix.to/#/#uniffi:mozilla.org) + room on Matrix. + +## Code of Conduct + +This project is governed by Mozilla's [Community Participation Guidelines](./CODE_OF_CONDUCT.md). diff --git a/third_party/rust/uniffi_bindgen/src/backend/filters.rs b/third_party/rust/uniffi_bindgen/src/backend/filters.rs index 0d2da8cab2..f4dde0e420 100644 --- a/third_party/rust/uniffi_bindgen/src/backend/filters.rs +++ b/third_party/rust/uniffi_bindgen/src/backend/filters.rs @@ -13,12 +13,12 @@ use std::fmt; // Need to define an error that implements std::error::Error, which neither String nor // anyhow::Error do. #[derive(Debug)] -struct UniFFIError { +pub struct UniFFIError { message: String, } impl UniFFIError { - fn new(message: String) -> Self { + pub fn new(message: String) -> Self { Self { message } } } diff --git a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/gen_kotlin/callback_interface.rs b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/gen_kotlin/callback_interface.rs index e20020e87c..ae4bffc973 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/gen_kotlin/callback_interface.rs +++ b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/gen_kotlin/callback_interface.rs @@ -26,6 +26,6 @@ impl CodeType for CallbackInterfaceCodeType { } fn initialization_fn(&self) -> Option { - Some(format!("{}.register", self.ffi_converter_name())) + Some(format!("uniffiCallbackInterface{}.register", self.id)) } } diff --git a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/gen_kotlin/compounds.rs b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/gen_kotlin/compounds.rs index 4329f32f4c..8d075bbedb 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/gen_kotlin/compounds.rs +++ b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/gen_kotlin/compounds.rs @@ -5,55 +5,81 @@ use super::{AsCodeType, CodeType}; use crate::backend::{Literal, Type}; use crate::ComponentInterface; -use paste::paste; -fn render_literal(literal: &Literal, inner: &Type, ci: &ComponentInterface) -> String { - match literal { - Literal::Null => "null".into(), - Literal::EmptySequence => "listOf()".into(), - Literal::EmptyMap => "mapOf()".into(), +#[derive(Debug)] +pub struct OptionalCodeType { + inner: Type, +} - // For optionals - _ => super::KotlinCodeOracle.find(inner).literal(literal, ci), +impl OptionalCodeType { + pub fn new(inner: Type) -> Self { + Self { inner } + } + fn inner(&self) -> &Type { + &self.inner } } -macro_rules! impl_code_type_for_compound { - ($T:ty, $type_label_pattern:literal, $canonical_name_pattern: literal) => { - paste! { - #[derive(Debug)] - pub struct $T { - inner: Type, - } - - impl $T { - pub fn new(inner: Type) -> Self { - Self { inner } - } - fn inner(&self) -> &Type { - &self.inner - } - } - - impl CodeType for $T { - fn type_label(&self, ci: &ComponentInterface) -> String { - format!($type_label_pattern, super::KotlinCodeOracle.find(self.inner()).type_label(ci)) - } - - fn canonical_name(&self) -> String { - format!($canonical_name_pattern, super::KotlinCodeOracle.find(self.inner()).canonical_name()) - } - - fn literal(&self, literal: &Literal, ci: &ComponentInterface) -> String { - render_literal(literal, self.inner(), ci) - } - } +impl CodeType for OptionalCodeType { + fn type_label(&self, ci: &ComponentInterface) -> String { + format!( + "{}?", + super::KotlinCodeOracle.find(self.inner()).type_label(ci) + ) + } + + fn canonical_name(&self) -> String { + format!( + "Optional{}", + super::KotlinCodeOracle.find(self.inner()).canonical_name() + ) + } + + fn literal(&self, literal: &Literal, ci: &ComponentInterface) -> String { + match literal { + Literal::None => "null".into(), + Literal::Some { inner } => super::KotlinCodeOracle.find(&self.inner).literal(inner, ci), + _ => panic!("Invalid literal for Optional type: {literal:?}"), } } - } +} -impl_code_type_for_compound!(OptionalCodeType, "{}?", "Optional{}"); -impl_code_type_for_compound!(SequenceCodeType, "List<{}>", "Sequence{}"); +#[derive(Debug)] +pub struct SequenceCodeType { + inner: Type, +} + +impl SequenceCodeType { + pub fn new(inner: Type) -> Self { + Self { inner } + } + fn inner(&self) -> &Type { + &self.inner + } +} + +impl CodeType for SequenceCodeType { + fn type_label(&self, ci: &ComponentInterface) -> String { + format!( + "List<{}>", + super::KotlinCodeOracle.find(self.inner()).type_label(ci) + ) + } + + fn canonical_name(&self) -> String { + format!( + "Sequence{}", + super::KotlinCodeOracle.find(self.inner()).canonical_name() + ) + } + + fn literal(&self, literal: &Literal, _ci: &ComponentInterface) -> String { + match literal { + Literal::EmptySequence => "listOf()".into(), + _ => panic!("Invalid literal for List type: {literal:?}"), + } + } +} #[derive(Debug)] pub struct MapCodeType { @@ -92,7 +118,10 @@ impl CodeType for MapCodeType { ) } - fn literal(&self, literal: &Literal, ci: &ComponentInterface) -> String { - render_literal(literal, &self.value, ci) + fn literal(&self, literal: &Literal, _ci: &ComponentInterface) -> String { + match literal { + Literal::EmptyMap => "mapOf()".into(), + _ => panic!("Invalid literal for Map type: {literal:?}"), + } } } diff --git a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/gen_kotlin/executor.rs b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/gen_kotlin/executor.rs deleted file mode 100644 index 154e12a381..0000000000 --- a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/gen_kotlin/executor.rs +++ /dev/null @@ -1,24 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -use super::CodeType; -use crate::ComponentInterface; - -#[derive(Debug)] -pub struct ForeignExecutorCodeType; - -impl CodeType for ForeignExecutorCodeType { - fn type_label(&self, _ci: &ComponentInterface) -> String { - // Kotlin uses a CoroutineScope for ForeignExecutor - "CoroutineScope".into() - } - - fn canonical_name(&self) -> String { - "ForeignExecutor".into() - } - - fn initialization_fn(&self) -> Option { - Some("FfiConverterForeignExecutor.register".into()) - } -} diff --git a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/gen_kotlin/external.rs b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/gen_kotlin/external.rs index 3ecf09d47f..d55c78f760 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/gen_kotlin/external.rs +++ b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/gen_kotlin/external.rs @@ -17,8 +17,8 @@ impl ExternalCodeType { } impl CodeType for ExternalCodeType { - fn type_label(&self, _ci: &ComponentInterface) -> String { - self.name.clone() + fn type_label(&self, ci: &ComponentInterface) -> String { + super::KotlinCodeOracle.class_name(ci, &self.name) } fn canonical_name(&self) -> String { diff --git a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/gen_kotlin/mod.rs b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/gen_kotlin/mod.rs index 1ed0575a9a..c4fc8e0ed6 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/gen_kotlin/mod.rs +++ b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/gen_kotlin/mod.rs @@ -7,20 +7,21 @@ use std::cell::RefCell; use std::collections::{BTreeSet, HashMap, HashSet}; use std::fmt::Debug; -use anyhow::{Context, Result}; +use anyhow::{bail, Context, Result}; use askama::Template; +use camino::Utf8Path; use heck::{ToLowerCamelCase, ToShoutySnakeCase, ToUpperCamelCase}; use serde::{Deserialize, Serialize}; use crate::backend::TemplateExpression; +use crate::bindings::kotlin; use crate::interface::*; -use crate::BindingsConfig; +use crate::{BindingGenerator, BindingsConfig}; mod callback_interface; mod compounds; mod custom; mod enum_; -mod executor; mod external; mod miscellany; mod object; @@ -28,6 +29,28 @@ mod primitives; mod record; mod variant; +pub struct KotlinBindingGenerator; +impl BindingGenerator for KotlinBindingGenerator { + type Config = Config; + + fn write_bindings( + &self, + ci: &ComponentInterface, + config: &Config, + out_dir: &Utf8Path, + try_format_code: bool, + ) -> Result<()> { + kotlin::write_bindings(config, ci, out_dir, try_format_code) + } + + fn check_library_path(&self, library_path: &Utf8Path, cdylib_name: Option<&str>) -> Result<()> { + if cdylib_name.is_none() { + bail!("Generate bindings for Kotlin requires a cdylib, but {library_path} was given"); + } + Ok(()) + } +} + trait CodeType: Debug { /// The language specific label used to reference this type. This will be used in /// method signatures and property declarations. @@ -73,10 +96,21 @@ trait CodeType: Debug { pub struct Config { package_name: Option, cdylib_name: Option, + generate_immutable_records: Option, #[serde(default)] custom_types: HashMap, #[serde(default)] external_packages: HashMap, + #[serde(default)] + android: bool, + #[serde(default)] + android_cleaner: Option, +} + +impl Config { + pub(crate) fn android_cleaner(&self) -> bool { + self.android_cleaner.unwrap_or(self.android) + } } #[derive(Debug, Default, Clone, Serialize, Deserialize)] @@ -103,6 +137,11 @@ impl Config { "uniffi".into() } } + + /// Whether to generate immutable records (`val` instead of `var`) + pub fn generate_immutable_records(&self) -> bool { + self.generate_immutable_records.unwrap_or(false) + } } impl BindingsConfig for Config { @@ -236,7 +275,6 @@ pub struct KotlinWrapper<'a> { ci: &'a ComponentInterface, type_helper_code: String, type_imports: BTreeSet, - has_async_fns: bool, } impl<'a> KotlinWrapper<'a> { @@ -249,7 +287,6 @@ impl<'a> KotlinWrapper<'a> { ci, type_helper_code, type_imports, - has_async_fns: ci.has_async_fns(), } } @@ -258,10 +295,6 @@ impl<'a> KotlinWrapper<'a> { .iter_types() .map(|t| KotlinCodeOracle.find(t)) .filter_map(|ct| ct.initialization_fn()) - .chain( - self.has_async_fns - .then(|| "uniffiRustFutureContinuationCallback.register".into()), - ) .collect() } @@ -301,7 +334,12 @@ impl KotlinCodeOracle { /// Get the idiomatic Kotlin rendering of a variable name. fn var_name(&self, nm: &str) -> String { - format!("`{}`", nm.to_string().to_lower_camel_case()) + format!("`{}`", self.var_name_raw(nm)) + } + + /// `var_name` without the backticks. Useful for using in `@Structure.FieldOrder`. + pub fn var_name_raw(&self, nm: &str) -> String { + nm.to_string().to_lower_camel_case() } /// Get the idiomatic Kotlin rendering of an individual enum variant. @@ -309,14 +347,78 @@ impl KotlinCodeOracle { nm.to_string().to_shouty_snake_case() } - fn ffi_type_label_by_value(ffi_type: &FfiType) -> String { + /// Get the idiomatic Kotlin rendering of an FFI callback function name + fn ffi_callback_name(&self, nm: &str) -> String { + format!("Uniffi{}", nm.to_upper_camel_case()) + } + + /// Get the idiomatic Kotlin rendering of an FFI struct name + fn ffi_struct_name(&self, nm: &str) -> String { + format!("Uniffi{}", nm.to_upper_camel_case()) + } + + fn ffi_type_label_by_value(&self, ffi_type: &FfiType) -> String { + match ffi_type { + FfiType::RustBuffer(_) => format!("{}.ByValue", self.ffi_type_label(ffi_type)), + FfiType::Struct(name) => format!("{}.UniffiByValue", self.ffi_struct_name(name)), + _ => self.ffi_type_label(ffi_type), + } + } + + /// FFI type name to use inside structs + /// + /// The main requirement here is that all types must have default values or else the struct + /// won't work in some JNA contexts. + fn ffi_type_label_for_ffi_struct(&self, ffi_type: &FfiType) -> String { + match ffi_type { + // Make callbacks function pointers nullable. This matches the semantics of a C + // function pointer better and allows for `null` as a default value. + FfiType::Callback(name) => format!("{}?", self.ffi_callback_name(name)), + _ => self.ffi_type_label_by_value(ffi_type), + } + } + + /// Default values for FFI + /// + /// This is used to: + /// - Set a default return value for error results + /// - Set a default for structs, which JNA sometimes requires + fn ffi_default_value(&self, ffi_type: &FfiType) -> String { + match ffi_type { + FfiType::UInt8 | FfiType::Int8 => "0.toByte()".to_owned(), + FfiType::UInt16 | FfiType::Int16 => "0.toShort()".to_owned(), + FfiType::UInt32 | FfiType::Int32 => "0".to_owned(), + FfiType::UInt64 | FfiType::Int64 => "0.toLong()".to_owned(), + FfiType::Float32 => "0.0f".to_owned(), + FfiType::Float64 => "0.0".to_owned(), + FfiType::RustArcPtr(_) => "Pointer.NULL".to_owned(), + FfiType::RustBuffer(_) => "RustBuffer.ByValue()".to_owned(), + FfiType::Callback(_) => "null".to_owned(), + FfiType::RustCallStatus => "UniffiRustCallStatus.ByValue()".to_owned(), + _ => unimplemented!("ffi_default_value: {ffi_type:?}"), + } + } + + fn ffi_type_label_by_reference(&self, ffi_type: &FfiType) -> String { match ffi_type { - FfiType::RustBuffer(_) => format!("{}.ByValue", Self::ffi_type_label(ffi_type)), - _ => Self::ffi_type_label(ffi_type), + FfiType::Int8 + | FfiType::UInt8 + | FfiType::Int16 + | FfiType::UInt16 + | FfiType::Int32 + | FfiType::UInt32 + | FfiType::Int64 + | FfiType::UInt64 + | FfiType::Float32 + | FfiType::Float64 => format!("{}ByReference", self.ffi_type_label(ffi_type)), + FfiType::RustArcPtr(_) => "PointerByReference".to_owned(), + // JNA structs default to ByReference + FfiType::RustBuffer(_) | FfiType::Struct(_) => self.ffi_type_label(ffi_type), + _ => panic!("{ffi_type:?} by reference is not implemented"), } } - fn ffi_type_label(ffi_type: &FfiType) -> String { + fn ffi_type_label(&self, ffi_type: &FfiType) -> String { match ffi_type { // Note that unsigned integers in Kotlin are currently experimental, but java.nio.ByteBuffer does not // support them yet. Thus, we use the signed variants to represent both signed and unsigned @@ -327,19 +429,35 @@ impl KotlinCodeOracle { FfiType::Int64 | FfiType::UInt64 => "Long".to_string(), FfiType::Float32 => "Float".to_string(), FfiType::Float64 => "Double".to_string(), + FfiType::Handle => "Long".to_string(), FfiType::RustArcPtr(_) => "Pointer".to_string(), FfiType::RustBuffer(maybe_suffix) => { format!("RustBuffer{}", maybe_suffix.as_deref().unwrap_or_default()) } + FfiType::RustCallStatus => "UniffiRustCallStatus.ByValue".to_string(), FfiType::ForeignBytes => "ForeignBytes.ByValue".to_string(), - FfiType::ForeignCallback => "ForeignCallback".to_string(), - FfiType::ForeignExecutorHandle => "USize".to_string(), - FfiType::ForeignExecutorCallback => "UniFfiForeignExecutorCallback".to_string(), - FfiType::RustFutureHandle => "Pointer".to_string(), - FfiType::RustFutureContinuationCallback => { - "UniFffiRustFutureContinuationCallbackType".to_string() - } - FfiType::RustFutureContinuationData => "USize".to_string(), + FfiType::Callback(name) => self.ffi_callback_name(name), + FfiType::Struct(name) => self.ffi_struct_name(name), + FfiType::Reference(inner) => self.ffi_type_label_by_reference(inner), + FfiType::VoidPointer => "Pointer".to_string(), + } + } + + /// Get the name of the interface and class name for an object. + /// + /// If we support callback interfaces, the interface name is the object name, and the class name is derived from that. + /// Otherwise, the class name is the object name and the interface name is derived from that. + /// + /// This split determines what types `FfiConverter.lower()` inputs. If we support callback + /// interfaces, `lower` must lower anything that implements the interface. If not, then lower + /// only lowers the concrete class. + fn object_names(&self, ci: &ComponentInterface, obj: &Object) -> (String, String) { + let class_name = self.class_name(ci, obj.name()); + if obj.has_callback_interface() { + let impl_name = format!("{class_name}Impl"); + (class_name, impl_name) + } else { + (format!("{class_name}Interface"), class_name) } } } @@ -376,12 +494,11 @@ impl AsCodeType for T { Type::Duration => Box::new(miscellany::DurationCodeType), Type::Enum { name, .. } => Box::new(enum_::EnumCodeType::new(name)), - Type::Object { name, .. } => Box::new(object::ObjectCodeType::new(name)), + Type::Object { name, imp, .. } => Box::new(object::ObjectCodeType::new(name, imp)), Type::Record { name, .. } => Box::new(record::RecordCodeType::new(name)), Type::CallbackInterface { name, .. } => { Box::new(callback_interface::CallbackInterfaceCodeType::new(name)) } - Type::ForeignExecutor => Box::new(executor::ForeignExecutorCodeType), Type::Optional { inner_type } => { Box::new(compounds::OptionalCodeType::new(*inner_type)) } @@ -401,6 +518,7 @@ impl AsCodeType for T { mod filters { use super::*; pub use crate::backend::filters::*; + use uniffi_meta::LiteralMetadata; pub(super) fn type_name( as_ct: &impl AsCodeType, @@ -454,8 +572,52 @@ mod filters { Ok(as_ct.as_codetype().literal(literal, ci)) } + // Get the idiomatic Kotlin rendering of an integer. + fn int_literal(t: &Option, base10: String) -> Result { + if let Some(t) = t { + match t { + Type::Int8 | Type::Int16 | Type::Int32 | Type::Int64 => Ok(base10), + Type::UInt8 | Type::UInt16 | Type::UInt32 | Type::UInt64 => Ok(base10 + "u"), + _ => Err(askama::Error::Custom(Box::new(UniFFIError::new( + "Only ints are supported.".to_string(), + )))), + } + } else { + Err(askama::Error::Custom(Box::new(UniFFIError::new( + "Enum hasn't defined a repr".to_string(), + )))) + } + } + + // Get the idiomatic Kotlin rendering of an individual enum variant's discriminant + pub fn variant_discr_literal(e: &Enum, index: &usize) -> Result { + let literal = e.variant_discr(*index).expect("invalid index"); + match literal { + // Kotlin doesn't convert between signed and unsigned by default + // so we'll need to make sure we define the type as appropriately + LiteralMetadata::UInt(v, _, _) => int_literal(e.variant_discr_type(), v.to_string()), + LiteralMetadata::Int(v, _, _) => int_literal(e.variant_discr_type(), v.to_string()), + _ => Err(askama::Error::Custom(Box::new(UniFFIError::new( + "Only ints are supported.".to_string(), + )))), + } + } + pub fn ffi_type_name_by_value(type_: &FfiType) -> Result { - Ok(KotlinCodeOracle::ffi_type_label_by_value(type_)) + Ok(KotlinCodeOracle.ffi_type_label_by_value(type_)) + } + + pub fn ffi_type_name_for_ffi_struct(type_: &FfiType) -> Result { + Ok(KotlinCodeOracle.ffi_type_label_for_ffi_struct(type_)) + } + + pub fn ffi_default_value(type_: FfiType) -> Result { + Ok(KotlinCodeOracle.ffi_default_value(&type_)) + } + + /// Get the idiomatic Kotlin rendering of a function name. + pub fn class_name(nm: &str, ci: &ComponentInterface) -> Result { + Ok(KotlinCodeOracle.class_name(ci, nm)) } /// Get the idiomatic Kotlin rendering of a function name. @@ -468,6 +630,11 @@ mod filters { Ok(KotlinCodeOracle.var_name(nm)) } + /// Get the idiomatic Kotlin rendering of a variable name. + pub fn var_name_raw(nm: &str) -> Result { + Ok(KotlinCodeOracle.var_name_raw(nm)) + } + /// Get a String representing the name used for an individual enum variant. pub fn variant_name(v: &Variant) -> Result { Ok(KotlinCodeOracle.enum_variant_name(v.name())) @@ -478,13 +645,30 @@ mod filters { Ok(KotlinCodeOracle.convert_error_suffix(&name)) } + /// Get the idiomatic Kotlin rendering of an FFI callback function name + pub fn ffi_callback_name(nm: &str) -> Result { + Ok(KotlinCodeOracle.ffi_callback_name(nm)) + } + + /// Get the idiomatic Kotlin rendering of an FFI struct name + pub fn ffi_struct_name(nm: &str) -> Result { + Ok(KotlinCodeOracle.ffi_struct_name(nm)) + } + + pub fn object_names( + obj: &Object, + ci: &ComponentInterface, + ) -> Result<(String, String), askama::Error> { + Ok(KotlinCodeOracle.object_names(ci, obj)) + } + pub fn async_poll( callable: impl Callable, ci: &ComponentInterface, ) -> Result { let ffi_func = callable.ffi_rust_future_poll(ci); Ok(format!( - "{{ future, continuation -> _UniFFILib.INSTANCE.{ffi_func}(future, continuation) }}" + "{{ future, callback, continuation -> UniffiLib.INSTANCE.{ffi_func}(future, callback, continuation) }}" )) } @@ -493,7 +677,7 @@ mod filters { ci: &ComponentInterface, ) -> Result { let ffi_func = callable.ffi_rust_future_complete(ci); - let call = format!("_UniFFILib.INSTANCE.{ffi_func}(future, continuation)"); + let call = format!("UniffiLib.INSTANCE.{ffi_func}(future, continuation)"); let call = match callable.return_type() { Some(Type::External { kind: ExternalKind::DataClass, @@ -502,7 +686,7 @@ mod filters { }) => { // Need to convert the RustBuffer from our package to the RustBuffer of the external package let suffix = KotlinCodeOracle.class_name(ci, &name); - format!("{call}.let {{ RustBuffer{suffix}.create(it.capacity, it.len, it.data) }}") + format!("{call}.let {{ RustBuffer{suffix}.create(it.capacity.toULong(), it.len.toULong(), it.data) }}") } _ => call, }; @@ -515,7 +699,7 @@ mod filters { ) -> Result { let ffi_func = callable.ffi_rust_future_free(ci); Ok(format!( - "{{ future -> _UniFFILib.INSTANCE.{ffi_func}(future) }}" + "{{ future -> UniffiLib.INSTANCE.{ffi_func}(future) }}" )) } @@ -527,4 +711,13 @@ mod filters { pub fn unquote(nm: &str) -> Result { Ok(nm.trim_matches('`').to_string()) } + + /// Get the idiomatic Kotlin rendering of docstring + pub fn docstring(docstring: &str, spaces: &i32) -> Result { + let middle = textwrap::indent(&textwrap::dedent(docstring), " * "); + let wrapped = format!("/**\n{middle}\n */"); + + let spaces = usize::try_from(*spaces).unwrap_or_default(); + Ok(textwrap::indent(&wrapped, &" ".repeat(spaces))) + } } diff --git a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/gen_kotlin/object.rs b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/gen_kotlin/object.rs index c39ae59cce..5a4305d14a 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/gen_kotlin/object.rs +++ b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/gen_kotlin/object.rs @@ -3,25 +3,32 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ use super::CodeType; -use crate::ComponentInterface; +use crate::{interface::ObjectImpl, ComponentInterface}; #[derive(Debug)] pub struct ObjectCodeType { - id: String, + name: String, + imp: ObjectImpl, } impl ObjectCodeType { - pub fn new(id: String) -> Self { - Self { id } + pub fn new(name: String, imp: ObjectImpl) -> Self { + Self { name, imp } } } impl CodeType for ObjectCodeType { fn type_label(&self, ci: &ComponentInterface) -> String { - super::KotlinCodeOracle.class_name(ci, &self.id) + super::KotlinCodeOracle.class_name(ci, &self.name) } fn canonical_name(&self) -> String { - format!("Type{}", self.id) + format!("Type{}", self.name) + } + + fn initialization_fn(&self) -> Option { + self.imp + .has_callback_interface() + .then(|| format!("uniffiCallbackInterface{}.register", self.name)) } } diff --git a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/gen_kotlin/primitives.rs b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/gen_kotlin/primitives.rs index 22495fa209..0bc5a5d99e 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/gen_kotlin/primitives.rs +++ b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/gen_kotlin/primitives.rs @@ -9,7 +9,11 @@ use paste::paste; fn render_literal(literal: &Literal, _ci: &ComponentInterface) -> String { fn typed_number(type_: &Type, num_str: String) -> String { - match type_ { + let unwrapped_type = match type_ { + Type::Optional { inner_type } => inner_type, + t => t, + }; + match unwrapped_type { // Bytes, Shorts and Ints can all be inferred from the type. Type::Int8 | Type::Int16 | Type::Int32 => num_str, Type::Int64 => format!("{num_str}L"), @@ -19,7 +23,7 @@ fn render_literal(literal: &Literal, _ci: &ComponentInterface) -> String { Type::Float32 => format!("{num_str}f"), Type::Float64 => num_str, - _ => panic!("Unexpected literal: {num_str} is not a number"), + _ => panic!("Unexpected literal: {num_str} for type: {type_:?}"), } } @@ -56,7 +60,7 @@ macro_rules! impl_code_type_for_primitive { impl CodeType for $T { fn type_label(&self, _ci: &ComponentInterface) -> String { - $class_name.into() + format!("kotlin.{}", $class_name) } fn canonical_name(&self) -> String { diff --git a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/Async.kt b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/Async.kt index c6a32655f2..b28fbd2c80 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/Async.kt +++ b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/Async.kt @@ -1,44 +1,117 @@ // Async return type handlers -internal const val UNIFFI_RUST_FUTURE_POLL_READY = 0.toShort() -internal const val UNIFFI_RUST_FUTURE_POLL_MAYBE_READY = 1.toShort() +internal const val UNIFFI_RUST_FUTURE_POLL_READY = 0.toByte() +internal const val UNIFFI_RUST_FUTURE_POLL_MAYBE_READY = 1.toByte() -internal val uniffiContinuationHandleMap = UniFfiHandleMap>() +internal val uniffiContinuationHandleMap = UniffiHandleMap>() // FFI type for Rust future continuations -internal object uniffiRustFutureContinuationCallback: UniFffiRustFutureContinuationCallbackType { - override fun callback(continuationHandle: USize, pollResult: Short) { - uniffiContinuationHandleMap.remove(continuationHandle)?.resume(pollResult) - } - - internal fun register(lib: _UniFFILib) { - lib.{{ ci.ffi_rust_future_continuation_callback_set().name() }}(this) +internal object uniffiRustFutureContinuationCallbackImpl: UniffiRustFutureContinuationCallback { + override fun callback(data: Long, pollResult: Byte) { + uniffiContinuationHandleMap.remove(data).resume(pollResult) } } internal suspend fun uniffiRustCallAsync( - rustFuture: Pointer, - pollFunc: (Pointer, USize) -> Unit, - completeFunc: (Pointer, RustCallStatus) -> F, - freeFunc: (Pointer) -> Unit, + rustFuture: Long, + pollFunc: (Long, UniffiRustFutureContinuationCallback, Long) -> Unit, + completeFunc: (Long, UniffiRustCallStatus) -> F, + freeFunc: (Long) -> Unit, liftFunc: (F) -> T, - errorHandler: CallStatusErrorHandler + errorHandler: UniffiRustCallStatusErrorHandler ): T { try { do { - val pollResult = suspendCancellableCoroutine { continuation -> + val pollResult = suspendCancellableCoroutine { continuation -> pollFunc( rustFuture, + uniffiRustFutureContinuationCallbackImpl, uniffiContinuationHandleMap.insert(continuation) ) } } while (pollResult != UNIFFI_RUST_FUTURE_POLL_READY); return liftFunc( - rustCallWithError(errorHandler, { status -> completeFunc(rustFuture, status) }) + uniffiRustCallWithError(errorHandler, { status -> completeFunc(rustFuture, status) }) ) } finally { freeFunc(rustFuture) } } +{%- if ci.has_async_callback_interface_definition() %} +internal inline fun uniffiTraitInterfaceCallAsync( + crossinline makeCall: suspend () -> T, + crossinline handleSuccess: (T) -> Unit, + crossinline handleError: (UniffiRustCallStatus.ByValue) -> Unit, +): UniffiForeignFuture { + // Using `GlobalScope` is labeled as a "delicate API" and generally discouraged in Kotlin programs, since it breaks structured concurrency. + // However, our parent task is a Rust future, so we're going to need to break structure concurrency in any case. + // + // Uniffi does its best to support structured concurrency across the FFI. + // If the Rust future is dropped, `uniffiForeignFutureFreeImpl` is called, which will cancel the Kotlin coroutine if it's still running. + @OptIn(DelicateCoroutinesApi::class) + val job = GlobalScope.launch { + try { + handleSuccess(makeCall()) + } catch(e: Exception) { + handleError( + UniffiRustCallStatus.create( + UNIFFI_CALL_UNEXPECTED_ERROR, + {{ Type::String.borrow()|lower_fn }}(e.toString()), + ) + ) + } + } + val handle = uniffiForeignFutureHandleMap.insert(job) + return UniffiForeignFuture(handle, uniffiForeignFutureFreeImpl) +} + +internal inline fun uniffiTraitInterfaceCallAsyncWithError( + crossinline makeCall: suspend () -> T, + crossinline handleSuccess: (T) -> Unit, + crossinline handleError: (UniffiRustCallStatus.ByValue) -> Unit, + crossinline lowerError: (E) -> RustBuffer.ByValue, +): UniffiForeignFuture { + // See uniffiTraitInterfaceCallAsync for details on `DelicateCoroutinesApi` + @OptIn(DelicateCoroutinesApi::class) + val job = GlobalScope.launch { + try { + handleSuccess(makeCall()) + } catch(e: Exception) { + if (e is E) { + handleError( + UniffiRustCallStatus.create( + UNIFFI_CALL_ERROR, + lowerError(e), + ) + ) + } else { + handleError( + UniffiRustCallStatus.create( + UNIFFI_CALL_UNEXPECTED_ERROR, + {{ Type::String.borrow()|lower_fn }}(e.toString()), + ) + ) + } + } + } + val handle = uniffiForeignFutureHandleMap.insert(job) + return UniffiForeignFuture(handle, uniffiForeignFutureFreeImpl) +} + +internal val uniffiForeignFutureHandleMap = UniffiHandleMap() + +internal object uniffiForeignFutureFreeImpl: UniffiForeignFutureFree { + override fun callback(handle: Long) { + val job = uniffiForeignFutureHandleMap.remove(handle) + if (!job.isCompleted) { + job.cancel() + } + } +} + +// For testing +public fun uniffiForeignFutureHandleCount() = uniffiForeignFutureHandleMap.size + +{%- endif %} diff --git a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/BooleanHelper.kt b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/BooleanHelper.kt index 8cfa2ce000..c6b266066d 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/BooleanHelper.kt +++ b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/BooleanHelper.kt @@ -11,7 +11,7 @@ public object FfiConverterBoolean: FfiConverter { return if (value) 1.toByte() else 0.toByte() } - override fun allocationSize(value: Boolean) = 1 + override fun allocationSize(value: Boolean) = 1UL override fun write(value: Boolean, buf: ByteBuffer) { buf.put(lower(value)) diff --git a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/ByteArrayHelper.kt b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/ByteArrayHelper.kt index 4840a199b4..c9449069e2 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/ByteArrayHelper.kt +++ b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/ByteArrayHelper.kt @@ -5,8 +5,8 @@ public object FfiConverterByteArray: FfiConverterRustBuffer { buf.get(byteArr) return byteArr } - override fun allocationSize(value: ByteArray): Int { - return 4 + value.size + override fun allocationSize(value: ByteArray): ULong { + return 4UL + value.size.toULong() } override fun write(value: ByteArray, buf: ByteBuffer) { buf.putInt(value.size) diff --git a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/CallbackInterfaceImpl.kt b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/CallbackInterfaceImpl.kt new file mode 100644 index 0000000000..30a39d9afb --- /dev/null +++ b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/CallbackInterfaceImpl.kt @@ -0,0 +1,117 @@ +{% if self.include_once_check("CallbackInterfaceRuntime.kt") %}{% include "CallbackInterfaceRuntime.kt" %}{% endif %} + +{%- let trait_impl=format!("uniffiCallbackInterface{}", name) %} + +// Put the implementation in an object so we don't pollute the top-level namespace +internal object {{ trait_impl }} { + {%- for (ffi_callback, meth) in vtable_methods.iter() %} + internal object {{ meth.name()|var_name }}: {{ ffi_callback.name()|ffi_callback_name }} { + override fun callback( + {%- for arg in ffi_callback.arguments() -%} + {{ arg.name().borrow()|var_name }}: {{ arg.type_().borrow()|ffi_type_name_by_value }}, + {%- endfor -%} + {%- if ffi_callback.has_rust_call_status_arg() -%} + uniffiCallStatus: UniffiRustCallStatus, + {%- endif -%} + ) + {%- match ffi_callback.return_type() %} + {%- when Some(return_type) %}: {{ return_type|ffi_type_name_by_value }}, + {%- when None %} + {%- endmatch %} { + val uniffiObj = {{ ffi_converter_name }}.handleMap.get(uniffiHandle) + val makeCall = {% if meth.is_async() %}suspend {% endif %}{ -> + uniffiObj.{{ meth.name()|fn_name() }}( + {%- for arg in meth.arguments() %} + {{ arg|lift_fn }}({{ arg.name()|var_name }}), + {%- endfor %} + ) + } + {%- if !meth.is_async() %} + + {%- match meth.return_type() %} + {%- when Some(return_type) %} + val writeReturn = { value: {{ return_type|type_name(ci) }} -> uniffiOutReturn.setValue({{ return_type|lower_fn }}(value)) } + {%- when None %} + val writeReturn = { _: Unit -> Unit } + {%- endmatch %} + + {%- match meth.throws_type() %} + {%- when None %} + uniffiTraitInterfaceCall(uniffiCallStatus, makeCall, writeReturn) + {%- when Some(error_type) %} + uniffiTraitInterfaceCallWithError( + uniffiCallStatus, + makeCall, + writeReturn, + { e: {{error_type|type_name(ci) }} -> {{ error_type|lower_fn }}(e) } + ) + {%- endmatch %} + + {%- else %} + val uniffiHandleSuccess = { {% if meth.return_type().is_some() %}returnValue{% else %}_{% endif %}: {% match meth.return_type() %}{%- when Some(return_type) %}{{ return_type|type_name(ci) }}{%- when None %}Unit{% endmatch %} -> + val uniffiResult = {{ meth.foreign_future_ffi_result_struct().name()|ffi_struct_name }}.UniffiByValue( + {%- match meth.return_type() %} + {%- when Some(return_type) %} + {{ return_type|lower_fn }}(returnValue), + {%- when None %} + {%- endmatch %} + UniffiRustCallStatus.ByValue() + ) + uniffiResult.write() + uniffiFutureCallback.callback(uniffiCallbackData, uniffiResult) + } + val uniffiHandleError = { callStatus: UniffiRustCallStatus.ByValue -> + uniffiFutureCallback.callback( + uniffiCallbackData, + {{ meth.foreign_future_ffi_result_struct().name()|ffi_struct_name }}.UniffiByValue( + {%- match meth.return_type() %} + {%- when Some(return_type) %} + {{ return_type.into()|ffi_default_value }}, + {%- when None %} + {%- endmatch %} + callStatus, + ), + ) + } + + uniffiOutReturn.uniffiSetValue( + {%- match meth.throws_type() %} + {%- when None %} + uniffiTraitInterfaceCallAsync( + makeCall, + uniffiHandleSuccess, + uniffiHandleError + ) + {%- when Some(error_type) %} + uniffiTraitInterfaceCallAsyncWithError( + makeCall, + uniffiHandleSuccess, + uniffiHandleError, + { e: {{error_type|type_name(ci) }} -> {{ error_type|lower_fn }}(e) } + ) + {%- endmatch %} + ) + {%- endif %} + } + } + {%- endfor %} + + internal object uniffiFree: {{ "CallbackInterfaceFree"|ffi_callback_name }} { + override fun callback(handle: Long) { + {{ ffi_converter_name }}.handleMap.remove(handle) + } + } + + internal var vtable = {{ vtable|ffi_type_name_by_value }}( + {%- for (ffi_callback, meth) in vtable_methods.iter() %} + {{ meth.name()|var_name() }}, + {%- endfor %} + uniffiFree, + ) + + // Registers the foreign callback with the Rust side. + // This method is generated for each callback interface. + internal fun register(lib: UniffiLib) { + lib.{{ ffi_init_callback.name() }}(vtable) + } +} diff --git a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/CallbackInterfaceRuntime.kt b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/CallbackInterfaceRuntime.kt index 62a71e02f1..d58a651e24 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/CallbackInterfaceRuntime.kt +++ b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/CallbackInterfaceRuntime.kt @@ -1,43 +1,3 @@ -internal typealias Handle = Long -internal class ConcurrentHandleMap( - private val leftMap: MutableMap = mutableMapOf(), - private val rightMap: MutableMap = mutableMapOf() -) { - private val lock = java.util.concurrent.locks.ReentrantLock() - private val currentHandle = AtomicLong(0L) - private val stride = 1L - - fun insert(obj: T): Handle = - lock.withLock { - rightMap[obj] ?: - currentHandle.getAndAdd(stride) - .also { handle -> - leftMap[handle] = obj - rightMap[obj] = handle - } - } - - fun get(handle: Handle) = lock.withLock { - leftMap[handle] - } - - fun delete(handle: Handle) { - this.remove(handle) - } - - fun remove(handle: Handle): T? = - lock.withLock { - leftMap.remove(handle)?.let { obj -> - rightMap.remove(obj) - obj - } - } -} - -interface ForeignCallback : com.sun.jna.Callback { - public fun callback(handle: Handle, method: Int, argsData: Pointer, argsLen: Int, outBuf: RustBufferByReference): Int -} - // Magic number for the Rust proxy to call using the same mechanism as every other method, // to free the callback once it's dropped by Rust. internal const val IDX_CALLBACK_FREE = 0 @@ -46,31 +6,22 @@ internal const val UNIFFI_CALLBACK_SUCCESS = 0 internal const val UNIFFI_CALLBACK_ERROR = 1 internal const val UNIFFI_CALLBACK_UNEXPECTED_ERROR = 2 -public abstract class FfiConverterCallbackInterface( - protected val foreignCallback: ForeignCallback -): FfiConverter { - private val handleMap = ConcurrentHandleMap() - - // Registers the foreign callback with the Rust side. - // This method is generated for each callback interface. - internal abstract fun register(lib: _UniFFILib) +public abstract class FfiConverterCallbackInterface: FfiConverter { + internal val handleMap = UniffiHandleMap() - fun drop(handle: Handle): RustBuffer.ByValue { - return handleMap.remove(handle).let { RustBuffer.ByValue() } + internal fun drop(handle: Long) { + handleMap.remove(handle) } - override fun lift(value: Handle): CallbackInterface { - return handleMap.get(value) ?: throw InternalException("No callback in handlemap; this is a Uniffi bug") + override fun lift(value: Long): CallbackInterface { + return handleMap.get(value) } override fun read(buf: ByteBuffer) = lift(buf.getLong()) - override fun lower(value: CallbackInterface) = - handleMap.insert(value).also { - assert(handleMap.get(it) === value) { "Handle map is not returning the object we just placed there. This is a bug in the HandleMap." } - } + override fun lower(value: CallbackInterface) = handleMap.insert(value) - override fun allocationSize(value: CallbackInterface) = 8 + override fun allocationSize(value: CallbackInterface) = 8UL override fun write(value: CallbackInterface, buf: ByteBuffer) { buf.putLong(lower(value)) diff --git a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/CallbackInterfaceTemplate.kt b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/CallbackInterfaceTemplate.kt index 5a29f0acc3..d2cdee4f33 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/CallbackInterfaceTemplate.kt +++ b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/CallbackInterfaceTemplate.kt @@ -1,129 +1,13 @@ {%- let cbi = ci|get_callback_interface_definition(name) %} -{%- let type_name = cbi|type_name(ci) %} -{%- let foreign_callback = format!("ForeignCallback{}", canonical_type_name) %} - -{% if self.include_once_check("CallbackInterfaceRuntime.kt") %}{% include "CallbackInterfaceRuntime.kt" %}{% endif %} -{{- self.add_import("java.util.concurrent.atomic.AtomicLong") }} -{{- self.add_import("java.util.concurrent.locks.ReentrantLock") }} -{{- self.add_import("kotlin.concurrent.withLock") }} - -// Declaration and FfiConverters for {{ type_name }} Callback Interface - -public interface {{ type_name }} { - {% for meth in cbi.methods() -%} - fun {{ meth.name()|fn_name }}({% call kt::arg_list_decl(meth) %}) - {%- match meth.return_type() -%} - {%- when Some with (return_type) %}: {{ return_type|type_name(ci) -}} - {%- else -%} - {%- endmatch %} - {% endfor %} - companion object -} - -// The ForeignCallback that is passed to Rust. -internal class {{ foreign_callback }} : ForeignCallback { - @Suppress("TooGenericExceptionCaught") - override fun callback(handle: Handle, method: Int, argsData: Pointer, argsLen: Int, outBuf: RustBufferByReference): Int { - val cb = {{ ffi_converter_name }}.lift(handle) - return when (method) { - IDX_CALLBACK_FREE -> { - {{ ffi_converter_name }}.drop(handle) - // Successful return - // See docs of ForeignCallback in `uniffi_core/src/ffi/foreigncallbacks.rs` - UNIFFI_CALLBACK_SUCCESS - } - {% for meth in cbi.methods() -%} - {% let method_name = format!("invoke_{}", meth.name())|fn_name -%} - {{ loop.index }} -> { - // Call the method, write to outBuf and return a status code - // See docs of ForeignCallback in `uniffi_core/src/ffi/foreigncallbacks.rs` for info - try { - this.{{ method_name }}(cb, argsData, argsLen, outBuf) - } catch (e: Throwable) { - // Unexpected error - try { - // Try to serialize the error into a string - outBuf.setValue({{ Type::String.borrow()|ffi_converter_name }}.lower(e.toString())) - } catch (e: Throwable) { - // If that fails, then it's time to give up and just return - } - UNIFFI_CALLBACK_UNEXPECTED_ERROR - } - } - {% endfor %} - else -> { - // An unexpected error happened. - // See docs of ForeignCallback in `uniffi_core/src/ffi/foreigncallbacks.rs` - try { - // Try to serialize the error into a string - outBuf.setValue({{ Type::String.borrow()|ffi_converter_name }}.lower("Invalid Callback index")) - } catch (e: Throwable) { - // If that fails, then it's time to give up and just return - } - UNIFFI_CALLBACK_UNEXPECTED_ERROR - } - } - } - - {% for meth in cbi.methods() -%} - {% let method_name = format!("invoke_{}", meth.name())|fn_name %} - @Suppress("UNUSED_PARAMETER") - private fun {{ method_name }}(kotlinCallbackInterface: {{ type_name }}, argsData: Pointer, argsLen: Int, outBuf: RustBufferByReference): Int { - {%- if meth.arguments().len() > 0 %} - val argsBuf = argsData.getByteBuffer(0, argsLen.toLong()).also { - it.order(ByteOrder.BIG_ENDIAN) - } - {%- endif %} - - {%- match meth.return_type() %} - {%- when Some with (return_type) %} - fun makeCall() : Int { - val returnValue = kotlinCallbackInterface.{{ meth.name()|fn_name }}( - {%- for arg in meth.arguments() %} - {{ arg|read_fn }}(argsBuf) - {% if !loop.last %}, {% endif %} - {%- endfor %} - ) - outBuf.setValue({{ return_type|ffi_converter_name }}.lowerIntoRustBuffer(returnValue)) - return UNIFFI_CALLBACK_SUCCESS - } - {%- when None %} - fun makeCall() : Int { - kotlinCallbackInterface.{{ meth.name()|fn_name }}( - {%- for arg in meth.arguments() %} - {{ arg|read_fn }}(argsBuf) - {%- if !loop.last %}, {% endif %} - {%- endfor %} - ) - return UNIFFI_CALLBACK_SUCCESS - } - {%- endmatch %} - - {%- match meth.throws_type() %} - {%- when None %} - fun makeCallAndHandleError() : Int = makeCall() - {%- when Some(error_type) %} - fun makeCallAndHandleError() : Int = try { - makeCall() - } catch (e: {{ error_type|type_name(ci) }}) { - // Expected error, serialize it into outBuf - outBuf.setValue({{ error_type|ffi_converter_name }}.lowerIntoRustBuffer(e)) - UNIFFI_CALLBACK_ERROR - } - {%- endmatch %} - - return makeCallAndHandleError() - } - {% endfor %} -} - -// The ffiConverter which transforms the Callbacks in to Handles to pass to Rust. -public object {{ ffi_converter_name }}: FfiConverterCallbackInterface<{{ type_name }}>( - foreignCallback = {{ foreign_callback }}() -) { - override fun register(lib: _UniFFILib) { - rustCall() { status -> - lib.{{ cbi.ffi_init_callback().name() }}(this.foreignCallback, status) - } - } -} +{%- let ffi_init_callback = cbi.ffi_init_callback() %} +{%- let interface_name = cbi|type_name(ci) %} +{%- let interface_docstring = cbi.docstring() %} +{%- let methods = cbi.methods() %} +{%- let vtable = cbi.vtable() %} +{%- let vtable_methods = cbi.vtable_methods() %} + +{% include "Interface.kt" %} +{% include "CallbackInterfaceImpl.kt" %} + +// The ffiConverter which transforms the Callbacks in to handles to pass to Rust. +public object {{ ffi_converter_name }}: FfiConverterCallbackInterface<{{ interface_name }}>() diff --git a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/CustomTypeTemplate.kt b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/CustomTypeTemplate.kt index 04150c5d78..aeb5f58002 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/CustomTypeTemplate.kt +++ b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/CustomTypeTemplate.kt @@ -49,7 +49,7 @@ public object {{ ffi_converter_name }}: FfiConverter<{{ name }}, {{ ffi_type_nam return {{ config.into_custom.render("builtinValue") }} } - override fun allocationSize(value: {{ name }}): Int { + override fun allocationSize(value: {{ name }}): ULong { val builtinValue = {{ config.from_custom.render("value") }} return {{ builtin|allocation_size_fn }}(builtinValue) } diff --git a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/DurationHelper.kt b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/DurationHelper.kt index 4237c6f9a8..62e02607f3 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/DurationHelper.kt +++ b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/DurationHelper.kt @@ -14,7 +14,7 @@ public object FfiConverterDuration: FfiConverterRustBuffer { } // 8 bytes for seconds, 4 bytes for nanoseconds - override fun allocationSize(value: java.time.Duration) = 12 + override fun allocationSize(value: java.time.Duration) = 12UL override fun write(value: java.time.Duration, buf: ByteBuffer) { if (value.seconds < 0) { diff --git a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/EnumTemplate.kt b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/EnumTemplate.kt index d4c4a1684a..8d1c2235ec 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/EnumTemplate.kt +++ b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/EnumTemplate.kt @@ -7,12 +7,25 @@ {%- if e.is_flat() %} +{%- call kt::docstring(e, 0) %} +{% match e.variant_discr_type() %} +{% when None %} enum class {{ type_name }} { {% for variant in e.variants() -%} + {%- call kt::docstring(variant, 4) %} {{ variant|variant_name }}{% if loop.last %};{% else %},{% endif %} {%- endfor %} companion object } +{% when Some with (variant_discr_type) %} +enum class {{ type_name }}(val value: {{ variant_discr_type|type_name(ci) }}) { + {% for variant in e.variants() -%} + {%- call kt::docstring(variant, 4) %} + {{ variant|variant_name }}({{ e|variant_discr_literal(loop.index0) }}){% if loop.last %};{% else %},{% endif %} + {%- endfor %} + companion object +} +{% endmatch %} public object {{ e|ffi_converter_name }}: FfiConverterRustBuffer<{{ type_name }}> { override fun read(buf: ByteBuffer) = try { @@ -21,7 +34,7 @@ public object {{ e|ffi_converter_name }}: FfiConverterRustBuffer<{{ type_name }} throw RuntimeException("invalid enum value, something is very wrong!!", e) } - override fun allocationSize(value: {{ type_name }}) = 4 + override fun allocationSize(value: {{ type_name }}) = 4UL override fun write(value: {{ type_name }}, buf: ByteBuffer) { buf.putInt(value.ordinal + 1) @@ -30,15 +43,18 @@ public object {{ e|ffi_converter_name }}: FfiConverterRustBuffer<{{ type_name }} {% else %} +{%- call kt::docstring(e, 0) %} sealed class {{ type_name }}{% if contains_object_references %}: Disposable {% endif %} { {% for variant in e.variants() -%} + {%- call kt::docstring(variant, 4) %} {% if !variant.has_fields() -%} object {{ variant|type_name(ci) }} : {{ type_name }}() {% else -%} data class {{ variant|type_name(ci) }}( - {% for field in variant.fields() -%} - val {{ field.name()|var_name }}: {{ field|type_name(ci) }}{% if loop.last %}{% else %}, {% endif %} - {% endfor -%} + {%- for field in variant.fields() -%} + {%- call kt::docstring(field, 8) %} + val {% call kt::field_name(field, loop.index) %}: {{ field|type_name(ci) }}{% if loop.last %}{% else %}, {% endif %} + {%- endfor -%} ) : {{ type_name }}() { companion object } @@ -83,9 +99,9 @@ public object {{ e|ffi_converter_name }} : FfiConverterRustBuffer<{{ type_name } is {{ type_name }}.{{ variant|type_name(ci) }} -> { // Add the size for the Int that specifies the variant plus the size needed for all fields ( - 4 + 4UL {%- for field in variant.fields() %} - + {{ field|allocation_size_fn }}(value.{{ field.name()|var_name }}) + + {{ field|allocation_size_fn }}(value.{%- call kt::field_name(field, loop.index) -%}) {%- endfor %} ) } @@ -98,7 +114,7 @@ public object {{ e|ffi_converter_name }} : FfiConverterRustBuffer<{{ type_name } is {{ type_name }}.{{ variant|type_name(ci) }} -> { buf.putInt({{ loop.index }}) {%- for field in variant.fields() %} - {{ field|write_fn }}(value.{{ field.name()|var_name }}, buf) + {{ field|write_fn }}(value.{%- call kt::field_name(field, loop.index) -%}, buf) {%- endfor %} Unit } diff --git a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/ErrorTemplate.kt b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/ErrorTemplate.kt index 986db5424d..4760c03fd6 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/ErrorTemplate.kt +++ b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/ErrorTemplate.kt @@ -3,24 +3,26 @@ {%- let canonical_type_name = type_|canonical_name %} {% if e.is_flat() %} +{%- call kt::docstring(e, 0) %} sealed class {{ type_name }}(message: String): Exception(message){% if contains_object_references %}, Disposable {% endif %} { - // Each variant is a nested class - // Flat enums carries a string error message, so no special implementation is necessary. {% for variant in e.variants() -%} + {%- call kt::docstring(variant, 4) %} class {{ variant|error_variant_name }}(message: String) : {{ type_name }}(message) {% endfor %} - companion object ErrorHandler : CallStatusErrorHandler<{{ type_name }}> { + companion object ErrorHandler : UniffiRustCallStatusErrorHandler<{{ type_name }}> { override fun lift(error_buf: RustBuffer.ByValue): {{ type_name }} = {{ ffi_converter_name }}.lift(error_buf) } } {%- else %} +{%- call kt::docstring(e, 0) %} sealed class {{ type_name }}: Exception(){% if contains_object_references %}, Disposable {% endif %} { - // Each variant is a nested class {% for variant in e.variants() -%} + {%- call kt::docstring(variant, 4) %} {%- let variant_name = variant|error_variant_name %} class {{ variant_name }}( {% for field in variant.fields() -%} + {%- call kt::docstring(field, 8) %} val {{ field.name()|var_name }}: {{ field|type_name(ci) }}{% if loop.last %}{% else %}, {% endif %} {% endfor -%} ) : {{ type_name }}() { @@ -29,7 +31,7 @@ sealed class {{ type_name }}: Exception(){% if contains_object_references %}, Di } {% endfor %} - companion object ErrorHandler : CallStatusErrorHandler<{{ type_name }}> { + companion object ErrorHandler : UniffiRustCallStatusErrorHandler<{{ type_name }}> { override fun lift(error_buf: RustBuffer.ByValue): {{ type_name }} = {{ ffi_converter_name }}.lift(error_buf) } @@ -76,15 +78,15 @@ public object {{ e|ffi_converter_name }} : FfiConverterRustBuffer<{{ type_name } {%- endif %} } - override fun allocationSize(value: {{ type_name }}): Int { + override fun allocationSize(value: {{ type_name }}): ULong { {%- if e.is_flat() %} - return 4 + return 4UL {%- else %} return when(value) { {%- for variant in e.variants() %} is {{ type_name }}.{{ variant|error_variant_name }} -> ( // Add the size for the Int that specifies the variant plus the size needed for all fields - 4 + 4UL {%- for field in variant.fields() %} + {{ field|allocation_size_fn }}(value.{{ field.name()|var_name }}) {%- endfor %} diff --git a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/ExternalTypeTemplate.kt b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/ExternalTypeTemplate.kt index 0fade7a0bc..b7e77f0b2d 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/ExternalTypeTemplate.kt +++ b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/ExternalTypeTemplate.kt @@ -1,5 +1,5 @@ {%- let package_name=self.external_type_package_name(module_path, namespace) %} -{%- let fully_qualified_type_name = "{}.{}"|format(package_name, name) %} +{%- let fully_qualified_type_name = "{}.{}"|format(package_name, name|class_name(ci)) %} {%- let fully_qualified_ffi_converter_name = "{}.FfiConverterType{}"|format(package_name, name) %} {%- let fully_qualified_rustbuffer_name = "{}.RustBuffer"|format(package_name) %} {%- let local_rustbuffer_name = "RustBuffer{}"|format(name) %} diff --git a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/FfiConverterTemplate.kt b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/FfiConverterTemplate.kt index 3b2c9d225a..0de90b9c4b 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/FfiConverterTemplate.kt +++ b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/FfiConverterTemplate.kt @@ -20,7 +20,7 @@ public interface FfiConverter { // encoding, so we pessimistically allocate the largest size possible (3 // bytes per codepoint). Allocating extra bytes is not really a big deal // because the `RustBuffer` is short-lived. - fun allocationSize(value: KotlinType): Int + fun allocationSize(value: KotlinType): ULong // Write a Kotlin type to a `ByteBuffer` fun write(value: KotlinType, buf: ByteBuffer) @@ -34,11 +34,11 @@ public interface FfiConverter { fun lowerIntoRustBuffer(value: KotlinType): RustBuffer.ByValue { val rbuf = RustBuffer.alloc(allocationSize(value)) try { - val bbuf = rbuf.data!!.getByteBuffer(0, rbuf.capacity.toLong()).also { + val bbuf = rbuf.data!!.getByteBuffer(0, rbuf.capacity).also { it.order(ByteOrder.BIG_ENDIAN) } write(value, bbuf) - rbuf.writeField("len", bbuf.position()) + rbuf.writeField("len", bbuf.position().toLong()) return rbuf } catch (e: Throwable) { RustBuffer.free(rbuf) diff --git a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/Float32Helper.kt b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/Float32Helper.kt index eafec5d122..be91ac8fcb 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/Float32Helper.kt +++ b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/Float32Helper.kt @@ -11,7 +11,7 @@ public object FfiConverterFloat: FfiConverter { return value } - override fun allocationSize(value: Float) = 4 + override fun allocationSize(value: Float) = 4UL override fun write(value: Float, buf: ByteBuffer) { buf.putFloat(value) diff --git a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/Float64Helper.kt b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/Float64Helper.kt index 9fc2892c95..5eb465f0df 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/Float64Helper.kt +++ b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/Float64Helper.kt @@ -11,7 +11,7 @@ public object FfiConverterDouble: FfiConverter { return value } - override fun allocationSize(value: Double) = 8 + override fun allocationSize(value: Double) = 8UL override fun write(value: Double, buf: ByteBuffer) { buf.putDouble(value) diff --git a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/ForeignExecutorTemplate.kt b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/ForeignExecutorTemplate.kt deleted file mode 100644 index 3544b2f9e6..0000000000 --- a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/ForeignExecutorTemplate.kt +++ /dev/null @@ -1,83 +0,0 @@ -{{ self.add_import("kotlinx.coroutines.CoroutineScope") }} -{{ self.add_import("kotlinx.coroutines.delay") }} -{{ self.add_import("kotlinx.coroutines.isActive") }} -{{ self.add_import("kotlinx.coroutines.launch") }} - -internal const val UNIFFI_RUST_TASK_CALLBACK_SUCCESS = 0.toByte() -internal const val UNIFFI_RUST_TASK_CALLBACK_CANCELLED = 1.toByte() -internal const val UNIFFI_FOREIGN_EXECUTOR_CALLBACK_SUCCESS = 0.toByte() -internal const val UNIFFI_FOREIGN_EXECUTOR_CALLBACK_CANCELLED = 1.toByte() -internal const val UNIFFI_FOREIGN_EXECUTOR_CALLBACK_ERROR = 2.toByte() - -// Callback function to execute a Rust task. The Kotlin code schedules these in a coroutine then -// invokes them. -internal interface UniFfiRustTaskCallback : com.sun.jna.Callback { - fun callback(rustTaskData: Pointer?, statusCode: Byte) -} - -internal object UniFfiForeignExecutorCallback : com.sun.jna.Callback { - fun callback(handle: USize, delayMs: Int, rustTask: UniFfiRustTaskCallback?, rustTaskData: Pointer?) : Byte { - if (rustTask == null) { - FfiConverterForeignExecutor.drop(handle) - return UNIFFI_FOREIGN_EXECUTOR_CALLBACK_SUCCESS - } else { - val coroutineScope = FfiConverterForeignExecutor.lift(handle) - if (coroutineScope.isActive) { - val job = coroutineScope.launch { - if (delayMs > 0) { - delay(delayMs.toLong()) - } - rustTask.callback(rustTaskData, UNIFFI_RUST_TASK_CALLBACK_SUCCESS) - } - job.invokeOnCompletion { cause -> - if (cause != null) { - rustTask.callback(rustTaskData, UNIFFI_RUST_TASK_CALLBACK_CANCELLED) - } - } - return UNIFFI_FOREIGN_EXECUTOR_CALLBACK_SUCCESS - } else { - return UNIFFI_FOREIGN_EXECUTOR_CALLBACK_CANCELLED - } - } - } -} - -public object FfiConverterForeignExecutor: FfiConverter { - internal val handleMap = UniFfiHandleMap() - - internal fun drop(handle: USize) { - handleMap.remove(handle) - } - - internal fun register(lib: _UniFFILib) { - {%- match ci.ffi_foreign_executor_callback_set() %} - {%- when Some with (fn) %} - lib.{{ fn.name() }}(UniFfiForeignExecutorCallback) - {%- when None %} - {#- No foreign executor, we don't set anything #} - {% endmatch %} - } - - // Number of live handles, exposed so we can test the memory management - public fun handleCount() : Int { - return handleMap.size - } - - override fun allocationSize(value: CoroutineScope) = USize.size - - override fun lift(value: USize): CoroutineScope { - return handleMap.get(value) ?: throw RuntimeException("unknown handle in FfiConverterForeignExecutor.lift") - } - - override fun read(buf: ByteBuffer): CoroutineScope { - return lift(USize.readFromBuffer(buf)) - } - - override fun lower(value: CoroutineScope): USize { - return handleMap.insert(value) - } - - override fun write(value: CoroutineScope, buf: ByteBuffer) { - lower(value).writeToBuffer(buf) - } -} diff --git a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/HandleMap.kt b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/HandleMap.kt new file mode 100644 index 0000000000..3a56648190 --- /dev/null +++ b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/HandleMap.kt @@ -0,0 +1,27 @@ +// Map handles to objects +// +// This is used pass an opaque 64-bit handle representing a foreign object to the Rust code. +internal class UniffiHandleMap { + private val map = ConcurrentHashMap() + private val counter = java.util.concurrent.atomic.AtomicLong(0) + + val size: Int + get() = map.size + + // Insert a new object into the handle map and get a handle for it + fun insert(obj: T): Long { + val handle = counter.getAndAdd(1) + map.put(handle, obj) + return handle + } + + // Get an object from the handle map + fun get(handle: Long): T { + return map.get(handle) ?: throw InternalException("UniffiHandleMap.get: Invalid handle") + } + + // Remove an entry from the handlemap and get the Kotlin object back + fun remove(handle: Long): T { + return map.remove(handle) ?: throw InternalException("UniffiHandleMap: Invalid handle") + } +} diff --git a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/Helpers.kt b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/Helpers.kt index 382a5f7413..1fdbd3ffc0 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/Helpers.kt +++ b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/Helpers.kt @@ -1,30 +1,43 @@ // A handful of classes and functions to support the generated data structures. // This would be a good candidate for isolating in its own ffi-support lib. -// Error runtime. + +internal const val UNIFFI_CALL_SUCCESS = 0.toByte() +internal const val UNIFFI_CALL_ERROR = 1.toByte() +internal const val UNIFFI_CALL_UNEXPECTED_ERROR = 2.toByte() + @Structure.FieldOrder("code", "error_buf") -internal open class RustCallStatus : Structure() { +internal open class UniffiRustCallStatus : Structure() { @JvmField var code: Byte = 0 @JvmField var error_buf: RustBuffer.ByValue = RustBuffer.ByValue() - class ByValue: RustCallStatus(), Structure.ByValue + class ByValue: UniffiRustCallStatus(), Structure.ByValue fun isSuccess(): Boolean { - return code == 0.toByte() + return code == UNIFFI_CALL_SUCCESS } fun isError(): Boolean { - return code == 1.toByte() + return code == UNIFFI_CALL_ERROR } fun isPanic(): Boolean { - return code == 2.toByte() + return code == UNIFFI_CALL_UNEXPECTED_ERROR + } + + companion object { + fun create(code: Byte, errorBuf: RustBuffer.ByValue): UniffiRustCallStatus.ByValue { + val callStatus = UniffiRustCallStatus.ByValue() + callStatus.code = code + callStatus.error_buf = errorBuf + return callStatus + } } } class InternalException(message: String) : Exception(message) // Each top-level error class has a companion object that can lift the error from the call status's rust buffer -interface CallStatusErrorHandler { +interface UniffiRustCallStatusErrorHandler { fun lift(error_buf: RustBuffer.ByValue): E; } @@ -33,15 +46,15 @@ interface CallStatusErrorHandler { // synchronize itself // Call a rust function that returns a Result<>. Pass in the Error class companion that corresponds to the Err -private inline fun rustCallWithError(errorHandler: CallStatusErrorHandler, callback: (RustCallStatus) -> U): U { - var status = RustCallStatus(); +private inline fun uniffiRustCallWithError(errorHandler: UniffiRustCallStatusErrorHandler, callback: (UniffiRustCallStatus) -> U): U { + var status = UniffiRustCallStatus(); val return_value = callback(status) - checkCallStatus(errorHandler, status) + uniffiCheckCallStatus(errorHandler, status) return return_value } -// Check RustCallStatus and throw an error if the call wasn't successful -private fun checkCallStatus(errorHandler: CallStatusErrorHandler, status: RustCallStatus) { +// Check UniffiRustCallStatus and throw an error if the call wasn't successful +private fun uniffiCheckCallStatus(errorHandler: UniffiRustCallStatusErrorHandler, status: UniffiRustCallStatus) { if (status.isSuccess()) { return } else if (status.isError()) { @@ -60,8 +73,8 @@ private fun checkCallStatus(errorHandler: CallStatusErrorHandler { +// UniffiRustCallStatusErrorHandler implementation for times when we don't expect a CALL_ERROR +object UniffiNullRustCallStatusErrorHandler: UniffiRustCallStatusErrorHandler { override fun lift(error_buf: RustBuffer.ByValue): InternalException { RustBuffer.free(error_buf) return InternalException("Unexpected CALL_ERROR") @@ -69,93 +82,38 @@ object NullCallStatusErrorHandler: CallStatusErrorHandler { } // Call a rust function that returns a plain value -private inline fun rustCall(callback: (RustCallStatus) -> U): U { - return rustCallWithError(NullCallStatusErrorHandler, callback); +private inline fun uniffiRustCall(callback: (UniffiRustCallStatus) -> U): U { + return uniffiRustCallWithError(UniffiNullRustCallStatusErrorHandler, callback); } -// IntegerType that matches Rust's `usize` / C's `size_t` -public class USize(value: Long = 0) : IntegerType(Native.SIZE_T_SIZE, value, true) { - // This is needed to fill in the gaps of IntegerType's implementation of Number for Kotlin. - override fun toByte() = toInt().toByte() - // Needed until https://youtrack.jetbrains.com/issue/KT-47902 is fixed. - @Deprecated("`toInt().toChar()` is deprecated") - override fun toChar() = toInt().toChar() - override fun toShort() = toInt().toShort() - - fun writeToBuffer(buf: ByteBuffer) { - // Make sure we always write usize integers using native byte-order, since they may be - // casted to pointer values - buf.order(ByteOrder.nativeOrder()) - try { - when (Native.SIZE_T_SIZE) { - 4 -> buf.putInt(toInt()) - 8 -> buf.putLong(toLong()) - else -> throw RuntimeException("Invalid SIZE_T_SIZE: ${Native.SIZE_T_SIZE}") - } - } finally { - buf.order(ByteOrder.BIG_ENDIAN) - } - } - - companion object { - val size: Int - get() = Native.SIZE_T_SIZE - - fun readFromBuffer(buf: ByteBuffer) : USize { - // Make sure we always read usize integers using native byte-order, since they may be - // casted from pointer values - buf.order(ByteOrder.nativeOrder()) - try { - return when (Native.SIZE_T_SIZE) { - 4 -> USize(buf.getInt().toLong()) - 8 -> USize(buf.getLong()) - else -> throw RuntimeException("Invalid SIZE_T_SIZE: ${Native.SIZE_T_SIZE}") - } - } finally { - buf.order(ByteOrder.BIG_ENDIAN) - } - } +internal inline fun uniffiTraitInterfaceCall( + callStatus: UniffiRustCallStatus, + makeCall: () -> T, + writeReturn: (T) -> Unit, +) { + try { + writeReturn(makeCall()) + } catch(e: Exception) { + callStatus.code = UNIFFI_CALL_UNEXPECTED_ERROR + callStatus.error_buf = {{ Type::String.borrow()|lower_fn }}(e.toString()) } } - -// Map handles to objects -// -// This is used when the Rust code expects an opaque pointer to represent some foreign object. -// Normally we would pass a pointer to the object, but JNA doesn't support getting a pointer from an -// object reference , nor does it support leaking a reference to Rust. -// -// Instead, this class maps USize values to objects so that we can pass a pointer-sized type to -// Rust when it needs an opaque pointer. -// -// TODO: refactor callbacks to use this class -internal class UniFfiHandleMap { - private val map = ConcurrentHashMap() - // Use AtomicInteger for our counter, since we may be on a 32-bit system. 4 billion possible - // values seems like enough. If somehow we generate 4 billion handles, then this will wrap - // around back to zero and we can assume the first handle generated will have been dropped by - // then. - private val counter = java.util.concurrent.atomic.AtomicInteger(0) - - val size: Int - get() = map.size - - fun insert(obj: T): USize { - val handle = USize(counter.getAndAdd(1).toLong()) - map.put(handle, obj) - return handle - } - - fun get(handle: USize): T? { - return map.get(handle) - } - - fun remove(handle: USize): T? { - return map.remove(handle) +internal inline fun uniffiTraitInterfaceCallWithError( + callStatus: UniffiRustCallStatus, + makeCall: () -> T, + writeReturn: (T) -> Unit, + lowerError: (E) -> RustBuffer.ByValue +) { + try { + writeReturn(makeCall()) + } catch(e: Exception) { + if (e is E) { + callStatus.code = UNIFFI_CALL_ERROR + callStatus.error_buf = lowerError(e) + } else { + callStatus.code = UNIFFI_CALL_UNEXPECTED_ERROR + callStatus.error_buf = {{ Type::String.borrow()|lower_fn }}(e.toString()) + } } } - -// FFI type for Rust future continuations -internal interface UniFffiRustFutureContinuationCallbackType : com.sun.jna.Callback { - fun callback(continuationHandle: USize, pollResult: Short); -} diff --git a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/Int16Helper.kt b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/Int16Helper.kt index 75564276be..de8296fff6 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/Int16Helper.kt +++ b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/Int16Helper.kt @@ -11,7 +11,7 @@ public object FfiConverterShort: FfiConverter { return value } - override fun allocationSize(value: Short) = 2 + override fun allocationSize(value: Short) = 2UL override fun write(value: Short, buf: ByteBuffer) { buf.putShort(value) diff --git a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/Int32Helper.kt b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/Int32Helper.kt index b7a8131c8b..171809a9c4 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/Int32Helper.kt +++ b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/Int32Helper.kt @@ -11,7 +11,7 @@ public object FfiConverterInt: FfiConverter { return value } - override fun allocationSize(value: Int) = 4 + override fun allocationSize(value: Int) = 4UL override fun write(value: Int, buf: ByteBuffer) { buf.putInt(value) diff --git a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/Int64Helper.kt b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/Int64Helper.kt index 601cfc7c2c..35cf8f3169 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/Int64Helper.kt +++ b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/Int64Helper.kt @@ -11,7 +11,7 @@ public object FfiConverterLong: FfiConverter { return value } - override fun allocationSize(value: Long) = 8 + override fun allocationSize(value: Long) = 8UL override fun write(value: Long, buf: ByteBuffer) { buf.putLong(value) diff --git a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/Int8Helper.kt b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/Int8Helper.kt index 9237768dbf..27c98a6659 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/Int8Helper.kt +++ b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/Int8Helper.kt @@ -11,7 +11,7 @@ public object FfiConverterByte: FfiConverter { return value } - override fun allocationSize(value: Byte) = 1 + override fun allocationSize(value: Byte) = 1UL override fun write(value: Byte, buf: ByteBuffer) { buf.put(value) diff --git a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/Interface.kt b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/Interface.kt new file mode 100644 index 0000000000..0b4249fb11 --- /dev/null +++ b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/Interface.kt @@ -0,0 +1,14 @@ +{%- call kt::docstring_value(interface_docstring, 0) %} +public interface {{ interface_name }} { + {% for meth in methods.iter() -%} + {%- call kt::docstring(meth, 4) %} + {% if meth.is_async() -%}suspend {% endif -%} + fun {{ meth.name()|fn_name }}({% call kt::arg_list(meth, true) %}) + {%- match meth.return_type() -%} + {%- when Some with (return_type) %}: {{ return_type|type_name(ci) -}} + {%- else -%} + {%- endmatch %} + {% endfor %} + companion object +} + diff --git a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/MapTemplate.kt b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/MapTemplate.kt index 776c402727..a80418eb00 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/MapTemplate.kt +++ b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/MapTemplate.kt @@ -12,8 +12,8 @@ public object {{ ffi_converter_name }}: FfiConverterRustBuffer): Int { - val spaceForMapSize = 4 + override fun allocationSize(value: Map<{{ key_type_name }}, {{ value_type_name }}>): ULong { + val spaceForMapSize = 4UL val spaceForChildren = value.map { (k, v) -> {{ key_type|allocation_size_fn }}(k) + {{ value_type|allocation_size_fn }}(v) diff --git a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/NamespaceLibraryTemplate.kt b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/NamespaceLibraryTemplate.kt index 6a3aeada35..1bac8a435c 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/NamespaceLibraryTemplate.kt +++ b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/NamespaceLibraryTemplate.kt @@ -13,14 +13,57 @@ private inline fun loadIndirect( return Native.load(findLibraryName(componentName), Lib::class.java) } +// Define FFI callback types +{%- for def in ci.ffi_definitions() %} +{%- match def %} +{%- when FfiDefinition::CallbackFunction(callback) %} +internal interface {{ callback.name()|ffi_callback_name }} : com.sun.jna.Callback { + fun callback( + {%- for arg in callback.arguments() -%} + {{ arg.name().borrow()|var_name }}: {{ arg.type_().borrow()|ffi_type_name_by_value }}, + {%- endfor -%} + {%- if callback.has_rust_call_status_arg() -%} + uniffiCallStatus: UniffiRustCallStatus, + {%- endif -%} + ) + {%- match callback.return_type() %} + {%- when Some(return_type) %}: {{ return_type|ffi_type_name_by_value }} + {%- when None %} + {%- endmatch %} +} +{%- when FfiDefinition::Struct(ffi_struct) %} +@Structure.FieldOrder({% for field in ffi_struct.fields() %}"{{ field.name()|var_name_raw }}"{% if !loop.last %}, {% endif %}{% endfor %}) +internal open class {{ ffi_struct.name()|ffi_struct_name }}( + {%- for field in ffi_struct.fields() %} + @JvmField internal var {{ field.name()|var_name }}: {{ field.type_().borrow()|ffi_type_name_for_ffi_struct }} = {{ field.type_()|ffi_default_value }}, + {%- endfor %} +) : Structure() { + class UniffiByValue( + {%- for field in ffi_struct.fields() %} + {{ field.name()|var_name }}: {{ field.type_().borrow()|ffi_type_name_for_ffi_struct }} = {{ field.type_()|ffi_default_value }}, + {%- endfor %} + ): {{ ffi_struct.name()|ffi_struct_name }}({%- for field in ffi_struct.fields() %}{{ field.name()|var_name }}, {%- endfor %}), Structure.ByValue + + internal fun uniffiSetValue(other: {{ ffi_struct.name()|ffi_struct_name }}) { + {%- for field in ffi_struct.fields() %} + {{ field.name()|var_name }} = other.{{ field.name()|var_name }} + {%- endfor %} + } + +} +{%- when FfiDefinition::Function(_) %} +{# functions are handled below #} +{%- endmatch %} +{%- endfor %} + // A JNA Library to expose the extern-C FFI definitions. // This is an implementation detail which will be called internally by the public API. -internal interface _UniFFILib : Library { +internal interface UniffiLib : Library { companion object { - internal val INSTANCE: _UniFFILib by lazy { - loadIndirect<_UniFFILib>(componentName = "{{ ci.namespace() }}") - .also { lib: _UniFFILib -> + internal val INSTANCE: UniffiLib by lazy { + loadIndirect(componentName = "{{ ci.namespace() }}") + .also { lib: UniffiLib -> uniffiCheckContractApiVersion(lib) uniffiCheckApiChecksums(lib) {% for fn in self.initialization_fns() -%} @@ -28,6 +71,12 @@ internal interface _UniFFILib : Library { {% endfor -%} } } + {% if ci.contains_object_types() %} + // The Cleaner for the whole library + internal val CLEANER: UniffiCleaner by lazy { + UniffiCleaner.create() + } + {%- endif %} } {% for func in ci.iter_ffi_function_definitions() -%} @@ -37,7 +86,7 @@ internal interface _UniFFILib : Library { {% endfor %} } -private fun uniffiCheckContractApiVersion(lib: _UniFFILib) { +private fun uniffiCheckContractApiVersion(lib: UniffiLib) { // Get the bindings contract version from our ComponentInterface val bindings_contract_version = {{ ci.uniffi_contract_version() }} // Get the scaffolding contract version by calling the into the dylib @@ -48,7 +97,7 @@ private fun uniffiCheckContractApiVersion(lib: _UniFFILib) { } @Suppress("UNUSED_PARAMETER") -private fun uniffiCheckApiChecksums(lib: _UniFFILib) { +private fun uniffiCheckApiChecksums(lib: UniffiLib) { {%- for (name, expected_checksum) in ci.iter_checksums() %} if (lib.{{ name }}() != {{ expected_checksum }}.toShort()) { throw RuntimeException("UniFFI API checksum mismatch: try cleaning and rebuilding your project") diff --git a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/ObjectCleanerHelper.kt b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/ObjectCleanerHelper.kt new file mode 100644 index 0000000000..e3e85544d7 --- /dev/null +++ b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/ObjectCleanerHelper.kt @@ -0,0 +1,40 @@ + +// The cleaner interface for Object finalization code to run. +// This is the entry point to any implementation that we're using. +// +// The cleaner registers objects and returns cleanables, so now we are +// defining a `UniffiCleaner` with a `UniffiClenaer.Cleanable` to abstract the +// different implmentations available at compile time. +interface UniffiCleaner { + interface Cleanable { + fun clean() + } + + fun register(value: Any, cleanUpTask: Runnable): UniffiCleaner.Cleanable + + companion object +} + +// The fallback Jna cleaner, which is available for both Android, and the JVM. +private class UniffiJnaCleaner : UniffiCleaner { + private val cleaner = com.sun.jna.internal.Cleaner.getCleaner() + + override fun register(value: Any, cleanUpTask: Runnable): UniffiCleaner.Cleanable = + UniffiJnaCleanable(cleaner.register(value, cleanUpTask)) +} + +private class UniffiJnaCleanable( + private val cleanable: com.sun.jna.internal.Cleaner.Cleanable, +) : UniffiCleaner.Cleanable { + override fun clean() = cleanable.clean() +} + +// We decide at uniffi binding generation time whether we were +// using Android or not. +// There are further runtime checks to chose the correct implementation +// of the cleaner. +{% if config.android_cleaner() %} +{%- include "ObjectCleanerHelperAndroid.kt" %} +{%- else %} +{%- include "ObjectCleanerHelperJvm.kt" %} +{%- endif %} diff --git a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/ObjectCleanerHelperAndroid.kt b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/ObjectCleanerHelperAndroid.kt new file mode 100644 index 0000000000..d025879848 --- /dev/null +++ b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/ObjectCleanerHelperAndroid.kt @@ -0,0 +1,26 @@ +{{- self.add_import("android.os.Build") }} +{{- self.add_import("androidx.annotation.RequiresApi") }} + +private fun UniffiCleaner.Companion.create(): UniffiCleaner = + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.UPSIDE_DOWN_CAKE) { + AndroidSystemCleaner() + } else { + UniffiJnaCleaner() + } + +// The SystemCleaner, available from API Level 33. +// Some API Level 33 OSes do not support using it, so we require API Level 34. +@RequiresApi(Build.VERSION_CODES.UPSIDE_DOWN_CAKE) +private class AndroidSystemCleaner : UniffiCleaner { + val cleaner = android.system.SystemCleaner.cleaner() + + override fun register(value: Any, cleanUpTask: Runnable): UniffiCleaner.Cleanable = + AndroidSystemCleanable(cleaner.register(value, cleanUpTask)) +} + +@RequiresApi(Build.VERSION_CODES.UPSIDE_DOWN_CAKE) +private class AndroidSystemCleanable( + private val cleanable: java.lang.ref.Cleaner.Cleanable, +) : UniffiCleaner.Cleanable { + override fun clean() = cleanable.clean() +} diff --git a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/ObjectCleanerHelperJvm.kt b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/ObjectCleanerHelperJvm.kt new file mode 100644 index 0000000000..c43bc167fc --- /dev/null +++ b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/ObjectCleanerHelperJvm.kt @@ -0,0 +1,25 @@ +private fun UniffiCleaner.Companion.create(): UniffiCleaner = + try { + // For safety's sake: if the library hasn't been run in android_cleaner = true + // mode, but is being run on Android, then we still need to think about + // Android API versions. + // So we check if java.lang.ref.Cleaner is there, and use that… + java.lang.Class.forName("java.lang.ref.Cleaner") + JavaLangRefCleaner() + } catch (e: ClassNotFoundException) { + // … otherwise, fallback to the JNA cleaner. + UniffiJnaCleaner() + } + +private class JavaLangRefCleaner : UniffiCleaner { + val cleaner = java.lang.ref.Cleaner.create() + + override fun register(value: Any, cleanUpTask: Runnable): UniffiCleaner.Cleanable = + JavaLangRefCleanable(cleaner.register(value, cleanUpTask)) +} + +private class JavaLangRefCleanable( + val cleanable: java.lang.ref.Cleaner.Cleanable +) : UniffiCleaner.Cleanable { + override fun clean() = cleanable.clean() +} diff --git a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/ObjectRuntime.kt b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/ObjectRuntime.kt deleted file mode 100644 index b9352c690f..0000000000 --- a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/ObjectRuntime.kt +++ /dev/null @@ -1,161 +0,0 @@ -// Interface implemented by anything that can contain an object reference. -// -// Such types expose a `destroy()` method that must be called to cleanly -// dispose of the contained objects. Failure to call this method may result -// in memory leaks. -// -// The easiest way to ensure this method is called is to use the `.use` -// helper method to execute a block and destroy the object at the end. -interface Disposable { - fun destroy() - companion object { - fun destroy(vararg args: Any?) { - args.filterIsInstance() - .forEach(Disposable::destroy) - } - } -} - -inline fun T.use(block: (T) -> R) = - try { - block(this) - } finally { - try { - // N.B. our implementation is on the nullable type `Disposable?`. - this?.destroy() - } catch (e: Throwable) { - // swallow - } - } - -// The base class for all UniFFI Object types. -// -// This class provides core operations for working with the Rust `Arc` pointer to -// the live Rust struct on the other side of the FFI. -// -// There's some subtlety here, because we have to be careful not to operate on a Rust -// struct after it has been dropped, and because we must expose a public API for freeing -// the Kotlin wrapper object in lieu of reliable finalizers. The core requirements are: -// -// * Each `FFIObject` instance holds an opaque pointer to the underlying Rust struct. -// Method calls need to read this pointer from the object's state and pass it in to -// the Rust FFI. -// -// * When an `FFIObject` is no longer needed, its pointer should be passed to a -// special destructor function provided by the Rust FFI, which will drop the -// underlying Rust struct. -// -// * Given an `FFIObject` instance, calling code is expected to call the special -// `destroy` method in order to free it after use, either by calling it explicitly -// or by using a higher-level helper like the `use` method. Failing to do so will -// leak the underlying Rust struct. -// -// * We can't assume that calling code will do the right thing, and must be prepared -// to handle Kotlin method calls executing concurrently with or even after a call to -// `destroy`, and to handle multiple (possibly concurrent!) calls to `destroy`. -// -// * We must never allow Rust code to operate on the underlying Rust struct after -// the destructor has been called, and must never call the destructor more than once. -// Doing so may trigger memory unsafety. -// -// If we try to implement this with mutual exclusion on access to the pointer, there is the -// possibility of a race between a method call and a concurrent call to `destroy`: -// -// * Thread A starts a method call, reads the value of the pointer, but is interrupted -// before it can pass the pointer over the FFI to Rust. -// * Thread B calls `destroy` and frees the underlying Rust struct. -// * Thread A resumes, passing the already-read pointer value to Rust and triggering -// a use-after-free. -// -// One possible solution would be to use a `ReadWriteLock`, with each method call taking -// a read lock (and thus allowed to run concurrently) and the special `destroy` method -// taking a write lock (and thus blocking on live method calls). However, we aim not to -// generate methods with any hidden blocking semantics, and a `destroy` method that might -// block if called incorrectly seems to meet that bar. -// -// So, we achieve our goals by giving each `FFIObject` an associated `AtomicLong` counter to track -// the number of in-flight method calls, and an `AtomicBoolean` flag to indicate whether `destroy` -// has been called. These are updated according to the following rules: -// -// * The initial value of the counter is 1, indicating a live object with no in-flight calls. -// The initial value for the flag is false. -// -// * At the start of each method call, we atomically check the counter. -// If it is 0 then the underlying Rust struct has already been destroyed and the call is aborted. -// If it is nonzero them we atomically increment it by 1 and proceed with the method call. -// -// * At the end of each method call, we atomically decrement and check the counter. -// If it has reached zero then we destroy the underlying Rust struct. -// -// * When `destroy` is called, we atomically flip the flag from false to true. -// If the flag was already true we silently fail. -// Otherwise we atomically decrement and check the counter. -// If it has reached zero then we destroy the underlying Rust struct. -// -// Astute readers may observe that this all sounds very similar to the way that Rust's `Arc` works, -// and indeed it is, with the addition of a flag to guard against multiple calls to `destroy`. -// -// The overall effect is that the underlying Rust struct is destroyed only when `destroy` has been -// called *and* all in-flight method calls have completed, avoiding violating any of the expectations -// of the underlying Rust code. -// -// In the future we may be able to replace some of this with automatic finalization logic, such as using -// the new "Cleaner" functionaility in Java 9. The above scheme has been designed to work even if `destroy` is -// invoked by garbage-collection machinery rather than by calling code (which by the way, it's apparently also -// possible for the JVM to finalize an object while there is an in-flight call to one of its methods [1], -// so there would still be some complexity here). -// -// Sigh...all of this for want of a robust finalization mechanism. -// -// [1] https://stackoverflow.com/questions/24376768/can-java-finalize-an-object-when-it-is-still-in-scope/24380219 -// -abstract class FFIObject( - protected val pointer: Pointer -): Disposable, AutoCloseable { - - private val wasDestroyed = AtomicBoolean(false) - private val callCounter = AtomicLong(1) - - open protected fun freeRustArcPtr() { - // To be overridden in subclasses. - } - - override fun destroy() { - // Only allow a single call to this method. - // TODO: maybe we should log a warning if called more than once? - if (this.wasDestroyed.compareAndSet(false, true)) { - // This decrement always matches the initial count of 1 given at creation time. - if (this.callCounter.decrementAndGet() == 0L) { - this.freeRustArcPtr() - } - } - } - - @Synchronized - override fun close() { - this.destroy() - } - - internal inline fun callWithPointer(block: (ptr: Pointer) -> R): R { - // Check and increment the call counter, to keep the object alive. - // This needs a compare-and-set retry loop in case of concurrent updates. - do { - val c = this.callCounter.get() - if (c == 0L) { - throw IllegalStateException("${this.javaClass.simpleName} object has already been destroyed") - } - if (c == Long.MAX_VALUE) { - throw IllegalStateException("${this.javaClass.simpleName} call counter would overflow") - } - } while (! this.callCounter.compareAndSet(c, c + 1L)) - // Now we can safely do the method call without the pointer being freed concurrently. - try { - return block(this.pointer) - } finally { - // This decrement always matches the increment we performed above. - if (this.callCounter.decrementAndGet() == 0L) { - this.freeRustArcPtr() - } - } - } -} diff --git a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/ObjectTemplate.kt b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/ObjectTemplate.kt index 8ce27a5d04..62cac7a4d0 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/ObjectTemplate.kt +++ b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/ObjectTemplate.kt @@ -1,125 +1,282 @@ +// This template implements a class for working with a Rust struct via a Pointer/Arc +// to the live Rust struct on the other side of the FFI. +// +// Each instance implements core operations for working with the Rust `Arc` and the +// Kotlin Pointer to work with the live Rust struct on the other side of the FFI. +// +// There's some subtlety here, because we have to be careful not to operate on a Rust +// struct after it has been dropped, and because we must expose a public API for freeing +// theq Kotlin wrapper object in lieu of reliable finalizers. The core requirements are: +// +// * Each instance holds an opaque pointer to the underlying Rust struct. +// Method calls need to read this pointer from the object's state and pass it in to +// the Rust FFI. +// +// * When an instance is no longer needed, its pointer should be passed to a +// special destructor function provided by the Rust FFI, which will drop the +// underlying Rust struct. +// +// * Given an instance, calling code is expected to call the special +// `destroy` method in order to free it after use, either by calling it explicitly +// or by using a higher-level helper like the `use` method. Failing to do so risks +// leaking the underlying Rust struct. +// +// * We can't assume that calling code will do the right thing, and must be prepared +// to handle Kotlin method calls executing concurrently with or even after a call to +// `destroy`, and to handle multiple (possibly concurrent!) calls to `destroy`. +// +// * We must never allow Rust code to operate on the underlying Rust struct after +// the destructor has been called, and must never call the destructor more than once. +// Doing so may trigger memory unsafety. +// +// * To mitigate many of the risks of leaking memory and use-after-free unsafety, a `Cleaner` +// is implemented to call the destructor when the Kotlin object becomes unreachable. +// This is done in a background thread. This is not a panacea, and client code should be aware that +// 1. the thread may starve if some there are objects that have poorly performing +// `drop` methods or do significant work in their `drop` methods. +// 2. the thread is shared across the whole library. This can be tuned by using `android_cleaner = true`, +// or `android = true` in the [`kotlin` section of the `uniffi.toml` file](https://mozilla.github.io/uniffi-rs/kotlin/configuration.html). +// +// If we try to implement this with mutual exclusion on access to the pointer, there is the +// possibility of a race between a method call and a concurrent call to `destroy`: +// +// * Thread A starts a method call, reads the value of the pointer, but is interrupted +// before it can pass the pointer over the FFI to Rust. +// * Thread B calls `destroy` and frees the underlying Rust struct. +// * Thread A resumes, passing the already-read pointer value to Rust and triggering +// a use-after-free. +// +// One possible solution would be to use a `ReadWriteLock`, with each method call taking +// a read lock (and thus allowed to run concurrently) and the special `destroy` method +// taking a write lock (and thus blocking on live method calls). However, we aim not to +// generate methods with any hidden blocking semantics, and a `destroy` method that might +// block if called incorrectly seems to meet that bar. +// +// So, we achieve our goals by giving each instance an associated `AtomicLong` counter to track +// the number of in-flight method calls, and an `AtomicBoolean` flag to indicate whether `destroy` +// has been called. These are updated according to the following rules: +// +// * The initial value of the counter is 1, indicating a live object with no in-flight calls. +// The initial value for the flag is false. +// +// * At the start of each method call, we atomically check the counter. +// If it is 0 then the underlying Rust struct has already been destroyed and the call is aborted. +// If it is nonzero them we atomically increment it by 1 and proceed with the method call. +// +// * At the end of each method call, we atomically decrement and check the counter. +// If it has reached zero then we destroy the underlying Rust struct. +// +// * When `destroy` is called, we atomically flip the flag from false to true. +// If the flag was already true we silently fail. +// Otherwise we atomically decrement and check the counter. +// If it has reached zero then we destroy the underlying Rust struct. +// +// Astute readers may observe that this all sounds very similar to the way that Rust's `Arc` works, +// and indeed it is, with the addition of a flag to guard against multiple calls to `destroy`. +// +// The overall effect is that the underlying Rust struct is destroyed only when `destroy` has been +// called *and* all in-flight method calls have completed, avoiding violating any of the expectations +// of the underlying Rust code. +// +// This makes a cleaner a better alternative to _not_ calling `destroy()` as +// and when the object is finished with, but the abstraction is not perfect: if the Rust object's `drop` +// method is slow, and/or there are many objects to cleanup, and it's on a low end Android device, then the cleaner +// thread may be starved, and the app will leak memory. +// +// In this case, `destroy`ing manually may be a better solution. +// +// The cleaner can live side by side with the manual calling of `destroy`. In the order of responsiveness, uniffi objects +// with Rust peers are reclaimed: +// +// 1. By calling the `destroy` method of the object, which calls `rustObject.free()`. If that doesn't happen: +// 2. When the object becomes unreachable, AND the Cleaner thread gets to call `rustObject.free()`. If the thread is starved then: +// 3. The memory is reclaimed when the process terminates. +// +// [1] https://stackoverflow.com/questions/24376768/can-java-finalize-an-object-when-it-is-still-in-scope/24380219 +// + +{{ self.add_import("java.util.concurrent.atomic.AtomicBoolean") }} +{%- if self.include_once_check("interface-support") %} + {%- include "ObjectCleanerHelper.kt" %} +{%- endif %} + {%- let obj = ci|get_object_definition(name) %} -{%- if self.include_once_check("ObjectRuntime.kt") %}{% include "ObjectRuntime.kt" %}{% endif %} -{{- self.add_import("java.util.concurrent.atomic.AtomicLong") }} -{{- self.add_import("java.util.concurrent.atomic.AtomicBoolean") }} +{%- let (interface_name, impl_class_name) = obj|object_names(ci) %} +{%- let methods = obj.methods() %} +{%- let interface_docstring = obj.docstring() %} +{%- let is_error = ci.is_name_used_as_error(name) %} +{%- let ffi_converter_name = obj|ffi_converter_name %} -public interface {{ type_name }}Interface { - {% for meth in obj.methods() -%} - {%- match meth.throws_type() -%} - {%- when Some with (throwable) -%} - @Throws({{ throwable|type_name(ci) }}::class) - {%- when None -%} - {%- endmatch %} - {% if meth.is_async() -%} - suspend fun {{ meth.name()|fn_name }}({% call kt::arg_list_decl(meth) %}) - {%- else -%} - fun {{ meth.name()|fn_name }}({% call kt::arg_list_decl(meth) %}) - {%- endif %} - {%- match meth.return_type() -%} - {%- when Some with (return_type) %}: {{ return_type|type_name(ci) -}} - {%- when None -%} - {%- endmatch -%} +{%- include "Interface.kt" %} - {% endfor %} - companion object -} +{%- call kt::docstring(obj, 0) %} +{% if (is_error) %} +open class {{ impl_class_name }} : Exception, Disposable, AutoCloseable, {{ interface_name }} { +{% else -%} +open class {{ impl_class_name }}: Disposable, AutoCloseable, {{ interface_name }} { +{%- endif %} -class {{ type_name }}( - pointer: Pointer -) : FFIObject(pointer), {{ type_name }}Interface { + constructor(pointer: Pointer) { + this.pointer = pointer + this.cleanable = UniffiLib.CLEANER.register(this, UniffiCleanAction(pointer)) + } + + /** + * This constructor can be used to instantiate a fake object. Only used for tests. Any + * attempt to actually use an object constructed this way will fail as there is no + * connected Rust object. + */ + @Suppress("UNUSED_PARAMETER") + constructor(noPointer: NoPointer) { + this.pointer = null + this.cleanable = UniffiLib.CLEANER.register(this, UniffiCleanAction(pointer)) + } {%- match obj.primary_constructor() %} - {%- when Some with (cons) %} - constructor({% call kt::arg_list_decl(cons) -%}) : + {%- when Some(cons) %} + {%- if cons.is_async() %} + // Note no constructor generated for this object as it is async. + {%- else %} + {%- call kt::docstring(cons, 4) %} + constructor({% call kt::arg_list(cons, true) -%}) : this({% call kt::to_ffi_call(cons) %}) + {%- endif %} {%- when None %} {%- endmatch %} - /** - * Disconnect the object from the underlying Rust object. - * - * It can be called more than once, but once called, interacting with the object - * causes an `IllegalStateException`. - * - * Clients **must** call this method once done with the object, or cause a memory leak. - */ - override protected fun freeRustArcPtr() { - rustCall() { status -> - _UniFFILib.INSTANCE.{{ obj.ffi_object_free().name() }}(this.pointer, status) + protected val pointer: Pointer? + protected val cleanable: UniffiCleaner.Cleanable + + private val wasDestroyed = AtomicBoolean(false) + private val callCounter = AtomicLong(1) + + override fun destroy() { + // Only allow a single call to this method. + // TODO: maybe we should log a warning if called more than once? + if (this.wasDestroyed.compareAndSet(false, true)) { + // This decrement always matches the initial count of 1 given at creation time. + if (this.callCounter.decrementAndGet() == 0L) { + cleanable.clean() + } } } - {% for meth in obj.methods() -%} - {%- match meth.throws_type() -%} - {%- when Some with (throwable) %} - @Throws({{ throwable|type_name(ci) }}::class) - {%- else -%} - {%- endmatch -%} - {%- if meth.is_async() %} - @Suppress("ASSIGNED_BUT_NEVER_ACCESSED_VARIABLE") - override suspend fun {{ meth.name()|fn_name }}({%- call kt::arg_list_decl(meth) -%}){% match meth.return_type() %}{% when Some with (return_type) %} : {{ return_type|type_name(ci) }}{% when None %}{%- endmatch %} { - return uniffiRustCallAsync( - callWithPointer { thisPtr -> - _UniFFILib.INSTANCE.{{ meth.ffi_func().name() }}( - thisPtr, - {% call kt::arg_list_lowered(meth) %} - ) - }, - {{ meth|async_poll(ci) }}, - {{ meth|async_complete(ci) }}, - {{ meth|async_free(ci) }}, - // lift function - {%- match meth.return_type() %} - {%- when Some(return_type) %} - { {{ return_type|lift_fn }}(it) }, - {%- when None %} - { Unit }, - {% endmatch %} - // Error FFI converter - {%- match meth.throws_type() %} - {%- when Some(e) %} - {{ e|type_name(ci) }}.ErrorHandler, - {%- when None %} - NullCallStatusErrorHandler, - {%- endmatch %} - ) - } - {%- else -%} - {%- match meth.return_type() -%} - {%- when Some with (return_type) -%} - override fun {{ meth.name()|fn_name }}({% call kt::arg_list_protocol(meth) %}): {{ return_type|type_name(ci) }} = - callWithPointer { - {%- call kt::to_ffi_call_with_prefix("it", meth) %} - }.let { - {{ return_type|lift_fn }}(it) + @Synchronized + override fun close() { + this.destroy() + } + + internal inline fun callWithPointer(block: (ptr: Pointer) -> R): R { + // Check and increment the call counter, to keep the object alive. + // This needs a compare-and-set retry loop in case of concurrent updates. + do { + val c = this.callCounter.get() + if (c == 0L) { + throw IllegalStateException("${this.javaClass.simpleName} object has already been destroyed") + } + if (c == Long.MAX_VALUE) { + throw IllegalStateException("${this.javaClass.simpleName} call counter would overflow") + } + } while (! this.callCounter.compareAndSet(c, c + 1L)) + // Now we can safely do the method call without the pointer being freed concurrently. + try { + return block(this.uniffiClonePointer()) + } finally { + // This decrement always matches the increment we performed above. + if (this.callCounter.decrementAndGet() == 0L) { + cleanable.clean() + } } + } - {%- when None -%} - override fun {{ meth.name()|fn_name }}({% call kt::arg_list_protocol(meth) %}) = - callWithPointer { - {%- call kt::to_ffi_call_with_prefix("it", meth) %} + // Use a static inner class instead of a closure so as not to accidentally + // capture `this` as part of the cleanable's action. + private class UniffiCleanAction(private val pointer: Pointer?) : Runnable { + override fun run() { + pointer?.let { ptr -> + uniffiRustCall { status -> + UniffiLib.INSTANCE.{{ obj.ffi_object_free().name() }}(ptr, status) + } + } } - {% endmatch %} - {% endif %} + } + + fun uniffiClonePointer(): Pointer { + return uniffiRustCall() { status -> + UniffiLib.INSTANCE.{{ obj.ffi_object_clone().name() }}(pointer!!, status) + } + } + + {% for meth in obj.methods() -%} + {%- call kt::func_decl("override", meth, 4) %} {% endfor %} + {%- for tm in obj.uniffi_traits() %} + {%- match tm %} + {% when UniffiTrait::Display { fmt } %} + override fun toString(): String { + return {{ fmt.return_type().unwrap()|lift_fn }}({% call kt::to_ffi_call(fmt) %}) + } + {% when UniffiTrait::Eq { eq, ne } %} + {# only equals used #} + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (other !is {{ impl_class_name}}) return false + return {{ eq.return_type().unwrap()|lift_fn }}({% call kt::to_ffi_call(eq) %}) + } + {% when UniffiTrait::Hash { hash } %} + override fun hashCode(): Int { + return {{ hash.return_type().unwrap()|lift_fn }}({%- call kt::to_ffi_call(hash) %}).toInt() + } + {%- else %} + {%- endmatch %} + {%- endfor %} + + {# XXX - "companion object" confusion? How to have alternate constructors *and* be an error? #} {% if !obj.alternate_constructors().is_empty() -%} companion object { {% for cons in obj.alternate_constructors() -%} - fun {{ cons.name()|fn_name }}({% call kt::arg_list_decl(cons) %}): {{ type_name }} = - {{ type_name }}({% call kt::to_ffi_call(cons) %}) + {% call kt::func_decl("", cons, 4) %} {% endfor %} } + {% else if is_error %} + companion object ErrorHandler : UniffiRustCallStatusErrorHandler<{{ impl_class_name }}> { + override fun lift(error_buf: RustBuffer.ByValue): {{ impl_class_name }} { + // Due to some mismatches in the ffi converter mechanisms, errors are a RustBuffer. + val bb = error_buf.asByteBuffer() + if (bb == null) { + throw InternalException("?") + } + return {{ ffi_converter_name }}.read(bb) + } + } {% else %} companion object {% endif %} } -public object {{ obj|ffi_converter_name }}: FfiConverter<{{ type_name }}, Pointer> { - override fun lower(value: {{ type_name }}): Pointer = value.callWithPointer { it } +{%- if obj.has_callback_interface() %} +{%- let vtable = obj.vtable().expect("trait interface should have a vtable") %} +{%- let vtable_methods = obj.vtable_methods() %} +{%- let ffi_init_callback = obj.ffi_init_callback() %} +{% include "CallbackInterfaceImpl.kt" %} +{%- endif %} + +public object {{ ffi_converter_name }}: FfiConverter<{{ type_name }}, Pointer> { + {%- if obj.has_callback_interface() %} + internal val handleMap = UniffiHandleMap<{{ type_name }}>() + {%- endif %} + + override fun lower(value: {{ type_name }}): Pointer { + {%- if obj.has_callback_interface() %} + return Pointer(handleMap.insert(value)) + {%- else %} + return value.uniffiClonePointer() + {%- endif %} + } override fun lift(value: Pointer): {{ type_name }} { - return {{ type_name }}(value) + return {{ impl_class_name }}(value) } override fun read(buf: ByteBuffer): {{ type_name }} { @@ -128,7 +285,7 @@ public object {{ obj|ffi_converter_name }}: FfiConverter<{{ type_name }}, Pointe return lift(Pointer(buf.getLong())) } - override fun allocationSize(value: {{ type_name }}) = 8 + override fun allocationSize(value: {{ type_name }}) = 8UL override fun write(value: {{ type_name }}, buf: ByteBuffer) { // The Rust code always expects pointers written as 8 bytes, diff --git a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/OptionalTemplate.kt b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/OptionalTemplate.kt index 56cb5f87a5..98451e1451 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/OptionalTemplate.kt +++ b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/OptionalTemplate.kt @@ -8,11 +8,11 @@ public object {{ ffi_converter_name }}: FfiConverterRustBuffer<{{ inner_type_nam return {{ inner_type|read_fn }}(buf) } - override fun allocationSize(value: {{ inner_type_name }}?): Int { + override fun allocationSize(value: {{ inner_type_name }}?): ULong { if (value == null) { - return 1 + return 1UL } else { - return 1 + {{ inner_type|allocation_size_fn }}(value) + return 1UL + {{ inner_type|allocation_size_fn }}(value) } } diff --git a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/README.md b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/README.md new file mode 100644 index 0000000000..0e770cb829 --- /dev/null +++ b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/README.md @@ -0,0 +1,13 @@ +# Rules for the Kotlin template code + +## Naming + +Private variables, classes, functions, etc. should be prefixed with `uniffi`, `Uniffi`, or `UNIFFI`. +This avoids naming collisions with user-defined items. +Users will not get name collisions as long as they don't use "uniffi", which is reserved for us. + +In particular, make sure to use the `uniffi` prefix for any variable names in generated functions. +If you name a variable something like `result` the code will probably work initially. +Then it will break later on when a user decides to define a function with a parameter named `result`. + +Note: this doesn't apply to items that we want to expose, for example users may want to catch `InternalException` so doesn't get the `Uniffi` prefix. diff --git a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/RecordTemplate.kt b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/RecordTemplate.kt index b588ca1398..bc3028c736 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/RecordTemplate.kt +++ b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/RecordTemplate.kt @@ -1,8 +1,11 @@ {%- let rec = ci|get_record_definition(name) %} +{%- if rec.has_fields() %} +{%- call kt::docstring(rec, 0) %} data class {{ type_name }} ( {%- for field in rec.fields() %} - var {{ field.name()|var_name }}: {{ field|type_name(ci) -}} + {%- call kt::docstring(field, 4) %} + {% if config.generate_immutable_records() %}val{% else %}var{% endif %} {{ field.name()|var_name }}: {{ field|type_name(ci) -}} {%- match field.default_value() %} {%- when Some with(literal) %} = {{ literal|render_literal(field, ci) }} {%- else %} @@ -18,21 +21,39 @@ data class {{ type_name }} ( {% endif %} companion object } +{%- else -%} +{%- call kt::docstring(rec, 0) %} +class {{ type_name }} { + override fun equals(other: Any?): Boolean { + return other is {{ type_name }} + } + + override fun hashCode(): Int { + return javaClass.hashCode() + } + + companion object +} +{%- endif %} public object {{ rec|ffi_converter_name }}: FfiConverterRustBuffer<{{ type_name }}> { override fun read(buf: ByteBuffer): {{ type_name }} { + {%- if rec.has_fields() %} return {{ type_name }}( {%- for field in rec.fields() %} {{ field|read_fn }}(buf), {%- endfor %} ) + {%- else %} + return {{ type_name }}() + {%- endif %} } - override fun allocationSize(value: {{ type_name }}) = ( + override fun allocationSize(value: {{ type_name }}) = {%- if rec.has_fields() %} ( {%- for field in rec.fields() %} - {{ field|allocation_size_fn }}(value.{{ field.name()|var_name }}){% if !loop.last %} +{% endif%} + {{ field|allocation_size_fn }}(value.{{ field.name()|var_name }}){% if !loop.last %} +{% endif %} {%- endfor %} - ) + ) {%- else %} 0UL {%- endif %} override fun write(value: {{ type_name }}, buf: ByteBuffer) { {%- for field in rec.fields() %} diff --git a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/RustBufferTemplate.kt b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/RustBufferTemplate.kt index dfbea24074..b28f25bfc3 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/RustBufferTemplate.kt +++ b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/RustBufferTemplate.kt @@ -4,32 +4,41 @@ @Structure.FieldOrder("capacity", "len", "data") open class RustBuffer : Structure() { - @JvmField var capacity: Int = 0 - @JvmField var len: Int = 0 + // Note: `capacity` and `len` are actually `ULong` values, but JVM only supports signed values. + // When dealing with these fields, make sure to call `toULong()`. + @JvmField var capacity: Long = 0 + @JvmField var len: Long = 0 @JvmField var data: Pointer? = null class ByValue: RustBuffer(), Structure.ByValue class ByReference: RustBuffer(), Structure.ByReference + internal fun setValue(other: RustBuffer) { + capacity = other.capacity + len = other.len + data = other.data + } + companion object { - internal fun alloc(size: Int = 0) = rustCall() { status -> - _UniFFILib.INSTANCE.{{ ci.ffi_rustbuffer_alloc().name() }}(size, status) + internal fun alloc(size: ULong = 0UL) = uniffiRustCall() { status -> + // Note: need to convert the size to a `Long` value to make this work with JVM. + UniffiLib.INSTANCE.{{ ci.ffi_rustbuffer_alloc().name() }}(size.toLong(), status) }.also { if(it.data == null) { throw RuntimeException("RustBuffer.alloc() returned null data pointer (size=${size})") } } - internal fun create(capacity: Int, len: Int, data: Pointer?): RustBuffer.ByValue { + internal fun create(capacity: ULong, len: ULong, data: Pointer?): RustBuffer.ByValue { var buf = RustBuffer.ByValue() - buf.capacity = capacity - buf.len = len + buf.capacity = capacity.toLong() + buf.len = len.toLong() buf.data = data return buf } - internal fun free(buf: RustBuffer.ByValue) = rustCall() { status -> - _UniFFILib.INSTANCE.{{ ci.ffi_rustbuffer_free().name() }}(buf, status) + internal fun free(buf: RustBuffer.ByValue) = uniffiRustCall() { status -> + UniffiLib.INSTANCE.{{ ci.ffi_rustbuffer_free().name() }}(buf, status) } } @@ -53,9 +62,9 @@ class RustBufferByReference : ByReference(16) { fun setValue(value: RustBuffer.ByValue) { // NOTE: The offsets are as they are in the C-like struct. val pointer = getPointer() - pointer.setInt(0, value.capacity) - pointer.setInt(4, value.len) - pointer.setPointer(8, value.data) + pointer.setLong(0, value.capacity) + pointer.setLong(8, value.len) + pointer.setPointer(16, value.data) } /** @@ -64,9 +73,9 @@ class RustBufferByReference : ByReference(16) { fun getValue(): RustBuffer.ByValue { val pointer = getPointer() val value = RustBuffer.ByValue() - value.writeField("capacity", pointer.getInt(0)) - value.writeField("len", pointer.getInt(4)) - value.writeField("data", pointer.getPointer(8)) + value.writeField("capacity", pointer.getLong(0)) + value.writeField("len", pointer.getLong(8)) + value.writeField("data", pointer.getLong(16)) return value } diff --git a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/SequenceTemplate.kt b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/SequenceTemplate.kt index 876d1bc05e..61f911cb0c 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/SequenceTemplate.kt +++ b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/SequenceTemplate.kt @@ -8,15 +8,15 @@ public object {{ ffi_converter_name }}: FfiConverterRustBuffer): Int { - val sizeForLength = 4 + override fun allocationSize(value: List<{{ inner_type_name }}>): ULong { + val sizeForLength = 4UL val sizeForItems = value.map { {{ inner_type|allocation_size_fn }}(it) }.sum() return sizeForLength + sizeForItems } override fun write(value: List<{{ inner_type_name }}>, buf: ByteBuffer) { buf.putInt(value.size) - value.forEach { + value.iterator().forEach { {{ inner_type|write_fn }}(it, buf) } } diff --git a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/StringHelper.kt b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/StringHelper.kt index 68324be4f9..b67435bd1a 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/StringHelper.kt +++ b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/StringHelper.kt @@ -4,7 +4,7 @@ public object FfiConverterString: FfiConverter { // store our length and avoid writing it out to the buffer. override fun lift(value: RustBuffer.ByValue): String { try { - val byteArr = ByteArray(value.len) + val byteArr = ByteArray(value.len.toInt()) value.asByteBuffer()!!.get(byteArr) return byteArr.toString(Charsets.UTF_8) } finally { @@ -31,7 +31,7 @@ public object FfiConverterString: FfiConverter { val byteBuf = toUtf8(value) // Ideally we'd pass these bytes to `ffi_bytebuffer_from_bytes`, but doing so would require us // to copy them into a JNA `Memory`. So we might as well directly copy them into a `RustBuffer`. - val rbuf = RustBuffer.alloc(byteBuf.limit()) + val rbuf = RustBuffer.alloc(byteBuf.limit().toULong()) rbuf.asByteBuffer()!!.put(byteBuf) return rbuf } @@ -39,9 +39,9 @@ public object FfiConverterString: FfiConverter { // We aren't sure exactly how many bytes our string will be once it's UTF-8 // encoded. Allocate 3 bytes per UTF-16 code unit which will always be // enough. - override fun allocationSize(value: String): Int { - val sizeForLength = 4 - val sizeForString = value.length * 3 + override fun allocationSize(value: String): ULong { + val sizeForLength = 4UL + val sizeForString = value.length.toULong() * 3UL return sizeForLength + sizeForString } diff --git a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/TimestampHelper.kt b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/TimestampHelper.kt index 21069d7ce8..10a450a4bd 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/TimestampHelper.kt +++ b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/TimestampHelper.kt @@ -14,7 +14,7 @@ public object FfiConverterTimestamp: FfiConverterRustBuffer { } // 8 bytes for seconds, 4 bytes for nanoseconds - override fun allocationSize(value: java.time.Instant) = 12 + override fun allocationSize(value: java.time.Instant) = 12UL override fun write(value: java.time.Instant, buf: ByteBuffer) { var epochOffset = java.time.Duration.between(java.time.Instant.EPOCH, value) diff --git a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/TopLevelFunctionTemplate.kt b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/TopLevelFunctionTemplate.kt index 6a841d3484..681c48093a 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/TopLevelFunctionTemplate.kt +++ b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/TopLevelFunctionTemplate.kt @@ -1,51 +1 @@ -{%- if func.is_async() %} -{%- match func.throws_type() -%} -{%- when Some with (throwable) %} -@Throws({{ throwable|type_name(ci) }}::class) -{%- else -%} -{%- endmatch %} - -@Suppress("ASSIGNED_BUT_NEVER_ACCESSED_VARIABLE") -suspend fun {{ func.name()|fn_name }}({%- call kt::arg_list_decl(func) -%}){% match func.return_type() %}{% when Some with (return_type) %} : {{ return_type|type_name(ci) }}{% when None %}{%- endmatch %} { - return uniffiRustCallAsync( - _UniFFILib.INSTANCE.{{ func.ffi_func().name() }}({% call kt::arg_list_lowered(func) %}), - {{ func|async_poll(ci) }}, - {{ func|async_complete(ci) }}, - {{ func|async_free(ci) }}, - // lift function - {%- match func.return_type() %} - {%- when Some(return_type) %} - { {{ return_type|lift_fn }}(it) }, - {%- when None %} - { Unit }, - {% endmatch %} - // Error FFI converter - {%- match func.throws_type() %} - {%- when Some(e) %} - {{ e|type_name(ci) }}.ErrorHandler, - {%- when None %} - NullCallStatusErrorHandler, - {%- endmatch %} - ) -} - -{%- else %} -{%- match func.throws_type() -%} -{%- when Some with (throwable) %} -@Throws({{ throwable|type_name(ci) }}::class) -{%- else -%} -{%- endmatch -%} - -{%- match func.return_type() -%} -{%- when Some with (return_type) %} - -fun {{ func.name()|fn_name }}({%- call kt::arg_list_decl(func) -%}): {{ return_type|type_name(ci) }} { - return {{ return_type|lift_fn }}({% call kt::to_ffi_call(func) %}) -} -{% when None %} - -fun {{ func.name()|fn_name }}({% call kt::arg_list_decl(func) %}) = - {% call kt::to_ffi_call(func) %} - -{% endmatch %} -{%- endif %} +{%- call kt::func_decl("", func, 8) %} diff --git a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/Types.kt b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/Types.kt index 103d444ea3..c27121b701 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/Types.kt +++ b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/Types.kt @@ -1,5 +1,38 @@ {%- import "macros.kt" as kt %} +// Interface implemented by anything that can contain an object reference. +// +// Such types expose a `destroy()` method that must be called to cleanly +// dispose of the contained objects. Failure to call this method may result +// in memory leaks. +// +// The easiest way to ensure this method is called is to use the `.use` +// helper method to execute a block and destroy the object at the end. +interface Disposable { + fun destroy() + companion object { + fun destroy(vararg args: Any?) { + args.filterIsInstance() + .forEach(Disposable::destroy) + } + } +} + +inline fun T.use(block: (T) -> R) = + try { + block(this) + } finally { + try { + // N.B. our implementation is on the nullable type `Disposable?`. + this?.destroy() + } catch (e: Throwable) { + // swallow + } + } + +/** Used to instantiate an interface without an actual pointer, for fakes in tests, mostly. */ +object NoPointer + {%- for type_ in ci.iter_types() %} {%- let type_name = type_|type_name(ci) %} {%- let ffi_converter_name = type_|ffi_converter_name %} @@ -82,9 +115,6 @@ {%- when Type::CallbackInterface { module_path, name } %} {% include "CallbackInterfaceTemplate.kt" %} -{%- when Type::ForeignExecutor %} -{% include "ForeignExecutorTemplate.kt" %} - {%- when Type::Timestamp %} {% include "TimestampHelper.kt" %} @@ -104,6 +134,10 @@ {%- if ci.has_async_fns() %} {# Import types needed for async support #} {{ self.add_import("kotlin.coroutines.resume") }} +{{ self.add_import("kotlinx.coroutines.launch") }} {{ self.add_import("kotlinx.coroutines.suspendCancellableCoroutine") }} {{ self.add_import("kotlinx.coroutines.CancellableContinuation") }} +{{ self.add_import("kotlinx.coroutines.DelicateCoroutinesApi") }} +{{ self.add_import("kotlinx.coroutines.Job") }} +{{ self.add_import("kotlinx.coroutines.GlobalScope") }} {%- endif %} diff --git a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/UInt16Helper.kt b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/UInt16Helper.kt index 279a8fa91b..b179145b62 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/UInt16Helper.kt +++ b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/UInt16Helper.kt @@ -11,7 +11,7 @@ public object FfiConverterUShort: FfiConverter { return value.toShort() } - override fun allocationSize(value: UShort) = 2 + override fun allocationSize(value: UShort) = 2UL override fun write(value: UShort, buf: ByteBuffer) { buf.putShort(value.toShort()) diff --git a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/UInt32Helper.kt b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/UInt32Helper.kt index da7b5b28d6..202d5bcd5b 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/UInt32Helper.kt +++ b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/UInt32Helper.kt @@ -11,7 +11,7 @@ public object FfiConverterUInt: FfiConverter { return value.toInt() } - override fun allocationSize(value: UInt) = 4 + override fun allocationSize(value: UInt) = 4UL override fun write(value: UInt, buf: ByteBuffer) { buf.putInt(value.toInt()) diff --git a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/UInt64Helper.kt b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/UInt64Helper.kt index 44d27ad36b..9be2a5a69d 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/UInt64Helper.kt +++ b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/UInt64Helper.kt @@ -11,7 +11,7 @@ public object FfiConverterULong: FfiConverter { return value.toLong() } - override fun allocationSize(value: ULong) = 8 + override fun allocationSize(value: ULong) = 8UL override fun write(value: ULong, buf: ByteBuffer) { buf.putLong(value.toLong()) diff --git a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/UInt8Helper.kt b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/UInt8Helper.kt index b6d176603e..ee360673e0 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/UInt8Helper.kt +++ b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/UInt8Helper.kt @@ -11,7 +11,7 @@ public object FfiConverterUByte: FfiConverter { return value.toByte() } - override fun allocationSize(value: UByte) = 1 + override fun allocationSize(value: UByte) = 1UL override fun write(value: UByte, buf: ByteBuffer) { buf.put(value.toByte()) diff --git a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/macros.kt b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/macros.kt index 6a95d6a66d..7acfdc8861 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/macros.kt +++ b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/macros.kt @@ -1,32 +1,91 @@ {# // Template to call into rust. Used in several places. -// Variable names in `arg_list_decl` should match up with arg lists +// Variable names in `arg_list` should match up with arg lists // passed to rust via `arg_list_lowered` #} {%- macro to_ffi_call(func) -%} - {%- match func.throws_type() %} - {%- when Some with (e) %} - rustCallWithError({{ e|type_name(ci) }}) - {%- else %} - rustCall() - {%- endmatch %} { _status -> - _UniFFILib.INSTANCE.{{ func.ffi_func().name() }}({% call arg_list_lowered(func) -%} _status) -} -{%- endmacro -%} + {%- if func.takes_self() %} + callWithPointer { + {%- call to_raw_ffi_call(func) %} + } + {% else %} + {%- call to_raw_ffi_call(func) %} + {% endif %} +{%- endmacro %} -{%- macro to_ffi_call_with_prefix(prefix, func) %} +{%- macro to_raw_ffi_call(func) -%} {%- match func.throws_type() %} {%- when Some with (e) %} - rustCallWithError({{ e|type_name(ci) }}) + uniffiRustCallWithError({{ e|type_name(ci) }}) {%- else %} - rustCall() + uniffiRustCall() {%- endmatch %} { _status -> - _UniFFILib.INSTANCE.{{ func.ffi_func().name() }}( - {{- prefix }}, - {% call arg_list_lowered(func) %} + UniffiLib.INSTANCE.{{ func.ffi_func().name() }}( + {% if func.takes_self() %}it, {% endif -%} + {% call arg_list_lowered(func) -%} _status) } +{%- endmacro -%} + +{%- macro func_decl(func_decl, callable, indent) %} + {%- call docstring(callable, indent) %} + {%- match callable.throws_type() -%} + {%- when Some(throwable) %} + @Throws({{ throwable|type_name(ci) }}::class) + {%- else -%} + {%- endmatch -%} + {%- if callable.is_async() %} + @Suppress("ASSIGNED_BUT_NEVER_ACCESSED_VARIABLE") + {{ func_decl }} suspend fun {{ callable.name()|fn_name }}( + {%- call arg_list(callable, !callable.takes_self()) -%} + ){% match callable.return_type() %}{% when Some with (return_type) %} : {{ return_type|type_name(ci) }}{% when None %}{%- endmatch %} { + return {% call call_async(callable) %} + } + {%- else -%} + {{ func_decl }} fun {{ callable.name()|fn_name }}( + {%- call arg_list(callable, !callable.takes_self()) -%} + ){%- match callable.return_type() -%} + {%- when Some with (return_type) -%} + : {{ return_type|type_name(ci) }} { + return {{ return_type|lift_fn }}({% call to_ffi_call(callable) %}) + } + {%- when None %} + = {% call to_ffi_call(callable) %} + {%- endmatch %} + {% endif %} +{% endmacro %} + +{%- macro call_async(callable) -%} + uniffiRustCallAsync( +{%- if callable.takes_self() %} + callWithPointer { thisPtr -> + UniffiLib.INSTANCE.{{ callable.ffi_func().name() }}( + thisPtr, + {% call arg_list_lowered(callable) %} + ) + }, +{%- else %} + UniffiLib.INSTANCE.{{ callable.ffi_func().name() }}({% call arg_list_lowered(callable) %}), +{%- endif %} + {{ callable|async_poll(ci) }}, + {{ callable|async_complete(ci) }}, + {{ callable|async_free(ci) }}, + // lift function + {%- match callable.return_type() %} + {%- when Some(return_type) %} + { {{ return_type|lift_fn }}(it) }, + {%- when None %} + { Unit }, + {% endmatch %} + // Error FFI converter + {%- match callable.throws_type() %} + {%- when Some(e) %} + {{ e|type_name(ci) }}.ErrorHandler, + {%- when None %} + UniffiNullRustCallStatusErrorHandler, + {%- endmatch %} + ) {%- endmacro %} {%- macro arg_list_lowered(func) %} @@ -37,37 +96,42 @@ {#- // Arglist as used in kotlin declarations of methods, functions and constructors. +// If is_decl, then default values be specified. // Note the var_name and type_name filters. -#} -{% macro arg_list_decl(func) %} - {%- for arg in func.arguments() -%} +{% macro arg_list(func, is_decl) %} +{%- for arg in func.arguments() -%} {{ arg.name()|var_name }}: {{ arg|type_name(ci) }} - {%- match arg.default_value() %} - {%- when Some with(literal) %} = {{ literal|render_literal(arg, ci) }} - {%- else %} - {%- endmatch %} - {%- if !loop.last %}, {% endif -%} - {%- endfor %} +{%- if is_decl %} +{%- match arg.default_value() %} +{%- when Some with(literal) %} = {{ literal|render_literal(arg, ci) }} +{%- else %} +{%- endmatch %} +{%- endif %} +{%- if !loop.last %}, {% endif -%} +{%- endfor %} {%- endmacro %} -{% macro arg_list_protocol(func) %} - {%- for arg in func.arguments() -%} - {{ arg.name()|var_name }}: {{ arg|type_name(ci) }} - {%- if !loop.last %}, {% endif -%} - {%- endfor %} -{%- endmacro %} {#- -// Arglist as used in the _UniFFILib function declarations. +// Arglist as used in the UniffiLib function declarations. // Note unfiltered name but ffi_type_name filters. -#} {%- macro arg_list_ffi_decl(func) %} {%- for arg in func.arguments() %} {{- arg.name()|var_name }}: {{ arg.type_().borrow()|ffi_type_name_by_value -}}, {%- endfor %} - {%- if func.has_rust_call_status_arg() %}_uniffi_out_err: RustCallStatus, {% endif %} + {%- if func.has_rust_call_status_arg() %}uniffi_out_err: UniffiRustCallStatus, {% endif %} {%- endmacro -%} +{% macro field_name(field, field_num) %} +{%- if field.name().is_empty() -%} +v{{- field_num -}} +{%- else -%} +{{ field.name()|var_name }} +{%- endif -%} +{%- endmacro %} + // Macro for destroying fields {%- macro destroy_fields(member) %} Disposable.destroy( @@ -75,3 +139,15 @@ this.{{ field.name()|var_name }}{%- if !loop.last %}, {% endif -%} {% endfor -%}) {%- endmacro -%} + +{%- macro docstring_value(maybe_docstring, indent_spaces) %} +{%- match maybe_docstring %} +{%- when Some(docstring) %} +{{ docstring|docstring(indent_spaces) }} +{%- else %} +{%- endmatch %} +{%- endmacro %} + +{%- macro docstring(defn, indent_spaces) %} +{%- call docstring_value(defn.docstring(), indent_spaces) %} +{%- endmacro %} diff --git a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/wrapper.kt b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/wrapper.kt index 9ee4229018..2cdc72a5e2 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/wrapper.kt +++ b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/templates/wrapper.kt @@ -1,6 +1,8 @@ // This file was autogenerated by some hot garbage in the `uniffi` crate. // Trust me, you don't want to mess with it! +{%- call kt::docstring_value(ci.namespace_docstring(), 0) %} + @file:Suppress("NAME_SHADOWING") package {{ config.package_name() }}; @@ -28,6 +30,7 @@ import java.nio.ByteBuffer import java.nio.ByteOrder import java.nio.CharBuffer import java.nio.charset.CodingErrorAction +import java.util.concurrent.atomic.AtomicLong import java.util.concurrent.ConcurrentHashMap {%- for req in self.imports() %} @@ -37,6 +40,7 @@ import java.util.concurrent.ConcurrentHashMap {% include "RustBufferTemplate.kt" %} {% include "FfiConverterTemplate.kt" %} {% include "Helpers.kt" %} +{% include "HandleMap.kt" %} // Contains loading, initialization code, // and the FFI Function declarations in a com.sun.jna.Library. diff --git a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/test.rs b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/test.rs index 7b78540741..0824015751 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/kotlin/test.rs +++ b/third_party/rust/uniffi_bindgen/src/bindings/kotlin/test.rs @@ -2,10 +2,8 @@ License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -use crate::{ - bindings::{RunScriptOptions, TargetLanguage}, - library_mode::generate_bindings, -}; +use crate::bindings::TargetLanguage; +use crate::{bindings::RunScriptOptions, library_mode::generate_bindings, BindingGeneratorDefault}; use anyhow::{bail, Context, Result}; use camino::{Utf8Path, Utf8PathBuf}; use std::env; @@ -33,14 +31,18 @@ pub fn run_script( args: Vec, options: &RunScriptOptions, ) -> Result<()> { - let script_path = Utf8Path::new(".").join(script_file); + let script_path = Utf8Path::new(script_file); let test_helper = UniFFITestHelper::new(crate_name)?; - let out_dir = test_helper.create_out_dir(tmp_dir, &script_path)?; + let out_dir = test_helper.create_out_dir(tmp_dir, script_path)?; let cdylib_path = test_helper.copy_cdylib_to_out_dir(&out_dir)?; generate_bindings( &cdylib_path, None, - &[TargetLanguage::Kotlin], + &BindingGeneratorDefault { + target_languages: vec![TargetLanguage::Kotlin], + try_format_code: false, + }, + None, &out_dir, false, )?; diff --git a/third_party/rust/uniffi_bindgen/src/bindings/python/gen_python/compounds.rs b/third_party/rust/uniffi_bindgen/src/bindings/python/gen_python/compounds.rs index b91bcbe18f..16adeca9a5 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/python/gen_python/compounds.rs +++ b/third_party/rust/uniffi_bindgen/src/bindings/python/gen_python/compounds.rs @@ -3,7 +3,10 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ use super::CodeType; -use crate::backend::{Literal, Type}; +use crate::{ + backend::{Literal, Type}, + bindings::python::gen_python::AsCodeType, +}; #[derive(Debug)] pub struct OptionalCodeType { @@ -33,8 +36,9 @@ impl CodeType for OptionalCodeType { fn literal(&self, literal: &Literal) -> String { match literal { - Literal::Null => "None".into(), - _ => super::PythonCodeOracle.find(&self.inner).literal(literal), + Literal::None => "None".into(), + Literal::Some { inner } => super::PythonCodeOracle.find(&self.inner).literal(inner), + _ => panic!("Invalid literal for Optional type: {literal:?}"), } } } @@ -88,7 +92,11 @@ impl MapCodeType { impl CodeType for MapCodeType { fn type_label(&self) -> String { - "dict".to_string() + format!( + "dict[{}, {}]", + self.key.as_codetype().type_label(), + self.value.as_codetype().type_label() + ) } fn canonical_name(&self) -> String { diff --git a/third_party/rust/uniffi_bindgen/src/bindings/python/gen_python/executor.rs b/third_party/rust/uniffi_bindgen/src/bindings/python/gen_python/executor.rs deleted file mode 100644 index be3ba1d791..0000000000 --- a/third_party/rust/uniffi_bindgen/src/bindings/python/gen_python/executor.rs +++ /dev/null @@ -1,18 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -use super::CodeType; - -#[derive(Debug)] -pub struct ForeignExecutorCodeType; - -impl CodeType for ForeignExecutorCodeType { - fn type_label(&self) -> String { - "asyncio.BaseEventLoop".into() - } - - fn canonical_name(&self) -> String { - "ForeignExecutor".into() - } -} diff --git a/third_party/rust/uniffi_bindgen/src/bindings/python/gen_python/external.rs b/third_party/rust/uniffi_bindgen/src/bindings/python/gen_python/external.rs index 0d19c4bb3c..0a46251d6d 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/python/gen_python/external.rs +++ b/third_party/rust/uniffi_bindgen/src/bindings/python/gen_python/external.rs @@ -17,7 +17,7 @@ impl ExternalCodeType { impl CodeType for ExternalCodeType { fn type_label(&self) -> String { - self.name.clone() + super::PythonCodeOracle.class_name(&self.name) } fn canonical_name(&self) -> String { diff --git a/third_party/rust/uniffi_bindgen/src/bindings/python/gen_python/mod.rs b/third_party/rust/uniffi_bindgen/src/bindings/python/gen_python/mod.rs index 8178fcc102..6a10a38e7f 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/python/gen_python/mod.rs +++ b/third_party/rust/uniffi_bindgen/src/bindings/python/gen_python/mod.rs @@ -2,8 +2,9 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -use anyhow::{Context, Result}; +use anyhow::{bail, Context, Result}; use askama::Template; +use camino::Utf8Path; use heck::{ToShoutySnakeCase, ToSnakeCase, ToUpperCamelCase}; use once_cell::sync::Lazy; use serde::{Deserialize, Serialize}; @@ -13,20 +14,43 @@ use std::collections::{BTreeSet, HashMap, HashSet}; use std::fmt::Debug; use crate::backend::TemplateExpression; +use crate::bindings::python; use crate::interface::*; -use crate::BindingsConfig; +use crate::{BindingGenerator, BindingsConfig}; mod callback_interface; mod compounds; mod custom; mod enum_; -mod executor; mod external; mod miscellany; mod object; mod primitives; mod record; +pub struct PythonBindingGenerator; + +impl BindingGenerator for PythonBindingGenerator { + type Config = Config; + + fn write_bindings( + &self, + ci: &ComponentInterface, + config: &Config, + out_dir: &Utf8Path, + try_format_code: bool, + ) -> Result<()> { + python::write_bindings(config, ci, out_dir, try_format_code) + } + + fn check_library_path(&self, library_path: &Utf8Path, cdylib_name: Option<&str>) -> Result<()> { + if cdylib_name.is_none() { + bail!("Generate bindings for Python requires a cdylib, but {library_path} was given"); + } + Ok(()) + } +} + /// A trait tor the implementation. trait CodeType: Debug { /// The language specific label used to reference this type. This will be used in @@ -114,6 +138,8 @@ pub struct Config { cdylib_name: Option, #[serde(default)] custom_types: HashMap, + #[serde(default)] + external_packages: HashMap, } #[derive(Debug, Clone, Default, Serialize, Deserialize)] @@ -133,6 +159,16 @@ impl Config { "uniffi".into() } } + + /// Get the package name for a given external namespace. + pub fn module_for_namespace(&self, ns: &str) -> String { + let ns = ns.to_string().to_snake_case(); + match self.external_packages.get(&ns) { + None => format!(".{ns}"), + Some(value) if value.is_empty() => ns, + Some(value) => format!("{value}.{ns}"), + } + } } impl BindingsConfig for Config { @@ -326,7 +362,19 @@ impl PythonCodeOracle { fixup_keyword(nm.to_string().to_shouty_snake_case()) } - fn ffi_type_label(ffi_type: &FfiType) -> String { + /// Get the idiomatic Python rendering of an FFI callback function name + fn ffi_callback_name(&self, nm: &str) -> String { + format!("UNIFFI_{}", nm.to_shouty_snake_case()) + } + + /// Get the idiomatic Python rendering of an FFI struct name + fn ffi_struct_name(&self, nm: &str) -> String { + // The ctypes docs use both SHOUTY_SNAKE_CASE AND UpperCamelCase for structs. Let's use + // UpperCamelCase and reserve shouting for global variables + format!("Uniffi{}", nm.to_upper_camel_case()) + } + + fn ffi_type_label(&self, ffi_type: &FfiType) -> String { match ffi_type { FfiType::Int8 => "ctypes.c_int8".to_string(), FfiType::UInt8 => "ctypes.c_uint8".to_string(), @@ -338,19 +386,64 @@ impl PythonCodeOracle { FfiType::UInt64 => "ctypes.c_uint64".to_string(), FfiType::Float32 => "ctypes.c_float".to_string(), FfiType::Float64 => "ctypes.c_double".to_string(), + FfiType::Handle => "ctypes.c_uint64".to_string(), FfiType::RustArcPtr(_) => "ctypes.c_void_p".to_string(), FfiType::RustBuffer(maybe_suffix) => match maybe_suffix { Some(suffix) => format!("_UniffiRustBuffer{suffix}"), None => "_UniffiRustBuffer".to_string(), }, + FfiType::RustCallStatus => "_UniffiRustCallStatus".to_string(), FfiType::ForeignBytes => "_UniffiForeignBytes".to_string(), - FfiType::ForeignCallback => "_UNIFFI_FOREIGN_CALLBACK_T".to_string(), + FfiType::Callback(name) => self.ffi_callback_name(name), + FfiType::Struct(name) => self.ffi_struct_name(name), // Pointer to an `asyncio.EventLoop` instance - FfiType::ForeignExecutorHandle => "ctypes.c_size_t".to_string(), - FfiType::ForeignExecutorCallback => "_UNIFFI_FOREIGN_EXECUTOR_CALLBACK_T".to_string(), - FfiType::RustFutureHandle => "ctypes.c_void_p".to_string(), - FfiType::RustFutureContinuationCallback => "_UNIFFI_FUTURE_CONTINUATION_T".to_string(), - FfiType::RustFutureContinuationData => "ctypes.c_size_t".to_string(), + FfiType::Reference(inner) => format!("ctypes.POINTER({})", self.ffi_type_label(inner)), + FfiType::VoidPointer => "ctypes.c_void_p".to_string(), + } + } + + /// Default values for FFI types + /// + /// Used to set a default return value when returning an error + fn ffi_default_value(&self, return_type: Option<&FfiType>) -> String { + match return_type { + Some(t) => match t { + FfiType::UInt8 + | FfiType::Int8 + | FfiType::UInt16 + | FfiType::Int16 + | FfiType::UInt32 + | FfiType::Int32 + | FfiType::UInt64 + | FfiType::Int64 => "0".to_owned(), + FfiType::Float32 | FfiType::Float64 => "0.0".to_owned(), + FfiType::RustArcPtr(_) => "ctypes.c_void_p()".to_owned(), + FfiType::RustBuffer(maybe_suffix) => match maybe_suffix { + Some(suffix) => format!("_UniffiRustBuffer{suffix}.default()"), + None => "_UniffiRustBuffer.default()".to_owned(), + }, + _ => unimplemented!("FFI return type: {t:?}"), + }, + // When we need to use a value for void returns, we use a `u8` placeholder + None => "0".to_owned(), + } + } + + /// Get the name of the protocol and class name for an object. + /// + /// If we support callback interfaces, the protocol name is the object name, and the class name is derived from that. + /// Otherwise, the class name is the object name and the protocol name is derived from that. + /// + /// This split determines what types `FfiConverter.lower()` inputs. If we support callback + /// interfaces, `lower` must lower anything that implements the protocol. If not, then lower + /// only lowers the concrete class. + fn object_names(&self, obj: &Object) -> (String, String) { + let class_name = self.class_name(obj.name()); + if obj.has_callback_interface() { + let impl_name = format!("{class_name}Impl"); + (class_name, impl_name) + } else { + (format!("{class_name}Protocol"), class_name) } } } @@ -392,7 +485,6 @@ impl AsCodeType for T { Type::CallbackInterface { name, .. } => { Box::new(callback_interface::CallbackInterfaceCodeType::new(name)) } - Type::ForeignExecutor => Box::new(executor::ForeignExecutorCodeType), Type::Optional { inner_type } => { Box::new(compounds::OptionalCodeType::new(*inner_type)) } @@ -429,6 +521,10 @@ pub mod filters { Ok(format!("{}.lift", ffi_converter_name(as_ct)?)) } + pub(super) fn check_lower_fn(as_ct: &impl AsCodeType) -> Result { + Ok(format!("{}.check_lower", ffi_converter_name(as_ct)?)) + } + pub(super) fn lower_fn(as_ct: &impl AsCodeType) -> Result { Ok(format!("{}.lower", ffi_converter_name(as_ct)?)) } @@ -448,8 +544,18 @@ pub mod filters { Ok(as_ct.as_codetype().literal(literal)) } + // Get the idiomatic Python rendering of an individual enum variant's discriminant + pub fn variant_discr_literal(e: &Enum, index: &usize) -> Result { + let literal = e.variant_discr(*index).expect("invalid index"); + Ok(Type::UInt64.as_codetype().literal(&literal)) + } + pub fn ffi_type_name(type_: &FfiType) -> Result { - Ok(PythonCodeOracle::ffi_type_label(type_)) + Ok(PythonCodeOracle.ffi_type_label(type_)) + } + + pub fn ffi_default_value(return_type: Option) -> Result { + Ok(PythonCodeOracle.ffi_default_value(return_type.as_ref())) } /// Get the idiomatic Python rendering of a class name (for enums, records, errors, etc). @@ -471,4 +577,51 @@ pub mod filters { pub fn enum_variant_py(nm: &str) -> Result { Ok(PythonCodeOracle.enum_variant_name(nm)) } + + /// Get the idiomatic Python rendering of an FFI callback function name + pub fn ffi_callback_name(nm: &str) -> Result { + Ok(PythonCodeOracle.ffi_callback_name(nm)) + } + + /// Get the idiomatic Python rendering of an FFI struct name + pub fn ffi_struct_name(nm: &str) -> Result { + Ok(PythonCodeOracle.ffi_struct_name(nm)) + } + + /// Get the idiomatic Python rendering of an individual enum variant. + pub fn object_names(obj: &Object) -> Result<(String, String), askama::Error> { + Ok(PythonCodeOracle.object_names(obj)) + } + + /// Get the idiomatic Python rendering of docstring + pub fn docstring(docstring: &str, spaces: &i32) -> Result { + let docstring = textwrap::dedent(docstring); + // Escape triple quotes to avoid syntax error + let escaped = docstring.replace(r#"""""#, r#"\"\"\""#); + + let wrapped = format!("\"\"\"\n{escaped}\n\"\"\""); + + let spaces = usize::try_from(*spaces).unwrap_or_default(); + Ok(textwrap::indent(&wrapped, &" ".repeat(spaces))) + } +} + +#[cfg(test)] +mod tests { + #[test] + fn test_docstring_escape() { + let docstring = r#""""This is a docstring beginning with triple quotes. +Contains "quotes" in it. +It also has a triple quote: """ +And a even longer quote: """"""#; + + let expected = r#"""" +\"\"\"This is a docstring beginning with triple quotes. +Contains "quotes" in it. +It also has a triple quote: \"\"\" +And a even longer quote: \"\"\""" +""""#; + + assert_eq!(super::filters::docstring(docstring, &0).unwrap(), expected); + } } diff --git a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/Async.py b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/Async.py index 82aa534b46..26daa9ba5c 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/Async.py +++ b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/Async.py @@ -3,13 +3,37 @@ _UNIFFI_RUST_FUTURE_POLL_READY = 0 _UNIFFI_RUST_FUTURE_POLL_MAYBE_READY = 1 # Stores futures for _uniffi_continuation_callback -_UniffiContinuationPointerManager = _UniffiPointerManager() +_UniffiContinuationHandleMap = _UniffiHandleMap() + +UNIFFI_GLOBAL_EVENT_LOOP = None + +""" +Set the event loop to use for async functions + +This is needed if some async functions run outside of the eventloop, for example: + - A non-eventloop thread is spawned, maybe from `EventLoop.run_in_executor` or maybe from the + Rust code spawning its own thread. + - The Rust code calls an async callback method from a sync callback function, using something + like `pollster` to block on the async call. + +In this case, we need an event loop to run the Python async function, but there's no eventloop set +for the thread. Use `uniffi_set_event_loop` to force an eventloop to be used in this case. +""" +def uniffi_set_event_loop(eventloop: asyncio.BaseEventLoop): + global UNIFFI_GLOBAL_EVENT_LOOP + UNIFFI_GLOBAL_EVENT_LOOP = eventloop + +def _uniffi_get_event_loop(): + if UNIFFI_GLOBAL_EVENT_LOOP is not None: + return UNIFFI_GLOBAL_EVENT_LOOP + else: + return asyncio.get_running_loop() # Continuation callback for async functions # lift the return value or error and resolve the future, causing the async function to resume. -@_UNIFFI_FUTURE_CONTINUATION_T +@UNIFFI_RUST_FUTURE_CONTINUATION_CALLBACK def _uniffi_continuation_callback(future_ptr, poll_code): - (eventloop, future) = _UniffiContinuationPointerManager.release_pointer(future_ptr) + (eventloop, future) = _UniffiContinuationHandleMap.remove(future_ptr) eventloop.call_soon_threadsafe(_uniffi_set_future_result, future, poll_code) def _uniffi_set_future_result(future, poll_code): @@ -18,14 +42,15 @@ def _uniffi_set_future_result(future, poll_code): async def _uniffi_rust_call_async(rust_future, ffi_poll, ffi_complete, ffi_free, lift_func, error_ffi_converter): try: - eventloop = asyncio.get_running_loop() + eventloop = _uniffi_get_event_loop() # Loop and poll until we see a _UNIFFI_RUST_FUTURE_POLL_READY value while True: future = eventloop.create_future() ffi_poll( rust_future, - _UniffiContinuationPointerManager.new_pointer((eventloop, future)), + _uniffi_continuation_callback, + _UniffiContinuationHandleMap.insert((eventloop, future)), ) poll_code = await future if poll_code == _UNIFFI_RUST_FUTURE_POLL_READY: @@ -37,4 +62,53 @@ async def _uniffi_rust_call_async(rust_future, ffi_poll, ffi_complete, ffi_free, finally: ffi_free(rust_future) -_UniffiLib.{{ ci.ffi_rust_future_continuation_callback_set().name() }}(_uniffi_continuation_callback) +{%- if ci.has_async_callback_interface_definition() %} +def uniffi_trait_interface_call_async(make_call, handle_success, handle_error): + async def make_call_and_call_callback(): + try: + handle_success(await make_call()) + except Exception as e: + print("UniFFI: Unhandled exception in trait interface call", file=sys.stderr) + traceback.print_exc(file=sys.stderr) + handle_error( + _UniffiRustCallStatus.CALL_UNEXPECTED_ERROR, + {{ Type::String.borrow()|lower_fn }}(repr(e)), + ) + eventloop = _uniffi_get_event_loop() + task = asyncio.run_coroutine_threadsafe(make_call_and_call_callback(), eventloop) + handle = UNIFFI_FOREIGN_FUTURE_HANDLE_MAP.insert((eventloop, task)) + return UniffiForeignFuture(handle, uniffi_foreign_future_free) + +def uniffi_trait_interface_call_async_with_error(make_call, handle_success, handle_error, error_type, lower_error): + async def make_call_and_call_callback(): + try: + try: + handle_success(await make_call()) + except error_type as e: + handle_error( + _UniffiRustCallStatus.CALL_ERROR, + lower_error(e), + ) + except Exception as e: + print("UniFFI: Unhandled exception in trait interface call", file=sys.stderr) + traceback.print_exc(file=sys.stderr) + handle_error( + _UniffiRustCallStatus.CALL_UNEXPECTED_ERROR, + {{ Type::String.borrow()|lower_fn }}(repr(e)), + ) + eventloop = _uniffi_get_event_loop() + task = asyncio.run_coroutine_threadsafe(make_call_and_call_callback(), eventloop) + handle = UNIFFI_FOREIGN_FUTURE_HANDLE_MAP.insert((eventloop, task)) + return UniffiForeignFuture(handle, uniffi_foreign_future_free) + +UNIFFI_FOREIGN_FUTURE_HANDLE_MAP = _UniffiHandleMap() + +@UNIFFI_FOREIGN_FUTURE_FREE +def uniffi_foreign_future_free(handle): + (eventloop, task) = UNIFFI_FOREIGN_FUTURE_HANDLE_MAP.remove(handle) + eventloop.call_soon(uniffi_foreign_future_do_free, task) + +def uniffi_foreign_future_do_free(task): + if not task.done(): + task.cancel() +{%- endif %} diff --git a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/BooleanHelper.py b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/BooleanHelper.py index 6775e9e132..3f8c5d1d4d 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/BooleanHelper.py +++ b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/BooleanHelper.py @@ -1,16 +1,20 @@ -class _UniffiConverterBool(_UniffiConverterPrimitive): +class _UniffiConverterBool: @classmethod - def check(cls, value): + def check_lower(cls, value): return not not value + @classmethod + def lower(cls, value): + return 1 if value else 0 + + @staticmethod + def lift(value): + return value != 0 + @classmethod def read(cls, buf): return cls.lift(buf.read_u8()) @classmethod - def write_unchecked(cls, value, buf): + def write(cls, value, buf): buf.write_u8(value) - - @staticmethod - def lift(value): - return value != 0 diff --git a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/BytesHelper.py b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/BytesHelper.py index 196b5b29fa..4d09531322 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/BytesHelper.py +++ b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/BytesHelper.py @@ -7,10 +7,13 @@ class _UniffiConverterBytes(_UniffiConverterRustBuffer): return buf.read(size) @staticmethod - def write(value, buf): + def check_lower(value): try: memoryview(value) except TypeError: raise TypeError("a bytes-like object is required, not {!r}".format(type(value).__name__)) + + @staticmethod + def write(value, buf): buf.write_i32(len(value)) buf.write(value) diff --git a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/CallbackInterfaceImpl.py b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/CallbackInterfaceImpl.py new file mode 100644 index 0000000000..676f01177a --- /dev/null +++ b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/CallbackInterfaceImpl.py @@ -0,0 +1,98 @@ +{% if self.include_once_check("CallbackInterfaceRuntime.py") %}{% include "CallbackInterfaceRuntime.py" %}{% endif %} +{%- let trait_impl=format!("UniffiTraitImpl{}", name) %} + +# Put all the bits inside a class to keep the top-level namespace clean +class {{ trait_impl }}: + # For each method, generate a callback function to pass to Rust + {%- for (ffi_callback, meth) in vtable_methods.iter() %} + + @{{ ffi_callback.name()|ffi_callback_name }} + def {{ meth.name()|fn_name }}( + {%- for arg in ffi_callback.arguments() %} + {{ arg.name()|var_name }}, + {%- endfor -%} + {%- if ffi_callback.has_rust_call_status_arg() %} + uniffi_call_status_ptr, + {%- endif %} + ): + uniffi_obj = {{ ffi_converter_name }}._handle_map.get(uniffi_handle) + def make_call(): + args = ({% for arg in meth.arguments() %}{{ arg|lift_fn }}({{ arg.name()|var_name }}), {% endfor %}) + method = uniffi_obj.{{ meth.name()|fn_name }} + return method(*args) + + {% if !meth.is_async() %} + {%- match meth.return_type() %} + {%- when Some(return_type) %} + def write_return_value(v): + uniffi_out_return[0] = {{ return_type|lower_fn }}(v) + {%- when None %} + write_return_value = lambda v: None + {%- endmatch %} + + {%- match meth.throws_type() %} + {%- when None %} + _uniffi_trait_interface_call( + uniffi_call_status_ptr.contents, + make_call, + write_return_value, + ) + {%- when Some(error) %} + _uniffi_trait_interface_call_with_error( + uniffi_call_status_ptr.contents, + make_call, + write_return_value, + {{ error|type_name }}, + {{ error|lower_fn }}, + ) + {%- endmatch %} + {%- else %} + def handle_success(return_value): + uniffi_future_callback( + uniffi_callback_data, + {{ meth.foreign_future_ffi_result_struct().name()|ffi_struct_name }}( + {%- match meth.return_type() %} + {%- when Some(return_type) %} + {{ return_type|lower_fn }}(return_value), + {%- when None %} + {%- endmatch %} + _UniffiRustCallStatus.default() + ) + ) + + def handle_error(status_code, rust_buffer): + uniffi_future_callback( + uniffi_callback_data, + {{ meth.foreign_future_ffi_result_struct().name()|ffi_struct_name }}( + {%- match meth.return_type() %} + {%- when Some(return_type) %} + {{ meth.return_type().map(FfiType::from)|ffi_default_value }}, + {%- when None %} + {%- endmatch %} + _UniffiRustCallStatus(status_code, rust_buffer), + ) + ) + + {%- match meth.throws_type() %} + {%- when None %} + uniffi_out_return[0] = uniffi_trait_interface_call_async(make_call, handle_success, handle_error) + {%- when Some(error) %} + uniffi_out_return[0] = uniffi_trait_interface_call_async_with_error(make_call, handle_success, handle_error, {{ error|type_name }}, {{ error|lower_fn }}) + {%- endmatch %} + {%- endif %} + {%- endfor %} + + @{{ "CallbackInterfaceFree"|ffi_callback_name }} + def uniffi_free(uniffi_handle): + {{ ffi_converter_name }}._handle_map.remove(uniffi_handle) + + # Generate the FFI VTable. This has a field for each callback interface method. + uniffi_vtable = {{ vtable|ffi_type_name }}( + {%- for (_, meth) in vtable_methods.iter() %} + {{ meth.name()|fn_name }}, + {%- endfor %} + uniffi_free + ) + # Send Rust a pointer to the VTable. Note: this means we need to keep the struct alive forever, + # or else bad things will happen when Rust tries to access it. + _UniffiLib.{{ ffi_init_callback.name() }}(ctypes.byref(uniffi_vtable)) diff --git a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/CallbackInterfaceRuntime.py b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/CallbackInterfaceRuntime.py index 0fe2ab8dc0..d802c90fde 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/CallbackInterfaceRuntime.py +++ b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/CallbackInterfaceRuntime.py @@ -1,42 +1,3 @@ -import threading - -class ConcurrentHandleMap: - """ - A map where inserting, getting and removing data is synchronized with a lock. - """ - - def __init__(self): - # type Handle = int - self._left_map = {} # type: Dict[Handle, Any] - self._right_map = {} # type: Dict[Any, Handle] - - self._lock = threading.Lock() - self._current_handle = 0 - self._stride = 1 - - - def insert(self, obj): - with self._lock: - if obj in self._right_map: - return self._right_map[obj] - else: - handle = self._current_handle - self._current_handle += self._stride - self._left_map[handle] = obj - self._right_map[obj] = handle - return handle - - def get(self, handle): - with self._lock: - return self._left_map.get(handle) - - def remove(self, handle): - with self._lock: - if handle in self._left_map: - obj = self._left_map.pop(handle) - del self._right_map[obj] - return obj - # Magic number for the Rust proxy to call using the same mechanism as every other method, # to free the callback once it's dropped by Rust. IDX_CALLBACK_FREE = 0 @@ -45,28 +6,22 @@ _UNIFFI_CALLBACK_SUCCESS = 0 _UNIFFI_CALLBACK_ERROR = 1 _UNIFFI_CALLBACK_UNEXPECTED_ERROR = 2 -class _UniffiConverterCallbackInterface: - _handle_map = ConcurrentHandleMap() - - def __init__(self, cb): - self._foreign_callback = cb - - def drop(self, handle): - self.__class__._handle_map.remove(handle) +class UniffiCallbackInterfaceFfiConverter: + _handle_map = _UniffiHandleMap() @classmethod def lift(cls, handle): - obj = cls._handle_map.get(handle) - if not obj: - raise InternalError("The object in the handle map has been dropped already") - - return obj + return cls._handle_map.get(handle) @classmethod def read(cls, buf): handle = buf.read_u64() cls.lift(handle) + @classmethod + def check_lower(cls, cb): + pass + @classmethod def lower(cls, cb): handle = cls._handle_map.insert(cb) diff --git a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/CallbackInterfaceTemplate.py b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/CallbackInterfaceTemplate.py index e0e926757a..a41e58e635 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/CallbackInterfaceTemplate.py +++ b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/CallbackInterfaceTemplate.py @@ -1,105 +1,13 @@ -{%- let cbi = ci|get_callback_interface_definition(id) %} -{%- let foreign_callback = format!("foreignCallback{}", canonical_type_name) %} +{%- let cbi = ci|get_callback_interface_definition(name) %} +{%- let ffi_init_callback = cbi.ffi_init_callback() %} +{%- let protocol_name = type_name.clone() %} +{%- let protocol_docstring = cbi.docstring() %} +{%- let vtable = cbi.vtable() %} +{%- let methods = cbi.methods() %} +{%- let vtable_methods = cbi.vtable_methods() %} -{% if self.include_once_check("CallbackInterfaceRuntime.py") %}{% include "CallbackInterfaceRuntime.py" %}{% endif %} - -# Declaration and _UniffiConverters for {{ type_name }} Callback Interface - -class {{ type_name }}: - {% for meth in cbi.methods() -%} - def {{ meth.name()|fn_name }}(self, {% call py::arg_list_decl(meth) %}): - raise NotImplementedError - - {% endfor %} - -def py_{{ foreign_callback }}(handle, method, args_data, args_len, buf_ptr): - {% for meth in cbi.methods() -%} - {% let method_name = format!("invoke_{}", meth.name())|fn_name %} - def {{ method_name }}(python_callback, args_stream, buf_ptr): - {#- Unpacking args from the _UniffiRustBuffer #} - def makeCall(): - {#- Calling the concrete callback object #} - {%- if meth.arguments().len() != 0 -%} - return python_callback.{{ meth.name()|fn_name }}( - {% for arg in meth.arguments() -%} - {{ arg|read_fn }}(args_stream) - {%- if !loop.last %}, {% endif %} - {% endfor -%} - ) - {%- else %} - return python_callback.{{ meth.name()|fn_name }}() - {%- endif %} - - def makeCallAndHandleReturn(): - {%- match meth.return_type() %} - {%- when Some(return_type) %} - rval = makeCall() - with _UniffiRustBuffer.alloc_with_builder() as builder: - {{ return_type|write_fn }}(rval, builder) - buf_ptr[0] = builder.finalize() - {%- when None %} - makeCall() - {%- endmatch %} - return _UNIFFI_CALLBACK_SUCCESS - - {%- match meth.throws_type() %} - {%- when None %} - return makeCallAndHandleReturn() - {%- when Some(err) %} - try: - return makeCallAndHandleReturn() - except {{ err|type_name }} as e: - # Catch errors declared in the UDL file - with _UniffiRustBuffer.alloc_with_builder() as builder: - {{ err|write_fn }}(e, builder) - buf_ptr[0] = builder.finalize() - return _UNIFFI_CALLBACK_ERROR - {%- endmatch %} - - {% endfor %} - - cb = {{ ffi_converter_name }}.lift(handle) - if not cb: - raise InternalError("No callback in handlemap; this is a uniffi bug") - - if method == IDX_CALLBACK_FREE: - {{ ffi_converter_name }}.drop(handle) - # Successfull return - # See docs of ForeignCallback in `uniffi_core/src/ffi/foreigncallbacks.rs` - return _UNIFFI_CALLBACK_SUCCESS - - {% for meth in cbi.methods() -%} - {% let method_name = format!("invoke_{}", meth.name())|fn_name -%} - if method == {{ loop.index }}: - # Call the method and handle any errors - # See docs of ForeignCallback in `uniffi_core/src/ffi/foreigncallbacks.rs` for details - try: - return {{ method_name }}(cb, _UniffiRustBufferStream(args_data, args_len), buf_ptr) - except BaseException as e: - # Catch unexpected errors - try: - # Try to serialize the exception into a String - buf_ptr[0] = {{ Type::String.borrow()|lower_fn }}(repr(e)) - except: - # If that fails, just give up - pass - return _UNIFFI_CALLBACK_UNEXPECTED_ERROR - {% endfor %} - - # This should never happen, because an out of bounds method index won't - # ever be used. Once we can catch errors, we should return an InternalException. - # https://github.com/mozilla/uniffi-rs/issues/351 - - # An unexpected error happened. - # See docs of ForeignCallback in `uniffi_core/src/ffi/foreigncallbacks.rs` - return _UNIFFI_CALLBACK_UNEXPECTED_ERROR - -# We need to keep this function reference alive: -# if they get GC'd while in use then UniFFI internals could attempt to call a function -# that is in freed memory. -# That would be...uh...bad. Yeah, that's the word. Bad. -{{ foreign_callback }} = _UNIFFI_FOREIGN_CALLBACK_T(py_{{ foreign_callback }}) -_rust_call(lambda err: _UniffiLib.{{ cbi.ffi_init_callback().name() }}({{ foreign_callback }}, err)) +{% include "Protocol.py" %} +{% include "CallbackInterfaceImpl.py" %} # The _UniffiConverter which transforms the Callbacks in to Handles to pass to Rust. -{{ ffi_converter_name }} = _UniffiConverterCallbackInterface({{ foreign_callback }}) +{{ ffi_converter_name }} = UniffiCallbackInterfaceFfiConverter() diff --git a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/CustomType.py b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/CustomType.py index 5be6155b84..f75a85dc27 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/CustomType.py +++ b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/CustomType.py @@ -17,6 +17,10 @@ class _UniffiConverterType{{ name }}: def lift(value): return {{ builtin|ffi_converter_name }}.lift(value) + @staticmethod + def check_lower(value): + return {{ builtin|ffi_converter_name }}.check_lower(value) + @staticmethod def lower(value): return {{ builtin|ffi_converter_name }}.lower(value) @@ -51,6 +55,11 @@ class _UniffiConverterType{{ name }}: builtin_value = {{ builtin|lift_fn }}(value) return {{ config.into_custom.render("builtin_value") }} + @staticmethod + def check_lower(value): + builtin_value = {{ config.from_custom.render("value") }} + return {{ builtin|check_lower_fn }}(builtin_value) + @staticmethod def lower(value): builtin_value = {{ config.from_custom.render("value") }} diff --git a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/DurationHelper.py b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/DurationHelper.py index 10974e009d..ecb035b7f4 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/DurationHelper.py +++ b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/DurationHelper.py @@ -12,10 +12,14 @@ class _UniffiConverterDuration(_UniffiConverterRustBuffer): return datetime.timedelta(seconds=seconds, microseconds=microseconds) @staticmethod - def write(value, buf): + def check_lower(value): seconds = value.seconds + value.days * 24 * 3600 - nanoseconds = value.microseconds * 1000 if seconds < 0: raise ValueError("Invalid duration, must be non-negative") + + @staticmethod + def write(value, buf): + seconds = value.seconds + value.days * 24 * 3600 + nanoseconds = value.microseconds * 1000 buf.write_i64(seconds) buf.write_u32(nanoseconds) diff --git a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/EnumTemplate.py b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/EnumTemplate.py index 84d089baf9..d07dd1c44a 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/EnumTemplate.py +++ b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/EnumTemplate.py @@ -7,31 +7,59 @@ {% if e.is_flat() %} class {{ type_name }}(enum.Enum): - {% for variant in e.variants() -%} - {{ variant.name()|enum_variant_py }} = {{ loop.index }} + {%- call py::docstring(e, 4) %} + {%- for variant in e.variants() %} + {{ variant.name()|enum_variant_py }} = {{ e|variant_discr_literal(loop.index0) }} + {%- call py::docstring(variant, 4) %} {% endfor %} {% else %} class {{ type_name }}: + {%- call py::docstring(e, 4) %} def __init__(self): raise RuntimeError("{{ type_name }} cannot be instantiated directly") # Each enum variant is a nested class of the enum itself. {% for variant in e.variants() -%} class {{ variant.name()|enum_variant_py }}: - {% for field in variant.fields() %} - {{- field.name()|var_name }}: "{{- field|type_name }}"; + {%- call py::docstring(variant, 8) %} + + {%- if variant.has_nameless_fields() %} + def __init__(self, *values): + if len(values) != {{ variant.fields().len() }}: + raise TypeError(f"Expected a tuple of len {{ variant.fields().len() }}, found len {len(values)}") + {%- for field in variant.fields() %} + if not isinstance(values[{{ loop.index0 }}], {{ field|type_name }}): + raise TypeError(f"unexpected type for tuple element {{ loop.index0 }} - expected '{{ field|type_name }}', got '{type(values[{{ loop.index0 }}])}'") + {%- endfor %} + self._values = values + + def __getitem__(self, index): + return self._values[index] + + def __str__(self): + return f"{{ type_name }}.{{ variant.name()|enum_variant_py }}{self._values!r}" + + def __eq__(self, other): + if not other.is_{{ variant.name()|var_name }}(): + return False + return self._values == other._values + + {%- else -%} + {%- for field in variant.fields() %} + {{ field.name()|var_name }}: "{{ field|type_name }}" + {%- call py::docstring(field, 8) %} {%- endfor %} @typing.no_type_check def __init__(self,{% for field in variant.fields() %}{{ field.name()|var_name }}: "{{- field|type_name }}"{% if loop.last %}{% else %}, {% endif %}{% endfor %}): - {% if variant.has_fields() %} + {%- if variant.has_fields() %} {%- for field in variant.fields() %} self.{{ field.name()|var_name }} = {{ field.name()|var_name }} {%- endfor %} - {% else %} + {%- else %} pass - {% endif %} + {%- endif %} def __str__(self): return "{{ type_name }}.{{ variant.name()|enum_variant_py }}({% for field in variant.fields() %}{{ field.name()|var_name }}={}{% if loop.last %}{% else %}, {% endif %}{% endfor %})".format({% for field in variant.fields() %}self.{{ field.name()|var_name }}{% if loop.last %}{% else %}, {% endif %}{% endfor %}) @@ -44,6 +72,7 @@ class {{ type_name }}: return False {%- endfor %} return True + {% endif %} {% endfor %} # For each variant, we have an `is_NAME` method for easily checking @@ -81,6 +110,30 @@ class {{ ffi_converter_name }}(_UniffiConverterRustBuffer): {%- endfor %} raise InternalError("Raw enum value doesn't match any cases") + @staticmethod + def check_lower(value): + {%- if e.variants().is_empty() %} + pass + {%- else %} + {%- for variant in e.variants() %} + {%- if e.is_flat() %} + if value == {{ type_name }}.{{ variant.name()|enum_variant_py }}: + {%- else %} + if value.is_{{ variant.name()|var_name }}(): + {%- endif %} + {%- for field in variant.fields() %} + {%- if variant.has_nameless_fields() %} + {{ field|check_lower_fn }}(value._values[{{ loop.index0 }}]) + {%- else %} + {{ field|check_lower_fn }}(value.{{ field.name()|var_name }}) + {%- endif %} + {%- endfor %} + return + {%- endfor %} + raise ValueError(value) + {%- endif %} + + @staticmethod def write(value, buf): {%- for variant in e.variants() %} {%- if e.is_flat() %} @@ -90,7 +143,11 @@ class {{ ffi_converter_name }}(_UniffiConverterRustBuffer): if value.is_{{ variant.name()|var_name }}(): buf.write_i32({{ loop.index }}) {%- for field in variant.fields() %} + {%- if variant.has_nameless_fields() %} + {{ field|write_fn }}(value._values[{{ loop.index0 }}], buf) + {%- else %} {{ field|write_fn }}(value.{{ field.name()|var_name }}, buf) + {%- endif %} {%- endfor %} {%- endif %} {%- endfor %} diff --git a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/ErrorTemplate.py b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/ErrorTemplate.py index 26a1e6452a..0911ff559a 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/ErrorTemplate.py +++ b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/ErrorTemplate.py @@ -5,6 +5,7 @@ # __dict__. All of this happens in dummy class to avoid polluting the module # namespace. class {{ type_name }}(Exception): + {%- call py::docstring(e, 4) %} pass _UniffiTemp{{ type_name }} = {{ type_name }} @@ -14,10 +15,14 @@ class {{ type_name }}: # type: ignore {%- let variant_type_name = variant.name()|class_name -%} {%- if e.is_flat() %} class {{ variant_type_name }}(_UniffiTemp{{ type_name }}): + {%- call py::docstring(variant, 8) %} + def __repr__(self): return "{{ type_name }}.{{ variant_type_name }}({})".format(repr(str(self))) {%- else %} class {{ variant_type_name }}(_UniffiTemp{{ type_name }}): + {%- call py::docstring(variant, 8) %} + def __init__(self{% for field in variant.fields() %}, {{ field.name()|var_name }}{% endfor %}): {%- if variant.has_fields() %} super().__init__(", ".join([ @@ -59,6 +64,20 @@ class {{ ffi_converter_name }}(_UniffiConverterRustBuffer): {%- endfor %} raise InternalError("Raw enum value doesn't match any cases") + @staticmethod + def check_lower(value): + {%- if e.variants().is_empty() %} + pass + {%- else %} + {%- for variant in e.variants() %} + if isinstance(value, {{ type_name }}.{{ variant.name()|class_name }}): + {%- for field in variant.fields() %} + {{ field|check_lower_fn }}(value.{{ field.name()|var_name }}) + {%- endfor %} + return + {%- endfor %} + {%- endif %} + @staticmethod def write(value, buf): {%- for variant in e.variants() %} diff --git a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/ExternalTemplate.py b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/ExternalTemplate.py index 71e05e8b06..6c0cee85ef 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/ExternalTemplate.py +++ b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/ExternalTemplate.py @@ -1,9 +1,9 @@ -{%- let ns = namespace|fn_name %} +{%- let module = python_config.module_for_namespace(namespace) -%} # External type {{name}} is in namespace "{{namespace}}", crate {{module_path}} {%- let ffi_converter_name = "_UniffiConverterType{}"|format(name) %} -{{ self.add_import_of(ns, ffi_converter_name) }} -{{ self.add_import_of(ns, name) }} {#- import the type alias itself -#} +{{ self.add_import_of(module, ffi_converter_name) }} +{{ self.add_import_of(module, name|class_name) }} {#- import the type alias itself -#} {%- let rustbuffer_local_name = "_UniffiRustBuffer{}"|format(name) %} -{{ self.add_import_of_as(ns, "_UniffiRustBuffer", rustbuffer_local_name) }} +{{ self.add_import_of_as(module, "_UniffiRustBuffer", rustbuffer_local_name) }} diff --git a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/Float32Helper.py b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/Float32Helper.py index a52107a638..49a1a7286e 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/Float32Helper.py +++ b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/Float32Helper.py @@ -4,5 +4,5 @@ class _UniffiConverterFloat(_UniffiConverterPrimitiveFloat): return buf.read_float() @staticmethod - def write_unchecked(value, buf): + def write(value, buf): buf.write_float(value) diff --git a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/Float64Helper.py b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/Float64Helper.py index 772f5080e9..e2084c7b13 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/Float64Helper.py +++ b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/Float64Helper.py @@ -4,5 +4,5 @@ class _UniffiConverterDouble(_UniffiConverterPrimitiveFloat): return buf.read_double() @staticmethod - def write_unchecked(value, buf): + def write(value, buf): buf.write_double(value) diff --git a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/ForeignExecutorTemplate.py b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/ForeignExecutorTemplate.py deleted file mode 100644 index 6a6932fed0..0000000000 --- a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/ForeignExecutorTemplate.py +++ /dev/null @@ -1,63 +0,0 @@ -# FFI code for the ForeignExecutor type - -{{ self.add_import("asyncio") }} - -_UNIFFI_RUST_TASK_CALLBACK_SUCCESS = 0 -_UNIFFI_RUST_TASK_CALLBACK_CANCELLED = 1 -_UNIFFI_FOREIGN_EXECUTOR_CALLBACK_SUCCESS = 0 -_UNIFFI_FOREIGN_EXECUTOR_CALLBACK_CANCELED = 1 -_UNIFFI_FOREIGN_EXECUTOR_CALLBACK_ERROR = 2 - -class {{ ffi_converter_name }}: - _pointer_manager = _UniffiPointerManager() - - @classmethod - def lower(cls, eventloop): - if not isinstance(eventloop, asyncio.BaseEventLoop): - raise TypeError("_uniffi_executor_callback: Expected EventLoop instance") - return cls._pointer_manager.new_pointer(eventloop) - - @classmethod - def write(cls, eventloop, buf): - buf.write_c_size_t(cls.lower(eventloop)) - - @classmethod - def read(cls, buf): - return cls.lift(buf.read_c_size_t()) - - @classmethod - def lift(cls, value): - return cls._pointer_manager.lookup(value) - -@_UNIFFI_FOREIGN_EXECUTOR_CALLBACK_T -def _uniffi_executor_callback(eventloop_address, delay, task_ptr, task_data): - if task_ptr is None: - {{ ffi_converter_name }}._pointer_manager.release_pointer(eventloop_address) - return _UNIFFI_FOREIGN_EXECUTOR_CALLBACK_SUCCESS - else: - eventloop = {{ ffi_converter_name }}._pointer_manager.lookup(eventloop_address) - if eventloop.is_closed(): - return _UNIFFI_FOREIGN_EXECUTOR_CALLBACK_CANCELED - - callback = _UNIFFI_RUST_TASK(task_ptr) - # FIXME: there's no easy way to get a callback when an eventloop is closed. This means that - # if eventloop is called before the `call_soon_threadsafe()` calls are invoked, the call - # will never happen and we will probably leak a resource. - if delay == 0: - # This can be called from any thread, so make sure to use `call_soon_threadsafe' - eventloop.call_soon_threadsafe(callback, task_data, - _UNIFFI_FOREIGN_EXECUTOR_CALLBACK_SUCCESS) - else: - # For delayed tasks, we use `call_soon_threadsafe()` + `call_later()` to make the - # operation threadsafe - eventloop.call_soon_threadsafe(eventloop.call_later, delay / 1000.0, callback, - task_data, _UNIFFI_FOREIGN_EXECUTOR_CALLBACK_SUCCESS) - return _UNIFFI_FOREIGN_EXECUTOR_CALLBACK_SUCCESS - -# Register the callback with the scaffolding -{%- match ci.ffi_foreign_executor_callback_set() %} -{%- when Some with (fn) %} -_UniffiLib.{{ fn.name() }}(_uniffi_executor_callback) -{%- when None %} -{#- No foreign executor, we don't set anything #} -{% endmatch %} diff --git a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/HandleMap.py b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/HandleMap.py new file mode 100644 index 0000000000..f7c13cf745 --- /dev/null +++ b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/HandleMap.py @@ -0,0 +1,33 @@ +class _UniffiHandleMap: + """ + A map where inserting, getting and removing data is synchronized with a lock. + """ + + def __init__(self): + # type Handle = int + self._map = {} # type: Dict[Handle, Any] + self._lock = threading.Lock() + self._counter = itertools.count() + + def insert(self, obj): + with self._lock: + handle = next(self._counter) + self._map[handle] = obj + return handle + + def get(self, handle): + try: + with self._lock: + return self._map[handle] + except KeyError: + raise InternalError("UniffiHandleMap.get: Invalid handle") + + def remove(self, handle): + try: + with self._lock: + return self._map.pop(handle) + except KeyError: + raise InternalError("UniffiHandleMap.remove: Invalid handle") + + def __len__(self): + return len(self._map) diff --git a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/Helpers.py b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/Helpers.py index dca962f176..5d4bcbba89 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/Helpers.py +++ b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/Helpers.py @@ -16,15 +16,19 @@ class _UniffiRustCallStatus(ctypes.Structure): # These match the values from the uniffi::rustcalls module CALL_SUCCESS = 0 CALL_ERROR = 1 - CALL_PANIC = 2 + CALL_UNEXPECTED_ERROR = 2 + + @staticmethod + def default(): + return _UniffiRustCallStatus(code=_UniffiRustCallStatus.CALL_SUCCESS, error_buf=_UniffiRustBuffer.default()) def __str__(self): if self.code == _UniffiRustCallStatus.CALL_SUCCESS: return "_UniffiRustCallStatus(CALL_SUCCESS)" elif self.code == _UniffiRustCallStatus.CALL_ERROR: return "_UniffiRustCallStatus(CALL_ERROR)" - elif self.code == _UniffiRustCallStatus.CALL_PANIC: - return "_UniffiRustCallStatus(CALL_PANIC)" + elif self.code == _UniffiRustCallStatus.CALL_UNEXPECTED_ERROR: + return "_UniffiRustCallStatus(CALL_UNEXPECTED_ERROR)" else: return "_UniffiRustCallStatus()" @@ -37,7 +41,7 @@ def _rust_call_with_error(error_ffi_converter, fn, *args): # # This function is used for rust calls that return Result<> and therefore can set the CALL_ERROR status code. # error_ffi_converter must be set to the _UniffiConverter for the error class that corresponds to the result. - call_status = _UniffiRustCallStatus(code=_UniffiRustCallStatus.CALL_SUCCESS, error_buf=_UniffiRustBuffer(0, 0, None)) + call_status = _UniffiRustCallStatus.default() args_with_error = args + (ctypes.byref(call_status),) result = fn(*args_with_error) @@ -53,7 +57,7 @@ def _uniffi_check_call_status(error_ffi_converter, call_status): raise InternalError("_rust_call_with_error: CALL_ERROR, but error_ffi_converter is None") else: raise error_ffi_converter.lift(call_status.error_buf) - elif call_status.code == _UniffiRustCallStatus.CALL_PANIC: + elif call_status.code == _UniffiRustCallStatus.CALL_UNEXPECTED_ERROR: # When the rust code sees a panic, it tries to construct a _UniffiRustBuffer # with the message. But if that code panics, then it just sends back # an empty buffer. @@ -66,10 +70,20 @@ def _uniffi_check_call_status(error_ffi_converter, call_status): raise InternalError("Invalid _UniffiRustCallStatus code: {}".format( call_status.code)) -# A function pointer for a callback as defined by UniFFI. -# Rust definition `fn(handle: u64, method: u32, args: _UniffiRustBuffer, buf_ptr: *mut _UniffiRustBuffer) -> int` -_UNIFFI_FOREIGN_CALLBACK_T = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_ulonglong, ctypes.c_ulong, ctypes.POINTER(ctypes.c_char), ctypes.c_int, ctypes.POINTER(_UniffiRustBuffer)) - -# UniFFI future continuation -_UNIFFI_FUTURE_CONTINUATION_T = ctypes.CFUNCTYPE(None, ctypes.c_size_t, ctypes.c_int8) +def _uniffi_trait_interface_call(call_status, make_call, write_return_value): + try: + return write_return_value(make_call()) + except Exception as e: + call_status.code = _UniffiRustCallStatus.CALL_UNEXPECTED_ERROR + call_status.error_buf = {{ Type::String.borrow()|lower_fn }}(repr(e)) +def _uniffi_trait_interface_call_with_error(call_status, make_call, write_return_value, error_type, lower_error): + try: + try: + return write_return_value(make_call()) + except error_type as e: + call_status.code = _UniffiRustCallStatus.CALL_ERROR + call_status.error_buf = lower_error(e) + except Exception as e: + call_status.code = _UniffiRustCallStatus.CALL_UNEXPECTED_ERROR + call_status.error_buf = {{ Type::String.borrow()|lower_fn }}(repr(e)) diff --git a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/Int16Helper.py b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/Int16Helper.py index 99f19dc1c0..befa563384 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/Int16Helper.py +++ b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/Int16Helper.py @@ -8,5 +8,5 @@ class _UniffiConverterInt16(_UniffiConverterPrimitiveInt): return buf.read_i16() @staticmethod - def write_unchecked(value, buf): + def write(value, buf): buf.write_i16(value) diff --git a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/Int32Helper.py b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/Int32Helper.py index 3b142c8749..70644f6717 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/Int32Helper.py +++ b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/Int32Helper.py @@ -8,5 +8,5 @@ class _UniffiConverterInt32(_UniffiConverterPrimitiveInt): return buf.read_i32() @staticmethod - def write_unchecked(value, buf): + def write(value, buf): buf.write_i32(value) diff --git a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/Int64Helper.py b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/Int64Helper.py index 6e94379cbf..232f127bd6 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/Int64Helper.py +++ b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/Int64Helper.py @@ -8,5 +8,5 @@ class _UniffiConverterInt64(_UniffiConverterPrimitiveInt): return buf.read_i64() @staticmethod - def write_unchecked(value, buf): + def write(value, buf): buf.write_i64(value) diff --git a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/Int8Helper.py b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/Int8Helper.py index 732530e3cb..c1de1625e7 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/Int8Helper.py +++ b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/Int8Helper.py @@ -8,5 +8,5 @@ class _UniffiConverterInt8(_UniffiConverterPrimitiveInt): return buf.read_i8() @staticmethod - def write_unchecked(value, buf): + def write(value, buf): buf.write_i8(value) diff --git a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/MapTemplate.py b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/MapTemplate.py index 387227ed09..a09ca28a30 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/MapTemplate.py +++ b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/MapTemplate.py @@ -2,6 +2,12 @@ {%- let value_ffi_converter = value_type|ffi_converter_name %} class {{ ffi_converter_name }}(_UniffiConverterRustBuffer): + @classmethod + def check_lower(cls, items): + for (key, value) in items.items(): + {{ key_ffi_converter }}.check_lower(key) + {{ value_ffi_converter }}.check_lower(value) + @classmethod def write(cls, items, buf): buf.write_i32(len(items)) diff --git a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/NamespaceLibraryTemplate.py b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/NamespaceLibraryTemplate.py index fac6cd5564..1929f9aad6 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/NamespaceLibraryTemplate.py +++ b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/NamespaceLibraryTemplate.py @@ -1,21 +1,5 @@ # Define some ctypes FFI types that we use in the library -""" -ctypes type for the foreign executor callback. This is a built-in interface for scheduling -tasks - -Args: - executor: opaque c_size_t value representing the eventloop - delay: delay in ms - task: function pointer to the task callback - task_data: void pointer to the task callback data - -Normally we should call task(task_data) after the detail. -However, when task is NULL this indicates that Rust has dropped the ForeignExecutor and we should -decrease the EventLoop refcount. -""" -_UNIFFI_FOREIGN_EXECUTOR_CALLBACK_T = ctypes.CFUNCTYPE(ctypes.c_int8, ctypes.c_size_t, ctypes.c_uint32, ctypes.c_void_p, ctypes.c_void_p) - """ Function pointer for a Rust task, which a callback function that takes a opaque pointer """ @@ -25,7 +9,7 @@ def _uniffi_future_callback_t(return_type): """ Factory function to create callback function types for async functions """ - return ctypes.CFUNCTYPE(None, ctypes.c_size_t, return_type, _UniffiRustCallStatus) + return ctypes.CFUNCTYPE(None, ctypes.c_uint64, return_type, _UniffiRustCallStatus) def _uniffi_load_indirect(): """ @@ -72,12 +56,37 @@ def _uniffi_check_api_checksums(lib): # This is an implementation detail which will be called internally by the public API. _UniffiLib = _uniffi_load_indirect() -{%- for func in ci.iter_ffi_function_definitions() %} + +{%- for def in ci.ffi_definitions() %} +{%- match def %} +{%- when FfiDefinition::CallbackFunction(callback) %} +{{ callback.name()|ffi_callback_name }} = ctypes.CFUNCTYPE( + {%- match callback.return_type() %} + {%- when Some(return_type) %}{{ return_type|ffi_type_name }}, + {%- when None %}None, + {%- endmatch %} + {%- for arg in callback.arguments() -%} + {{ arg.type_().borrow()|ffi_type_name }}, + {%- endfor -%} + {%- if callback.has_rust_call_status_arg() %} + ctypes.POINTER(_UniffiRustCallStatus), + {%- endif %} +) +{%- when FfiDefinition::Struct(ffi_struct) %} +class {{ ffi_struct.name()|ffi_struct_name }}(ctypes.Structure): + _fields_ = [ + {%- for field in ffi_struct.fields() %} + ("{{ field.name()|var_name }}", {{ field.type_().borrow()|ffi_type_name }}), + {%- endfor %} + ] +{%- when FfiDefinition::Function(func) %} _UniffiLib.{{ func.name() }}.argtypes = ( {%- call py::arg_list_ffi_decl(func) -%} ) _UniffiLib.{{ func.name() }}.restype = {% match func.return_type() %}{% when Some with (type_) %}{{ type_|ffi_type_name }}{% when None %}None{% endmatch %} +{%- endmatch %} {%- endfor %} + {# Ensure to call the contract verification only after we defined all functions. -#} _uniffi_check_contract_api_version(_UniffiLib) _uniffi_check_api_checksums(_UniffiLib) diff --git a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/ObjectTemplate.py b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/ObjectTemplate.py index 7e98f7c46f..18dca4743a 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/ObjectTemplate.py +++ b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/ObjectTemplate.py @@ -1,14 +1,33 @@ {%- let obj = ci|get_object_definition(name) %} - -class {{ type_name }}: +{%- let (protocol_name, impl_name) = obj|object_names %} +{%- let methods = obj.methods() %} +{%- let protocol_docstring = obj.docstring() %} + +{% include "Protocol.py" %} + +{% if ci.is_name_used_as_error(name) %} +class {{ impl_name }}(Exception): +{%- else %} +class {{ impl_name }}: +{%- endif %} + {%- call py::docstring(obj, 4) %} _pointer: ctypes.c_void_p {%- match obj.primary_constructor() %} {%- when Some with (cons) %} +{%- if cons.is_async() %} + def __init__(self, *args, **kw): + raise ValueError("async constructors not supported.") +{%- else %} def __init__(self, {% call py::arg_list_decl(cons) -%}): + {%- call py::docstring(cons, 8) %} {%- call py::setup_args_extra_indent(cons) %} self._pointer = {% call py::to_ffi_call(cons) %} +{%- endif %} {%- when None %} + {# no __init__ means simple construction without a pointer works, which can confuse #} + def __init__(self, *args, **kwargs): + raise ValueError("This class has no default constructor") {%- endmatch %} def __del__(self): @@ -17,6 +36,9 @@ class {{ type_name }}: if pointer is not None: _rust_call(_UniffiLib.{{ obj.ffi_object_free().name() }}, pointer) + def _uniffi_clone_pointer(self): + return _rust_call(_UniffiLib.{{ obj.ffi_object_clone().name() }}, self._pointer) + # Used by alternative constructors or any methods which return this type. @classmethod def _make_instance_(cls, pointer): @@ -29,17 +51,32 @@ class {{ type_name }}: {%- for cons in obj.alternate_constructors() %} @classmethod +{%- if cons.is_async() %} + async def {{ cons.name()|fn_name }}(cls, {% call py::arg_list_decl(cons) %}): + {%- call py::docstring(cons, 8) %} + {%- call py::setup_args_extra_indent(cons) %} + + return await _uniffi_rust_call_async( + _UniffiLib.{{ cons.ffi_func().name() }}({% call py::arg_list_lowered(cons) %}), + _UniffiLib.{{ cons.ffi_rust_future_poll(ci) }}, + _UniffiLib.{{ cons.ffi_rust_future_complete(ci) }}, + _UniffiLib.{{ cons.ffi_rust_future_free(ci) }}, + {{ ffi_converter_name }}.lift, + {% call py::error_ffi_converter(cons) %} + ) +{%- else %} def {{ cons.name()|fn_name }}(cls, {% call py::arg_list_decl(cons) %}): + {%- call py::docstring(cons, 8) %} {%- call py::setup_args_extra_indent(cons) %} # Call the (fallible) function before creating any half-baked object instances. pointer = {% call py::to_ffi_call(cons) %} return cls._make_instance_(pointer) +{%- endif %} {% endfor %} {%- for meth in obj.methods() -%} {%- call py::method_decl(meth.name()|fn_name, meth) %} -{% endfor %} - +{%- endfor %} {%- for tm in obj.uniffi_traits() -%} {%- match tm %} {%- when UniffiTrait::Debug { fmt } %} @@ -51,37 +88,84 @@ class {{ type_name }}: if not isinstance(other, {{ type_name }}): return NotImplemented - return {{ eq.return_type().unwrap()|lift_fn }}({% call py::to_ffi_call_with_prefix("self._pointer", eq) %}) + return {{ eq.return_type().unwrap()|lift_fn }}({% call py::to_ffi_call_with_prefix("self._uniffi_clone_pointer()", eq) %}) def __ne__(self, other: object) -> {{ ne.return_type().unwrap()|type_name }}: if not isinstance(other, {{ type_name }}): return NotImplemented - return {{ ne.return_type().unwrap()|lift_fn }}({% call py::to_ffi_call_with_prefix("self._pointer", ne) %}) + return {{ ne.return_type().unwrap()|lift_fn }}({% call py::to_ffi_call_with_prefix("self._uniffi_clone_pointer()", ne) %}) {%- when UniffiTrait::Hash { hash } %} {%- call py::method_decl("__hash__", hash) %} -{% endmatch %} -{% endfor %} - - -class {{ ffi_converter_name }}: +{%- endmatch %} +{%- endfor %} + +{%- if obj.has_callback_interface() %} +{%- let ffi_init_callback = obj.ffi_init_callback() %} +{%- let vtable = obj.vtable().expect("trait interface should have a vtable") %} +{%- let vtable_methods = obj.vtable_methods() %} +{% include "CallbackInterfaceImpl.py" %} +{%- endif %} + +{# Objects as error #} +{%- if ci.is_name_used_as_error(name) %} +{# Due to some mismatches in the ffi converter mechanisms, errors are forced to be a RustBuffer #} +class {{ ffi_converter_name }}__as_error(_UniffiConverterRustBuffer): @classmethod def read(cls, buf): - ptr = buf.read_u64() - if ptr == 0: - raise InternalError("Raw pointer value was null") - return cls.lift(ptr) + raise NotImplementedError() @classmethod def write(cls, value, buf): - if not isinstance(value, {{ type_name }}): - raise TypeError("Expected {{ type_name }} instance, {} found".format(type(value).__name__)) - buf.write_u64(cls.lower(value)) + raise NotImplementedError() @staticmethod def lift(value): - return {{ type_name }}._make_instance_(value) + # Errors are always a rust buffer holding a pointer - which is a "read" + with value.consume_with_stream() as stream: + return {{ ffi_converter_name }}.read(stream) @staticmethod def lower(value): - return value._pointer + raise NotImplementedError() + +{%- endif %} + +class {{ ffi_converter_name }}: + {%- if obj.has_callback_interface() %} + _handle_map = _UniffiHandleMap() + {%- endif %} + + @staticmethod + def lift(value: int): + return {{ impl_name }}._make_instance_(value) + + @staticmethod + def check_lower(value: {{ type_name }}): + {%- if obj.has_callback_interface() %} + pass + {%- else %} + if not isinstance(value, {{ impl_name }}): + raise TypeError("Expected {{ impl_name }} instance, {} found".format(type(value).__name__)) + {%- endif %} + + @staticmethod + def lower(value: {{ protocol_name }}): + {%- if obj.has_callback_interface() %} + return {{ ffi_converter_name }}._handle_map.insert(value) + {%- else %} + if not isinstance(value, {{ impl_name }}): + raise TypeError("Expected {{ impl_name }} instance, {} found".format(type(value).__name__)) + return value._uniffi_clone_pointer() + {%- endif %} + + @classmethod + def read(cls, buf: _UniffiRustBuffer): + ptr = buf.read_u64() + if ptr == 0: + raise InternalError("Raw pointer value was null") + return cls.lift(ptr) + + @classmethod + def write(cls, value: {{ protocol_name }}, buf: _UniffiRustBuffer): + buf.write_u64(cls.lower(value)) diff --git a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/OptionalTemplate.py b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/OptionalTemplate.py index e406c51d49..4c07ae3e34 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/OptionalTemplate.py +++ b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/OptionalTemplate.py @@ -1,6 +1,11 @@ {%- let inner_ffi_converter = inner_type|ffi_converter_name %} class {{ ffi_converter_name }}(_UniffiConverterRustBuffer): + @classmethod + def check_lower(cls, value): + if value is not None: + {{ inner_ffi_converter }}.check_lower(value) + @classmethod def write(cls, value, buf): if value is None: diff --git a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/PointerManager.py b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/PointerManager.py deleted file mode 100644 index 23aa28eab4..0000000000 --- a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/PointerManager.py +++ /dev/null @@ -1,68 +0,0 @@ -class _UniffiPointerManagerCPython: - """ - Manage giving out pointers to Python objects on CPython - - This class is used to generate opaque pointers that reference Python objects to pass to Rust. - It assumes a CPython platform. See _UniffiPointerManagerGeneral for the alternative. - """ - - def new_pointer(self, obj): - """ - Get a pointer for an object as a ctypes.c_size_t instance - - Each call to new_pointer() must be balanced with exactly one call to release_pointer() - - This returns a ctypes.c_size_t. This is always the same size as a pointer and can be - interchanged with pointers for FFI function arguments and return values. - """ - # IncRef the object since we're going to pass a pointer to Rust - ctypes.pythonapi.Py_IncRef(ctypes.py_object(obj)) - # id() is the object address on CPython - # (https://docs.python.org/3/library/functions.html#id) - return id(obj) - - def release_pointer(self, address): - py_obj = ctypes.cast(address, ctypes.py_object) - obj = py_obj.value - ctypes.pythonapi.Py_DecRef(py_obj) - return obj - - def lookup(self, address): - return ctypes.cast(address, ctypes.py_object).value - -class _UniffiPointerManagerGeneral: - """ - Manage giving out pointers to Python objects on non-CPython platforms - - This has the same API as _UniffiPointerManagerCPython, but doesn't assume we're running on - CPython and is slightly slower. - - Instead of using real pointers, it maps integer values to objects and returns the keys as - c_size_t values. - """ - - def __init__(self): - self._map = {} - self._lock = threading.Lock() - self._current_handle = 0 - - def new_pointer(self, obj): - with self._lock: - handle = self._current_handle - self._current_handle += 1 - self._map[handle] = obj - return handle - - def release_pointer(self, handle): - with self._lock: - return self._map.pop(handle) - - def lookup(self, handle): - with self._lock: - return self._map[handle] - -# Pick an pointer manager implementation based on the platform -if platform.python_implementation() == 'CPython': - _UniffiPointerManager = _UniffiPointerManagerCPython # type: ignore -else: - _UniffiPointerManager = _UniffiPointerManagerGeneral # type: ignore diff --git a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/Protocol.py b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/Protocol.py new file mode 100644 index 0000000000..3b7e93596a --- /dev/null +++ b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/Protocol.py @@ -0,0 +1,9 @@ +class {{ protocol_name }}(typing.Protocol): + {%- call py::docstring_value(protocol_docstring, 4) %} + {%- for meth in methods.iter() %} + def {{ meth.name()|fn_name }}(self, {% call py::arg_list_decl(meth) %}): + {%- call py::docstring(meth, 8) %} + raise NotImplementedError + {%- else %} + pass + {%- endfor %} diff --git a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/RecordTemplate.py b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/RecordTemplate.py index 99a30e120f..0b5634eb52 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/RecordTemplate.py +++ b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/RecordTemplate.py @@ -1,11 +1,14 @@ {%- let rec = ci|get_record_definition(name) %} class {{ type_name }}: - {% for field in rec.fields() %} - {{- field.name()|var_name }}: "{{- field|type_name }}"; + {%- call py::docstring(rec, 4) %} + {%- for field in rec.fields() %} + {{ field.name()|var_name }}: "{{ field|type_name }}" + {%- call py::docstring(field, 4) %} {%- endfor %} + {%- if rec.has_fields() %} @typing.no_type_check - def __init__(self, {% for field in rec.fields() %} + def __init__(self, *, {% for field in rec.fields() %} {{- field.name()|var_name }}: "{{- field|type_name }}" {%- if field.default_value().is_some() %} = _DEFAULT{% endif %} {%- if !loop.last %}, {% endif %} @@ -22,6 +25,7 @@ class {{ type_name }}: self.{{ field_name }} = {{ field_name }} {%- endmatch %} {%- endfor %} + {%- endif %} def __str__(self): return "{{ type_name }}({% for field in rec.fields() %}{{ field.name()|var_name }}={}{% if loop.last %}{% else %}, {% endif %}{% endfor %})".format({% for field in rec.fields() %}self.{{ field.name()|var_name }}{% if loop.last %}{% else %}, {% endif %}{% endfor %}) @@ -42,8 +46,22 @@ class {{ ffi_converter_name }}(_UniffiConverterRustBuffer): {%- endfor %} ) + @staticmethod + def check_lower(value): + {%- if rec.fields().is_empty() %} + pass + {%- else %} + {%- for field in rec.fields() %} + {{ field|check_lower_fn }}(value.{{ field.name()|var_name }}) + {%- endfor %} + {%- endif %} + @staticmethod def write(value, buf): + {%- if rec.has_fields() %} {%- for field in rec.fields() %} {{ field|write_fn }}(value.{{ field.name()|var_name }}, buf) {%- endfor %} + {%- else %} + pass + {%- endif %} diff --git a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/RustBufferHelper.py b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/RustBufferHelper.py index daabd5b4b9..4db74fb157 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/RustBufferHelper.py +++ b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/RustBufferHelper.py @@ -1,28 +1,16 @@ # Types conforming to `_UniffiConverterPrimitive` pass themselves directly over the FFI. class _UniffiConverterPrimitive: - @classmethod - def check(cls, value): - return value - @classmethod def lift(cls, value): return value @classmethod def lower(cls, value): - return cls.lowerUnchecked(cls.check(value)) - - @classmethod - def lowerUnchecked(cls, value): return value - @classmethod - def write(cls, value, buf): - cls.write_unchecked(cls.check(value), buf) - class _UniffiConverterPrimitiveInt(_UniffiConverterPrimitive): @classmethod - def check(cls, value): + def check_lower(cls, value): try: value = value.__index__() except Exception: @@ -31,18 +19,16 @@ class _UniffiConverterPrimitiveInt(_UniffiConverterPrimitive): raise TypeError("__index__ returned non-int (type {})".format(type(value).__name__)) if not cls.VALUE_MIN <= value < cls.VALUE_MAX: raise ValueError("{} requires {} <= value < {}".format(cls.CLASS_NAME, cls.VALUE_MIN, cls.VALUE_MAX)) - return super().check(value) class _UniffiConverterPrimitiveFloat(_UniffiConverterPrimitive): @classmethod - def check(cls, value): + def check_lower(cls, value): try: value = value.__float__() except Exception: raise TypeError("must be real number, not {}".format(type(value).__name__)) if not isinstance(value, float): raise TypeError("__float__ returned non-float (type {})".format(type(value).__name__)) - return super().check(value) # Helper class for wrapper types that will always go through a _UniffiRustBuffer. # Classes should inherit from this and implement the `read` and `write` static methods. diff --git a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/RustBufferTemplate.py b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/RustBufferTemplate.py index c317a632fc..44e0ba1001 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/RustBufferTemplate.py +++ b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/RustBufferTemplate.py @@ -1,11 +1,15 @@ class _UniffiRustBuffer(ctypes.Structure): _fields_ = [ - ("capacity", ctypes.c_int32), - ("len", ctypes.c_int32), + ("capacity", ctypes.c_uint64), + ("len", ctypes.c_uint64), ("data", ctypes.POINTER(ctypes.c_char)), ] + @staticmethod + def default(): + return _UniffiRustBuffer(0, 0, None) + @staticmethod def alloc(size): return _rust_call(_UniffiLib.{{ ci.ffi_rustbuffer_alloc().name() }}, size) @@ -137,9 +141,6 @@ class _UniffiRustBufferStream: def read_double(self): return self._unpack_from(8, ">d") - def read_c_size_t(self): - return self._unpack_from(ctypes.sizeof(ctypes.c_size_t) , "@N") - class _UniffiRustBufferBuilder: """ Helper for structured writing of bytes into a _UniffiRustBuffer. diff --git a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/SequenceTemplate.py b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/SequenceTemplate.py index 3c9f5a4596..3c30d9f9f5 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/SequenceTemplate.py +++ b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/SequenceTemplate.py @@ -1,6 +1,11 @@ {%- let inner_ffi_converter = inner_type|ffi_converter_name %} class {{ ffi_converter_name}}(_UniffiConverterRustBuffer): + @classmethod + def check_lower(cls, value): + for item in value: + {{ inner_ffi_converter }}.check_lower(item) + @classmethod def write(cls, value, buf): items = len(value) diff --git a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/StringHelper.py b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/StringHelper.py index 40890b6abc..d574edd09f 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/StringHelper.py +++ b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/StringHelper.py @@ -1,6 +1,6 @@ class _UniffiConverterString: @staticmethod - def check(value): + def check_lower(value): if not isinstance(value, str): raise TypeError("argument must be str, not {}".format(type(value).__name__)) return value @@ -15,7 +15,6 @@ class _UniffiConverterString: @staticmethod def write(value, buf): - value = _UniffiConverterString.check(value) utf8_bytes = value.encode("utf-8") buf.write_i32(len(utf8_bytes)) buf.write(utf8_bytes) @@ -27,7 +26,6 @@ class _UniffiConverterString: @staticmethod def lower(value): - value = _UniffiConverterString.check(value) with _UniffiRustBuffer.alloc_with_builder() as builder: builder.write(value.encode("utf-8")) return builder.finalize() diff --git a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/TimestampHelper.py b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/TimestampHelper.py index 8402f6095d..76d7f8bcdb 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/TimestampHelper.py +++ b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/TimestampHelper.py @@ -17,6 +17,10 @@ class _UniffiConverterTimestamp(_UniffiConverterRustBuffer): else: return datetime.datetime.fromtimestamp(0, tz=datetime.timezone.utc) - datetime.timedelta(seconds=-seconds, microseconds=microseconds) + @staticmethod + def check_lower(value): + pass + @staticmethod def write(value, buf): if value >= datetime.datetime.fromtimestamp(0, datetime.timezone.utc): diff --git a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/TopLevelFunctionTemplate.py b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/TopLevelFunctionTemplate.py index f258b60a1c..230b9e853f 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/TopLevelFunctionTemplate.py +++ b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/TopLevelFunctionTemplate.py @@ -1,7 +1,15 @@ {%- if func.is_async() %} -def {{ func.name()|fn_name }}({%- call py::arg_list_decl(func) -%}): - return _uniffi_rust_call_async( +{%- match func.return_type() -%} +{%- when Some with (return_type) %} +async def {{ func.name()|fn_name }}({%- call py::arg_list_decl(func) -%}) -> "{{ return_type|type_name }}": +{% when None %} +async def {{ func.name()|fn_name }}({%- call py::arg_list_decl(func) -%}) -> None: +{% endmatch %} + + {%- call py::docstring(func, 4) %} + {%- call py::setup_args(func) %} + return await _uniffi_rust_call_async( _UniffiLib.{{ func.ffi_func().name() }}({% call py::arg_list_lowered(func) %}), _UniffiLib.{{func.ffi_rust_future_poll(ci) }}, _UniffiLib.{{func.ffi_rust_future_complete(ci) }}, @@ -13,13 +21,7 @@ def {{ func.name()|fn_name }}({%- call py::arg_list_decl(func) -%}): {%- when None %} lambda val: None, {% endmatch %} - # Error FFI converter - {%- match func.throws_type() %} - {%- when Some(e) %} - {{ e|ffi_converter_name }}, - {%- when None %} - None, - {%- endmatch %} + {% call py::error_ffi_converter(func) %} ) {%- else %} @@ -27,11 +29,13 @@ def {{ func.name()|fn_name }}({%- call py::arg_list_decl(func) -%}): {%- when Some with (return_type) %} def {{ func.name()|fn_name }}({%- call py::arg_list_decl(func) -%}) -> "{{ return_type|type_name }}": + {%- call py::docstring(func, 4) %} {%- call py::setup_args(func) %} return {{ return_type|lift_fn }}({% call py::to_ffi_call(func) %}) {% when None %} -def {{ func.name()|fn_name }}({%- call py::arg_list_decl(func) -%}): +def {{ func.name()|fn_name }}({%- call py::arg_list_decl(func) -%}) -> None: + {%- call py::docstring(func, 4) %} {%- call py::setup_args(func) %} {% call py::to_ffi_call(func) %} {% endmatch %} diff --git a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/Types.py b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/Types.py index 5e05314c37..4aaed253e0 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/Types.py +++ b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/Types.py @@ -85,7 +85,7 @@ {%- when Type::Map { key_type, value_type } %} {%- include "MapTemplate.py" %} -{%- when Type::CallbackInterface { name: id, module_path } %} +{%- when Type::CallbackInterface { name, module_path } %} {%- include "CallbackInterfaceTemplate.py" %} {%- when Type::Custom { name, module_path, builtin } %} @@ -94,9 +94,6 @@ {%- when Type::External { name, module_path, namespace, kind, tagged } %} {%- include "ExternalTemplate.py" %} -{%- when Type::ForeignExecutor %} -{%- include "ForeignExecutorTemplate.py" %} - {%- else %} {%- endmatch %} {%- endfor %} diff --git a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/UInt16Helper.py b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/UInt16Helper.py index 081c6731ce..039bf76162 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/UInt16Helper.py +++ b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/UInt16Helper.py @@ -8,5 +8,5 @@ class _UniffiConverterUInt16(_UniffiConverterPrimitiveInt): return buf.read_u16() @staticmethod - def write_unchecked(value, buf): + def write(value, buf): buf.write_u16(value) diff --git a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/UInt32Helper.py b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/UInt32Helper.py index b80e75177d..1650bf9b60 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/UInt32Helper.py +++ b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/UInt32Helper.py @@ -8,5 +8,5 @@ class _UniffiConverterUInt32(_UniffiConverterPrimitiveInt): return buf.read_u32() @staticmethod - def write_unchecked(value, buf): + def write(value, buf): buf.write_u32(value) diff --git a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/UInt64Helper.py b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/UInt64Helper.py index 4b87e58547..f354545e26 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/UInt64Helper.py +++ b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/UInt64Helper.py @@ -8,5 +8,5 @@ class _UniffiConverterUInt64(_UniffiConverterPrimitiveInt): return buf.read_u64() @staticmethod - def write_unchecked(value, buf): + def write(value, buf): buf.write_u64(value) diff --git a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/UInt8Helper.py b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/UInt8Helper.py index 33026706f2..cee130b4d9 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/UInt8Helper.py +++ b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/UInt8Helper.py @@ -8,5 +8,5 @@ class _UniffiConverterUInt8(_UniffiConverterPrimitiveInt): return buf.read_u8() @staticmethod - def write_unchecked(value, buf): + def write(value, buf): buf.write_u8(value) diff --git a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/macros.py b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/macros.py index ef3b1bb94d..6818a8c107 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/macros.py +++ b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/macros.py @@ -5,27 +5,29 @@ #} {%- macro to_ffi_call(func) -%} - {%- match func.throws_type() -%} - {%- when Some with (e) -%} -_rust_call_with_error({{ e|ffi_converter_name }}, - {%- else -%} -_rust_call( - {%- endmatch -%} - _UniffiLib.{{ func.ffi_func().name() }}, - {%- call arg_list_lowered(func) -%} -) +{%- call _to_ffi_call_with_prefix_arg("", func) %} {%- endmacro -%} {%- macro to_ffi_call_with_prefix(prefix, func) -%} - {%- match func.throws_type() -%} - {%- when Some with (e) -%} -_rust_call_with_error( - {{ e|ffi_converter_name }}, - {%- else -%} +{%- call _to_ffi_call_with_prefix_arg(format!("{},", prefix), func) %} +{%- endmacro -%} + +{%- macro _to_ffi_call_with_prefix_arg(prefix, func) -%} +{%- match func.throws_type() -%} +{%- when Some with (e) -%} +{%- match e -%} +{%- when Type::Enum { name, module_path } -%} +_rust_call_with_error({{ e|ffi_converter_name }}, +{%- when Type::Object { name, module_path, imp } -%} +_rust_call_with_error({{ e|ffi_converter_name }}__as_error, +{%- else %} +# unsupported error type! +{%- endmatch %} +{%- else -%} _rust_call( - {%- endmatch -%} +{%- endmatch -%} _UniffiLib.{{ func.ffi_func().name() }}, - {{- prefix }}, + {{- prefix }} {%- call arg_list_lowered(func) -%} ) {%- endmacro -%} @@ -37,6 +39,19 @@ _rust_call( {%- endfor %} {%- endmacro -%} +{%- macro docstring_value(maybe_docstring, indent_spaces) %} +{%- match maybe_docstring %} +{%- when Some(docstring) %} +{{ docstring|docstring(indent_spaces) }} +{{ "" }} +{%- else %} +{%- endmatch %} +{%- endmacro %} + +{%- macro docstring(defn, indent_spaces) %} +{%- call docstring_value(defn.docstring(), indent_spaces) %} +{%- endmacro %} + {#- // Arglist as used in Python declarations of methods, functions and constructors. // Note the var_name and type_name filters. @@ -76,6 +91,7 @@ _rust_call( if {{ arg.name()|var_name }} is _DEFAULT: {{ arg.name()|var_name }} = {{ literal|literal_py(arg.as_type().borrow()) }} {%- endmatch %} + {{ arg|check_lower_fn }}({{ arg.name()|var_name }}) {% endfor -%} {%- endmacro -%} @@ -91,6 +107,7 @@ _rust_call( if {{ arg.name()|var_name }} is _DEFAULT: {{ arg.name()|var_name }} = {{ literal|literal_py(arg.as_type().borrow()) }} {%- endmatch %} + {{ arg|check_lower_fn }}({{ arg.name()|var_name }}) {% endfor -%} {%- endmacro -%} @@ -100,11 +117,18 @@ _rust_call( {%- macro method_decl(py_method_name, meth) %} {% if meth.is_async() %} - def {{ py_method_name }}(self, {% call arg_list_decl(meth) %}): +{%- match meth.return_type() %} +{%- when Some with (return_type) %} + async def {{ py_method_name }}(self, {% call arg_list_decl(meth) %}) -> "{{ return_type|type_name }}": +{%- when None %} + async def {{ py_method_name }}(self, {% call arg_list_decl(meth) %}) -> None: +{% endmatch %} + + {%- call docstring(meth, 8) %} {%- call setup_args_extra_indent(meth) %} - return _uniffi_rust_call_async( + return await _uniffi_rust_call_async( _UniffiLib.{{ meth.ffi_func().name() }}( - self._pointer, {% call arg_list_lowered(meth) %} + self._uniffi_clone_pointer(), {% call arg_list_lowered(meth) %} ), _UniffiLib.{{ meth.ffi_rust_future_poll(ci) }}, _UniffiLib.{{ meth.ffi_rust_future_complete(ci) }}, @@ -116,13 +140,7 @@ _rust_call( {%- when None %} lambda val: None, {% endmatch %} - # Error FFI converter - {%- match meth.throws_type() %} - {%- when Some(e) %} - {{ e|ffi_converter_name }}, - {%- when None %} - None, - {%- endmatch %} + {% call error_ffi_converter(meth) %} ) {%- else -%} @@ -131,17 +149,36 @@ _rust_call( {%- when Some with (return_type) %} def {{ py_method_name }}(self, {% call arg_list_decl(meth) %}) -> "{{ return_type|type_name }}": + {%- call docstring(meth, 8) %} {%- call setup_args_extra_indent(meth) %} return {{ return_type|lift_fn }}( - {% call to_ffi_call_with_prefix("self._pointer", meth) %} + {% call to_ffi_call_with_prefix("self._uniffi_clone_pointer()", meth) %} ) {%- when None %} - def {{ py_method_name }}(self, {% call arg_list_decl(meth) %}): + def {{ py_method_name }}(self, {% call arg_list_decl(meth) %}) -> None: + {%- call docstring(meth, 8) %} {%- call setup_args_extra_indent(meth) %} - {% call to_ffi_call_with_prefix("self._pointer", meth) %} + {% call to_ffi_call_with_prefix("self._uniffi_clone_pointer()", meth) %} {% endmatch %} {% endif %} {% endmacro %} + +{%- macro error_ffi_converter(func) %} + # Error FFI converter +{% match func.throws_type() %} +{%- when Some(e) %} +{%- match e -%} +{%- when Type::Enum { name, module_path } -%} + {{ e|ffi_converter_name }}, +{%- when Type::Object { name, module_path, imp } -%} + {{ e|ffi_converter_name }}__as_error, +{%- else %} + # unsupported error type! +{%- endmatch %} +{%- when None %} + None, +{%- endmatch %} +{% endmacro %} diff --git a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/wrapper.py b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/wrapper.py index 24c3290ff7..1ccd6821c0 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/python/templates/wrapper.py +++ b/third_party/rust/uniffi_bindgen/src/bindings/python/templates/wrapper.py @@ -1,3 +1,5 @@ +{%- call py::docstring_value(ci.namespace_docstring(), 0) %} + # This file was autogenerated by some hot garbage in the `uniffi` crate. # Trust me, you don't want to mess with it! @@ -13,6 +15,7 @@ # compile the rust component. The easiest way to ensure this is to bundle the Python # helpers directly inline like we're doing here. +from __future__ import annotations import os import sys import ctypes @@ -20,6 +23,9 @@ import enum import struct import contextlib import datetime +import threading +import itertools +import traceback import typing {%- if ci.has_async_fns() %} import asyncio @@ -34,20 +40,20 @@ _DEFAULT = object() {% include "RustBufferTemplate.py" %} {% include "Helpers.py" %} -{% include "PointerManager.py" %} +{% include "HandleMap.py" %} {% include "RustBufferHelper.py" %} # Contains loading, initialization code, and the FFI Function declarations. {% include "NamespaceLibraryTemplate.py" %} +# Public interface members begin here. +{{ type_helper_code }} + # Async support {%- if ci.has_async_fns() %} {%- include "Async.py" %} {%- endif %} -# Public interface members begin here. -{{ type_helper_code }} - {%- for func in ci.function_definitions() %} {%- include "TopLevelFunctionTemplate.py" %} {%- endfor %} @@ -69,6 +75,9 @@ __all__ = [ {%- for c in ci.callback_interface_definitions() %} "{{ c.name()|class_name }}", {%- endfor %} + {%- if ci.has_async_fns() %} + "uniffi_set_event_loop", + {%- endif %} ] {% import "macros.py" as py %} diff --git a/third_party/rust/uniffi_bindgen/src/bindings/python/test.rs b/third_party/rust/uniffi_bindgen/src/bindings/python/test.rs index 0fcf09996f..0c23140b33 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/python/test.rs +++ b/third_party/rust/uniffi_bindgen/src/bindings/python/test.rs @@ -2,10 +2,8 @@ License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -use crate::{ - bindings::{RunScriptOptions, TargetLanguage}, - library_mode::generate_bindings, -}; +use crate::bindings::TargetLanguage; +use crate::{bindings::RunScriptOptions, library_mode::generate_bindings, BindingGeneratorDefault}; use anyhow::{Context, Result}; use camino::Utf8Path; use std::env; @@ -34,14 +32,18 @@ pub fn run_script( args: Vec, _options: &RunScriptOptions, ) -> Result<()> { - let script_path = Utf8Path::new(".").join(script_file).canonicalize_utf8()?; + let script_path = Utf8Path::new(script_file).canonicalize_utf8()?; let test_helper = UniFFITestHelper::new(crate_name)?; let out_dir = test_helper.create_out_dir(tmp_dir, &script_path)?; let cdylib_path = test_helper.copy_cdylib_to_out_dir(&out_dir)?; generate_bindings( &cdylib_path, None, - &[TargetLanguage::Python], + &BindingGeneratorDefault { + target_languages: vec![TargetLanguage::Python], + try_format_code: false, + }, + None, &out_dir, false, )?; diff --git a/third_party/rust/uniffi_bindgen/src/bindings/ruby/gen_ruby/mod.rs b/third_party/rust/uniffi_bindgen/src/bindings/ruby/gen_ruby/mod.rs index 1f1bf8e299..04841b459c 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/ruby/gen_ruby/mod.rs +++ b/third_party/rust/uniffi_bindgen/src/bindings/ruby/gen_ruby/mod.rs @@ -2,15 +2,39 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -use anyhow::Result; +use anyhow::{bail, Result}; use askama::Template; +use camino::Utf8Path; use heck::{ToShoutySnakeCase, ToSnakeCase, ToUpperCamelCase}; use serde::{Deserialize, Serialize}; use std::borrow::Borrow; use std::collections::HashMap; +use crate::bindings::ruby; use crate::interface::*; -use crate::BindingsConfig; +use crate::{BindingGenerator, BindingsConfig}; + +pub struct RubyBindingGenerator; +impl BindingGenerator for RubyBindingGenerator { + type Config = Config; + + fn write_bindings( + &self, + ci: &ComponentInterface, + config: &Config, + out_dir: &Utf8Path, + try_format_code: bool, + ) -> Result<()> { + ruby::write_bindings(config, ci, out_dir, try_format_code) + } + + fn check_library_path(&self, library_path: &Utf8Path, cdylib_name: Option<&str>) -> Result<()> { + if cdylib_name.is_none() { + bail!("Generate bindings for Ruby requires a cdylib, but {library_path} was given"); + } + Ok(()) + } +} const RESERVED_WORDS: &[&str] = &[ "alias", "and", "BEGIN", "begin", "break", "case", "class", "def", "defined?", "do", "else", @@ -57,7 +81,6 @@ pub fn canonical_name(t: &Type) -> String { Type::CallbackInterface { name, .. } => format!("CallbackInterface{name}"), Type::Timestamp => "Timestamp".into(), Type::Duration => "Duration".into(), - Type::ForeignExecutor => "ForeignExecutor".into(), // Recursive types. // These add a prefix to the name of the underlying type. // The component API definition cannot give names to recursive types, so as long as the @@ -150,20 +173,20 @@ mod filters { FfiType::UInt64 => ":uint64".to_string(), FfiType::Float32 => ":float".to_string(), FfiType::Float64 => ":double".to_string(), + FfiType::Handle => ":uint64".to_string(), FfiType::RustArcPtr(_) => ":pointer".to_string(), FfiType::RustBuffer(_) => "RustBuffer.by_value".to_string(), + FfiType::RustCallStatus => "RustCallStatus".to_string(), FfiType::ForeignBytes => "ForeignBytes".to_string(), - FfiType::ForeignCallback => unimplemented!("Callback interfaces are not implemented"), - FfiType::ForeignExecutorCallback => { - unimplemented!("Foreign executors are not implemented") - } - FfiType::ForeignExecutorHandle => { - unimplemented!("Foreign executors are not implemented") - } - FfiType::RustFutureHandle - | FfiType::RustFutureContinuationCallback - | FfiType::RustFutureContinuationData => { - unimplemented!("Async functions are not implemented") + FfiType::Callback(_) => unimplemented!("FFI Callbacks not implemented"), + // Note: this can't just be `unimplemented!()` because some of the FFI function + // definitions use references. Those FFI functions aren't actually used, so we just + // pick something that runs and makes some sense. Revisit this once the references + // are actually implemented. + FfiType::Reference(_) => ":pointer".to_string(), + FfiType::VoidPointer => ":pointer".to_string(), + FfiType::Struct(_) => { + unimplemented!("Structs are not implemented") } }) } @@ -179,7 +202,8 @@ mod filters { } // use the double-quote form to match with the other languages, and quote escapes. Literal::String(s) => format!("\"{s}\""), - Literal::Null => "nil".into(), + Literal::None => "nil".into(), + Literal::Some { inner } => literal_rb(inner)?, Literal::EmptySequence => "[]".into(), Literal::EmptyMap => "{}".into(), Literal::Enum(v, type_) => match type_ { @@ -264,7 +288,24 @@ mod filters { } Type::External { .. } => panic!("No support for external types, yet"), Type::Custom { .. } => panic!("No support for custom types, yet"), - Type::ForeignExecutor => unimplemented!("Foreign executors are not implemented"), + }) + } + + pub fn check_lower_rb(nm: &str, type_: &Type) -> Result { + Ok(match type_ { + Type::Object { name, .. } => { + format!("({}.uniffi_check_lower {nm})", class_name_rb(name)?) + } + Type::Enum { .. } + | Type::Record { .. } + | Type::Optional { .. } + | Type::Sequence { .. } + | Type::Map { .. } => format!( + "RustBuffer.check_lower_{}({})", + class_name_rb(&canonical_name(type_))?, + nm + ), + _ => "".to_owned(), }) } @@ -283,7 +324,7 @@ mod filters { Type::Boolean => format!("({nm} ? 1 : 0)"), Type::String => format!("RustBuffer.allocFromString({nm})"), Type::Bytes => format!("RustBuffer.allocFromBytes({nm})"), - Type::Object { name, .. } => format!("({}._uniffi_lower {nm})", class_name_rb(name)?), + Type::Object { name, .. } => format!("({}.uniffi_lower {nm})", class_name_rb(name)?), Type::CallbackInterface { .. } => { panic!("No support for lowering callback interfaces yet") } @@ -300,7 +341,6 @@ mod filters { ), Type::External { .. } => panic!("No support for lowering external types, yet"), Type::Custom { .. } => panic!("No support for lowering custom types, yet"), - Type::ForeignExecutor => unimplemented!("Foreign executors are not implemented"), }) } @@ -318,7 +358,7 @@ mod filters { Type::Boolean => format!("1 == {nm}"), Type::String => format!("{nm}.consumeIntoString"), Type::Bytes => format!("{nm}.consumeIntoBytes"), - Type::Object { name, .. } => format!("{}._uniffi_allocate({nm})", class_name_rb(name)?), + Type::Object { name, .. } => format!("{}.uniffi_allocate({nm})", class_name_rb(name)?), Type::CallbackInterface { .. } => { panic!("No support for lifting callback interfaces, yet") } @@ -341,7 +381,6 @@ mod filters { ), Type::External { .. } => panic!("No support for lifting external types, yet"), Type::Custom { .. } => panic!("No support for lifting custom types, yet"), - Type::ForeignExecutor => unimplemented!("Foreign executors are not implemented"), }) } } diff --git a/third_party/rust/uniffi_bindgen/src/bindings/ruby/templates/ObjectTemplate.rb b/third_party/rust/uniffi_bindgen/src/bindings/ruby/templates/ObjectTemplate.rb index 677c5c729b..ba2caf7380 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/ruby/templates/ObjectTemplate.rb +++ b/third_party/rust/uniffi_bindgen/src/bindings/ruby/templates/ObjectTemplate.rb @@ -2,18 +2,18 @@ class {{ obj.name()|class_name_rb }} # A private helper for initializing instances of the class from a raw pointer, # bypassing any initialization logic and ensuring they are GC'd properly. - def self._uniffi_allocate(pointer) + def self.uniffi_allocate(pointer) pointer.autorelease = false inst = allocate inst.instance_variable_set :@pointer, pointer - ObjectSpace.define_finalizer(inst, _uniffi_define_finalizer_by_pointer(pointer, inst.object_id)) + ObjectSpace.define_finalizer(inst, uniffi_define_finalizer_by_pointer(pointer, inst.object_id)) return inst end # A private helper for registering an object finalizer. # N.B. it's important that this does not capture a reference # to the actual instance, only its underlying pointer. - def self._uniffi_define_finalizer_by_pointer(pointer, object_id) + def self.uniffi_define_finalizer_by_pointer(pointer, object_id) Proc.new do |_id| {{ ci.namespace()|class_name_rb }}.rust_call( :{{ obj.ffi_object_free().name() }}, @@ -25,31 +25,41 @@ class {{ obj.name()|class_name_rb }} # A private helper for lowering instances into a raw pointer. # This does an explicit typecheck, because accidentally lowering a different type of # object in a place where this type is expected, could lead to memory unsafety. - def self._uniffi_lower(inst) + def self.uniffi_check_lower(inst) if not inst.is_a? self raise TypeError.new "Expected a {{ obj.name()|class_name_rb }} instance, got #{inst}" end - return inst.instance_variable_get :@pointer + end + + def uniffi_clone_pointer() + return {{ ci.namespace()|class_name_rb }}.rust_call( + :{{ obj.ffi_object_clone().name() }}, + @pointer + ) + end + + def self.uniffi_lower(inst) + return inst.uniffi_clone_pointer() end {%- match obj.primary_constructor() %} {%- when Some with (cons) %} def initialize({% call rb::arg_list_decl(cons) -%}) - {%- call rb::coerce_args_extra_indent(cons) %} + {%- call rb::setup_args_extra_indent(cons) %} pointer = {% call rb::to_ffi_call(cons) %} @pointer = pointer - ObjectSpace.define_finalizer(self, self.class._uniffi_define_finalizer_by_pointer(pointer, self.object_id)) + ObjectSpace.define_finalizer(self, self.class.uniffi_define_finalizer_by_pointer(pointer, self.object_id)) end {%- when None %} {%- endmatch %} {% for cons in obj.alternate_constructors() -%} def self.{{ cons.name()|fn_name_rb }}({% call rb::arg_list_decl(cons) %}) - {%- call rb::coerce_args_extra_indent(cons) %} + {%- call rb::setup_args_extra_indent(cons) %} # Call the (fallible) function before creating any half-baked object instances. # Lightly yucky way to bypass the usual "initialize" logic # and just create a new instance with the required pointer. - return _uniffi_allocate({% call rb::to_ffi_call(cons) %}) + return uniffi_allocate({% call rb::to_ffi_call(cons) %}) end {% endfor %} @@ -58,15 +68,15 @@ class {{ obj.name()|class_name_rb }} {%- when Some with (return_type) -%} def {{ meth.name()|fn_name_rb }}({% call rb::arg_list_decl(meth) %}) - {%- call rb::coerce_args_extra_indent(meth) %} - result = {% call rb::to_ffi_call_with_prefix("@pointer", meth) %} + {%- call rb::setup_args_extra_indent(meth) %} + result = {% call rb::to_ffi_call_with_prefix("uniffi_clone_pointer()", meth) %} return {{ "result"|lift_rb(return_type) }} end {%- when None -%} def {{ meth.name()|fn_name_rb }}({% call rb::arg_list_decl(meth) %}) - {%- call rb::coerce_args_extra_indent(meth) %} - {% call rb::to_ffi_call_with_prefix("@pointer", meth) %} + {%- call rb::setup_args_extra_indent(meth) %} + {% call rb::to_ffi_call_with_prefix("uniffi_clone_pointer()", meth) %} end {% endmatch %} {% endfor %} diff --git a/third_party/rust/uniffi_bindgen/src/bindings/ruby/templates/RecordTemplate.rb b/third_party/rust/uniffi_bindgen/src/bindings/ruby/templates/RecordTemplate.rb index c940b31060..b5a201b248 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/ruby/templates/RecordTemplate.rb +++ b/third_party/rust/uniffi_bindgen/src/bindings/ruby/templates/RecordTemplate.rb @@ -2,7 +2,12 @@ class {{ rec.name()|class_name_rb }} attr_reader {% for field in rec.fields() %}:{{ field.name()|var_name_rb }}{% if loop.last %}{% else %}, {% endif %}{%- endfor %} - def initialize({% for field in rec.fields() %}{{ field.name()|var_name_rb }}{% if loop.last %}{% else %}, {% endif %}{% endfor %}) + def initialize({% for field in rec.fields() %}{{ field.name()|var_name_rb -}}: + {%- match field.default_value() %} + {%- when Some with(literal) %} {{ literal|literal_rb }} + {%- else %} + {%- endmatch %} + {%- if loop.last %}{% else %}, {% endif -%}{% endfor %}) {%- for field in rec.fields() %} @{{ field.name()|var_name_rb }} = {{ field.name()|var_name_rb }} {%- endfor %} diff --git a/third_party/rust/uniffi_bindgen/src/bindings/ruby/templates/RustBufferBuilder.rb b/third_party/rust/uniffi_bindgen/src/bindings/ruby/templates/RustBufferBuilder.rb index 8749139116..d15c0bbe76 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/ruby/templates/RustBufferBuilder.rb +++ b/third_party/rust/uniffi_bindgen/src/bindings/ruby/templates/RustBufferBuilder.rb @@ -163,7 +163,7 @@ class RustBufferBuilder # The Object type {{ object_name }}. def write_{{ canonical_type_name }}(obj) - pointer = {{ object_name|class_name_rb}}._uniffi_lower obj + pointer = {{ object_name|class_name_rb}}.uniffi_lower obj pack_into(8, 'Q>', pointer.address) end diff --git a/third_party/rust/uniffi_bindgen/src/bindings/ruby/templates/RustBufferStream.rb b/third_party/rust/uniffi_bindgen/src/bindings/ruby/templates/RustBufferStream.rb index b085dddf15..f9b0806abc 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/ruby/templates/RustBufferStream.rb +++ b/third_party/rust/uniffi_bindgen/src/bindings/ruby/templates/RustBufferStream.rb @@ -155,7 +155,7 @@ class RustBufferStream def read{{ canonical_type_name }} pointer = FFI::Pointer.new unpack_from 8, 'Q>' - return {{ object_name|class_name_rb }}._uniffi_allocate(pointer) + return {{ object_name|class_name_rb }}.uniffi_allocate(pointer) end {% when Type::Enum { name, module_path } -%} @@ -237,7 +237,7 @@ class RustBufferStream def read{{ canonical_type_name }} {{ rec.name()|class_name_rb }}.new( {%- for field in rec.fields() %} - read{{ canonical_name(field.as_type().borrow()).borrow()|class_name_rb }}{% if loop.last %}{% else %},{% endif %} + {{ field.name()|var_name_rb }}: read{{ canonical_name(field.as_type().borrow()).borrow()|class_name_rb }}{% if loop.last %}{% else %},{% endif %} {%- endfor %} ) end diff --git a/third_party/rust/uniffi_bindgen/src/bindings/ruby/templates/RustBufferTemplate.rb b/third_party/rust/uniffi_bindgen/src/bindings/ruby/templates/RustBufferTemplate.rb index 0194c9666d..452d9831cd 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/ruby/templates/RustBufferTemplate.rb +++ b/third_party/rust/uniffi_bindgen/src/bindings/ruby/templates/RustBufferTemplate.rb @@ -1,6 +1,6 @@ class RustBuffer < FFI::Struct - layout :capacity, :int32, - :len, :int32, + layout :capacity, :uint64, + :len, :uint64, :data, :pointer def self.alloc(size) @@ -128,6 +128,12 @@ class RustBuffer < FFI::Struct {%- let rec = ci|get_record_definition(record_name) -%} # The Record type {{ record_name }}. + def self.check_lower_{{ canonical_type_name }}(v) + {%- for field in rec.fields() %} + {{ "v.{}"|format(field.name()|var_name_rb)|check_lower_rb(field.as_type().borrow()) }} + {%- endfor %} + end + def self.alloc_from_{{ canonical_type_name }}(v) RustBuffer.allocWithBuilder do |builder| builder.write_{{ canonical_type_name }}(v) @@ -146,6 +152,19 @@ class RustBuffer < FFI::Struct {%- let e = ci|get_enum_definition(enum_name) -%} # The Enum type {{ enum_name }}. + def self.check_lower_{{ canonical_type_name }}(v) + {%- if !e.is_flat() %} + {%- for variant in e.variants() %} + if v.{{ variant.name()|var_name_rb }}? + {%- for field in variant.fields() %} + {{ "v.{}"|format(field.name())|check_lower_rb(field.as_type().borrow()) }} + {%- endfor %} + return + end + {%- endfor %} + {%- endif %} + end + def self.alloc_from_{{ canonical_type_name }}(v) RustBuffer.allocWithBuilder do |builder| builder.write_{{ canonical_type_name }}(v) @@ -163,6 +182,12 @@ class RustBuffer < FFI::Struct {% when Type::Optional { inner_type } -%} # The Optional type for {{ canonical_name(inner_type) }}. + def self.check_lower_{{ canonical_type_name }}(v) + if not v.nil? + {{ "v"|check_lower_rb(inner_type.borrow()) }} + end + end + def self.alloc_from_{{ canonical_type_name }}(v) RustBuffer.allocWithBuilder do |builder| builder.write_{{ canonical_type_name }}(v) @@ -179,6 +204,12 @@ class RustBuffer < FFI::Struct {% when Type::Sequence { inner_type } -%} # The Sequence type for {{ canonical_name(inner_type) }}. + def self.check_lower_{{ canonical_type_name }}(v) + v.each do |item| + {{ "item"|check_lower_rb(inner_type.borrow()) }} + end + end + def self.alloc_from_{{ canonical_type_name }}(v) RustBuffer.allocWithBuilder do |builder| builder.write_{{ canonical_type_name }}(v) @@ -195,6 +226,13 @@ class RustBuffer < FFI::Struct {% when Type::Map { key_type: k, value_type: inner_type } -%} # The Map type for {{ canonical_name(inner_type) }}. + def self.check_lower_{{ canonical_type_name }}(v) + v.each do |k, v| + {{ "k"|check_lower_rb(k.borrow()) }} + {{ "v"|check_lower_rb(inner_type.borrow()) }} + end + end + def self.alloc_from_{{ canonical_type_name }}(v) RustBuffer.allocWithBuilder do |builder| builder.write_{{ canonical_type_name }}(v) diff --git a/third_party/rust/uniffi_bindgen/src/bindings/ruby/templates/TopLevelFunctionTemplate.rb b/third_party/rust/uniffi_bindgen/src/bindings/ruby/templates/TopLevelFunctionTemplate.rb index 13214cf31b..b6dce0effa 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/ruby/templates/TopLevelFunctionTemplate.rb +++ b/third_party/rust/uniffi_bindgen/src/bindings/ruby/templates/TopLevelFunctionTemplate.rb @@ -2,7 +2,7 @@ {%- when Some with (return_type) %} def self.{{ func.name()|fn_name_rb }}({%- call rb::arg_list_decl(func) -%}) - {%- call rb::coerce_args(func) %} + {%- call rb::setup_args(func) %} result = {% call rb::to_ffi_call(func) %} return {{ "result"|lift_rb(return_type) }} end @@ -10,7 +10,7 @@ end {% when None %} def self.{{ func.name()|fn_name_rb }}({%- call rb::arg_list_decl(func) -%}) - {%- call rb::coerce_args(func) %} + {%- call rb::setup_args(func) %} {% call rb::to_ffi_call(func) %} end {% endmatch %} diff --git a/third_party/rust/uniffi_bindgen/src/bindings/ruby/templates/macros.rb b/third_party/rust/uniffi_bindgen/src/bindings/ruby/templates/macros.rb index 8dc3e5e613..59fa4ef4cc 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/ruby/templates/macros.rb +++ b/third_party/rust/uniffi_bindgen/src/bindings/ruby/templates/macros.rb @@ -60,14 +60,16 @@ [{%- for arg in func.arguments() -%}{{ arg.type_().borrow()|type_ffi }}, {% endfor -%} RustCallStatus.by_ref] {%- endmacro -%} -{%- macro coerce_args(func) %} +{%- macro setup_args(func) %} {%- for arg in func.arguments() %} - {{ arg.name() }} = {{ arg.name()|coerce_rb(ci.namespace()|class_name_rb, arg.as_type().borrow()) -}} + {{ arg.name() }} = {{ arg.name()|coerce_rb(ci.namespace()|class_name_rb, arg.as_type().borrow()) }} + {{ arg.name()|check_lower_rb(arg.as_type().borrow()) }} {% endfor -%} {%- endmacro -%} -{%- macro coerce_args_extra_indent(func) %} - {%- for arg in func.arguments() %} +{%- macro setup_args_extra_indent(meth) %} + {%- for arg in meth.arguments() %} {{ arg.name() }} = {{ arg.name()|coerce_rb(ci.namespace()|class_name_rb, arg.as_type().borrow()) }} + {{ arg.name()|check_lower_rb(arg.as_type().borrow()) }} {%- endfor %} {%- endmacro -%} diff --git a/third_party/rust/uniffi_bindgen/src/bindings/ruby/test.rs b/third_party/rust/uniffi_bindgen/src/bindings/ruby/test.rs index 03da37d567..88f770b9dc 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/ruby/test.rs +++ b/third_party/rust/uniffi_bindgen/src/bindings/ruby/test.rs @@ -4,6 +4,7 @@ License, v. 2.0. If a copy of the MPL was not distributed with this use crate::bindings::TargetLanguage; use crate::library_mode::generate_bindings; +use crate::BindingGeneratorDefault; use anyhow::{bail, Context, Result}; use camino::Utf8Path; use std::env; @@ -30,11 +31,21 @@ pub fn test_script_command( fixture_name: &str, script_file: &str, ) -> Result { - let script_path = Utf8Path::new(".").join(script_file).canonicalize_utf8()?; + let script_path = Utf8Path::new(script_file).canonicalize_utf8()?; let test_helper = UniFFITestHelper::new(fixture_name)?; let out_dir = test_helper.create_out_dir(tmp_dir, &script_path)?; let cdylib_path = test_helper.copy_cdylib_to_out_dir(&out_dir)?; - generate_bindings(&cdylib_path, None, &[TargetLanguage::Ruby], &out_dir, false)?; + generate_bindings( + &cdylib_path, + None, + &BindingGeneratorDefault { + target_languages: vec![TargetLanguage::Ruby], + try_format_code: false, + }, + None, + &out_dir, + false, + )?; let rubypath = env::var_os("RUBYLIB").unwrap_or_else(|| OsString::from("")); let rubypath = env::join_paths( diff --git a/third_party/rust/uniffi_bindgen/src/bindings/swift/gen_swift/callback_interface.rs b/third_party/rust/uniffi_bindgen/src/bindings/swift/gen_swift/callback_interface.rs index 5d8b37e0af..dab89e0259 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/swift/gen_swift/callback_interface.rs +++ b/third_party/rust/uniffi_bindgen/src/bindings/swift/gen_swift/callback_interface.rs @@ -6,21 +6,25 @@ use super::CodeType; #[derive(Debug)] pub struct CallbackInterfaceCodeType { - id: String, + name: String, } impl CallbackInterfaceCodeType { - pub fn new(id: String) -> Self { - Self { id } + pub fn new(name: String) -> Self { + Self { name } } } impl CodeType for CallbackInterfaceCodeType { fn type_label(&self) -> String { - super::SwiftCodeOracle.class_name(&self.id) + super::SwiftCodeOracle.class_name(&self.name) } fn canonical_name(&self) -> String { format!("CallbackInterface{}", self.type_label()) } + + fn initialization_fn(&self) -> Option { + Some(format!("uniffiCallbackInit{}", self.name)) + } } diff --git a/third_party/rust/uniffi_bindgen/src/bindings/swift/gen_swift/compounds.rs b/third_party/rust/uniffi_bindgen/src/bindings/swift/gen_swift/compounds.rs index 8e6dddf3f9..d89fdfd386 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/swift/gen_swift/compounds.rs +++ b/third_party/rust/uniffi_bindgen/src/bindings/swift/gen_swift/compounds.rs @@ -30,8 +30,9 @@ impl CodeType for OptionalCodeType { fn literal(&self, literal: &Literal) -> String { match literal { - Literal::Null => "nil".into(), - _ => super::SwiftCodeOracle.find(&self.inner).literal(literal), + Literal::None => "nil".into(), + Literal::Some { inner } => super::SwiftCodeOracle.find(&self.inner).literal(inner), + _ => panic!("Invalid literal for Optional type: {literal:?}"), } } } diff --git a/third_party/rust/uniffi_bindgen/src/bindings/swift/gen_swift/executor.rs b/third_party/rust/uniffi_bindgen/src/bindings/swift/gen_swift/executor.rs deleted file mode 100644 index b488b004cf..0000000000 --- a/third_party/rust/uniffi_bindgen/src/bindings/swift/gen_swift/executor.rs +++ /dev/null @@ -1,23 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -use super::CodeType; - -#[derive(Debug)] -pub struct ForeignExecutorCodeType; - -impl CodeType for ForeignExecutorCodeType { - fn type_label(&self) -> String { - // On Swift, we define a struct to represent a ForeignExecutor - "UniFfiForeignExecutor".into() - } - - fn canonical_name(&self) -> String { - "ForeignExecutor".into() - } - - fn initialization_fn(&self) -> Option { - Some("uniffiInitForeignExecutor".into()) - } -} diff --git a/third_party/rust/uniffi_bindgen/src/bindings/swift/gen_swift/external.rs b/third_party/rust/uniffi_bindgen/src/bindings/swift/gen_swift/external.rs index 0b6728ba84..3960b7aae1 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/swift/gen_swift/external.rs +++ b/third_party/rust/uniffi_bindgen/src/bindings/swift/gen_swift/external.rs @@ -17,7 +17,7 @@ impl ExternalCodeType { impl CodeType for ExternalCodeType { fn type_label(&self) -> String { - self.name.clone() + super::SwiftCodeOracle.class_name(&self.name) } fn canonical_name(&self) -> String { diff --git a/third_party/rust/uniffi_bindgen/src/bindings/swift/gen_swift/mod.rs b/third_party/rust/uniffi_bindgen/src/bindings/swift/gen_swift/mod.rs index 12db4afc66..16c1625123 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/swift/gen_swift/mod.rs +++ b/third_party/rust/uniffi_bindgen/src/bindings/swift/gen_swift/mod.rs @@ -10,25 +10,49 @@ use std::fmt::Debug; use anyhow::{Context, Result}; use askama::Template; -use heck::{ToLowerCamelCase, ToUpperCamelCase}; +use camino::Utf8Path; +use heck::{ToLowerCamelCase, ToShoutySnakeCase, ToUpperCamelCase}; use serde::{Deserialize, Serialize}; use super::Bindings; use crate::backend::TemplateExpression; +use crate::bindings::swift; use crate::interface::*; -use crate::BindingsConfig; +use crate::{BindingGenerator, BindingsConfig}; mod callback_interface; mod compounds; mod custom; mod enum_; -mod executor; mod external; mod miscellany; mod object; mod primitives; mod record; +pub struct SwiftBindingGenerator; +impl BindingGenerator for SwiftBindingGenerator { + type Config = Config; + + fn write_bindings( + &self, + ci: &ComponentInterface, + config: &Config, + out_dir: &Utf8Path, + try_format_code: bool, + ) -> Result<()> { + swift::write_bindings(config, ci, out_dir, try_format_code) + } + + fn check_library_path( + &self, + _library_path: &Utf8Path, + _cdylib_name: Option<&str>, + ) -> Result<()> { + Ok(()) + } +} + /// A trait tor the implementation. trait CodeType: Debug { /// The language specific label used to reference this type. This will be used in @@ -196,6 +220,8 @@ pub struct Config { ffi_module_filename: Option, generate_module_map: Option, omit_argument_labels: Option, + generate_immutable_records: Option, + experimental_sendable_value_types: Option, #[serde(default)] custom_types: HashMap, } @@ -261,6 +287,16 @@ impl Config { pub fn omit_argument_labels(&self) -> bool { self.omit_argument_labels.unwrap_or(false) } + + /// Whether to generate immutable records (`let` instead of `var`) + pub fn generate_immutable_records(&self) -> bool { + self.generate_immutable_records.unwrap_or(false) + } + + /// Whether to mark value types as 'Sendable' + pub fn experimental_sendable_value_types(&self) -> bool { + self.experimental_sendable_value_types.unwrap_or(false) + } } impl BindingsConfig for Config { @@ -400,7 +436,6 @@ pub struct SwiftWrapper<'a> { ci: &'a ComponentInterface, type_helper_code: String, type_imports: BTreeSet, - has_async_fns: bool, } impl<'a> SwiftWrapper<'a> { pub fn new(config: Config, ci: &'a ComponentInterface) -> Self { @@ -412,7 +447,6 @@ impl<'a> SwiftWrapper<'a> { ci, type_helper_code, type_imports, - has_async_fns: ci.has_async_fns(), } } @@ -425,10 +459,6 @@ impl<'a> SwiftWrapper<'a> { .iter_types() .map(|t| SwiftCodeOracle.find(t)) .filter_map(|ct| ct.initialization_fn()) - .chain( - self.has_async_fns - .then(|| "uniffiInitContinuationCallback".into()), - ) .collect() } } @@ -464,12 +494,11 @@ impl SwiftCodeOracle { Type::Duration => Box::new(miscellany::DurationCodeType), Type::Enum { name, .. } => Box::new(enum_::EnumCodeType::new(name)), - Type::Object { name, .. } => Box::new(object::ObjectCodeType::new(name)), + Type::Object { name, imp, .. } => Box::new(object::ObjectCodeType::new(name, imp)), Type::Record { name, .. } => Box::new(record::RecordCodeType::new(name)), Type::CallbackInterface { name, .. } => { Box::new(callback_interface::CallbackInterfaceCodeType::new(name)) } - Type::ForeignExecutor => Box::new(executor::ForeignExecutorCodeType), Type::Optional { inner_type } => { Box::new(compounds::OptionalCodeType::new(*inner_type)) } @@ -509,7 +538,22 @@ impl SwiftCodeOracle { nm.to_string().to_lower_camel_case() } - fn ffi_type_label_raw(&self, ffi_type: &FfiType) -> String { + /// Get the idiomatic Swift rendering of an FFI callback function name + fn ffi_callback_name(&self, nm: &str) -> String { + format!("Uniffi{}", nm.to_upper_camel_case()) + } + + /// Get the idiomatic Swift rendering of an FFI struct name + fn ffi_struct_name(&self, nm: &str) -> String { + format!("Uniffi{}", nm.to_upper_camel_case()) + } + + /// Get the idiomatic Swift rendering of an if guard name + fn if_guard_name(&self, nm: &str) -> String { + format!("UNIFFI_FFIDEF_{}", nm.to_shouty_snake_case()) + } + + fn ffi_type_label(&self, ffi_type: &FfiType) -> String { match ffi_type { FfiType::Int8 => "Int8".into(), FfiType::UInt8 => "UInt8".into(), @@ -521,40 +565,74 @@ impl SwiftCodeOracle { FfiType::UInt64 => "UInt64".into(), FfiType::Float32 => "Float".into(), FfiType::Float64 => "Double".into(), + FfiType::Handle => "UInt64".into(), FfiType::RustArcPtr(_) => "UnsafeMutableRawPointer".into(), FfiType::RustBuffer(_) => "RustBuffer".into(), + FfiType::RustCallStatus => "RustCallStatus".into(), FfiType::ForeignBytes => "ForeignBytes".into(), - FfiType::ForeignCallback => "ForeignCallback".into(), - FfiType::ForeignExecutorHandle => "Int".into(), - FfiType::ForeignExecutorCallback => "ForeignExecutorCallback".into(), - FfiType::RustFutureContinuationCallback => "UniFfiRustFutureContinuation".into(), - FfiType::RustFutureHandle | FfiType::RustFutureContinuationData => { - "UnsafeMutableRawPointer".into() + // Note: @escaping is required for Swift versions before 5.7 for callbacks passed into + // async functions. Swift 5.7 and later does not require it. We should probably remove + // it once we upgrade our minimum requirement to 5.7 or later. + FfiType::Callback(name) => format!("@escaping {}", self.ffi_callback_name(name)), + FfiType::Struct(name) => self.ffi_struct_name(name), + FfiType::Reference(inner) => { + format!("UnsafeMutablePointer<{}>", self.ffi_type_label(inner)) } + FfiType::VoidPointer => "UnsafeMutableRawPointer".into(), } } - fn ffi_type_label(&self, ffi_type: &FfiType) -> String { - match ffi_type { - FfiType::ForeignCallback - | FfiType::ForeignExecutorCallback - | FfiType::RustFutureHandle - | FfiType::RustFutureContinuationCallback - | FfiType::RustFutureContinuationData => { - format!("{} _Nonnull", self.ffi_type_label_raw(ffi_type)) - } - _ => self.ffi_type_label_raw(ffi_type), + /// Default values for FFI types + /// + /// Used to set a default return value when returning an error + fn ffi_default_value(&self, return_type: Option<&FfiType>) -> String { + match return_type { + Some(t) => match t { + FfiType::UInt8 + | FfiType::Int8 + | FfiType::UInt16 + | FfiType::Int16 + | FfiType::UInt32 + | FfiType::Int32 + | FfiType::UInt64 + | FfiType::Int64 => "0".to_owned(), + FfiType::Float32 | FfiType::Float64 => "0.0".to_owned(), + FfiType::RustArcPtr(_) => "nil".to_owned(), + FfiType::RustBuffer(_) => "RustBuffer.empty()".to_owned(), + _ => unimplemented!("FFI return type: {t:?}"), + }, + // When we need to use a value for void returns, we use a `u8` placeholder + None => "0".to_owned(), } } fn ffi_canonical_name(&self, ffi_type: &FfiType) -> String { - self.ffi_type_label_raw(ffi_type) + self.ffi_type_label(ffi_type) + } + + /// Get the name of the protocol and class name for an object. + /// + /// If we support callback interfaces, the protocol name is the object name, and the class name is derived from that. + /// Otherwise, the class name is the object name and the protocol name is derived from that. + /// + /// This split determines what types `FfiConverter.lower()` inputs. If we support callback + /// interfaces, `lower` must lower anything that implements the protocol. If not, then lower + /// only lowers the concrete class. + fn object_names(&self, obj: &Object) -> (String, String) { + let class_name = self.class_name(obj.name()); + if obj.has_callback_interface() { + let impl_name = format!("{class_name}Impl"); + (class_name, impl_name) + } else { + (format!("{class_name}Protocol"), class_name) + } } } pub mod filters { use super::*; pub use crate::backend::filters::*; + use uniffi_meta::LiteralMetadata; fn oracle() -> &'static SwiftCodeOracle { &SwiftCodeOracle @@ -564,6 +642,13 @@ pub mod filters { Ok(oracle().find(&as_type.as_type()).type_label()) } + pub fn return_type_name(as_type: Option<&impl AsType>) -> Result { + Ok(match as_type { + Some(as_type) => oracle().find(&as_type.as_type()).type_label(), + None => "()".to_owned(), + }) + } + pub fn canonical_name(as_type: &impl AsType) -> Result { Ok(oracle().find(&as_type.as_type()).canonical_name()) } @@ -572,6 +657,15 @@ pub mod filters { Ok(oracle().find(&as_type.as_type()).ffi_converter_name()) } + pub fn ffi_error_converter_name(as_type: &impl AsType) -> Result { + // special handling for types used as errors. + let mut name = oracle().find(&as_type.as_type()).ffi_converter_name(); + if matches!(&as_type.as_type(), Type::Object { .. }) { + name.push_str("__as_error") + } + Ok(name) + } + pub fn lower_fn(as_type: &impl AsType) -> Result { Ok(oracle().find(&as_type.as_type()).lower()) } @@ -595,6 +689,16 @@ pub mod filters { Ok(oracle().find(&as_type.as_type()).literal(literal)) } + // Get the idiomatic Swift rendering of an individual enum variant's discriminant + pub fn variant_discr_literal(e: &Enum, index: &usize) -> Result { + let literal = e.variant_discr(*index).expect("invalid index"); + match literal { + LiteralMetadata::UInt(v, _, _) => Ok(v.to_string()), + LiteralMetadata::Int(v, _, _) => Ok(v.to_string()), + _ => unreachable!("expected an UInt!"), + } + } + /// Get the Swift type for an FFIType pub fn ffi_type_name(ffi_type: &FfiType) -> Result { Ok(oracle().ffi_type_label(ffi_type)) @@ -604,6 +708,10 @@ pub mod filters { Ok(oracle().ffi_canonical_name(ffi_type)) } + pub fn ffi_default_value(return_type: Option) -> Result { + Ok(oracle().ffi_default_value(return_type.as_ref())) + } + /// Like `ffi_type_name`, but used in `BridgingHeaderTemplate.h` which uses a slightly different /// names. pub fn header_ffi_type_name(ffi_type: &FfiType) -> Result { @@ -618,18 +726,17 @@ pub mod filters { FfiType::UInt64 => "uint64_t".into(), FfiType::Float32 => "float".into(), FfiType::Float64 => "double".into(), + FfiType::Handle => "uint64_t".into(), FfiType::RustArcPtr(_) => "void*_Nonnull".into(), FfiType::RustBuffer(_) => "RustBuffer".into(), + FfiType::RustCallStatus => "RustCallStatus".into(), FfiType::ForeignBytes => "ForeignBytes".into(), - FfiType::ForeignCallback => "ForeignCallback _Nonnull".into(), - FfiType::ForeignExecutorCallback => "UniFfiForeignExecutorCallback _Nonnull".into(), - FfiType::ForeignExecutorHandle => "size_t".into(), - FfiType::RustFutureContinuationCallback => { - "UniFfiRustFutureContinuation _Nonnull".into() - } - FfiType::RustFutureHandle | FfiType::RustFutureContinuationData => { - "void* _Nonnull".into() + FfiType::Callback(name) => { + format!("{} _Nonnull", SwiftCodeOracle.ffi_callback_name(name)) } + FfiType::Struct(name) => SwiftCodeOracle.ffi_struct_name(name), + FfiType::Reference(inner) => format!("{}* _Nonnull", header_ffi_type_name(inner)?), + FfiType::VoidPointer => "void* _Nonnull".into(), }) } @@ -664,6 +771,30 @@ pub mod filters { Ok(oracle().enum_variant_name(nm)) } + /// Get the idiomatic Swift rendering of an FFI callback function name + pub fn ffi_callback_name(nm: &str) -> Result { + Ok(oracle().ffi_callback_name(nm)) + } + + /// Get the idiomatic Swift rendering of an FFI struct name + pub fn ffi_struct_name(nm: &str) -> Result { + Ok(oracle().ffi_struct_name(nm)) + } + + /// Get the idiomatic Swift rendering of an if guard name + pub fn if_guard_name(nm: &str) -> Result { + Ok(oracle().if_guard_name(nm)) + } + + /// Get the idiomatic Swift rendering of docstring + pub fn docstring(docstring: &str, spaces: &i32) -> Result { + let middle = textwrap::indent(&textwrap::dedent(docstring), " * "); + let wrapped = format!("/**\n{middle}\n */"); + + let spaces = usize::try_from(*spaces).unwrap_or_default(); + Ok(textwrap::indent(&wrapped, &" ".repeat(spaces))) + } + pub fn error_handler(result: &ResultType) -> Result { Ok(match &result.throws_type { Some(t) => format!("{}.lift", ffi_converter_name(t)?), @@ -685,4 +816,8 @@ pub mod filters { } )) } + + pub fn object_names(obj: &Object) -> Result<(String, String), askama::Error> { + Ok(SwiftCodeOracle.object_names(obj)) + } } diff --git a/third_party/rust/uniffi_bindgen/src/bindings/swift/gen_swift/object.rs b/third_party/rust/uniffi_bindgen/src/bindings/swift/gen_swift/object.rs index ea140c998d..d4497a7b19 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/swift/gen_swift/object.rs +++ b/third_party/rust/uniffi_bindgen/src/bindings/swift/gen_swift/object.rs @@ -3,24 +3,32 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ use super::CodeType; +use crate::interface::ObjectImpl; #[derive(Debug)] pub struct ObjectCodeType { - id: String, + name: String, + imp: ObjectImpl, } impl ObjectCodeType { - pub fn new(id: String) -> Self { - Self { id } + pub fn new(name: String, imp: ObjectImpl) -> Self { + Self { name, imp } } } impl CodeType for ObjectCodeType { fn type_label(&self) -> String { - super::SwiftCodeOracle.class_name(&self.id) + super::SwiftCodeOracle.class_name(&self.name) } fn canonical_name(&self) -> String { - format!("Type{}", self.id) + format!("Type{}", self.name) + } + + fn initialization_fn(&self) -> Option { + self.imp + .has_callback_interface() + .then(|| format!("uniffiCallbackInit{}", self.name)) } } diff --git a/third_party/rust/uniffi_bindgen/src/bindings/swift/gen_swift/primitives.rs b/third_party/rust/uniffi_bindgen/src/bindings/swift/gen_swift/primitives.rs index 86424658a3..e0c670520e 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/swift/gen_swift/primitives.rs +++ b/third_party/rust/uniffi_bindgen/src/bindings/swift/gen_swift/primitives.rs @@ -9,7 +9,11 @@ use paste::paste; fn render_literal(literal: &Literal) -> String { fn typed_number(type_: &Type, num_str: String) -> String { - match type_ { + let unwrapped_type = match type_ { + Type::Optional { inner_type } => inner_type, + t => t, + }; + match unwrapped_type { // special case Int32. Type::Int32 => num_str, // otherwise use constructor e.g. UInt8(x) @@ -29,7 +33,7 @@ fn render_literal(literal: &Literal) -> String { super::SwiftCodeOracle.find(type_).type_label() ) } - _ => panic!("Unexpected literal: {num_str} is not a number"), + _ => panic!("Unexpected literal: {num_str} for type: {type_:?}"), } } diff --git a/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/Async.swift b/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/Async.swift index 695208861d..e16f3108e1 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/Async.swift +++ b/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/Async.swift @@ -1,11 +1,13 @@ private let UNIFFI_RUST_FUTURE_POLL_READY: Int8 = 0 private let UNIFFI_RUST_FUTURE_POLL_MAYBE_READY: Int8 = 1 +fileprivate let uniffiContinuationHandleMap = UniffiHandleMap>() + fileprivate func uniffiRustCallAsync( - rustFutureFunc: () -> UnsafeMutableRawPointer, - pollFunc: (UnsafeMutableRawPointer, UnsafeMutableRawPointer) -> (), - completeFunc: (UnsafeMutableRawPointer, UnsafeMutablePointer) -> F, - freeFunc: (UnsafeMutableRawPointer) -> (), + rustFutureFunc: () -> UInt64, + pollFunc: (UInt64, @escaping UniffiRustFutureContinuationCallback, UInt64) -> (), + completeFunc: (UInt64, UnsafeMutablePointer) -> F, + freeFunc: (UInt64) -> (), liftFunc: (F) throws -> T, errorHandler: ((RustBuffer) throws -> Error)? ) async throws -> T { @@ -19,7 +21,11 @@ fileprivate func uniffiRustCallAsync( var pollResult: Int8; repeat { pollResult = await withUnsafeContinuation { - pollFunc(rustFuture, ContinuationHolder($0).toOpaque()) + pollFunc( + rustFuture, + uniffiFutureContinuationCallback, + uniffiContinuationHandleMap.insert(obj: $0) + ) } } while pollResult != UNIFFI_RUST_FUTURE_POLL_READY @@ -31,32 +37,80 @@ fileprivate func uniffiRustCallAsync( // Callback handlers for an async calls. These are invoked by Rust when the future is ready. They // lift the return value or error and resume the suspended function. -fileprivate func uniffiFutureContinuationCallback(ptr: UnsafeMutableRawPointer, pollResult: Int8) { - ContinuationHolder.fromOpaque(ptr).resume(pollResult) +fileprivate func uniffiFutureContinuationCallback(handle: UInt64, pollResult: Int8) { + if let continuation = try? uniffiContinuationHandleMap.remove(handle: handle) { + continuation.resume(returning: pollResult) + } else { + print("uniffiFutureContinuationCallback invalid handle") + } } -// Wraps UnsafeContinuation in a class so that we can use reference counting when passing it across -// the FFI -fileprivate class ContinuationHolder { - let continuation: UnsafeContinuation - - init(_ continuation: UnsafeContinuation) { - self.continuation = continuation +{%- if ci.has_async_callback_interface_definition() %} +private func uniffiTraitInterfaceCallAsync( + makeCall: @escaping () async throws -> T, + handleSuccess: @escaping (T) -> (), + handleError: @escaping (Int8, RustBuffer) -> () +) -> UniffiForeignFuture { + let task = Task { + do { + handleSuccess(try await makeCall()) + } catch { + handleError(CALL_UNEXPECTED_ERROR, {{ Type::String.borrow()|lower_fn }}(String(describing: error))) + } } + let handle = UNIFFI_FOREIGN_FUTURE_HANDLE_MAP.insert(obj: task) + return UniffiForeignFuture(handle: handle, free: uniffiForeignFutureFree) - func resume(_ pollResult: Int8) { - self.continuation.resume(returning: pollResult) - } +} - func toOpaque() -> UnsafeMutableRawPointer { - return Unmanaged.passRetained(self).toOpaque() +private func uniffiTraitInterfaceCallAsyncWithError( + makeCall: @escaping () async throws -> T, + handleSuccess: @escaping (T) -> (), + handleError: @escaping (Int8, RustBuffer) -> (), + lowerError: @escaping (E) -> RustBuffer +) -> UniffiForeignFuture { + let task = Task { + do { + handleSuccess(try await makeCall()) + } catch let error as E { + handleError(CALL_ERROR, lowerError(error)) + } catch { + handleError(CALL_UNEXPECTED_ERROR, {{ Type::String.borrow()|lower_fn }}(String(describing: error))) + } } + let handle = UNIFFI_FOREIGN_FUTURE_HANDLE_MAP.insert(obj: task) + return UniffiForeignFuture(handle: handle, free: uniffiForeignFutureFree) +} + +// Borrow the callback handle map implementation to store foreign future handles +// TODO: consolidate the handle-map code (https://github.com/mozilla/uniffi-rs/pull/1823) +fileprivate var UNIFFI_FOREIGN_FUTURE_HANDLE_MAP = UniffiHandleMap() + +// Protocol for tasks that handle foreign futures. +// +// Defining a protocol allows all tasks to be stored in the same handle map. This can't be done +// with the task object itself, since has generic parameters. +protocol UniffiForeignFutureTask { + func cancel() +} + +extension Task: UniffiForeignFutureTask {} - static func fromOpaque(_ ptr: UnsafeRawPointer) -> ContinuationHolder { - return Unmanaged.fromOpaque(ptr).takeRetainedValue() +private func uniffiForeignFutureFree(handle: UInt64) { + do { + let task = try UNIFFI_FOREIGN_FUTURE_HANDLE_MAP.remove(handle: handle) + // Set the cancellation flag on the task. If it's still running, the code can check the + // cancellation flag or call `Task.checkCancellation()`. If the task has completed, this is + // a no-op. + task.cancel() + } catch { + print("uniffiForeignFutureFree: handle missing from handlemap") } } -fileprivate func uniffiInitContinuationCallback() { - {{ ci.ffi_rust_future_continuation_callback_set().name() }}(uniffiFutureContinuationCallback) +// For testing +public func uniffiForeignFutureHandleCount{{ ci.namespace()|class_name }}() -> Int { + UNIFFI_FOREIGN_FUTURE_HANDLE_MAP.count } + +{%- endif %} diff --git a/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/BridgingHeaderTemplate.h b/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/BridgingHeaderTemplate.h index 87698e359f..89d98594d3 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/BridgingHeaderTemplate.h +++ b/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/BridgingHeaderTemplate.h @@ -24,25 +24,11 @@ typedef struct RustBuffer { - int32_t capacity; - int32_t len; + uint64_t capacity; + uint64_t len; uint8_t *_Nullable data; } RustBuffer; -typedef int32_t (*ForeignCallback)(uint64_t, int32_t, const uint8_t *_Nonnull, int32_t, RustBuffer *_Nonnull); - -// Task defined in Rust that Swift executes -typedef void (*UniFfiRustTaskCallback)(const void * _Nullable, int8_t); - -// Callback to execute Rust tasks using a Swift Task -// -// Args: -// executor: ForeignExecutor lowered into a size_t value -// delay: Delay in MS -// task: UniFfiRustTaskCallback to call -// task_data: data to pass the task callback -typedef int8_t (*UniFfiForeignExecutorCallback)(size_t, uint32_t, UniFfiRustTaskCallback _Nullable, const void * _Nullable); - typedef struct ForeignBytes { int32_t len; @@ -59,11 +45,29 @@ typedef struct RustCallStatus { // ⚠️ increment the version suffix in all instances of UNIFFI_SHARED_HEADER_V4 in this file. ⚠️ #endif // def UNIFFI_SHARED_H -// Continuation callback for UniFFI Futures -typedef void (*UniFfiRustFutureContinuation)(void * _Nonnull, int8_t); - -// Scaffolding functions -{%- for func in ci.iter_ffi_function_definitions() %} +{%- for def in ci.ffi_definitions() %} +#ifndef {{ def.name()|if_guard_name }} +#define {{ def.name()|if_guard_name }} +{%- match def %} +{% when FfiDefinition::CallbackFunction(callback) %} +typedef + {%- match callback.return_type() %}{% when Some(return_type) %} {{ return_type|header_ffi_type_name }} {% when None %} void {% endmatch -%} + (*{{ callback.name()|ffi_callback_name }})( + {%- for arg in callback.arguments() -%} + {{ arg.type_().borrow()|header_ffi_type_name }} + {%- if !loop.last || callback.has_rust_call_status_arg() %}, {% endif %} + {%- endfor -%} + {%- if callback.has_rust_call_status_arg() %} + RustCallStatus *_Nonnull uniffiCallStatus + {%- endif %} + ); +{% when FfiDefinition::Struct(struct) %} +typedef struct {{ struct.name()|ffi_struct_name }} { + {%- for field in struct.fields() %} + {{ field.type_().borrow()|header_ffi_type_name }} {{ field.name()|var_name }}; + {%- endfor %} +} {{ struct.name()|ffi_struct_name }}; +{% when FfiDefinition::Function(func) %} {% match func.return_type() -%}{%- when Some with (type_) %}{{ type_|header_ffi_type_name }}{% when None %}void{% endmatch %} {{ func.name() }}( {%- if func.arguments().len() > 0 %} {%- for arg in func.arguments() %} @@ -74,6 +78,8 @@ typedef void (*UniFfiRustFutureContinuation)(void * _Nonnull, int8_t); {%- if func.has_rust_call_status_arg() %}RustCallStatus *_Nonnull out_status{%- else %}void{% endif %} {% endif %} ); +{%- endmatch %} +#endif {%- endfor %} {% import "macros.swift" as swift %} diff --git a/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/CallbackInterfaceImpl.swift b/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/CallbackInterfaceImpl.swift new file mode 100644 index 0000000000..74ee372642 --- /dev/null +++ b/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/CallbackInterfaceImpl.swift @@ -0,0 +1,113 @@ +{%- if self.include_once_check("CallbackInterfaceRuntime.swift") %}{%- include "CallbackInterfaceRuntime.swift" %}{%- endif %} +{%- let trait_impl=format!("UniffiCallbackInterface{}", name) %} + +// Put the implementation in a struct so we don't pollute the top-level namespace +fileprivate struct {{ trait_impl }} { + + // Create the VTable using a series of closures. + // Swift automatically converts these into C callback functions. + static var vtable: {{ vtable|ffi_type_name }} = {{ vtable|ffi_type_name }}( + {%- for (ffi_callback, meth) in vtable_methods %} + {{ meth.name()|fn_name }}: { ( + {%- for arg in ffi_callback.arguments() %} + {{ arg.name()|var_name }}: {{ arg.type_().borrow()|ffi_type_name }}{% if !loop.last || ffi_callback.has_rust_call_status_arg() %},{% endif %} + {%- endfor -%} + {%- if ffi_callback.has_rust_call_status_arg() %} + uniffiCallStatus: UnsafeMutablePointer + {%- endif %} + ) in + let makeCall = { + () {% if meth.is_async() %}async {% endif %}throws -> {% match meth.return_type() %}{% when Some(t) %}{{ t|type_name }}{% when None %}(){% endmatch %} in + guard let uniffiObj = try? {{ ffi_converter_name }}.handleMap.get(handle: uniffiHandle) else { + throw UniffiInternalError.unexpectedStaleHandle + } + return {% if meth.throws() %}try {% endif %}{% if meth.is_async() %}await {% endif %}uniffiObj.{{ meth.name()|fn_name }}( + {%- for arg in meth.arguments() %} + {% if !config.omit_argument_labels() %} {{ arg.name()|arg_name }}: {% endif %}try {{ arg|lift_fn }}({{ arg.name()|var_name }}){% if !loop.last %},{% endif %} + {%- endfor %} + ) + } + {%- if !meth.is_async() %} + + {% match meth.return_type() %} + {%- when Some(t) %} + let writeReturn = { uniffiOutReturn.pointee = {{ t|lower_fn }}($0) } + {%- when None %} + let writeReturn = { () } + {%- endmatch %} + + {%- match meth.throws_type() %} + {%- when None %} + uniffiTraitInterfaceCall( + callStatus: uniffiCallStatus, + makeCall: makeCall, + writeReturn: writeReturn + ) + {%- when Some(error_type) %} + uniffiTraitInterfaceCallWithError( + callStatus: uniffiCallStatus, + makeCall: makeCall, + writeReturn: writeReturn, + lowerError: {{ error_type|lower_fn }} + ) + {%- endmatch %} + {%- else %} + + let uniffiHandleSuccess = { (returnValue: {{ meth.return_type()|return_type_name }}) in + uniffiFutureCallback( + uniffiCallbackData, + {{ meth.foreign_future_ffi_result_struct().name()|ffi_struct_name }}( + {%- match meth.return_type() %} + {%- when Some(return_type) %} + returnValue: {{ return_type|lower_fn }}(returnValue), + {%- when None %} + {%- endmatch %} + callStatus: RustCallStatus() + ) + ) + } + let uniffiHandleError = { (statusCode, errorBuf) in + uniffiFutureCallback( + uniffiCallbackData, + {{ meth.foreign_future_ffi_result_struct().name()|ffi_struct_name }}( + {%- match meth.return_type() %} + {%- when Some(return_type) %} + returnValue: {{ meth.return_type().map(FfiType::from)|ffi_default_value }}, + {%- when None %} + {%- endmatch %} + callStatus: RustCallStatus(code: statusCode, errorBuf: errorBuf) + ) + ) + } + + {%- match meth.throws_type() %} + {%- when None %} + let uniffiForeignFuture = uniffiTraitInterfaceCallAsync( + makeCall: makeCall, + handleSuccess: uniffiHandleSuccess, + handleError: uniffiHandleError + ) + {%- when Some(error_type) %} + let uniffiForeignFuture = uniffiTraitInterfaceCallAsyncWithError( + makeCall: makeCall, + handleSuccess: uniffiHandleSuccess, + handleError: uniffiHandleError, + lowerError: {{ error_type|lower_fn }} + ) + {%- endmatch %} + uniffiOutReturn.pointee = uniffiForeignFuture + {%- endif %} + }, + {%- endfor %} + uniffiFree: { (uniffiHandle: UInt64) -> () in + let result = try? {{ ffi_converter_name }}.handleMap.remove(handle: uniffiHandle) + if result == nil { + print("Uniffi callback interface {{ name }}: handle missing in uniffiFree") + } + } + ) +} + +private func {{ callback_init }}() { + {{ ffi_init_callback.name() }}(&{{ trait_impl }}.vtable) +} diff --git a/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/CallbackInterfaceRuntime.swift b/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/CallbackInterfaceRuntime.swift index 9ae62d1667..5863c2ad41 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/CallbackInterfaceRuntime.swift +++ b/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/CallbackInterfaceRuntime.swift @@ -1,60 +1,3 @@ -fileprivate extension NSLock { - func withLock(f: () throws -> T) rethrows -> T { - self.lock() - defer { self.unlock() } - return try f() - } -} - -fileprivate typealias UniFFICallbackHandle = UInt64 -fileprivate class UniFFICallbackHandleMap { - private var leftMap: [UniFFICallbackHandle: T] = [:] - private var counter: [UniFFICallbackHandle: UInt64] = [:] - private var rightMap: [ObjectIdentifier: UniFFICallbackHandle] = [:] - - private let lock = NSLock() - private var currentHandle: UniFFICallbackHandle = 0 - private let stride: UniFFICallbackHandle = 1 - - func insert(obj: T) -> UniFFICallbackHandle { - lock.withLock { - let id = ObjectIdentifier(obj as AnyObject) - let handle = rightMap[id] ?? { - currentHandle += stride - let handle = currentHandle - leftMap[handle] = obj - rightMap[id] = handle - return handle - }() - counter[handle] = (counter[handle] ?? 0) + 1 - return handle - } - } - - func get(handle: UniFFICallbackHandle) -> T? { - lock.withLock { - leftMap[handle] - } - } - - func delete(handle: UniFFICallbackHandle) { - remove(handle: handle) - } - - @discardableResult - func remove(handle: UniFFICallbackHandle) -> T? { - lock.withLock { - defer { counter[handle] = (counter[handle] ?? 1) - 1 } - guard counter[handle] == 1 else { return leftMap[handle] } - let obj = leftMap.removeValue(forKey: handle) - if let obj = obj { - rightMap.removeValue(forKey: ObjectIdentifier(obj as AnyObject)) - } - return obj - } - } -} - // Magic number for the Rust proxy to call using the same mechanism as every other method, // to free the callback once it's dropped by Rust. private let IDX_CALLBACK_FREE: Int32 = 0 diff --git a/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/CallbackInterfaceTemplate.swift b/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/CallbackInterfaceTemplate.swift index aec8ded930..7aa1cca9b2 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/CallbackInterfaceTemplate.swift +++ b/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/CallbackInterfaceTemplate.swift @@ -1,150 +1,39 @@ {%- let cbi = ci|get_callback_interface_definition(name) %} -{%- let foreign_callback = format!("foreignCallback{}", canonical_type_name) %} -{%- if self.include_once_check("CallbackInterfaceRuntime.swift") %}{%- include "CallbackInterfaceRuntime.swift" %}{%- endif %} - -// Declaration and FfiConverters for {{ type_name }} Callback Interface - -public protocol {{ type_name }} : AnyObject { - {% for meth in cbi.methods() -%} - func {{ meth.name()|fn_name }}({% call swift::arg_list_protocol(meth) %}) {% call swift::throws(meth) -%} - {%- match meth.return_type() -%} - {%- when Some with (return_type) %} -> {{ return_type|type_name -}} - {%- else -%} - {%- endmatch %} - {% endfor %} -} - -// The ForeignCallback that is passed to Rust. -fileprivate let {{ foreign_callback }} : ForeignCallback = - { (handle: UniFFICallbackHandle, method: Int32, argsData: UnsafePointer, argsLen: Int32, out_buf: UnsafeMutablePointer) -> Int32 in - {% for meth in cbi.methods() -%} - {%- let method_name = format!("invoke_{}", meth.name())|fn_name %} - - func {{ method_name }}(_ swiftCallbackInterface: {{ type_name }}, _ argsData: UnsafePointer, _ argsLen: Int32, _ out_buf: UnsafeMutablePointer) throws -> Int32 { - {%- if meth.arguments().len() > 0 %} - var reader = createReader(data: Data(bytes: argsData, count: Int(argsLen))) - {%- endif %} - - {%- match meth.return_type() %} - {%- when Some(return_type) %} - func makeCall() throws -> Int32 { - let result = {% if meth.throws() %} try{% endif %} swiftCallbackInterface.{{ meth.name()|fn_name }}( - {% for arg in meth.arguments() -%} - {% if !config.omit_argument_labels() %}{{ arg.name()|var_name }}: {% endif %} try {{ arg|read_fn }}(from: &reader) - {%- if !loop.last %}, {% endif %} - {% endfor -%} - ) - var writer = [UInt8]() - {{ return_type|write_fn }}(result, into: &writer) - out_buf.pointee = RustBuffer(bytes: writer) - return UNIFFI_CALLBACK_SUCCESS - } - {%- when None %} - func makeCall() throws -> Int32 { - try swiftCallbackInterface.{{ meth.name()|fn_name }}( - {% for arg in meth.arguments() -%} - {% if !config.omit_argument_labels() %}{{ arg.name()|var_name }}: {% endif %} try {{ arg|read_fn }}(from: &reader) - {%- if !loop.last %}, {% endif %} - {% endfor -%} - ) - return UNIFFI_CALLBACK_SUCCESS - } - {%- endmatch %} - - {%- match meth.throws_type() %} - {%- when None %} - return try makeCall() - {%- when Some(error_type) %} - do { - return try makeCall() - } catch let error as {{ error_type|type_name }} { - out_buf.pointee = {{ error_type|lower_fn }}(error) - return UNIFFI_CALLBACK_ERROR - } - {%- endmatch %} - } - {%- endfor %} - - - switch method { - case IDX_CALLBACK_FREE: - {{ ffi_converter_name }}.drop(handle: handle) - // Sucessful return - // See docs of ForeignCallback in `uniffi_core/src/ffi/foreigncallbacks.rs` - return UNIFFI_CALLBACK_SUCCESS - {% for meth in cbi.methods() -%} - {% let method_name = format!("invoke_{}", meth.name())|fn_name -%} - case {{ loop.index }}: - let cb: {{ cbi|type_name }} - do { - cb = try {{ ffi_converter_name }}.lift(handle) - } catch { - out_buf.pointee = {{ Type::String.borrow()|lower_fn }}("{{ cbi.name() }}: Invalid handle") - return UNIFFI_CALLBACK_UNEXPECTED_ERROR - } - do { - return try {{ method_name }}(cb, argsData, argsLen, out_buf) - } catch let error { - out_buf.pointee = {{ Type::String.borrow()|lower_fn }}(String(describing: error)) - return UNIFFI_CALLBACK_UNEXPECTED_ERROR - } - {% endfor %} - // This should never happen, because an out of bounds method index won't - // ever be used. Once we can catch errors, we should return an InternalError. - // https://github.com/mozilla/uniffi-rs/issues/351 - default: - // An unexpected error happened. - // See docs of ForeignCallback in `uniffi_core/src/ffi/foreigncallbacks.rs` - return UNIFFI_CALLBACK_UNEXPECTED_ERROR - } -} +{%- let callback_handler = format!("uniffiCallbackHandler{}", name) %} +{%- let callback_init = format!("uniffiCallbackInit{}", name) %} +{%- let methods = cbi.methods() %} +{%- let protocol_name = type_name.clone() %} +{%- let protocol_docstring = cbi.docstring() %} +{%- let vtable = cbi.vtable() %} +{%- let vtable_methods = cbi.vtable_methods() %} +{%- let ffi_init_callback = cbi.ffi_init_callback() %} + +{% include "Protocol.swift" %} +{% include "CallbackInterfaceImpl.swift" %} // FfiConverter protocol for callback interfaces fileprivate struct {{ ffi_converter_name }} { - private static let initCallbackOnce: () = { - // Swift ensures this initializer code will once run once, even when accessed by multiple threads. - try! rustCall { (err: UnsafeMutablePointer) in - {{ cbi.ffi_init_callback().name() }}({{ foreign_callback }}, err) - } - }() - - private static func ensureCallbackinitialized() { - _ = initCallbackOnce - } - - static func drop(handle: UniFFICallbackHandle) { - handleMap.remove(handle: handle) - } - - private static var handleMap = UniFFICallbackHandleMap<{{ type_name }}>() + fileprivate static var handleMap = UniffiHandleMap<{{ type_name }}>() } extension {{ ffi_converter_name }} : FfiConverter { typealias SwiftType = {{ type_name }} - // We can use Handle as the FfiType because it's a typealias to UInt64 - typealias FfiType = UniFFICallbackHandle + typealias FfiType = UInt64 - public static func lift(_ handle: UniFFICallbackHandle) throws -> SwiftType { - ensureCallbackinitialized(); - guard let callback = handleMap.get(handle: handle) else { - throw UniffiInternalError.unexpectedStaleHandle - } - return callback + public static func lift(_ handle: UInt64) throws -> SwiftType { + try handleMap.get(handle: handle) } public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> SwiftType { - ensureCallbackinitialized(); - let handle: UniFFICallbackHandle = try readInt(&buf) + let handle: UInt64 = try readInt(&buf) return try lift(handle) } - public static func lower(_ v: SwiftType) -> UniFFICallbackHandle { - ensureCallbackinitialized(); + public static func lower(_ v: SwiftType) -> UInt64 { return handleMap.insert(obj: v) } public static func write(_ v: SwiftType, into buf: inout [UInt8]) { - ensureCallbackinitialized(); writeInt(&buf, lower(v)) } } diff --git a/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/EnumTemplate.swift b/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/EnumTemplate.swift index 99f45290cc..1d8b3cf500 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/EnumTemplate.swift +++ b/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/EnumTemplate.swift @@ -1,10 +1,26 @@ // Note that we don't yet support `indirect` for enums. // See https://github.com/mozilla/uniffi-rs/issues/396 for further discussion. +{%- call swift::docstring(e, 0) %} +{% match e.variant_discr_type() %} +{% when None %} public enum {{ type_name }} { {% for variant in e.variants() %} - case {{ variant.name()|enum_variant_swift_quoted }}{% if variant.fields().len() > 0 %}({% call swift::field_list_decl(variant) %}){% endif -%} + {%- call swift::docstring(variant, 4) %} + case {{ variant.name()|enum_variant_swift_quoted }}{% if variant.fields().len() > 0 %}( + {%- call swift::field_list_decl(variant, variant.has_nameless_fields()) %} + ){% endif -%} {% endfor %} } +{% when Some with (variant_discr_type) %} +public enum {{ type_name }} : {{ variant_discr_type|type_name }} { + {% for variant in e.variants() %} + {%- call swift::docstring(variant, 4) %} + case {{ variant.name()|enum_variant_swift_quoted }} = {{ e|variant_discr_literal(loop.index0) }}{% if variant.fields().len() > 0 %}( + {%- call swift::field_list_decl(variant, variant.has_nameless_fields()) %} + ){% endif -%} + {% endfor %} +} +{% endmatch %} public struct {{ ffi_converter_name }}: FfiConverterRustBuffer { typealias SwiftType = {{ type_name }} @@ -15,7 +31,11 @@ public struct {{ ffi_converter_name }}: FfiConverterRustBuffer { {% for variant in e.variants() %} case {{ loop.index }}: return .{{ variant.name()|enum_variant_swift_quoted }}{% if variant.has_fields() %}( {%- for field in variant.fields() %} + {%- if variant.has_nameless_fields() -%} + try {{ field|read_fn }}(from: &buf) + {%- else -%} {{ field.name()|arg_name }}: try {{ field|read_fn }}(from: &buf) + {%- endif -%} {%- if !loop.last %}, {% endif %} {%- endfor %} ){%- endif %} @@ -28,10 +48,10 @@ public struct {{ ffi_converter_name }}: FfiConverterRustBuffer { switch value { {% for variant in e.variants() %} {% if variant.has_fields() %} - case let .{{ variant.name()|enum_variant_swift_quoted }}({% for field in variant.fields() %}{{ field.name()|var_name }}{%- if loop.last -%}{%- else -%},{%- endif -%}{% endfor %}): + case let .{{ variant.name()|enum_variant_swift_quoted }}({% for field in variant.fields() %}{%- call swift::field_name(field, loop.index) -%}{%- if loop.last -%}{%- else -%},{%- endif -%}{% endfor %}): writeInt(&buf, Int32({{ loop.index }})) {% for field in variant.fields() -%} - {{ field|write_fn }}({{ field.name()|var_name }}, into: &buf) + {{ field|write_fn }}({% call swift::field_name(field, loop.index) %}, into: &buf) {% endfor -%} {% else %} case .{{ variant.name()|enum_variant_swift_quoted }}: @@ -55,5 +75,6 @@ public func {{ ffi_converter_name }}_lower(_ value: {{ type_name }}) -> RustBuff } {% if !contains_object_references %} +{% if config.experimental_sendable_value_types() %}extension {{ type_name }}: Sendable {} {% endif %} extension {{ type_name }}: Equatable, Hashable {} {% endif %} diff --git a/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/ErrorTemplate.swift b/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/ErrorTemplate.swift index 786091395b..0702c477e9 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/ErrorTemplate.swift +++ b/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/ErrorTemplate.swift @@ -1,21 +1,21 @@ +{%- call swift::docstring(e, 0) %} public enum {{ type_name }} { {% if e.is_flat() %} {% for variant in e.variants() %} - // Simple error enums only carry a message + {%- call swift::docstring(variant, 4) %} case {{ variant.name()|class_name }}(message: String) {% endfor %} {%- else %} {% for variant in e.variants() %} - case {{ variant.name()|class_name }}{% if variant.fields().len() > 0 %}({% call swift::field_list_decl(variant) %}){% endif -%} + {%- call swift::docstring(variant, 4) %} + case {{ variant.name()|class_name }}{% if variant.fields().len() > 0 %}( + {%- call swift::field_list_decl(variant, variant.has_nameless_fields()) %} + ){% endif -%} {% endfor %} {%- endif %} - - fileprivate static func uniffiErrorHandler(_ error: RustBuffer) throws -> Error { - return try {{ ffi_converter_name }}.lift(error) - } } diff --git a/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/ForeignExecutorTemplate.swift b/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/ForeignExecutorTemplate.swift deleted file mode 100644 index 167e4c7546..0000000000 --- a/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/ForeignExecutorTemplate.swift +++ /dev/null @@ -1,69 +0,0 @@ -private let UNIFFI_RUST_TASK_CALLBACK_SUCCESS: Int8 = 0 -private let UNIFFI_RUST_TASK_CALLBACK_CANCELLED: Int8 = 1 -private let UNIFFI_FOREIGN_EXECUTOR_CALLBACK_SUCCESS: Int8 = 0 -private let UNIFFI_FOREIGN_EXECUTOR_CALLBACK_CANCELED: Int8 = 1 -private let UNIFFI_FOREIGN_EXECUTOR_CALLBACK_ERROR: Int8 = 2 - -// Encapsulates an executor that can run Rust tasks -// -// On Swift, `Task.detached` can handle this we just need to know what priority to send it. -public struct UniFfiForeignExecutor { - var priority: TaskPriority - - public init(priority: TaskPriority) { - self.priority = priority - } - - public init() { - self.priority = Task.currentPriority - } -} - -fileprivate struct FfiConverterForeignExecutor: FfiConverter { - typealias SwiftType = UniFfiForeignExecutor - // Rust uses a pointer to represent the FfiConverterForeignExecutor, but we only need a u8. - // let's use `Int`, which is equivalent to `size_t` - typealias FfiType = Int - - public static func lift(_ value: FfiType) throws -> SwiftType { - UniFfiForeignExecutor(priority: TaskPriority(rawValue: numericCast(value))) - } - public static func lower(_ value: SwiftType) -> FfiType { - numericCast(value.priority.rawValue) - } - - public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> SwiftType { - fatalError("FfiConverterForeignExecutor.read not implemented yet") - } - public static func write(_ value: SwiftType, into buf: inout [UInt8]) { - fatalError("FfiConverterForeignExecutor.read not implemented yet") - } -} - - -fileprivate func uniffiForeignExecutorCallback(executorHandle: Int, delayMs: UInt32, rustTask: UniFfiRustTaskCallback?, taskData: UnsafeRawPointer?) -> Int8 { - if let rustTask = rustTask { - let executor = try! FfiConverterForeignExecutor.lift(executorHandle) - Task.detached(priority: executor.priority) { - if delayMs != 0 { - let nanoseconds: UInt64 = numericCast(delayMs * 1000000) - try! await Task.sleep(nanoseconds: nanoseconds) - } - rustTask(taskData, UNIFFI_RUST_TASK_CALLBACK_SUCCESS) - } - return UNIFFI_FOREIGN_EXECUTOR_CALLBACK_SUCCESS - } else { - // When rustTask is null, we should drop the foreign executor. - // However, since its just a value type, we don't need to do anything here. - return UNIFFI_FOREIGN_EXECUTOR_CALLBACK_SUCCESS - } -} - -fileprivate func uniffiInitForeignExecutor() { - {%- match ci.ffi_foreign_executor_callback_set() %} - {%- when Some with (fn) %} - {{ fn.name() }}(uniffiForeignExecutorCallback) - {%- when None %} - {#- No foreign executor, we don't set anything #} - {% endmatch %} -} diff --git a/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/HandleMap.swift b/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/HandleMap.swift new file mode 100644 index 0000000000..6de9f085d6 --- /dev/null +++ b/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/HandleMap.swift @@ -0,0 +1,40 @@ +fileprivate class UniffiHandleMap { + private var map: [UInt64: T] = [:] + private let lock = NSLock() + private var currentHandle: UInt64 = 1 + + func insert(obj: T) -> UInt64 { + lock.withLock { + let handle = currentHandle + currentHandle += 1 + map[handle] = obj + return handle + } + } + + func get(handle: UInt64) throws -> T { + try lock.withLock { + guard let obj = map[handle] else { + throw UniffiInternalError.unexpectedStaleHandle + } + return obj + } + } + + @discardableResult + func remove(handle: UInt64) throws -> T { + try lock.withLock { + guard let obj = map.removeValue(forKey: handle) else { + throw UniffiInternalError.unexpectedStaleHandle + } + return obj + } + } + + var count: Int { + get { + map.count + } + } +} + diff --git a/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/Helpers.swift b/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/Helpers.swift index a34b128e23..cfddf7b313 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/Helpers.swift +++ b/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/Helpers.swift @@ -26,9 +26,17 @@ fileprivate enum UniffiInternalError: LocalizedError { } } +fileprivate extension NSLock { + func withLock(f: () throws -> T) rethrows -> T { + self.lock() + defer { self.unlock() } + return try f() + } +} + fileprivate let CALL_SUCCESS: Int8 = 0 fileprivate let CALL_ERROR: Int8 = 1 -fileprivate let CALL_PANIC: Int8 = 2 +fileprivate let CALL_UNEXPECTED_ERROR: Int8 = 2 fileprivate let CALL_CANCELLED: Int8 = 3 fileprivate extension RustCallStatus { @@ -81,7 +89,7 @@ private func uniffiCheckCallStatus( throw UniffiInternalError.unexpectedRustCallError } - case CALL_PANIC: + case CALL_UNEXPECTED_ERROR: // When the rust code sees a panic, it tries to construct a RustBuffer // with the message. But if that code panics, then it just sends back // an empty buffer. @@ -93,9 +101,39 @@ private func uniffiCheckCallStatus( } case CALL_CANCELLED: - throw CancellationError() + fatalError("Cancellation not supported yet") default: throw UniffiInternalError.unexpectedRustCallStatusCode } } + +private func uniffiTraitInterfaceCall( + callStatus: UnsafeMutablePointer, + makeCall: () throws -> T, + writeReturn: (T) -> () +) { + do { + try writeReturn(makeCall()) + } catch let error { + callStatus.pointee.code = CALL_UNEXPECTED_ERROR + callStatus.pointee.errorBuf = {{ Type::String.borrow()|lower_fn }}(String(describing: error)) + } +} + +private func uniffiTraitInterfaceCallWithError( + callStatus: UnsafeMutablePointer, + makeCall: () throws -> T, + writeReturn: (T) -> (), + lowerError: (E) -> RustBuffer +) { + do { + try writeReturn(makeCall()) + } catch let error as E { + callStatus.pointee.code = CALL_ERROR + callStatus.pointee.errorBuf = lowerError(error) + } catch { + callStatus.pointee.code = CALL_UNEXPECTED_ERROR + callStatus.pointee.errorBuf = {{ Type::String.borrow()|lower_fn }}(String(describing: error)) + } +} diff --git a/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/ObjectTemplate.swift b/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/ObjectTemplate.swift index 57a77ca6df..0c28bc4c09 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/ObjectTemplate.swift +++ b/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/ObjectTemplate.swift @@ -1,104 +1,146 @@ {%- let obj = ci|get_object_definition(name) %} -public protocol {{ obj.name() }}Protocol { - {% for meth in obj.methods() -%} - func {{ meth.name()|fn_name }}({% call swift::arg_list_protocol(meth) %}) {% call swift::async(meth) %} {% call swift::throws(meth) -%} - {%- match meth.return_type() -%} - {%- when Some with (return_type) %} -> {{ return_type|type_name -}} - {%- else -%} - {%- endmatch %} - {% endfor %} -} - -public class {{ type_name }}: {{ obj.name() }}Protocol { - fileprivate let pointer: UnsafeMutableRawPointer +{%- let (protocol_name, impl_class_name) = obj|object_names %} +{%- let methods = obj.methods() %} +{%- let protocol_docstring = obj.docstring() %} + +{%- let is_error = ci.is_name_used_as_error(name) %} + +{% include "Protocol.swift" %} + +{%- call swift::docstring(obj, 0) %} +open class {{ impl_class_name }}: + {%- for tm in obj.uniffi_traits() %} + {%- match tm %} + {%- when UniffiTrait::Display { fmt } %} + CustomStringConvertible, + {%- when UniffiTrait::Debug { fmt } %} + CustomDebugStringConvertible, + {%- when UniffiTrait::Eq { eq, ne } %} + Equatable, + {%- when UniffiTrait::Hash { hash } %} + Hashable, + {%- else %} + {%- endmatch %} + {%- endfor %} + {%- if is_error %} + Error, + {% endif %} + {{ protocol_name }} { + fileprivate let pointer: UnsafeMutableRawPointer! + + /// Used to instantiate a [FFIObject] without an actual pointer, for fakes in tests, mostly. + public struct NoPointer { + public init() {} + } // TODO: We'd like this to be `private` but for Swifty reasons, // we can't implement `FfiConverter` without making this `required` and we can't // make it `required` without making it `public`. - required init(unsafeFromRawPointer pointer: UnsafeMutableRawPointer) { + required public init(unsafeFromRawPointer pointer: UnsafeMutableRawPointer) { self.pointer = pointer } + /// This constructor can be used to instantiate a fake object. + /// - Parameter noPointer: Placeholder value so we can have a constructor separate from the default empty one that may be implemented for classes extending [FFIObject]. + /// + /// - Warning: + /// Any object instantiated with this constructor cannot be passed to an actual Rust-backed object. Since there isn't a backing [Pointer] the FFI lower functions will crash. + public init(noPointer: NoPointer) { + self.pointer = nil + } + + public func uniffiClonePointer() -> UnsafeMutableRawPointer { + return try! rustCall { {{ obj.ffi_object_clone().name() }}(self.pointer, $0) } + } + {%- match obj.primary_constructor() %} {%- when Some with (cons) %} - public convenience init({% call swift::arg_list_decl(cons) -%}) {% call swift::throws(cons) %} { - self.init(unsafeFromRawPointer: {% call swift::to_ffi_call(cons) %}) - } + {%- call swift::ctor_decl(cons, 4) %} {%- when None %} + // No primary constructor declared for this class. {%- endmatch %} deinit { + guard let pointer = pointer else { + return + } + try! rustCall { {{ obj.ffi_object_free().name() }}(pointer, $0) } } {% for cons in obj.alternate_constructors() %} - - public static func {{ cons.name()|fn_name }}({% call swift::arg_list_decl(cons) %}) {% call swift::throws(cons) %} -> {{ type_name }} { - return {{ type_name }}(unsafeFromRawPointer: {% call swift::to_ffi_call(cons) %}) - } - + {%- call swift::func_decl("public static func", cons, 4) %} {% endfor %} - {# // TODO: Maybe merge the two templates (i.e the one with a return type and the one without) #} {% for meth in obj.methods() -%} - {%- if meth.is_async() %} - - public func {{ meth.name()|fn_name }}({%- call swift::arg_list_decl(meth) -%}) async {% call swift::throws(meth) %}{% match meth.return_type() %}{% when Some with (return_type) %} -> {{ return_type|type_name }}{% when None %}{% endmatch %} { - return {% call swift::try(meth) %} await uniffiRustCallAsync( - rustFutureFunc: { - {{ meth.ffi_func().name() }}( - self.pointer - {%- for arg in meth.arguments() -%} - , - {{ arg|lower_fn }}({{ arg.name()|var_name }}) - {%- endfor %} - ) - }, - pollFunc: {{ meth.ffi_rust_future_poll(ci) }}, - completeFunc: {{ meth.ffi_rust_future_complete(ci) }}, - freeFunc: {{ meth.ffi_rust_future_free(ci) }}, - {%- match meth.return_type() %} - {%- when Some(return_type) %} - liftFunc: {{ return_type|lift_fn }}, - {%- when None %} - liftFunc: { $0 }, - {%- endmatch %} - {%- match meth.throws_type() %} - {%- when Some with (e) %} - errorHandler: {{ e|ffi_converter_name }}.lift - {%- else %} - errorHandler: nil - {% endmatch %} + {%- call swift::func_decl("open func", meth, 4) %} + {% endfor %} + + {%- for tm in obj.uniffi_traits() %} + {%- match tm %} + {%- when UniffiTrait::Display { fmt } %} + open var description: String { + return {% call swift::try(fmt) %} {{ fmt.return_type().unwrap()|lift_fn }}( + {% call swift::to_ffi_call(fmt) %} ) } - - {% else -%} - - {%- match meth.return_type() -%} - - {%- when Some with (return_type) %} - - public func {{ meth.name()|fn_name }}({% call swift::arg_list_decl(meth) %}) {% call swift::throws(meth) %} -> {{ return_type|type_name }} { - return {% call swift::try(meth) %} {{ return_type|lift_fn }}( - {% call swift::to_ffi_call_with_prefix("self.pointer", meth) %} + {%- when UniffiTrait::Debug { fmt } %} + open var debugDescription: String { + return {% call swift::try(fmt) %} {{ fmt.return_type().unwrap()|lift_fn }}( + {% call swift::to_ffi_call(fmt) %} ) } - - {%- when None %} - - public func {{ meth.name()|fn_name }}({% call swift::arg_list_decl(meth) %}) {% call swift::throws(meth) %} { - {% call swift::to_ffi_call_with_prefix("self.pointer", meth) %} + {%- when UniffiTrait::Eq { eq, ne } %} + public static func == (self: {{ impl_class_name }}, other: {{ impl_class_name }}) -> Bool { + return {% call swift::try(eq) %} {{ eq.return_type().unwrap()|lift_fn }}( + {% call swift::to_ffi_call(eq) %} + ) + } + {%- when UniffiTrait::Hash { hash } %} + open func hash(into hasher: inout Hasher) { + let val = {% call swift::try(hash) %} {{ hash.return_type().unwrap()|lift_fn }}( + {% call swift::to_ffi_call(hash) %} + ) + hasher.combine(val) } + {%- else %} + {%- endmatch %} + {%- endfor %} - {%- endmatch -%} - {%- endif -%} - {% endfor %} } +{%- if obj.has_callback_interface() %} +{%- let callback_handler = format!("uniffiCallbackInterface{}", name) %} +{%- let callback_init = format!("uniffiCallbackInit{}", name) %} +{%- let vtable = obj.vtable().expect("trait interface should have a vtable") %} +{%- let vtable_methods = obj.vtable_methods() %} +{%- let ffi_init_callback = obj.ffi_init_callback() %} +{% include "CallbackInterfaceImpl.swift" %} +{%- endif %} + public struct {{ ffi_converter_name }}: FfiConverter { + {%- if obj.has_callback_interface() %} + fileprivate static var handleMap = UniffiHandleMap<{{ type_name }}>() + {%- endif %} + typealias FfiType = UnsafeMutableRawPointer typealias SwiftType = {{ type_name }} + public static func lift(_ pointer: UnsafeMutableRawPointer) throws -> {{ type_name }} { + return {{ impl_class_name }}(unsafeFromRawPointer: pointer) + } + + public static func lower(_ value: {{ type_name }}) -> UnsafeMutableRawPointer { + {%- if obj.has_callback_interface() %} + guard let ptr = UnsafeMutableRawPointer(bitPattern: UInt(truncatingIfNeeded: handleMap.insert(obj: value))) else { + fatalError("Cast to UnsafeMutableRawPointer failed") + } + return ptr + {%- else %} + return value.uniffiClonePointer() + {%- endif %} + } + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> {{ type_name }} { let v: UInt64 = try readInt(&buf) // The Rust code won't compile if a pointer won't fit in a UInt64. @@ -115,15 +157,30 @@ public struct {{ ffi_converter_name }}: FfiConverter { // The Rust code won't compile if a pointer won't fit in a `UInt64`. writeInt(&buf, UInt64(bitPattern: Int64(Int(bitPattern: lower(value))))) } +} - public static func lift(_ pointer: UnsafeMutableRawPointer) throws -> {{ type_name }} { - return {{ type_name}}(unsafeFromRawPointer: pointer) +{# Objects as error #} +{%- if is_error %} +{# Due to some mismatches in the ffi converter mechanisms, errors are a RustBuffer holding a pointer #} +public struct {{ ffi_converter_name }}__as_error: FfiConverterRustBuffer { + public static func lift(_ buf: RustBuffer) throws -> {{ type_name }} { + var reader = createReader(data: Data(rustBuffer: buf)) + return try {{ ffi_converter_name }}.read(from: &reader) } - public static func lower(_ value: {{ type_name }}) -> UnsafeMutableRawPointer { - return value.pointer + public static func lower(_ value: {{ type_name }}) -> RustBuffer { + fatalError("not implemented") + } + + public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> {{ type_name }} { + fatalError("not implemented") + } + + public static func write(_ value: {{ type_name }}, into buf: inout [UInt8]) { + fatalError("not implemented") } } +{%- endif %} {# We always write these public functions just in case the enum is used as diff --git a/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/Protocol.swift b/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/Protocol.swift new file mode 100644 index 0000000000..7df953558a --- /dev/null +++ b/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/Protocol.swift @@ -0,0 +1,12 @@ +{%- call swift::docstring_value(protocol_docstring, 0) %} +public protocol {{ protocol_name }} : AnyObject { + {% for meth in methods.iter() -%} + {%- call swift::docstring(meth, 4) %} + func {{ meth.name()|fn_name }}({% call swift::arg_list_protocol(meth) %}) {% call swift::async(meth) -%}{% call swift::throws(meth) -%} + {%- match meth.return_type() -%} + {%- when Some with (return_type) %} -> {{ return_type|type_name -}} + {%- else -%} + {%- endmatch %} + {% endfor %} +} + diff --git a/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/RecordTemplate.swift b/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/RecordTemplate.swift index 44de9dd358..c262a7a216 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/RecordTemplate.swift +++ b/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/RecordTemplate.swift @@ -1,12 +1,14 @@ {%- let rec = ci|get_record_definition(name) %} +{%- call swift::docstring(rec, 0) %} public struct {{ type_name }} { {%- for field in rec.fields() %} - public var {{ field.name()|var_name }}: {{ field|type_name }} + {%- call swift::docstring(field, 4) %} + public {% if config.generate_immutable_records() %}let{% else %}var{% endif %} {{ field.name()|var_name }}: {{ field|type_name }} {%- endfor %} // Default memberwise initializers are never public by default, so we // declare one manually. - public init({% call swift::field_list_decl(rec) %}) { + public init({% call swift::field_list_decl(rec, false) %}) { {%- for field in rec.fields() %} self.{{ field.name()|var_name }} = {{ field.name()|var_name }} {%- endfor %} @@ -14,6 +16,7 @@ public struct {{ type_name }} { } {% if !contains_object_references %} +{% if config.experimental_sendable_value_types() %}extension {{ type_name }}: Sendable {} {% endif %} extension {{ type_name }}: Equatable, Hashable { public static func ==(lhs: {{ type_name }}, rhs: {{ type_name }}) -> Bool { {%- for field in rec.fields() %} @@ -34,12 +37,16 @@ extension {{ type_name }}: Equatable, Hashable { public struct {{ ffi_converter_name }}: FfiConverterRustBuffer { public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> {{ type_name }} { - return try {{ type_name }}( + return {%- if rec.has_fields() %} + try {{ type_name }}( {%- for field in rec.fields() %} - {{ field.name()|arg_name }}: {{ field|read_fn }}(from: &buf) - {%- if !loop.last %}, {% endif %} + {{ field.name()|arg_name }}: {{ field|read_fn }}(from: &buf) + {%- if !loop.last %}, {% endif %} {%- endfor %} ) + {%- else %} + {{ type_name }}() + {%- endif %} } public static func write(_ value: {{ type_name }}, into buf: inout [UInt8]) { diff --git a/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/RustBufferTemplate.swift b/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/RustBufferTemplate.swift index 2f737b6635..a053334a30 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/RustBufferTemplate.swift +++ b/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/RustBufferTemplate.swift @@ -7,6 +7,10 @@ fileprivate extension RustBuffer { self.init(capacity: rbuf.capacity, len: rbuf.len, data: rbuf.data) } + static func empty() -> RustBuffer { + RustBuffer(capacity: 0, len:0, data: nil) + } + static func from(_ ptr: UnsafeBufferPointer) -> RustBuffer { try! rustCall { {{ ci.ffi_rustbuffer_from_bytes().name() }}(ForeignBytes(bufferPointer: ptr), $0) } } diff --git a/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/TopLevelFunctionTemplate.swift b/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/TopLevelFunctionTemplate.swift index a2c6311931..ce946076f7 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/TopLevelFunctionTemplate.swift +++ b/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/TopLevelFunctionTemplate.swift @@ -1,48 +1 @@ -{%- if func.is_async() %} - -public func {{ func.name()|fn_name }}({%- call swift::arg_list_decl(func) -%}) async {% call swift::throws(func) %}{% match func.return_type() %}{% when Some with (return_type) %} -> {{ return_type|type_name }}{% when None %}{% endmatch %} { - return {% call swift::try(func) %} await uniffiRustCallAsync( - rustFutureFunc: { - {{ func.ffi_func().name() }}( - {%- for arg in func.arguments() %} - {{ arg|lower_fn }}({{ arg.name()|var_name }}){% if !loop.last %},{% endif %} - {%- endfor %} - ) - }, - pollFunc: {{ func.ffi_rust_future_poll(ci) }}, - completeFunc: {{ func.ffi_rust_future_complete(ci) }}, - freeFunc: {{ func.ffi_rust_future_free(ci) }}, - {%- match func.return_type() %} - {%- when Some(return_type) %} - liftFunc: {{ return_type|lift_fn }}, - {%- when None %} - liftFunc: { $0 }, - {%- endmatch %} - {%- match func.throws_type() %} - {%- when Some with (e) %} - errorHandler: {{ e|ffi_converter_name }}.lift - {%- else %} - errorHandler: nil - {% endmatch %} - ) -} - -{% else %} - -{%- match func.return_type() -%} -{%- when Some with (return_type) %} - -public func {{ func.name()|fn_name }}({%- call swift::arg_list_decl(func) -%}) {% call swift::throws(func) %} -> {{ return_type|type_name }} { - return {% call swift::try(func) %} {{ return_type|lift_fn }}( - {% call swift::to_ffi_call(func) %} - ) -} - -{%- when None %} - -public func {{ func.name()|fn_name }}({% call swift::arg_list_decl(func) %}) {% call swift::throws(func) %} { - {% call swift::to_ffi_call(func) %} -} - -{% endmatch %} -{%- endif %} +{%- call swift::func_decl("public func", func, 0) %} diff --git a/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/Types.swift b/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/Types.swift index aba34f4b0b..5e26758f3c 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/Types.swift +++ b/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/Types.swift @@ -64,9 +64,6 @@ {%- when Type::CallbackInterface { name, module_path } %} {%- include "CallbackInterfaceTemplate.swift" %} -{%- when Type::ForeignExecutor %} -{%- include "ForeignExecutorTemplate.swift" %} - {%- when Type::Custom { name, module_path, builtin } %} {%- include "CustomType.swift" %} diff --git a/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/macros.swift b/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/macros.swift index 0a125e6f61..8692cd6ff0 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/macros.swift +++ b/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/macros.swift @@ -8,28 +8,97 @@ {%- call try(func) -%} {%- match func.throws_type() -%} {%- when Some with (e) -%} - rustCallWithError({{ e|ffi_converter_name }}.lift) { + rustCallWithError({{ e|ffi_error_converter_name }}.lift) { {%- else -%} rustCall() { {%- endmatch %} - {{ func.ffi_func().name() }}({% call arg_list_lowered(func) -%} $0) + {{ func.ffi_func().name() }}( + {%- if func.takes_self() %}self.uniffiClonePointer(),{% endif %} + {%- call arg_list_lowered(func) -%} $0 + ) } {%- endmacro -%} -{%- macro to_ffi_call_with_prefix(prefix, func) -%} -{% call try(func) %} - {%- match func.throws_type() %} - {%- when Some with (e) %} - rustCallWithError({{ e|ffi_converter_name }}.lift) { +// eg, `public func foo_bar() { body }` +{%- macro func_decl(func_decl, callable, indent) %} +{%- call docstring(callable, indent) %} +{{ func_decl }} {{ callable.name()|fn_name }}( + {%- call arg_list_decl(callable) -%}) + {%- call async(callable) %} + {%- call throws(callable) %} + {%- match callable.return_type() %} + {%- when Some with (return_type) %} -> {{ return_type|type_name }} + {%- when None %} + {%- endmatch %} { + {%- call call_body(callable) %} +} +{%- endmacro %} + +// primary ctor - no name, no return-type. +{%- macro ctor_decl(callable, indent) %} +{%- call docstring(callable, indent) %} +public convenience init( + {%- call arg_list_decl(callable) -%}) {%- call async(callable) %} {%- call throws(callable) %} { + {%- if callable.is_async() %} + let pointer = + {%- call call_async(callable) %} + {# The async mechanism returns an already constructed self. + We work around that by cloning the pointer from that object, then + assune the old object dies as there are no other references possible. + #} + .uniffiClonePointer() {%- else %} - rustCall() { - {% endmatch %} - {{ func.ffi_func().name() }}( - {{- prefix }}, {% call arg_list_lowered(func) -%} $0 - ) + let pointer = + {% call to_ffi_call(callable) %} + {%- endif %} + self.init(unsafeFromRawPointer: pointer) } {%- endmacro %} +{%- macro call_body(callable) %} +{%- if callable.is_async() %} + return {%- call call_async(callable) %} +{%- else %} +{%- match callable.return_type() -%} +{%- when Some with (return_type) %} + return {% call try(callable) %} {{ return_type|lift_fn }}({% call to_ffi_call(callable) %}) +{%- when None %} +{%- call to_ffi_call(callable) %} +{%- endmatch %} +{%- endif %} + +{%- endmacro %} + +{%- macro call_async(callable) %} + {% call try(callable) %} await uniffiRustCallAsync( + rustFutureFunc: { + {{ callable.ffi_func().name() }}( + {%- if callable.takes_self() %} + self.uniffiClonePointer(){% if !callable.arguments().is_empty() %},{% endif %} + {% endif %} + {%- for arg in callable.arguments() -%} + {{ arg|lower_fn }}({{ arg.name()|var_name }}){% if !loop.last %},{% endif %} + {%- endfor %} + ) + }, + pollFunc: {{ callable.ffi_rust_future_poll(ci) }}, + completeFunc: {{ callable.ffi_rust_future_complete(ci) }}, + freeFunc: {{ callable.ffi_rust_future_free(ci) }}, + {%- match callable.return_type() %} + {%- when Some(return_type) %} + liftFunc: {{ return_type|lift_fn }}, + {%- when None %} + liftFunc: { $0 }, + {%- endmatch %} + {%- match callable.throws_type() %} + {%- when Some with (e) %} + errorHandler: {{ e|ffi_error_converter_name }}.lift + {%- else %} + errorHandler: nil + {% endmatch %} + ) +{%- endmacro %} + {%- macro arg_list_lowered(func) %} {%- for arg in func.arguments() %} {{ arg|lower_fn }}({{ arg.name()|var_name }}), @@ -56,17 +125,30 @@ // Field lists as used in Swift declarations of Records and Enums. // Note the var_name and type_name filters. -#} -{% macro field_list_decl(item) %} +{% macro field_list_decl(item, has_nameless_fields) %} {%- for field in item.fields() -%} + {%- call docstring(field, 8) %} + {%- if has_nameless_fields %} + {{- field|type_name -}} + {%- if !loop.last -%}, {%- endif -%} + {%- else -%} {{ field.name()|var_name }}: {{ field|type_name -}} {%- match field.default_value() %} {%- when Some with(literal) %} = {{ literal|literal_swift(field) }} {%- else %} {%- endmatch -%} {% if !loop.last %}, {% endif %} + {%- endif -%} {%- endfor %} {%- endmacro %} +{% macro field_name(field, field_num) %} +{%- if field.name().is_empty() -%} +v{{- field_num -}} +{%- else -%} +{{ field.name()|var_name }} +{%- endif -%} +{%- endmacro %} {% macro arg_list_protocol(func) %} {%- for arg in func.arguments() -%} @@ -75,15 +157,26 @@ {%- endfor %} {%- endmacro %} - {%- macro async(func) %} -{%- if func.is_async() %}async{% endif %} +{%- if func.is_async() %}async {% endif %} {%- endmacro -%} {%- macro throws(func) %} -{%- if func.throws() %}throws{% endif %} +{%- if func.throws() %}throws {% endif %} {%- endmacro -%} {%- macro try(func) %} {%- if func.throws() %}try {% else %}try! {% endif %} {%- endmacro -%} + +{%- macro docstring_value(maybe_docstring, indent_spaces) %} +{%- match maybe_docstring %} +{%- when Some(docstring) %} +{{ docstring|docstring(indent_spaces) }} +{%- else %} +{%- endmatch %} +{%- endmacro %} + +{%- macro docstring(defn, indent_spaces) %} +{%- call docstring_value(defn.docstring(), indent_spaces) %} +{%- endmacro %} diff --git a/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/wrapper.swift b/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/wrapper.swift index c34d348efb..17fdde74e0 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/wrapper.swift +++ b/third_party/rust/uniffi_bindgen/src/bindings/swift/templates/wrapper.swift @@ -1,5 +1,10 @@ // This file was autogenerated by some hot garbage in the `uniffi` crate. // Trust me, you don't want to mess with it! + +// swiftlint:disable all + +{%- call swift::docstring_value(ci.namespace_docstring(), 0) %} + {%- import "macros.swift" as swift %} import Foundation {%- for imported_class in self.imports() %} @@ -15,6 +20,7 @@ import {{ config.ffi_module_name() }} {% include "RustBufferTemplate.swift" %} {% include "Helpers.swift" %} +{% include "HandleMap.swift" %} // Public interface members begin here. {{ type_helper_code }} @@ -66,3 +72,5 @@ private func uniffiEnsureInitialized() { fatalError("UniFFI API checksum mismatch: try cleaning and rebuilding your project") } } + +// swiftlint:enable all diff --git a/third_party/rust/uniffi_bindgen/src/bindings/swift/test.rs b/third_party/rust/uniffi_bindgen/src/bindings/swift/test.rs index c3b2f15277..195a77696b 100644 --- a/third_party/rust/uniffi_bindgen/src/bindings/swift/test.rs +++ b/third_party/rust/uniffi_bindgen/src/bindings/swift/test.rs @@ -2,10 +2,7 @@ License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -use crate::{ - bindings::{RunScriptOptions, TargetLanguage}, - library_mode::generate_bindings, -}; +use crate::{bindings::RunScriptOptions, library_mode::generate_bindings, BindingGeneratorDefault}; use anyhow::{bail, Context, Result}; use camino::{Utf8Path, Utf8PathBuf}; use std::env::consts::{DLL_PREFIX, DLL_SUFFIX}; @@ -15,6 +12,8 @@ use std::io::Write; use std::process::{Command, Stdio}; use uniffi_testing::UniFFITestHelper; +use crate::bindings::TargetLanguage; + /// Run Swift tests for a UniFFI test fixture pub fn run_test(tmp_dir: &str, fixture_name: &str, script_file: &str) -> Result<()> { run_script( @@ -36,7 +35,7 @@ pub fn run_script( args: Vec, options: &RunScriptOptions, ) -> Result<()> { - let script_path = Utf8Path::new(".").join(script_file).canonicalize_utf8()?; + let script_path = Utf8Path::new(script_file).canonicalize_utf8()?; let test_helper = UniFFITestHelper::new(crate_name)?; let out_dir = test_helper.create_out_dir(tmp_dir, &script_path)?; let cdylib_path = test_helper.copy_cdylib_to_out_dir(&out_dir)?; @@ -126,8 +125,17 @@ struct GeneratedSources { impl GeneratedSources { fn new(crate_name: &str, cdylib_path: &Utf8Path, out_dir: &Utf8Path) -> Result { - let sources = - generate_bindings(cdylib_path, None, &[TargetLanguage::Swift], out_dir, false)?; + let sources = generate_bindings( + cdylib_path, + None, + &BindingGeneratorDefault { + target_languages: vec![TargetLanguage::Swift], + try_format_code: false, + }, + None, + out_dir, + false, + )?; let main_source = sources .iter() .find(|s| s.package.name == crate_name) @@ -169,7 +177,7 @@ fn create_command(program: &str, options: &RunScriptOptions) -> Command { if !options.show_compiler_messages { // This prevents most compiler messages, but not remarks command.arg("-suppress-warnings"); - // This gets the remarks. Note: swift will eventually get a `-supress-remarks` argument, + // This gets the remarks. Note: swift will eventually get a `-suppress-remarks` argument, // maybe we can eventually move to that command.stderr(Stdio::null()); } diff --git a/third_party/rust/uniffi_bindgen/src/interface/callbacks.rs b/third_party/rust/uniffi_bindgen/src/interface/callbacks.rs index e3bca4f966..f176a7a684 100644 --- a/third_party/rust/uniffi_bindgen/src/interface/callbacks.rs +++ b/third_party/rust/uniffi_bindgen/src/interface/callbacks.rs @@ -33,9 +33,12 @@ //! # Ok::<(), anyhow::Error>(()) //! ``` +use std::iter; + +use heck::ToUpperCamelCase; use uniffi_meta::Checksum; -use super::ffi::{FfiArgument, FfiFunction, FfiType}; +use super::ffi::{FfiArgument, FfiCallbackFunction, FfiField, FfiFunction, FfiStruct, FfiType}; use super::object::Method; use super::{AsType, Type, TypeIterator}; @@ -52,18 +55,11 @@ pub struct CallbackInterface { // avoids a weird circular dependency in the calculation. #[checksum_ignore] pub(super) ffi_init_callback: FfiFunction, + #[checksum_ignore] + pub(super) docstring: Option, } impl CallbackInterface { - pub fn new(name: String, module_path: String) -> CallbackInterface { - CallbackInterface { - name, - module_path, - methods: Default::default(), - ffi_init_callback: Default::default(), - } - } - pub fn name(&self) -> &str { &self.name } @@ -77,18 +73,45 @@ impl CallbackInterface { } pub(super) fn derive_ffi_funcs(&mut self) { - self.ffi_init_callback.name = - uniffi_meta::init_callback_fn_symbol_name(&self.module_path, &self.name); - self.ffi_init_callback.arguments = vec![FfiArgument { - name: "callback_stub".to_string(), - type_: FfiType::ForeignCallback, - }]; - self.ffi_init_callback.return_type = None; + self.ffi_init_callback = + FfiFunction::callback_init(&self.module_path, &self.name, vtable_name(&self.name)); + } + + /// FfiCallbacks to define for our methods. + pub fn ffi_callbacks(&self) -> Vec { + ffi_callbacks(&self.name, &self.methods) + } + + /// The VTable FFI type + pub fn vtable(&self) -> FfiType { + FfiType::Struct(vtable_name(&self.name)) + } + + /// the VTable struct to define. + pub fn vtable_definition(&self) -> FfiStruct { + vtable_struct(&self.name, &self.methods) + } + + /// Vec of (ffi_callback, method) pairs + pub fn vtable_methods(&self) -> Vec<(FfiCallbackFunction, &Method)> { + self.methods + .iter() + .enumerate() + .map(|(i, method)| (method_ffi_callback(&self.name, method, i), method)) + .collect() } pub fn iter_types(&self) -> TypeIterator<'_> { Box::new(self.methods.iter().flat_map(Method::iter_types)) } + + pub fn docstring(&self) -> Option<&str> { + self.docstring.as_deref() + } + + pub fn has_async_method(&self) -> bool { + self.methods.iter().any(Method::is_async) + } } impl AsType for CallbackInterface { @@ -100,6 +123,139 @@ impl AsType for CallbackInterface { } } +impl TryFrom for CallbackInterface { + type Error = anyhow::Error; + + fn try_from(meta: uniffi_meta::CallbackInterfaceMetadata) -> anyhow::Result { + Ok(Self { + name: meta.name, + module_path: meta.module_path, + methods: Default::default(), + ffi_init_callback: Default::default(), + docstring: meta.docstring.clone(), + }) + } +} + +/// [FfiCallbackFunction] functions for the methods of a callback/trait interface +pub fn ffi_callbacks(trait_name: &str, methods: &[Method]) -> Vec { + methods + .iter() + .enumerate() + .map(|(i, method)| method_ffi_callback(trait_name, method, i)) + .collect() +} + +pub fn method_ffi_callback(trait_name: &str, method: &Method, index: usize) -> FfiCallbackFunction { + if !method.is_async() { + FfiCallbackFunction { + name: method_ffi_callback_name(trait_name, index), + arguments: iter::once(FfiArgument::new("uniffi_handle", FfiType::UInt64)) + .chain(method.arguments().into_iter().map(Into::into)) + .chain(iter::once(match method.return_type() { + Some(t) => FfiArgument::new("uniffi_out_return", FfiType::from(t).reference()), + None => FfiArgument::new("uniffi_out_return", FfiType::VoidPointer), + })) + .collect(), + has_rust_call_status_arg: true, + return_type: None, + } + } else { + let completion_callback = + ffi_foreign_future_complete(method.return_type().map(FfiType::from)); + FfiCallbackFunction { + name: method_ffi_callback_name(trait_name, index), + arguments: iter::once(FfiArgument::new("uniffi_handle", FfiType::UInt64)) + .chain(method.arguments().into_iter().map(Into::into)) + .chain([ + FfiArgument::new( + "uniffi_future_callback", + FfiType::Callback(completion_callback.name), + ), + FfiArgument::new("uniffi_callback_data", FfiType::UInt64), + FfiArgument::new( + "uniffi_out_return", + FfiType::Struct("ForeignFuture".to_owned()).reference(), + ), + ]) + .collect(), + has_rust_call_status_arg: false, + return_type: None, + } + } +} + +/// Result struct to pass to the completion callback for async methods +pub fn foreign_future_ffi_result_struct(return_ffi_type: Option) -> FfiStruct { + let return_type_name = + FfiType::return_type_name(return_ffi_type.as_ref()).to_upper_camel_case(); + FfiStruct { + name: format!("ForeignFutureStruct{return_type_name}"), + fields: match return_ffi_type { + Some(return_ffi_type) => vec![ + FfiField::new("return_value", return_ffi_type), + FfiField::new("call_status", FfiType::RustCallStatus), + ], + None => vec![ + // In Rust, `return_value` is `()` -- a ZST. + // ZSTs are not valid in `C`, but they also take up 0 space. + // Skip the `return_value` field to make the layout correct. + FfiField::new("call_status", FfiType::RustCallStatus), + ], + }, + } +} + +/// Definition for callback functions to complete an async callback interface method +pub fn ffi_foreign_future_complete(return_ffi_type: Option) -> FfiCallbackFunction { + let return_type_name = + FfiType::return_type_name(return_ffi_type.as_ref()).to_upper_camel_case(); + FfiCallbackFunction { + name: format!("ForeignFutureComplete{return_type_name}"), + arguments: vec![ + FfiArgument::new("callback_data", FfiType::UInt64), + FfiArgument::new( + "result", + FfiType::Struct(format!("ForeignFutureStruct{return_type_name}")), + ), + ], + return_type: None, + has_rust_call_status_arg: false, + } +} + +/// [FfiStruct] for a callback/trait interface VTable +/// +/// This struct has a FfiCallbackFunction field for each method, plus extra fields for special +/// methods +pub fn vtable_struct(trait_name: &str, methods: &[Method]) -> FfiStruct { + FfiStruct { + name: vtable_name(trait_name), + fields: methods + .iter() + .enumerate() + .map(|(i, method)| { + FfiField::new( + method.name(), + FfiType::Callback(format!("CallbackInterface{trait_name}Method{i}")), + ) + }) + .chain([FfiField::new( + "uniffi_free", + FfiType::Callback("CallbackInterfaceFree".to_owned()), + )]) + .collect(), + } +} + +pub fn method_ffi_callback_name(trait_name: &str, index: usize) -> String { + format!("CallbackInterface{trait_name}Method{index}") +} + +pub fn vtable_name(trait_name: &str) -> String { + format!("VTableCallbackInterface{trait_name}") +} + #[cfg(test)] mod test { use super::super::ComponentInterface; @@ -146,4 +302,21 @@ mod test { assert_eq!(callbacks_two.methods()[0].name(), "two"); assert_eq!(callbacks_two.methods()[1].name(), "too"); } + + #[test] + fn test_docstring_callback_interface() { + const UDL: &str = r#" + namespace test{}; + /// informative docstring + callback interface Testing { }; + "#; + let ci = ComponentInterface::from_webidl(UDL, "crate_name").unwrap(); + assert_eq!( + ci.get_callback_interface_definition("Testing") + .unwrap() + .docstring() + .unwrap(), + "informative docstring" + ); + } } diff --git a/third_party/rust/uniffi_bindgen/src/interface/enum_.rs b/third_party/rust/uniffi_bindgen/src/interface/enum_.rs index 82baf1dd50..a666cc3605 100644 --- a/third_party/rust/uniffi_bindgen/src/interface/enum_.rs +++ b/third_party/rust/uniffi_bindgen/src/interface/enum_.rs @@ -94,7 +94,9 @@ //! //! ``` //! # let ci = uniffi_bindgen::interface::ComponentInterface::from_webidl(r##" -//! # namespace example {}; +//! # namespace example { +//! # [Throws=Example] void func(); +//! # }; //! # [Error] //! # enum Example { //! # "one", @@ -130,7 +132,9 @@ //! //! ``` //! # let ci = uniffi_bindgen::interface::ComponentInterface::from_webidl(r##" -//! # namespace example {}; +//! # namespace example { +//! # [Throws=Example] void func(); +//! # }; //! # [Error] //! # interface Example { //! # one(); @@ -159,7 +163,7 @@ use anyhow::Result; use uniffi_meta::Checksum; use super::record::Field; -use super::{AsType, Type, TypeIterator}; +use super::{AsType, Literal, Type, TypeIterator}; /// Represents an enum with named variants, each of which may have named /// and typed fields. @@ -170,6 +174,7 @@ use super::{AsType, Type, TypeIterator}; pub struct Enum { pub(super) name: String, pub(super) module_path: String, + pub(super) discr_type: Option, pub(super) variants: Vec, // NOTE: `flat` is a misleading name and to make matters worse, has 2 different // meanings depending on the context :( @@ -189,6 +194,9 @@ pub struct Enum { // * For an Enum not used as an error but which has no variants with data, `flat` will be // false when generating the scaffolding but `true` when generating bindings. pub(super) flat: bool, + pub(super) non_exhaustive: bool, + #[checksum_ignore] + pub(super) docstring: Option, } impl Enum { @@ -200,14 +208,61 @@ impl Enum { &self.variants } + // Get the literal value to use for the specified variant's discriminant. + // Follows Rust's rules when mixing specified and unspecified values; please + // file a bug if you find a case where it does not. + // However, it *does not* attempt to handle error cases - either cases where + // a discriminant is not unique, or where a discriminant would overflow the + // repr. The intention is that the Rust compiler itself will fail to build + // in those cases, so by the time this get's run we can be confident these + // error cases can't exist. + pub fn variant_discr(&self, variant_index: usize) -> Result { + if variant_index >= self.variants.len() { + anyhow::bail!("Invalid variant index {variant_index}"); + } + let mut next = 0; + let mut this; + let mut this_lit = Literal::new_uint(0); + for v in self.variants().iter().take(variant_index + 1) { + (this, this_lit) = match v.discr { + None => ( + next, + if (next as i64) < 0 { + Literal::new_int(next as i64) + } else { + Literal::new_uint(next) + }, + ), + Some(Literal::UInt(v, _, _)) => (v, Literal::new_uint(v)), + // in-practice, Literal::Int == a negative number. + Some(Literal::Int(v, _, _)) => (v as u64, Literal::new_int(v)), + _ => anyhow::bail!("Invalid literal type {v:?}"), + }; + next = this.wrapping_add(1); + } + Ok(this_lit) + } + + pub fn variant_discr_type(&self) -> &Option { + &self.discr_type + } + pub fn is_flat(&self) -> bool { self.flat } + pub fn is_non_exhaustive(&self) -> bool { + self.non_exhaustive + } + pub fn iter_types(&self) -> TypeIterator<'_> { Box::new(self.variants.iter().flat_map(Variant::iter_types)) } + pub fn docstring(&self) -> Option<&str> { + self.docstring.as_deref() + } + // Sadly can't use TryFrom due to the 'is_flat' complication. pub fn try_from_meta(meta: uniffi_meta::EnumMetadata, flat: bool) -> Result { // This is messy - error enums are considered "flat" if the user @@ -218,12 +273,15 @@ impl Enum { Ok(Self { name: meta.name, module_path: meta.module_path, + discr_type: meta.discr_type, variants: meta .variants .into_iter() .map(TryInto::try_into) .collect::>()?, flat, + non_exhaustive: meta.non_exhaustive, + docstring: meta.docstring.clone(), }) } } @@ -243,7 +301,10 @@ impl AsType for Enum { #[derive(Debug, Clone, Default, PartialEq, Eq, Checksum)] pub struct Variant { pub(super) name: String, + pub(super) discr: Option, pub(super) fields: Vec, + #[checksum_ignore] + pub(super) docstring: Option, } impl Variant { @@ -259,6 +320,14 @@ impl Variant { !self.fields.is_empty() } + pub fn has_nameless_fields(&self) -> bool { + self.fields.iter().any(|f| f.name.is_empty()) + } + + pub fn docstring(&self) -> Option<&str> { + self.docstring.as_deref() + } + pub fn iter_types(&self) -> TypeIterator<'_> { Box::new(self.fields.iter().flat_map(Field::iter_types)) } @@ -270,11 +339,13 @@ impl TryFrom for Variant { fn try_from(meta: uniffi_meta::VariantMetadata) -> Result { Ok(Self { name: meta.name, + discr: meta.discr, fields: meta .fields .into_iter() .map(TryInto::try_into) .collect::>()?, + docstring: meta.docstring.clone(), }) } } @@ -447,7 +518,10 @@ mod test { #[test] fn test_variants() { const UDL: &str = r#" - namespace test{}; + namespace test{ + [Throws=Testing] + void func(); + }; [Error] enum Testing { "one", "two", "three" }; "#; @@ -486,7 +560,10 @@ mod test { #[test] fn test_variant_data() { const UDL: &str = r#" - namespace test{}; + namespace test{ + [Throws=Testing] + void func(); + }; [Error] interface Testing { @@ -564,4 +641,141 @@ mod test { vec!["Normal", "Error"] ); } + + fn variant(val: Option) -> Variant { + Variant { + name: "v".to_string(), + discr: val.map(Literal::new_uint), + fields: vec![], + docstring: None, + } + } + + fn check_discrs(e: &mut Enum, vs: Vec) -> Vec { + e.variants = vs; + (0..e.variants.len()) + .map(|i| e.variant_discr(i).unwrap()) + .map(|l| match l { + Literal::UInt(v, _, _) => v, + _ => unreachable!(), + }) + .collect() + } + + #[test] + fn test_variant_values() { + let mut e = Enum { + module_path: "test".to_string(), + name: "test".to_string(), + discr_type: None, + variants: vec![], + flat: false, + non_exhaustive: false, + docstring: None, + }; + + assert!(e.variant_discr(0).is_err()); + + // single values + assert_eq!(check_discrs(&mut e, vec![variant(None)]), vec![0]); + assert_eq!(check_discrs(&mut e, vec![variant(Some(3))]), vec![3]); + + // no values + assert_eq!( + check_discrs(&mut e, vec![variant(None), variant(None)]), + vec![0, 1] + ); + + // values + assert_eq!( + check_discrs(&mut e, vec![variant(Some(1)), variant(Some(3))]), + vec![1, 3] + ); + + // mixed values + assert_eq!( + check_discrs(&mut e, vec![variant(None), variant(Some(3)), variant(None)]), + vec![0, 3, 4] + ); + + assert_eq!( + check_discrs( + &mut e, + vec![variant(Some(4)), variant(None), variant(Some(1))] + ), + vec![4, 5, 1] + ); + } + + #[test] + fn test_docstring_enum() { + const UDL: &str = r#" + namespace test{}; + /// informative docstring + enum Testing { "foo" }; + "#; + let ci = ComponentInterface::from_webidl(UDL, "crate_name").unwrap(); + assert_eq!( + ci.get_enum_definition("Testing") + .unwrap() + .docstring() + .unwrap(), + "informative docstring" + ); + } + + #[test] + fn test_docstring_enum_variant() { + const UDL: &str = r#" + namespace test{}; + enum Testing { + /// informative docstring + "foo" + }; + "#; + let ci = ComponentInterface::from_webidl(UDL, "crate_name").unwrap(); + assert_eq!( + ci.get_enum_definition("Testing").unwrap().variants()[0] + .docstring() + .unwrap(), + "informative docstring" + ); + } + + #[test] + fn test_docstring_associated_enum() { + const UDL: &str = r#" + namespace test{}; + /// informative docstring + [Enum] + interface Testing { }; + "#; + let ci = ComponentInterface::from_webidl(UDL, "crate_name").unwrap(); + assert_eq!( + ci.get_enum_definition("Testing") + .unwrap() + .docstring() + .unwrap(), + "informative docstring" + ); + } + + #[test] + fn test_docstring_associated_enum_variant() { + const UDL: &str = r#" + namespace test{}; + [Enum] + interface Testing { + /// informative docstring + testing(); + }; + "#; + let ci = ComponentInterface::from_webidl(UDL, "crate_name").unwrap(); + assert_eq!( + ci.get_enum_definition("Testing").unwrap().variants()[0] + .docstring() + .unwrap(), + "informative docstring" + ); + } } diff --git a/third_party/rust/uniffi_bindgen/src/interface/ffi.rs b/third_party/rust/uniffi_bindgen/src/interface/ffi.rs index d18aaf8262..b27cb78477 100644 --- a/third_party/rust/uniffi_bindgen/src/interface/ffi.rs +++ b/third_party/rust/uniffi_bindgen/src/interface/ffi.rs @@ -47,20 +47,49 @@ pub enum FfiType { /// A borrowed reference to some raw bytes owned by foreign language code. /// The provider of this reference must keep it alive for the duration of the receiving call. ForeignBytes, - /// Pointer to a callback function that handles all callbacks on the foreign language side. - ForeignCallback, - /// Pointer-sized opaque handle that represents a foreign executor. Foreign bindings can - /// either use an actual pointer or a usized integer. - ForeignExecutorHandle, - /// Pointer to the callback function that's invoked to schedule calls with a ForeignExecutor - ForeignExecutorCallback, - /// Pointer to a Rust future - RustFutureHandle, - /// Continuation function for a Rust future - RustFutureContinuationCallback, - RustFutureContinuationData, - // TODO: you can imagine a richer structural typesystem here, e.g. `Ref` or something. - // We don't need that yet and it's possible we never will, so it isn't here for now. + /// Pointer to a callback function. The inner value which matches one of the callback + /// definitions in [crate::ComponentInterface::ffi_definitions]. + Callback(String), + /// Pointer to a FFI struct (e.g. a VTable). The inner value matches one of the struct + /// definitions in [crate::ComponentInterface::ffi_definitions]. + Struct(String), + /// Opaque 64-bit handle + /// + /// These are used to pass objects across the FFI. + Handle, + RustCallStatus, + /// Pointer to an FfiType. + Reference(Box), + /// Opaque pointer + VoidPointer, +} + +impl FfiType { + pub fn reference(self) -> FfiType { + FfiType::Reference(Box::new(self)) + } + + /// Unique name for an FFI return type + pub fn return_type_name(return_type: Option<&FfiType>) -> String { + match return_type { + Some(t) => match t { + FfiType::UInt8 => "u8".to_owned(), + FfiType::Int8 => "i8".to_owned(), + FfiType::UInt16 => "u16".to_owned(), + FfiType::Int16 => "i16".to_owned(), + FfiType::UInt32 => "u32".to_owned(), + FfiType::Int32 => "i32".to_owned(), + FfiType::UInt64 => "u64".to_owned(), + FfiType::Int64 => "i64".to_owned(), + FfiType::Float32 => "f32".to_owned(), + FfiType::Float64 => "f64".to_owned(), + FfiType::RustArcPtr(_) => "pointer".to_owned(), + FfiType::RustBuffer(_) => "rust_buffer".to_owned(), + _ => unimplemented!("FFI return type: {t:?}"), + }, + None => "void".to_owned(), + } + } } /// When passing data across the FFI, each `Type` value will be lowered into a corresponding @@ -94,7 +123,6 @@ impl From<&Type> for FfiType { Type::Object { name, .. } => FfiType::RustArcPtr(name.to_owned()), // Callback interfaces are passed as opaque integer handles. Type::CallbackInterface { .. } => FfiType::UInt64, - Type::ForeignExecutor => FfiType::ForeignExecutorHandle, // Other types are serialized into a bytebuffer and deserialized on the other side. Type::Enum { .. } | Type::Record { .. } @@ -107,6 +135,11 @@ impl From<&Type> for FfiType { name, kind: ExternalKind::Interface, .. + } + | Type::External { + name, + kind: ExternalKind::Trait, + .. } => FfiType::RustArcPtr(name.clone()), Type::External { name, @@ -131,6 +164,24 @@ impl From<&&Type> for FfiType { } } +/// An Ffi definition +#[derive(Debug, Clone)] +pub enum FfiDefinition { + Function(FfiFunction), + CallbackFunction(FfiCallbackFunction), + Struct(FfiStruct), +} + +impl FfiDefinition { + pub fn name(&self) -> &str { + match self { + Self::Function(f) => f.name(), + Self::CallbackFunction(f) => f.name(), + Self::Struct(s) => s.name(), + } + } +} + /// Represents an "extern C"-style function that will be part of the FFI. /// /// These can't be declared explicitly in the UDL, but rather, are derived automatically @@ -150,6 +201,19 @@ pub struct FfiFunction { } impl FfiFunction { + pub fn callback_init(module_path: &str, trait_name: &str, vtable_name: String) -> Self { + Self { + name: uniffi_meta::init_callback_vtable_fn_symbol_name(module_path, trait_name), + arguments: vec![FfiArgument { + name: "vtable".to_string(), + type_: FfiType::Struct(vtable_name).reference(), + }], + return_type: None, + has_rust_call_status_arg: false, + ..Self::default() + } + } + pub fn name(&self) -> &str { &self.name } @@ -181,7 +245,7 @@ impl FfiFunction { ) { self.arguments = args.into_iter().collect(); if self.is_async() { - self.return_type = Some(FfiType::RustFutureHandle); + self.return_type = Some(FfiType::Handle); self.has_rust_call_status_arg = false; } else { self.return_type = return_type; @@ -212,14 +276,113 @@ pub struct FfiArgument { } impl FfiArgument { + pub fn new(name: impl Into, type_: FfiType) -> Self { + Self { + name: name.into(), + type_, + } + } + pub fn name(&self) -> &str { &self.name } + pub fn type_(&self) -> FfiType { self.type_.clone() } } +/// Represents an "extern C"-style callback function +/// +/// These are defined in the foreign code and passed to Rust as a function pointer. +#[derive(Debug, Default, Clone)] +pub struct FfiCallbackFunction { + // Name for this function type. This matches the value inside `FfiType::Callback` + pub(super) name: String, + pub(super) arguments: Vec, + pub(super) return_type: Option, + pub(super) has_rust_call_status_arg: bool, +} + +impl FfiCallbackFunction { + pub fn name(&self) -> &str { + &self.name + } + + pub fn arguments(&self) -> Vec<&FfiArgument> { + self.arguments.iter().collect() + } + + pub fn return_type(&self) -> Option<&FfiType> { + self.return_type.as_ref() + } + + pub fn has_rust_call_status_arg(&self) -> bool { + self.has_rust_call_status_arg + } +} + +/// Represents a repr(C) struct used in the FFI +#[derive(Debug, Default, Clone)] +pub struct FfiStruct { + pub(super) name: String, + pub(super) fields: Vec, +} + +impl FfiStruct { + /// Get the name of this struct + pub fn name(&self) -> &str { + &self.name + } + + /// Get the fields for this struct + pub fn fields(&self) -> &[FfiField] { + &self.fields + } +} + +/// Represents a field of an [FfiStruct] +#[derive(Debug, Clone)] +pub struct FfiField { + pub(super) name: String, + pub(super) type_: FfiType, +} + +impl FfiField { + pub fn new(name: impl Into, type_: FfiType) -> Self { + Self { + name: name.into(), + type_, + } + } + + pub fn name(&self) -> &str { + &self.name + } + + pub fn type_(&self) -> FfiType { + self.type_.clone() + } +} + +impl From for FfiDefinition { + fn from(value: FfiFunction) -> FfiDefinition { + FfiDefinition::Function(value) + } +} + +impl From for FfiDefinition { + fn from(value: FfiStruct) -> FfiDefinition { + FfiDefinition::Struct(value) + } +} + +impl From for FfiDefinition { + fn from(value: FfiCallbackFunction) -> FfiDefinition { + FfiDefinition::CallbackFunction(value) + } +} + #[cfg(test)] mod test { // There's not really much to test here to be honest, diff --git a/third_party/rust/uniffi_bindgen/src/interface/function.rs b/third_party/rust/uniffi_bindgen/src/interface/function.rs index 2d18288c1c..8effc4c876 100644 --- a/third_party/rust/uniffi_bindgen/src/interface/function.rs +++ b/third_party/rust/uniffi_bindgen/src/interface/function.rs @@ -59,6 +59,8 @@ pub struct Function { // avoids a weird circular dependency in the calculation. #[checksum_ignore] pub(super) ffi_func: FfiFunction, + #[checksum_ignore] + pub(super) docstring: Option, pub(super) throws: Option, pub(super) checksum_fn_name: String, // Force a checksum value, or we'll fallback to the trait. @@ -128,6 +130,10 @@ impl Function { .chain(self.return_type.iter().flat_map(Type::iter_types)), ) } + + pub fn docstring(&self) -> Option<&str> { + self.docstring.as_deref() + } } impl From for Argument { @@ -163,6 +169,7 @@ impl From for Function { arguments, return_type, ffi_func, + docstring: meta.docstring.clone(), throws: meta.throws, checksum_fn_name, checksum: meta.checksum, @@ -242,6 +249,9 @@ pub trait Callable { fn return_type(&self) -> Option; fn throws_type(&self) -> Option; fn is_async(&self) -> bool; + fn takes_self(&self) -> bool { + false + } fn result_type(&self) -> ResultType { ResultType { return_type: self.return_type(), @@ -311,6 +321,10 @@ impl Callable for &T { fn is_async(&self) -> bool { (*self).is_async() } + + fn takes_self(&self) -> bool { + (*self).takes_self() + } } #[cfg(test)] @@ -364,4 +378,22 @@ mod test { ); Ok(()) } + + #[test] + fn test_docstring_function() { + const UDL: &str = r#" + namespace test { + /// informative docstring + void testing(); + }; + "#; + let ci = ComponentInterface::from_webidl(UDL, "crate_name").unwrap(); + assert_eq!( + ci.get_function_definition("testing") + .unwrap() + .docstring() + .unwrap(), + "informative docstring" + ); + } } diff --git a/third_party/rust/uniffi_bindgen/src/interface/mod.rs b/third_party/rust/uniffi_bindgen/src/interface/mod.rs index 8e4df2149b..90a941637a 100644 --- a/third_party/rust/uniffi_bindgen/src/interface/mod.rs +++ b/third_party/rust/uniffi_bindgen/src/interface/mod.rs @@ -67,7 +67,9 @@ mod record; pub use record::{Field, Record}; pub mod ffi; -pub use ffi::{FfiArgument, FfiFunction, FfiType}; +pub use ffi::{ + FfiArgument, FfiCallbackFunction, FfiDefinition, FfiField, FfiFunction, FfiStruct, FfiType, +}; pub use uniffi_meta::Radix; use uniffi_meta::{ ConstructorMetadata, LiteralMetadata, NamespaceMetadata, ObjectMetadata, TraitMethodMetadata, @@ -139,6 +141,11 @@ impl ComponentInterface { self.types.namespace ); } + + if group.namespace_docstring.is_some() { + self.types.namespace_docstring = group.namespace_docstring.clone(); + } + // Unconditionally add the String type, which is used by the panic handling self.types.add_known_type(&uniffi_meta::Type::String)?; crate::macro_metadata::add_group_to_ci(self, group)?; @@ -153,6 +160,10 @@ impl ComponentInterface { &self.types.namespace.name } + pub fn namespace_docstring(&self) -> Option<&str> { + self.types.namespace_docstring.as_deref() + } + pub fn uniffi_contract_version(&self) -> u32 { // This is set by the scripts in the version-mismatch fixture let force_version = std::env::var("UNIFFI_FORCE_CONTRACT_VERSION"); @@ -204,6 +215,29 @@ impl ComponentInterface { self.objects.iter().find(|o| o.name == name) } + fn callback_interface_callback_definitions( + &self, + ) -> impl IntoIterator + '_ { + self.callback_interfaces + .iter() + .flat_map(|cbi| cbi.ffi_callbacks()) + .chain(self.objects.iter().flat_map(|o| o.ffi_callbacks())) + } + + /// Get the definitions for callback FFI functions + /// + /// These are defined by the foreign code and invoked by Rust. + fn callback_interface_vtable_definitions(&self) -> impl IntoIterator + '_ { + self.callback_interface_definitions() + .iter() + .map(|cbi| cbi.vtable_definition()) + .chain( + self.object_definitions() + .iter() + .flat_map(|o| o.vtable_definition()), + ) + } + /// Get the definitions for every Callback Interface type in the interface. pub fn callback_interface_definitions(&self) -> &[CallbackInterface] { &self.callback_interfaces @@ -215,6 +249,17 @@ impl ComponentInterface { self.callback_interfaces.iter().find(|o| o.name == name) } + /// Get the definitions for every Callback Interface type in the interface. + pub fn has_async_callback_interface_definition(&self) -> bool { + self.callback_interfaces + .iter() + .any(|cbi| cbi.has_async_method()) + || self + .objects + .iter() + .any(|o| o.has_callback_interface() && o.has_async_method()) + } + /// Get the definitions for every Method type in the interface. pub fn iter_callables(&self) -> impl Iterator { // Each of the `as &dyn Callable` casts is a trivial cast, but it seems like the clearest @@ -241,13 +286,19 @@ impl ComponentInterface { let fielded = !e.is_flat(); // For flat errors, we should only generate read() methods if we need them to support // callback interface errors - let used_in_callback_interface = self + let used_in_foreign_interface = self .callback_interface_definitions() .iter() .flat_map(|cb| cb.methods()) + .chain( + self.object_definitions() + .iter() + .filter(|o| o.has_callback_interface()) + .flat_map(|o| o.methods()), + ) .any(|m| m.throws_type() == Some(&e.as_type())); - self.is_name_used_as_error(&e.name) && (fielded || used_in_callback_interface) + self.is_name_used_as_error(&e.name) && (fielded || used_in_foreign_interface) } /// Get details about all `Type::External` types. @@ -304,8 +355,17 @@ impl ComponentInterface { /// This is important to know in language bindings that cannot integrate object types /// tightly with the host GC, and hence need to perform manual destruction of objects. pub fn item_contains_object_references(&self, item: &Type) -> bool { - self.iter_types_in_item(item) - .any(|t| matches!(t, Type::Object { .. })) + // this is surely broken for external records with object refs? + self.iter_types_in_item(item).any(|t| { + matches!( + t, + Type::Object { .. } + | Type::External { + kind: ExternalKind::Interface, + .. + } + ) + }) } /// Check whether the given item contains any (possibly nested) unsigned types @@ -335,6 +395,13 @@ impl ComponentInterface { .any(|t| matches!(t, Type::Map { .. })) } + /// Check whether the interface contains any object types + pub fn contains_object_types(&self) -> bool { + self.types + .iter_known_types() + .any(|t| matches!(t, Type::Object { .. })) + } + // The namespace to use in crate-level FFI function definitions. Not used as the ffi // namespace for types - each type has its own `module_path` which is used for them. fn ffi_namespace(&self) -> &str { @@ -364,7 +431,7 @@ impl ComponentInterface { is_async: false, arguments: vec![FfiArgument { name: "size".to_string(), - type_: FfiType::Int32, + type_: FfiType::UInt64, }], return_type: Some(FfiType::RustBuffer(None)), has_rust_call_status_arg: true, @@ -420,7 +487,7 @@ impl ComponentInterface { }, FfiArgument { name: "additional".to_string(), - type_: FfiType::Int32, + type_: FfiType::UInt64, }, ], return_type: Some(FfiType::RustBuffer(None)), @@ -429,24 +496,6 @@ impl ComponentInterface { } } - /// Builtin FFI function to set the Rust Future continuation callback - pub fn ffi_rust_future_continuation_callback_set(&self) -> FfiFunction { - FfiFunction { - name: format!( - "ffi_{}_rust_future_continuation_callback_set", - self.ffi_namespace() - ), - arguments: vec![FfiArgument { - name: "callback".to_owned(), - type_: FfiType::RustFutureContinuationCallback, - }], - return_type: None, - is_async: false, - has_rust_call_status_arg: false, - is_object_free_function: false, - } - } - /// Builtin FFI function to poll a Rust future. pub fn ffi_rust_future_poll(&self, return_ffi_type: Option) -> FfiFunction { FfiFunction { @@ -455,12 +504,15 @@ impl ComponentInterface { arguments: vec![ FfiArgument { name: "handle".to_owned(), - type_: FfiType::RustFutureHandle, + type_: FfiType::Handle, + }, + FfiArgument { + name: "callback".to_owned(), + type_: FfiType::Callback("RustFutureContinuationCallback".to_owned()), }, - // Data to pass to the continuation FfiArgument { - name: "uniffi_callback".to_owned(), - type_: FfiType::RustFutureContinuationData, + name: "callback_data".to_owned(), + type_: FfiType::Handle, }, ], return_type: None, @@ -478,7 +530,7 @@ impl ComponentInterface { is_async: false, arguments: vec![FfiArgument { name: "handle".to_owned(), - type_: FfiType::RustFutureHandle, + type_: FfiType::Handle, }], return_type: return_ffi_type, has_rust_call_status_arg: true, @@ -493,7 +545,7 @@ impl ComponentInterface { is_async: false, arguments: vec![FfiArgument { name: "handle".to_owned(), - type_: FfiType::RustFutureHandle, + type_: FfiType::Handle, }], return_type: None, has_rust_call_status_arg: false, @@ -508,7 +560,7 @@ impl ComponentInterface { is_async: false, arguments: vec![FfiArgument { name: "handle".to_owned(), - type_: FfiType::RustFutureHandle, + type_: FfiType::Handle, }], return_type: None, has_rust_call_status_arg: false, @@ -518,29 +570,17 @@ impl ComponentInterface { fn rust_future_ffi_fn_name(&self, base_name: &str, return_ffi_type: Option) -> String { let namespace = self.ffi_namespace(); - match return_ffi_type { - Some(t) => match t { - FfiType::UInt8 => format!("ffi_{namespace}_{base_name}_u8"), - FfiType::Int8 => format!("ffi_{namespace}_{base_name}_i8"), - FfiType::UInt16 => format!("ffi_{namespace}_{base_name}_u16"), - FfiType::Int16 => format!("ffi_{namespace}_{base_name}_i16"), - FfiType::UInt32 => format!("ffi_{namespace}_{base_name}_u32"), - FfiType::Int32 => format!("ffi_{namespace}_{base_name}_i32"), - FfiType::UInt64 => format!("ffi_{namespace}_{base_name}_u64"), - FfiType::Int64 => format!("ffi_{namespace}_{base_name}_i64"), - FfiType::Float32 => format!("ffi_{namespace}_{base_name}_f32"), - FfiType::Float64 => format!("ffi_{namespace}_{base_name}_f64"), - FfiType::RustArcPtr(_) => format!("ffi_{namespace}_{base_name}_pointer"), - FfiType::RustBuffer(_) => format!("ffi_{namespace}_{base_name}_rust_buffer"), - _ => unimplemented!("FFI return type: {t:?}"), - }, - None => format!("ffi_{namespace}_{base_name}_void"), - } + let return_type_name = FfiType::return_type_name(return_ffi_type.as_ref()); + format!("ffi_{namespace}_{base_name}_{return_type_name}") } /// Does this interface contain async functions? pub fn has_async_fns(&self) -> bool { self.iter_ffi_function_definitions().any(|f| f.is_async()) + || self + .callback_interfaces + .iter() + .any(CallbackInterface::has_async_method) } /// Iterate over `T` parameters of the `FutureCallback` callbacks in this interface @@ -561,6 +601,73 @@ impl ComponentInterface { unique_results.into_iter() } + /// Iterate over all Ffi definitions + pub fn ffi_definitions(&self) -> impl Iterator + '_ { + // Note: for languages like Python it's important to keep things in dependency order. + // For example some FFI function definitions depend on FFI struct definitions, so the + // function definitions come last. + self.builtin_ffi_definitions() + .into_iter() + .chain( + self.callback_interface_callback_definitions() + .into_iter() + .map(Into::into), + ) + .chain( + self.callback_interface_vtable_definitions() + .into_iter() + .map(Into::into), + ) + .chain(self.iter_ffi_function_definitions().map(Into::into)) + } + + fn builtin_ffi_definitions(&self) -> impl IntoIterator + '_ { + [ + FfiCallbackFunction { + name: "RustFutureContinuationCallback".to_owned(), + arguments: vec![ + FfiArgument::new("data", FfiType::UInt64), + FfiArgument::new("poll_result", FfiType::Int8), + ], + return_type: None, + has_rust_call_status_arg: false, + } + .into(), + FfiCallbackFunction { + name: "ForeignFutureFree".to_owned(), + arguments: vec![FfiArgument::new("handle", FfiType::UInt64)], + return_type: None, + has_rust_call_status_arg: false, + } + .into(), + FfiCallbackFunction { + name: "CallbackInterfaceFree".to_owned(), + arguments: vec![FfiArgument::new("handle", FfiType::UInt64)], + return_type: None, + has_rust_call_status_arg: false, + } + .into(), + FfiStruct { + name: "ForeignFuture".to_owned(), + fields: vec![ + FfiField::new("handle", FfiType::UInt64), + FfiField::new("free", FfiType::Callback("ForeignFutureFree".to_owned())), + ], + } + .into(), + ] + .into_iter() + .chain( + self.all_possible_return_ffi_types() + .flat_map(|return_type| { + [ + callbacks::foreign_future_ffi_result_struct(return_type.clone()).into(), + callbacks::ffi_foreign_future_complete(return_type).into(), + ] + }), + ) + } + /// List the definitions of all FFI functions in the interface. /// /// The set of FFI functions is derived automatically from the set of higher-level types @@ -569,9 +676,8 @@ impl ComponentInterface { self.iter_user_ffi_function_definitions() .cloned() .chain(self.iter_rust_buffer_ffi_function_definitions()) - .chain(self.iter_futures_ffi_function_definitons()) + .chain(self.iter_futures_ffi_function_definitions()) .chain(self.iter_checksum_ffi_functions()) - .chain(self.ffi_foreign_executor_callback_set()) .chain([self.ffi_uniffi_contract_version()]) } @@ -618,9 +724,8 @@ impl ComponentInterface { .into_iter() } - /// List all FFI functions definitions for async functionality. - pub fn iter_futures_ffi_function_definitons(&self) -> impl Iterator + '_ { - let all_possible_return_ffi_types = [ + fn all_possible_return_ffi_types(&self) -> impl Iterator> { + [ Some(FfiType::UInt8), Some(FfiType::Int8), Some(FfiType::UInt16), @@ -631,46 +736,26 @@ impl ComponentInterface { Some(FfiType::Int64), Some(FfiType::Float32), Some(FfiType::Float64), - // RustBuffer and RustArcPtr have an inner field which doesn't affect the rust future - // complete scaffolding function, so we just use a placeholder value here. + // RustBuffer and RustArcPtr have an inner field which we have to fill in with a + // placeholder value. Some(FfiType::RustArcPtr("".to_owned())), Some(FfiType::RustBuffer(None)), None, - ]; - - iter::once(self.ffi_rust_future_continuation_callback_set()).chain( - all_possible_return_ffi_types - .into_iter() - .flat_map(|return_type| { - [ - self.ffi_rust_future_poll(return_type.clone()), - self.ffi_rust_future_cancel(return_type.clone()), - self.ffi_rust_future_free(return_type.clone()), - self.ffi_rust_future_complete(return_type), - ] - }), - ) + ] + .into_iter() } - /// The ffi_foreign_executor_callback_set FFI function - /// - /// We only include this in the FFI if the `ForeignExecutor` type is actually used - pub fn ffi_foreign_executor_callback_set(&self) -> Option { - if self.types.contains(&Type::ForeignExecutor) { - Some(FfiFunction { - name: format!("ffi_{}_foreign_executor_callback_set", self.ffi_namespace()), - arguments: vec![FfiArgument { - name: "callback".into(), - type_: FfiType::ForeignExecutorCallback, - }], - return_type: None, - is_async: false, - has_rust_call_status_arg: false, - is_object_free_function: false, + /// List all FFI functions definitions for async functionality. + pub fn iter_futures_ffi_function_definitions(&self) -> impl Iterator + '_ { + self.all_possible_return_ffi_types() + .flat_map(|return_type| { + [ + self.ffi_rust_future_poll(return_type.clone()), + self.ffi_rust_future_cancel(return_type.clone()), + self.ffi_rust_future_free(return_type.clone()), + self.ffi_rust_future_complete(return_type), + ] }) - } else { - None - } } /// List all API checksums to check @@ -778,6 +863,8 @@ impl ComponentInterface { bail!("Conflicting type definition for \"{}\"", defn.name()); } self.types.add_known_types(defn.iter_types())?; + defn.throws_name() + .map(|n| self.errors.insert(n.to_string())); self.functions.push(defn); Ok(()) @@ -789,6 +876,8 @@ impl ComponentInterface { let defn: Constructor = meta.into(); self.types.add_known_types(defn.iter_types())?; + defn.throws_name() + .map(|n| self.errors.insert(n.to_string())); object.constructors.push(defn); Ok(()) @@ -800,6 +889,9 @@ impl ComponentInterface { .ok_or_else(|| anyhow!("add_method_meta: object {} not found", &method.object_name))?; self.types.add_known_types(method.iter_types())?; + method + .throws_name() + .map(|n| self.errors.insert(n.to_string())); method.object_impl = object.imp; object.methods.push(method); Ok(()) @@ -825,10 +917,6 @@ impl ComponentInterface { Ok(()) } - pub(super) fn note_name_used_as_error(&mut self, name: &str) { - self.errors.insert(name.to_string()); - } - pub fn is_name_used_as_error(&self, name: &str) -> bool { self.errors.contains(name) } @@ -856,6 +944,9 @@ impl ComponentInterface { self.callback_interface_throws_types.insert(error.clone()); } self.types.add_known_types(method.iter_types())?; + method + .throws_name() + .map(|n| self.errors.insert(n.to_string())); cbi.methods.push(method); } else { self.add_method_meta(meta)?; @@ -880,31 +971,6 @@ impl ComponentInterface { bail!("Conflicting type definition for \"{}\"", f.name()); } } - - for ty in self.iter_types() { - match ty { - Type::Object { name, .. } => { - ensure!( - self.objects.iter().any(|o| o.name == *name), - "Object `{name}` has no definition" - ); - } - Type::Record { name, .. } => { - ensure!( - self.records.contains_key(name), - "Record `{name}` has no definition", - ); - } - Type::Enum { name, .. } => { - ensure!( - self.enums.contains_key(name), - "Enum `{name}` has no definition", - ); - } - _ => {} - } - } - Ok(()) } @@ -1047,7 +1113,7 @@ fn throws_name(throws: &Option) -> Option<&str> { // Type has no `name()` method, just `canonical_name()` which isn't what we want. match throws { None => None, - Some(Type::Enum { name, .. }) => Some(name), + Some(Type::Enum { name, .. }) | Some(Type::Object { name, .. }) => Some(name), _ => panic!("unknown throw type: {throws:?}"), } } @@ -1089,35 +1155,50 @@ mod test { let err = ComponentInterface::from_webidl(UDL2, "crate_name").unwrap_err(); assert_eq!( err.to_string(), - "Mismatching definition for enum `Testing`!\nexisting definition: Enum { + "Mismatching definition for enum `Testing`! +existing definition: Enum { name: \"Testing\", module_path: \"crate_name\", + discr_type: None, variants: [ Variant { name: \"one\", + discr: None, fields: [], + docstring: None, }, Variant { name: \"two\", + discr: None, fields: [], + docstring: None, }, ], flat: true, + non_exhaustive: false, + docstring: None, }, new definition: Enum { name: \"Testing\", module_path: \"crate_name\", + discr_type: None, variants: [ Variant { name: \"three\", + discr: None, fields: [], + docstring: None, }, Variant { name: \"four\", + discr: None, fields: [], + docstring: None, }, ], flat: true, + non_exhaustive: false, + docstring: None, }", ); @@ -1231,4 +1312,25 @@ new definition: Enum { imp: ObjectImpl::Struct, })); } + + #[test] + fn test_docstring_namespace() { + const UDL: &str = r#" + /// informative docstring + namespace test{}; + "#; + let ci = ComponentInterface::from_webidl(UDL, "crate_name").unwrap(); + assert_eq!(ci.namespace_docstring().unwrap(), "informative docstring"); + } + + #[test] + fn test_multiline_docstring() { + const UDL: &str = r#" + /// informative + /// docstring + namespace test{}; + "#; + let ci = ComponentInterface::from_webidl(UDL, "crate_name").unwrap(); + assert_eq!(ci.namespace_docstring().unwrap(), "informative\ndocstring"); + } } diff --git a/third_party/rust/uniffi_bindgen/src/interface/object.rs b/third_party/rust/uniffi_bindgen/src/interface/object.rs index 942032b3c6..2b86e54a45 100644 --- a/third_party/rust/uniffi_bindgen/src/interface/object.rs +++ b/third_party/rust/uniffi_bindgen/src/interface/object.rs @@ -57,12 +57,11 @@ //! # Ok::<(), anyhow::Error>(()) //! ``` -use std::iter; - use anyhow::Result; use uniffi_meta::Checksum; -use super::ffi::{FfiArgument, FfiFunction, FfiType}; +use super::callbacks; +use super::ffi::{FfiArgument, FfiCallbackFunction, FfiFunction, FfiStruct, FfiType}; use super::function::{Argument, Callable}; use super::{AsType, ObjectImpl, Type, TypeIterator}; @@ -92,14 +91,24 @@ pub struct Object { // a regular method (albeit with a generated name) // XXX - this should really be a HashSet, but not enough transient types support hash to make it worthwhile now. pub(super) uniffi_traits: Vec, - // We don't include the FfiFunc in the hash calculation, because: + // We don't include the FfiFuncs in the hash calculation, because: // - it is entirely determined by the other fields, // so excluding it is safe. // - its `name` property includes a checksum derived from the very // hash value we're trying to calculate here, so excluding it // avoids a weird circular dependency in the calculation. + + // FFI function to clone a pointer for this object + #[checksum_ignore] + pub(super) ffi_func_clone: FfiFunction, + // FFI function to free a pointer for this object #[checksum_ignore] pub(super) ffi_func_free: FfiFunction, + // Ffi function to initialize the foreign callback for trait interfaces + #[checksum_ignore] + pub(super) ffi_init_callback: Option, + #[checksum_ignore] + pub(super) docstring: Option, } impl Object { @@ -118,6 +127,18 @@ impl Object { &self.imp } + pub fn is_trait_interface(&self) -> bool { + self.imp.is_trait_interface() + } + + pub fn has_callback_interface(&self) -> bool { + self.imp.has_callback_interface() + } + + pub fn has_async_method(&self) -> bool { + self.methods.iter().any(Method::is_async) + } + pub fn constructors(&self) -> Vec<&Constructor> { self.constructors.iter().collect() } @@ -151,12 +172,28 @@ impl Object { self.uniffi_traits.iter().collect() } + pub fn ffi_object_clone(&self) -> &FfiFunction { + &self.ffi_func_clone + } + pub fn ffi_object_free(&self) -> &FfiFunction { &self.ffi_func_free } + pub fn ffi_init_callback(&self) -> &FfiFunction { + self.ffi_init_callback + .as_ref() + .unwrap_or_else(|| panic!("No ffi_init_callback set for {}", &self.name)) + } + + pub fn docstring(&self) -> Option<&str> { + self.docstring.as_deref() + } + pub fn iter_ffi_function_definitions(&self) -> impl Iterator { - iter::once(&self.ffi_func_free) + [&self.ffi_func_clone, &self.ffi_func_free] + .into_iter() + .chain(&self.ffi_init_callback) .chain(self.constructors.iter().map(|f| &f.ffi_func)) .chain(self.methods.iter().map(|f| &f.ffi_func)) .chain( @@ -173,13 +210,26 @@ impl Object { } pub fn derive_ffi_funcs(&mut self) -> Result<()> { + assert!(!self.ffi_func_clone.name().is_empty()); assert!(!self.ffi_func_free.name().is_empty()); + self.ffi_func_clone.arguments = vec![FfiArgument { + name: "ptr".to_string(), + type_: FfiType::RustArcPtr(self.name.to_string()), + }]; + self.ffi_func_clone.return_type = Some(FfiType::RustArcPtr(self.name.to_string())); self.ffi_func_free.arguments = vec![FfiArgument { name: "ptr".to_string(), type_: FfiType::RustArcPtr(self.name.to_string()), }]; self.ffi_func_free.return_type = None; self.ffi_func_free.is_object_free_function = true; + if self.has_callback_interface() { + self.ffi_init_callback = Some(FfiFunction::callback_init( + &self.module_path, + &self.name, + callbacks::vtable_name(&self.name), + )); + } for cons in self.constructors.iter_mut() { cons.derive_ffi_func(); @@ -194,6 +244,41 @@ impl Object { Ok(()) } + /// For trait interfaces, FfiCallbacks to define for our methods, otherwise an empty vec. + pub fn ffi_callbacks(&self) -> Vec { + if self.is_trait_interface() { + callbacks::ffi_callbacks(&self.name, &self.methods) + } else { + vec![] + } + } + + /// For trait interfaces, the VTable FFI type + pub fn vtable(&self) -> Option { + self.is_trait_interface() + .then(|| FfiType::Struct(callbacks::vtable_name(&self.name))) + } + + /// For trait interfaces, the VTable struct to define. Otherwise None. + pub fn vtable_definition(&self) -> Option { + self.is_trait_interface() + .then(|| callbacks::vtable_struct(&self.name, &self.methods)) + } + + /// Vec of (ffi_callback_name, method) pairs + pub fn vtable_methods(&self) -> Vec<(FfiCallbackFunction, &Method)> { + self.methods + .iter() + .enumerate() + .map(|(i, method)| { + ( + callbacks::method_ffi_callback(&self.name, method, i), + method, + ) + }) + .collect() + } + pub fn iter_types(&self) -> TypeIterator<'_> { Box::new( self.methods @@ -218,6 +303,7 @@ impl AsType for Object { impl From for Object { fn from(meta: uniffi_meta::ObjectMetadata) -> Self { + let ffi_clone_name = meta.clone_ffi_symbol_name(); let ffi_free_name = meta.free_ffi_symbol_name(); Object { module_path: meta.module_path, @@ -226,10 +312,16 @@ impl From for Object { constructors: Default::default(), methods: Default::default(), uniffi_traits: Default::default(), + ffi_func_clone: FfiFunction { + name: ffi_clone_name, + ..Default::default() + }, ffi_func_free: FfiFunction { name: ffi_free_name, ..Default::default() }, + ffi_init_callback: None, + docstring: meta.docstring.clone(), } } } @@ -263,6 +355,7 @@ pub struct Constructor { pub(super) name: String, pub(super) object_name: String, pub(super) object_module_path: String, + pub(super) is_async: bool, pub(super) arguments: Vec, // We don't include the FFIFunc in the hash calculation, because: // - it is entirely determined by the other fields, @@ -272,6 +365,8 @@ pub struct Constructor { // avoids a weird circular dependency in the calculation. #[checksum_ignore] pub(super) ffi_func: FfiFunction, + #[checksum_ignore] + pub(super) docstring: Option, pub(super) throws: Option, pub(super) checksum_fn_name: String, // Force a checksum value, or we'll fallback to the trait. @@ -316,14 +411,20 @@ impl Constructor { self.throws.as_ref() } + pub fn docstring(&self) -> Option<&str> { + self.docstring.as_deref() + } + pub fn is_primary_constructor(&self) -> bool { self.name == "new" } fn derive_ffi_func(&mut self) { assert!(!self.ffi_func.name().is_empty()); - self.ffi_func.arguments = self.arguments.iter().map(Into::into).collect(); - self.ffi_func.return_type = Some(FfiType::RustArcPtr(self.object_name.clone())); + self.ffi_func.init( + Some(FfiType::RustArcPtr(self.object_name.clone())), + self.arguments.iter().map(Into::into), + ); } pub fn iter_types(&self) -> TypeIterator<'_> { @@ -339,14 +440,17 @@ impl From for Constructor { let ffi_func = FfiFunction { name: ffi_name, + is_async: meta.is_async, ..FfiFunction::default() }; Self { name: meta.name, object_name: meta.self_name, + is_async: meta.is_async, object_module_path: meta.module_path, arguments, ffi_func, + docstring: meta.docstring.clone(), throws: meta.throws.map(Into::into), checksum_fn_name, checksum: meta.checksum, @@ -375,6 +479,8 @@ pub struct Method { // avoids a weird circular dependency in the calculation. #[checksum_ignore] pub(super) ffi_func: FfiFunction, + #[checksum_ignore] + pub(super) docstring: Option, pub(super) throws: Option, pub(super) takes_self_by_arc: bool, pub(super) checksum_fn_name: String, @@ -445,6 +551,10 @@ impl Method { self.throws.as_ref() } + pub fn docstring(&self) -> Option<&str> { + self.docstring.as_deref() + } + pub fn takes_self_by_arc(&self) -> bool { self.takes_self_by_arc } @@ -466,6 +576,11 @@ impl Method { .chain(self.return_type.iter().flat_map(Type::iter_types)), ) } + + /// For async callback interface methods, the FFI struct to pass to the completion function. + pub fn foreign_future_ffi_result_struct(&self) -> FfiStruct { + callbacks::foreign_future_ffi_result_struct(self.return_type.as_ref().map(FfiType::from)) + } } impl From for Method { @@ -491,6 +606,7 @@ impl From for Method { arguments, return_type, ffi_func, + docstring: meta.docstring.clone(), throws: meta.throws.map(Into::into), takes_self_by_arc: meta.takes_self_by_arc, checksum_fn_name, @@ -503,19 +619,22 @@ impl From for Method { fn from(meta: uniffi_meta::TraitMethodMetadata) -> Self { let ffi_name = meta.ffi_symbol_name(); let checksum_fn_name = meta.checksum_symbol_name(); + let is_async = meta.is_async; let return_type = meta.return_type.map(Into::into); let arguments = meta.inputs.into_iter().map(Into::into).collect(); let ffi_func = FfiFunction { name: ffi_name, + is_async, ..FfiFunction::default() }; Self { name: meta.name, object_name: meta.trait_name, object_module_path: meta.module_path, - is_async: false, + is_async, arguments, return_type, + docstring: meta.docstring.clone(), throws: meta.throws.map(Into::into), takes_self_by_arc: meta.takes_self_by_arc, checksum_fn_name, @@ -583,7 +702,7 @@ impl Callable for Constructor { } fn is_async(&self) -> bool { - false + self.is_async } } @@ -603,6 +722,10 @@ impl Callable for Method { fn is_async(&self) -> bool { self.is_async } + + fn takes_self(&self) -> bool { + true + } } #[cfg(test)] @@ -770,4 +893,62 @@ mod test { "Trait interfaces can not have constructors: \"new\"" ); } + + #[test] + fn test_docstring_object() { + const UDL: &str = r#" + namespace test{}; + /// informative docstring + interface Testing { }; + "#; + let ci = ComponentInterface::from_webidl(UDL, "crate_name").unwrap(); + assert_eq!( + ci.get_object_definition("Testing") + .unwrap() + .docstring() + .unwrap(), + "informative docstring" + ); + } + + #[test] + fn test_docstring_constructor() { + const UDL: &str = r#" + namespace test{}; + interface Testing { + /// informative docstring + constructor(); + }; + "#; + let ci = ComponentInterface::from_webidl(UDL, "crate_name").unwrap(); + assert_eq!( + ci.get_object_definition("Testing") + .unwrap() + .primary_constructor() + .unwrap() + .docstring() + .unwrap(), + "informative docstring" + ); + } + + #[test] + fn test_docstring_method() { + const UDL: &str = r#" + namespace test{}; + interface Testing { + /// informative docstring + void testing(); + }; + "#; + let ci = ComponentInterface::from_webidl(UDL, "crate_name").unwrap(); + assert_eq!( + ci.get_object_definition("Testing") + .unwrap() + .get_method("testing") + .docstring() + .unwrap(), + "informative docstring" + ); + } } diff --git a/third_party/rust/uniffi_bindgen/src/interface/record.rs b/third_party/rust/uniffi_bindgen/src/interface/record.rs index 17d3774a49..e9a6004189 100644 --- a/third_party/rust/uniffi_bindgen/src/interface/record.rs +++ b/third_party/rust/uniffi_bindgen/src/interface/record.rs @@ -60,6 +60,8 @@ pub struct Record { pub(super) name: String, pub(super) module_path: String, pub(super) fields: Vec, + #[checksum_ignore] + pub(super) docstring: Option, } impl Record { @@ -71,9 +73,17 @@ impl Record { &self.fields } + pub fn docstring(&self) -> Option<&str> { + self.docstring.as_deref() + } + pub fn iter_types(&self) -> TypeIterator<'_> { Box::new(self.fields.iter().flat_map(Field::iter_types)) } + + pub fn has_fields(&self) -> bool { + !self.fields.is_empty() + } } impl AsType for Record { @@ -97,6 +107,7 @@ impl TryFrom for Record { .into_iter() .map(TryInto::try_into) .collect::>()?, + docstring: meta.docstring.clone(), }) } } @@ -107,6 +118,8 @@ pub struct Field { pub(super) name: String, pub(super) type_: Type, pub(super) default: Option, + #[checksum_ignore] + pub(super) docstring: Option, } impl Field { @@ -118,6 +131,10 @@ impl Field { self.default.as_ref() } + pub fn docstring(&self) -> Option<&str> { + self.docstring.as_deref() + } + pub fn iter_types(&self) -> TypeIterator<'_> { self.type_.iter_types() } @@ -140,6 +157,7 @@ impl TryFrom for Field { name, type_, default, + docstring: meta.docstring.clone(), }) } } @@ -227,4 +245,39 @@ mod test { .iter_types() .any(|t| matches!(t, Type::Record { name, .. } if name == "Testing"))); } + + #[test] + fn test_docstring_record() { + const UDL: &str = r#" + namespace test{}; + /// informative docstring + dictionary Testing { }; + "#; + let ci = ComponentInterface::from_webidl(UDL, "crate_name").unwrap(); + assert_eq!( + ci.get_record_definition("Testing") + .unwrap() + .docstring() + .unwrap(), + "informative docstring" + ); + } + + #[test] + fn test_docstring_record_field() { + const UDL: &str = r#" + namespace test{}; + dictionary Testing { + /// informative docstring + i32 testing; + }; + "#; + let ci = ComponentInterface::from_webidl(UDL, "crate_name").unwrap(); + assert_eq!( + ci.get_record_definition("Testing").unwrap().fields()[0] + .docstring() + .unwrap(), + "informative docstring" + ); + } } diff --git a/third_party/rust/uniffi_bindgen/src/interface/universe.rs b/third_party/rust/uniffi_bindgen/src/interface/universe.rs index e69d86e44f..70bc61f8a9 100644 --- a/third_party/rust/uniffi_bindgen/src/interface/universe.rs +++ b/third_party/rust/uniffi_bindgen/src/interface/universe.rs @@ -25,6 +25,7 @@ pub use uniffi_meta::{AsType, ExternalKind, NamespaceMetadata, ObjectImpl, Type, pub(crate) struct TypeUniverse { /// The unique prefixes that we'll use for namespacing when exposing this component's API. pub namespace: NamespaceMetadata, + pub namespace_docstring: Option, // Named type definitions (including aliases). type_definitions: HashMap, @@ -83,9 +84,6 @@ impl TypeUniverse { Type::Bytes => self.add_type_definition("bytes", type_)?, Type::Timestamp => self.add_type_definition("timestamp", type_)?, Type::Duration => self.add_type_definition("duration", type_)?, - Type::ForeignExecutor => { - self.add_type_definition("ForeignExecutor", type_)?; - } Type::Object { name, .. } | Type::Record { name, .. } | Type::Enum { name, .. } @@ -118,6 +116,7 @@ impl TypeUniverse { Ok(()) } + #[cfg(test)] /// Check if a [Type] is present pub fn contains(&self, type_: &Type) -> bool { self.all_known_types.contains(type_) diff --git a/third_party/rust/uniffi_bindgen/src/lib.rs b/third_party/rust/uniffi_bindgen/src/lib.rs index 019b24022f..dfc90b32a6 100644 --- a/third_party/rust/uniffi_bindgen/src/lib.rs +++ b/third_party/rust/uniffi_bindgen/src/lib.rs @@ -58,9 +58,8 @@ //! //! ### 3) Generate and include component scaffolding from the UDL file //! -//! First you will need to install `uniffi-bindgen` on your system using `cargo install uniffi_bindgen`. -//! Then add to your crate `uniffi_build` under `[build-dependencies]`. -//! Finally, add a `build.rs` script to your crate and have it call `uniffi_build::generate_scaffolding` +//! Add to your crate `uniffi_build` under `[build-dependencies]`, +//! then add a `build.rs` script to your crate and have it call `uniffi_build::generate_scaffolding` //! to process your `.udl` file. This will generate some Rust code to be included in the top-level source //! code of your crate. If your UDL file is named `example.udl`, then your build script would call: //! @@ -77,12 +76,13 @@ //! //! ### 4) Generate foreign language bindings for the library //! -//! The `uniffi-bindgen` utility provides a command-line tool that can produce code to +//! You will need ensure a local `uniffi-bindgen` - see +//! This utility provides a command-line tool that can produce code to //! consume the Rust library in any of several supported languages. //! It is done by calling (in kotlin for example): //! //! ```text -//! uniffi-bindgen --language kotlin ./src/example.udl +//! cargo run --bin -p uniffi-bindgen --language kotlin ./src/example.udl //! ``` //! //! This will produce a file `example.kt` in the same directory as the .udl file, containing kotlin bindings @@ -160,15 +160,16 @@ pub trait BindingGenerator: Sized { ci: &ComponentInterface, config: &Self::Config, out_dir: &Utf8Path, + try_format_code: bool, ) -> Result<()>; /// Check if `library_path` used by library mode is valid for this generator fn check_library_path(&self, library_path: &Utf8Path, cdylib_name: Option<&str>) -> Result<()>; } -struct BindingGeneratorDefault { - target_languages: Vec, - try_format_code: bool, +pub struct BindingGeneratorDefault { + pub target_languages: Vec, + pub try_format_code: bool, } impl BindingGenerator for BindingGeneratorDefault { @@ -179,6 +180,7 @@ impl BindingGenerator for BindingGeneratorDefault { ci: &ComponentInterface, config: &Self::Config, out_dir: &Utf8Path, + _try_format_code: bool, ) -> Result<()> { for &language in &self.target_languages { bindings::write_bindings( @@ -219,12 +221,13 @@ impl BindingGenerator for BindingGeneratorDefault { /// - `library_file`: The path to a dynamic library to attempt to extract the definitions from and extend the component interface with. No extensions to component interface occur if it's [`None`] /// - `crate_name`: Override the default crate name that is guessed from UDL file path. pub fn generate_external_bindings( - binding_generator: T, + binding_generator: &T, udl_file: impl AsRef, config_file_override: Option>, out_dir_override: Option>, library_file: Option>, crate_name: Option<&str>, + try_format_code: bool, ) -> Result<()> { let crate_name = crate_name .map(|c| Ok(c.to_string())) @@ -253,7 +256,7 @@ pub fn generate_external_bindings( udl_file.as_ref(), out_dir_override.as_ref().map(|p| p.as_ref()), )?; - binding_generator.write_bindings(&component, &config, &out_dir) + binding_generator.write_bindings(&component, &config, &out_dir, try_format_code) } // Generate the infrastructural Rust code for implementing the UDL interface, @@ -301,25 +304,23 @@ fn generate_component_scaffolding_inner( // Generate the bindings in the target languages that call the scaffolding // Rust code. -pub fn generate_bindings( +pub fn generate_bindings( udl_file: &Utf8Path, config_file_override: Option<&Utf8Path>, - target_languages: Vec, + binding_generator: T, out_dir_override: Option<&Utf8Path>, library_file: Option<&Utf8Path>, crate_name: Option<&str>, try_format_code: bool, ) -> Result<()> { generate_external_bindings( - BindingGeneratorDefault { - target_languages, - try_format_code, - }, + &binding_generator, udl_file, config_file_override, out_dir_override, library_file, crate_name, + try_format_code, ) } @@ -417,22 +418,53 @@ fn format_code_with_rustfmt(path: &Utf8Path) -> Result<()> { Ok(()) } +/// Load TOML from file if the file exists. +fn load_toml_file(source: Option<&Utf8Path>) -> Result> { + if let Some(source) = source { + if source.exists() { + let contents = + fs::read_to_string(source).with_context(|| format!("read file: {:?}", source))?; + return Ok(Some( + toml::de::from_str(&contents) + .with_context(|| format!("parse toml: {:?}", source))?, + )); + } + } + + Ok(None) +} + +/// Load the default `uniffi.toml` config, merge TOML trees with `config_file_override` if specified. fn load_initial_config( crate_root: &Utf8Path, config_file_override: Option<&Utf8Path>, ) -> Result { - let path = match config_file_override { - Some(cfg) => Some(cfg.to_owned()), - None => crate_root.join("uniffi.toml").canonicalize_utf8().ok(), - }; - let toml_config = match path { - Some(path) => { - let contents = fs::read_to_string(path).context("Failed to read config file")?; - toml::de::from_str(&contents)? + let mut config = load_toml_file(Some(crate_root.join("uniffi.toml").as_path())) + .context("default config")? + .unwrap_or(toml::value::Table::default()); + + let override_config = load_toml_file(config_file_override).context("override config")?; + if let Some(override_config) = override_config { + merge_toml(&mut config, override_config); + } + + Ok(toml::Value::from(config).try_into()?) +} + +fn merge_toml(a: &mut toml::value::Table, b: toml::value::Table) { + for (key, value) in b.into_iter() { + match a.get_mut(&key) { + Some(existing_value) => match (existing_value, value) { + (toml::Value::Table(ref mut t0), toml::Value::Table(t1)) => { + merge_toml(t0, t1); + } + (v, value) => *v = value, + }, + None => { + a.insert(key, value); + } } - None => toml::Value::from(toml::value::Table::default()), - }; - Ok(toml_config.try_into()?) + } } #[derive(Debug, Clone, Default, Serialize, Deserialize)] @@ -516,4 +548,56 @@ mod test { let not_a_crate_root = &this_crate_root.join("src/templates"); assert!(guess_crate_root(¬_a_crate_root.join("src/example.udl")).is_err()); } + + #[test] + fn test_merge_toml() { + let default = r#" + foo = "foo" + bar = "bar" + + [table1] + foo = "foo" + bar = "bar" + "#; + let mut default = toml::de::from_str(default).unwrap(); + + let override_toml = r#" + # update key + bar = "BAR" + # insert new key + baz = "BAZ" + + [table1] + # update key + bar = "BAR" + # insert new key + baz = "BAZ" + + # new table + [table1.table2] + bar = "BAR" + baz = "BAZ" + "#; + let override_toml = toml::de::from_str(override_toml).unwrap(); + + let expected = r#" + foo = "foo" + bar = "BAR" + baz = "BAZ" + + [table1] + foo = "foo" + bar = "BAR" + baz = "BAZ" + + [table1.table2] + bar = "BAR" + baz = "BAZ" + "#; + let expected: toml::value::Table = toml::de::from_str(expected).unwrap(); + + merge_toml(&mut default, override_toml); + + assert_eq!(&expected, &default); + } } diff --git a/third_party/rust/uniffi_bindgen/src/library_mode.rs b/third_party/rust/uniffi_bindgen/src/library_mode.rs index f170ea5e91..c460c03d9f 100644 --- a/third_party/rust/uniffi_bindgen/src/library_mode.rs +++ b/third_party/rust/uniffi_bindgen/src/library_mode.rs @@ -16,8 +16,8 @@ /// - UniFFI can figure out the package/module names for each crate, eliminating the external /// package maps. use crate::{ - bindings::TargetLanguage, load_initial_config, macro_metadata, BindingGenerator, - BindingGeneratorDefault, BindingsConfig, ComponentInterface, Result, + load_initial_config, macro_metadata, BindingGenerator, BindingsConfig, ComponentInterface, + Result, }; use anyhow::{bail, Context}; use camino::Utf8Path; @@ -33,21 +33,21 @@ use uniffi_meta::{ /// Generate foreign bindings /// /// Returns the list of sources used to generate the bindings, in no particular order. -pub fn generate_bindings( +pub fn generate_bindings( library_path: &Utf8Path, crate_name: Option, - target_languages: &[TargetLanguage], + binding_generator: &T, + config_file_override: Option<&Utf8Path>, out_dir: &Utf8Path, try_format_code: bool, -) -> Result>> { +) -> Result>> { generate_external_bindings( - BindingGeneratorDefault { - target_languages: target_languages.into(), - try_format_code, - }, + binding_generator, library_path, - crate_name, + crate_name.clone(), + config_file_override, out_dir, + try_format_code, ) } @@ -55,10 +55,12 @@ pub fn generate_bindings( /// /// Returns the list of sources used to generate the bindings, in no particular order. pub fn generate_external_bindings( - binding_generator: T, + binding_generator: &T, library_path: &Utf8Path, crate_name: Option, + config_file_override: Option<&Utf8Path>, out_dir: &Utf8Path, + try_format_code: bool, ) -> Result>> { let cargo_metadata = MetadataCommand::new() .exec() @@ -66,7 +68,12 @@ pub fn generate_external_bindings( let cdylib_name = calc_cdylib_name(library_path); binding_generator.check_library_path(library_path, cdylib_name)?; - let mut sources = find_sources(&cargo_metadata, library_path, cdylib_name)?; + let mut sources = find_sources( + &cargo_metadata, + library_path, + cdylib_name, + config_file_override, + )?; for i in 0..sources.len() { // Partition up the sources list because we're eventually going to call // `update_from_dependency_configs()` which requires an exclusive reference to one source and @@ -101,7 +108,7 @@ pub fn generate_external_bindings( } for source in sources.iter() { - binding_generator.write_bindings(&source.ci, &source.config, out_dir)?; + binding_generator.write_bindings(&source.ci, &source.config, out_dir, try_format_code)?; } Ok(sources) @@ -118,10 +125,10 @@ pub struct Source { // If `library_path` is a C dynamic library, return its name pub fn calc_cdylib_name(library_path: &Utf8Path) -> Option<&str> { - let cdylib_extentions = [".so", ".dll", ".dylib"]; + let cdylib_extensions = [".so", ".dll", ".dylib"]; let filename = library_path.file_name()?; let filename = filename.strip_prefix("lib").unwrap_or(filename); - for ext in cdylib_extentions { + for ext in cdylib_extensions { if let Some(f) = filename.strip_suffix(ext) { return Some(f); } @@ -133,6 +140,7 @@ fn find_sources( cargo_metadata: &cargo_metadata::Metadata, library_path: &Utf8Path, cdylib_name: Option<&str>, + config_file_override: Option<&Utf8Path>, ) -> Result>> { let items = macro_metadata::extract_from_library(library_path)?; let mut metadata_groups = create_metadata_groups(&items); @@ -178,7 +186,7 @@ fn find_sources( ci.add_metadata(metadata)?; }; ci.add_metadata(group)?; - let mut config = load_initial_config::(crate_root, None)?; + let mut config = load_initial_config::(crate_root, config_file_override)?; if let Some(cdylib_name) = cdylib_name { config.update_from_cdylib_name(cdylib_name); } diff --git a/third_party/rust/uniffi_bindgen/src/macro_metadata/ci.rs b/third_party/rust/uniffi_bindgen/src/macro_metadata/ci.rs index 7ce6c3a70b..69fad1980e 100644 --- a/third_party/rust/uniffi_bindgen/src/macro_metadata/ci.rs +++ b/third_party/rust/uniffi_bindgen/src/macro_metadata/ci.rs @@ -4,9 +4,7 @@ use crate::interface::{CallbackInterface, ComponentInterface, Enum, Record, Type}; use anyhow::{bail, Context}; -use uniffi_meta::{ - create_metadata_groups, group_metadata, EnumMetadata, ErrorMetadata, Metadata, MetadataGroup, -}; +use uniffi_meta::{create_metadata_groups, group_metadata, EnumMetadata, Metadata, MetadataGroup}; /// Add Metadata items to the ComponentInterface /// @@ -98,7 +96,9 @@ fn add_item_to_ci(iface: &mut ComponentInterface, item: Metadata) -> anyhow::Res iface.add_record_definition(record)?; } Metadata::Enum(meta) => { - let flat = meta.variants.iter().all(|v| v.fields.is_empty()); + let flat = meta + .forced_flatness + .unwrap_or_else(|| meta.variants.iter().all(|v| v.fields.is_empty())); add_enum_to_ci(iface, meta, flat)?; } Metadata::Object(meta) => { @@ -117,22 +117,11 @@ fn add_item_to_ci(iface: &mut ComponentInterface, item: Metadata) -> anyhow::Res module_path: meta.module_path.clone(), name: meta.name.clone(), })?; - iface.add_callback_interface_definition(CallbackInterface::new( - meta.name, - meta.module_path, - )); + iface.add_callback_interface_definition(CallbackInterface::try_from(meta)?); } Metadata::TraitMethod(meta) => { iface.add_trait_method_meta(meta)?; } - Metadata::Error(meta) => { - iface.note_name_used_as_error(meta.name()); - match meta { - ErrorMetadata::Enum { enum_, is_flat } => { - add_enum_to_ci(iface, enum_, is_flat)?; - } - }; - } Metadata::CustomType(meta) => { iface.types.add_known_type(&Type::Custom { module_path: meta.module_path.clone(), diff --git a/third_party/rust/uniffi_bindgen/src/macro_metadata/extract.rs b/third_party/rust/uniffi_bindgen/src/macro_metadata/extract.rs index 25b5ef17ba..6d440919f1 100644 --- a/third_party/rust/uniffi_bindgen/src/macro_metadata/extract.rs +++ b/third_party/rust/uniffi_bindgen/src/macro_metadata/extract.rs @@ -30,7 +30,7 @@ fn extract_from_bytes(file_data: &[u8]) -> anyhow::Result> { Object::PE(pe) => extract_from_pe(pe, file_data), Object::Mach(mach) => extract_from_mach(mach, file_data), Object::Archive(archive) => extract_from_archive(archive, file_data), - Object::Unknown(_) => bail!("Unknown library format"), + _ => bail!("Unknown library format"), } } diff --git a/third_party/rust/uniffi_bindgen/src/scaffolding/mod.rs b/third_party/rust/uniffi_bindgen/src/scaffolding/mod.rs index f3759cf6fa..7fd81831aa 100644 --- a/third_party/rust/uniffi_bindgen/src/scaffolding/mod.rs +++ b/third_party/rust/uniffi_bindgen/src/scaffolding/mod.rs @@ -45,7 +45,6 @@ mod filters { format!("std::sync::Arc<{}>", imp.rust_name_for(name)) } Type::CallbackInterface { name, .. } => format!("Box"), - Type::ForeignExecutor => "::uniffi::ForeignExecutor".into(), Type::Optional { inner_type } => { format!("std::option::Option<{}>", type_rs(inner_type)?) } @@ -64,41 +63,12 @@ mod filters { kind: ExternalKind::Interface, .. } => format!("::std::sync::Arc"), - Type::External { name, .. } => format!("r#{name}"), - }) - } - - // Map a type to Rust code that specifies the FfiConverter implementation. - // - // This outputs something like `>` - pub fn ffi_trait(type_: &Type, trait_name: &str) -> Result { - Ok(match type_ { Type::External { name, - kind: ExternalKind::Interface, + kind: ExternalKind::Trait, .. - } => { - format!("<::std::sync::Arc as ::uniffi::{trait_name}>") - } - _ => format!( - "<{} as ::uniffi::{trait_name}>", - type_rs(type_)? - ), - }) - } - - pub fn return_type(callable: &T) -> Result { - let return_type = match callable.return_type() { - Some(t) => type_rs(&t)?, - None => "()".to_string(), - }; - match callable.throws_type() { - Some(t) => type_rs(&t)?, - None => "()".to_string(), - }; - Ok(match callable.throws_type() { - Some(e) => format!("::std::result::Result<{return_type}, {}>", type_rs(&e)?), - None => return_type, + } => format!("::std::sync::Arc"), + Type::External { name, .. } => format!("r#{name}"), }) } diff --git a/third_party/rust/uniffi_bindgen/src/scaffolding/templates/CallbackInterfaceTemplate.rs b/third_party/rust/uniffi_bindgen/src/scaffolding/templates/CallbackInterfaceTemplate.rs index 64c69e4d8e..658f4c8de5 100644 --- a/third_party/rust/uniffi_bindgen/src/scaffolding/templates/CallbackInterfaceTemplate.rs +++ b/third_party/rust/uniffi_bindgen/src/scaffolding/templates/CallbackInterfaceTemplate.rs @@ -1,82 +1,17 @@ -{# -// For each Callback Interface definition, we assume that there is a corresponding trait defined in Rust client code. -// If the UDL callback interface and Rust trait's methods don't match, the Rust compiler will complain. -// We generate: -// * an init function to accept that `ForeignCallback` from the foreign language, and stores it. -// * a holder for a `ForeignCallback`, of type `uniffi::ForeignCallbackInternals`. -// * a proxy `struct` which implements the `trait` that the Callback Interface corresponds to. This -// is the object that client code interacts with. -// - for each method, arguments will be packed into a `RustBuffer` and sent over the `ForeignCallback` to be -// unpacked and called. The return value is packed into another `RustBuffer` and sent back to Rust. -// - a `Drop` `impl`, which tells the foreign language to forget about the real callback object. -#} -{% let trait_name = cbi.name() -%} -{% let trait_impl = format!("UniFFICallbackHandler{}", trait_name) %} -{% let foreign_callback_internals = format!("foreign_callback_{}_internals", trait_name)|upper -%} - -// Register a foreign callback for getting across the FFI. -#[doc(hidden)] -static {{ foreign_callback_internals }}: uniffi::ForeignCallbackInternals = uniffi::ForeignCallbackInternals::new(); - -#[doc(hidden)] -#[no_mangle] -pub extern "C" fn {{ cbi.ffi_init_callback().name() }}(callback: uniffi::ForeignCallback, _: &mut uniffi::RustCallStatus) { - {{ foreign_callback_internals }}.set_callback(callback); - // The call status should be initialized to CALL_SUCCESS, so no need to modify it. -} - -// Make an implementation which will shell out to the foreign language. -#[doc(hidden)] -#[derive(Debug)] -struct {{ trait_impl }} { - handle: u64 -} - -impl {{ trait_impl }} { - fn new(handle: u64) -> Self { - Self { handle } - } -} - -impl Drop for {{ trait_impl }} { - fn drop(&mut self) { - {{ foreign_callback_internals }}.invoke_callback::<(), crate::UniFfiTag>( - self.handle, uniffi::IDX_CALLBACK_FREE, Default::default() - ) - } -} - -uniffi::deps::static_assertions::assert_impl_all!({{ trait_impl }}: Send); - -impl r#{{ trait_name }} for {{ trait_impl }} { +#[::uniffi::export_for_udl(callback_interface)] +pub trait r#{{ cbi.name() }} { {%- for meth in cbi.methods() %} - - {#- Method declaration #} - fn r#{{ meth.name() -}} - ({% call rs::arg_list_decl_with_prefix("&self", meth) %}) - {%- match (meth.return_type(), meth.throws_type()) %} - {%- when (Some(return_type), None) %} -> {{ return_type.borrow()|type_rs }} - {%- when (Some(return_type), Some(err)) %} -> ::std::result::Result<{{ return_type.borrow()|type_rs }}, {{ err|type_rs }}> - {%- when (None, Some(err)) %} -> ::std::result::Result<(), {{ err|type_rs }}> - {% else -%} - {%- endmatch -%} { - {#- Method body #} - - {#- Packing args into a RustBuffer #} - {% if meth.arguments().len() == 0 -%} - let args_buf = Vec::new(); - {% else -%} - let mut args_buf = Vec::new(); - {% endif -%} + fn r#{{ meth.name() }}( + {% if meth.takes_self_by_arc()%}self: Arc{% else %}&self{% endif %}, {%- for arg in meth.arguments() %} - {{ arg.as_type().borrow()|ffi_trait("Lower") }}::write(r#{{ arg.name() }}, &mut args_buf); - {%- endfor -%} - let args_rbuf = uniffi::RustBuffer::from_vec(args_buf); - - {#- Calling into foreign code. #} - {{ foreign_callback_internals }}.invoke_callback::<{{ meth|return_type }}, crate::UniFfiTag>(self.handle, {{ loop.index }}, args_rbuf) - } - {%- endfor %} + r#{{ arg.name() }}: {% if arg.by_ref() %}&{% endif %}{{ arg.as_type().borrow()|type_rs }}, + {%- endfor %} + ) + {%- match (meth.return_type(), meth.throws_type()) %} + {%- when (Some(return_type), None) %} -> {{ return_type|type_rs }}; + {%- when (Some(return_type), Some(error_type)) %} -> ::std::result::Result::<{{ return_type|type_rs }}, {{ error_type|type_rs }}>; + {%- when (None, Some(error_type)) %} -> ::std::result::Result::<(), {{ error_type|type_rs }}>; + {%- when (None, None) %}; + {%- endmatch %} + {% endfor %} } - -::uniffi::scaffolding_ffi_converter_callback_interface!(r#{{ trait_name }}, {{ trait_impl }}); diff --git a/third_party/rust/uniffi_bindgen/src/scaffolding/templates/EnumTemplate.rs b/third_party/rust/uniffi_bindgen/src/scaffolding/templates/EnumTemplate.rs index 6b9f96f224..f918ef2f3a 100644 --- a/third_party/rust/uniffi_bindgen/src/scaffolding/templates/EnumTemplate.rs +++ b/third_party/rust/uniffi_bindgen/src/scaffolding/templates/EnumTemplate.rs @@ -1,13 +1,12 @@ {# -// For each enum declared in the UDL, we assume the caller has provided a corresponding -// rust `enum`. We provide the traits for sending it across the FFI, which will fail to -// compile if the provided struct has a different shape to the one declared in the UDL. -// -// We define a unit-struct to implement the trait to sidestep Rust's orphan rule (ADR-0006). It's -// public so other crates can refer to it via an `[External='crate'] typedef` +// Forward work to `uniffi_macros` This keeps macro-based and UDL-based generated code consistent. #} -#[::uniffi::derive_enum_for_udl] +#[::uniffi::derive_enum_for_udl( + {%- if e.is_non_exhaustive() -%} + non_exhaustive, + {%- endif %} +)] enum r#{{ e.name() }} { {%- for variant in e.variants() %} r#{{ variant.name() }} { diff --git a/third_party/rust/uniffi_bindgen/src/scaffolding/templates/ErrorTemplate.rs b/third_party/rust/uniffi_bindgen/src/scaffolding/templates/ErrorTemplate.rs index 94538ecaa8..64f48e2334 100644 --- a/third_party/rust/uniffi_bindgen/src/scaffolding/templates/ErrorTemplate.rs +++ b/third_party/rust/uniffi_bindgen/src/scaffolding/templates/ErrorTemplate.rs @@ -1,10 +1,5 @@ {# -// For each error declared in the UDL, we assume the caller has provided a corresponding -// rust `enum`. We provide the traits for sending it across the FFI, which will fail to -// compile if the provided struct has a different shape to the one declared in the UDL. -// -// We define a unit-struct to implement the trait to sidestep Rust's orphan rule (ADR-0006). It's -// public so other crates can refer to it via an `[External='crate'] typedef` +// Forward work to `uniffi_macros` This keeps macro-based and UDL-based generated code consistent. #} #[::uniffi::derive_error_for_udl( @@ -14,6 +9,9 @@ with_try_read, {%- endif %} {%- endif %} + {%- if e.is_non_exhaustive() -%} + non_exhaustive, + {%- endif %} )] enum r#{{ e.name() }} { {%- for variant in e.variants() %} diff --git a/third_party/rust/uniffi_bindgen/src/scaffolding/templates/ExternalTypesTemplate.rs b/third_party/rust/uniffi_bindgen/src/scaffolding/templates/ExternalTypesTemplate.rs index ade1578897..d67e172cc2 100644 --- a/third_party/rust/uniffi_bindgen/src/scaffolding/templates/ExternalTypesTemplate.rs +++ b/third_party/rust/uniffi_bindgen/src/scaffolding/templates/ExternalTypesTemplate.rs @@ -10,6 +10,8 @@ ::uniffi::ffi_converter_forward!(r#{{ name }}, ::{{ crate_name|crate_name_rs }}::UniFfiTag, crate::UniFfiTag); {%- when ExternalKind::Interface %} ::uniffi::ffi_converter_arc_forward!(r#{{ name }}, ::{{ crate_name|crate_name_rs }}::UniFfiTag, crate::UniFfiTag); +{%- when ExternalKind::Trait %} +::uniffi::ffi_converter_arc_forward!(dyn r#{{ name }}, ::{{ crate_name|crate_name_rs }}::UniFfiTag, crate::UniFfiTag); {%- endmatch %} {% endif %} {%- endfor %} diff --git a/third_party/rust/uniffi_bindgen/src/scaffolding/templates/ObjectTemplate.rs b/third_party/rust/uniffi_bindgen/src/scaffolding/templates/ObjectTemplate.rs index e2445c670d..e752878af5 100644 --- a/third_party/rust/uniffi_bindgen/src/scaffolding/templates/ObjectTemplate.rs +++ b/third_party/rust/uniffi_bindgen/src/scaffolding/templates/ObjectTemplate.rs @@ -1,24 +1,15 @@ -// For each Object definition, we assume the caller has provided an appropriately-shaped `struct T` -// with an `impl` for each method on the object. We create an `Arc` for "safely" handing out -// references to these structs to foreign language code, and we provide a `pub extern "C"` function -// corresponding to each method. -// -// (Note that "safely" is in "scare quotes" - that's because we use functions on an `Arc` that -// that are inherently unsafe, but the code we generate is safe in practice.) -// -// If the caller's implementation of the struct does not match with the methods or types specified -// in the UDL, then the rust compiler will complain with a (hopefully at least somewhat helpful!) -// error message when processing this generated code. +{# +// Forward work to `uniffi_macros` This keeps macro-based and UDL-based generated code consistent. +#} -{%- match obj.imp() -%} -{%- when ObjectImpl::Trait %} -#[::uniffi::export_for_udl] +{%- if obj.is_trait_interface() %} +#[::uniffi::export_for_udl{% if obj.has_callback_interface() %}(with_foreign){% endif %}] pub trait r#{{ obj.name() }} { {%- for meth in obj.methods() %} - fn {{ meth.name() }}( + {% if meth.is_async() %}async {% endif %}fn r#{{ meth.name() }}( {% if meth.takes_self_by_arc()%}self: Arc{% else %}&self{% endif %}, {%- for arg in meth.arguments() %} - {{ arg.name() }}: {% if arg.by_ref() %}&{% endif %}{{ arg.as_type().borrow()|type_rs }}, + r#{{ arg.name() }}: {% if arg.by_ref() %}&{% endif %}{{ arg.as_type().borrow()|type_rs }}, {%- endfor %} ) {%- match (meth.return_type(), meth.throws_type()) %} @@ -29,7 +20,7 @@ pub trait r#{{ obj.name() }} { {%- endmatch %} {% endfor %} } -{% when ObjectImpl::Struct %} +{%- else %} {%- for tm in obj.uniffi_traits() %} {% match tm %} {% when UniffiTrait::Debug { fmt }%} @@ -46,9 +37,10 @@ pub trait r#{{ obj.name() }} { struct {{ obj.rust_name() }} { } {%- for cons in obj.constructors() %} -#[::uniffi::export_for_udl(constructor)] +#[::uniffi::export_for_udl] impl {{ obj.rust_name() }} { - pub fn r#{{ cons.name() }}( + #[uniffi::constructor] + pub {% if cons.is_async() %}async {% endif %}fn r#{{ cons.name() }}( {%- for arg in cons.arguments() %} r#{{ arg.name() }}: {% if arg.by_ref() %}&{% endif %}{{ arg.as_type().borrow()|type_rs }}, {%- endfor %} @@ -68,7 +60,7 @@ impl {{ obj.rust_name() }} { {%- for meth in obj.methods() %} #[::uniffi::export_for_udl] impl {{ obj.rust_name() }} { - pub fn r#{{ meth.name() }}( + pub {% if meth.is_async() %}async {% endif %}fn r#{{ meth.name() }}( {% if meth.takes_self_by_arc()%}self: Arc{% else %}&self{% endif %}, {%- for arg in meth.arguments() %} r#{{ arg.name() }}: {% if arg.by_ref() %}&{% endif %}{{ arg.as_type().borrow()|type_rs }}, @@ -86,4 +78,4 @@ impl {{ obj.rust_name() }} { } {%- endfor %} -{% endmatch %} +{% endif %} diff --git a/third_party/rust/uniffi_bindgen/src/scaffolding/templates/RecordTemplate.rs b/third_party/rust/uniffi_bindgen/src/scaffolding/templates/RecordTemplate.rs index 85e131dd8c..a7affdf7b8 100644 --- a/third_party/rust/uniffi_bindgen/src/scaffolding/templates/RecordTemplate.rs +++ b/third_party/rust/uniffi_bindgen/src/scaffolding/templates/RecordTemplate.rs @@ -1,11 +1,5 @@ {# -// For each record declared in the UDL, we assume the caller has provided a corresponding -// rust `struct` with the declared fields. We provide the traits for sending it across the FFI. -// If the caller's struct does not match the shape and types declared in the UDL then the rust -// compiler will complain with a type error. -// -// We define a unit-struct to implement the trait to sidestep Rust's orphan rule (ADR-0006). It's -// public so other crates can refer to it via an `[External='crate'] typedef` +// Forward work to `uniffi_macros` This keeps macro-based and UDL-based generated code consistent. #} #[::uniffi::derive_record_for_udl] diff --git a/third_party/rust/uniffi_bindgen/src/scaffolding/templates/TopLevelFunctionTemplate.rs b/third_party/rust/uniffi_bindgen/src/scaffolding/templates/TopLevelFunctionTemplate.rs index eeee0f5ee2..27f3686b9f 100644 --- a/third_party/rust/uniffi_bindgen/src/scaffolding/templates/TopLevelFunctionTemplate.rs +++ b/third_party/rust/uniffi_bindgen/src/scaffolding/templates/TopLevelFunctionTemplate.rs @@ -1,5 +1,8 @@ +{# +// Forward work to `uniffi_macros` This keeps macro-based and UDL-based generated code consistent. +#} #[::uniffi::export_for_udl] -pub fn r#{{ func.name() }}( +pub {% if func.is_async() %}async {% endif %}fn r#{{ func.name() }}( {%- for arg in func.arguments() %} r#{{ arg.name() }}: {% if arg.by_ref() %}&{% endif %}{{ arg.as_type().borrow()|type_rs }}, {%- endfor %} diff --git a/third_party/rust/uniffi_build/.cargo-checksum.json b/third_party/rust/uniffi_build/.cargo-checksum.json index 8e585bfa95..a9005a0f58 100644 --- a/third_party/rust/uniffi_build/.cargo-checksum.json +++ b/third_party/rust/uniffi_build/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"Cargo.toml":"a6db989e5a3d597219df0a9c94541130b7db607efd8606043cd1187971020639","src/lib.rs":"47ff3d1a18456164414af1c20cd5df129401e5257cc15552ecc39afed8970707"},"package":"001964dd3682d600084b3aaf75acf9c3426699bc27b65e96bb32d175a31c74e9"} \ No newline at end of file +{"files":{"Cargo.toml":"8fcf43ff5e6c1281a1ee5f9ed796b0f8115bd39ca9ce5b2d0c32e88d9eb2038f","README.md":"37c1af00ec81a9f1bc206ab3578356e5f9ad4077dc46dd1bb623d81d804948b8","src/lib.rs":"47ff3d1a18456164414af1c20cd5df129401e5257cc15552ecc39afed8970707"},"package":"45cba427aeb7b3a8b54830c4c915079a7a3c62608dd03dddba1d867a8a023eb4"} \ No newline at end of file diff --git a/third_party/rust/uniffi_build/Cargo.toml b/third_party/rust/uniffi_build/Cargo.toml index 3fe7ee5cf0..fed51c34ca 100644 --- a/third_party/rust/uniffi_build/Cargo.toml +++ b/third_party/rust/uniffi_build/Cargo.toml @@ -12,11 +12,12 @@ [package] edition = "2021" name = "uniffi_build" -version = "0.25.3" +version = "0.27.1" authors = ["Firefox Sync Team "] description = "a multi-language bindings generator for rust (build script helpers)" homepage = "https://mozilla.github.io/uniffi-rs" documentation = "https://mozilla.github.io/uniffi-rs" +readme = "README.md" keywords = [ "ffi", "bindgen", @@ -31,7 +32,7 @@ version = "1" version = "1.0.8" [dependencies.uniffi_bindgen] -version = "=0.25.3" +version = "=0.27.1" default-features = false [features] diff --git a/third_party/rust/uniffi_build/README.md b/third_party/rust/uniffi_build/README.md new file mode 100644 index 0000000000..64ac3486a3 --- /dev/null +++ b/third_party/rust/uniffi_build/README.md @@ -0,0 +1,81 @@ +# UniFFI - a multi-language bindings generator for Rust + +UniFFI is a toolkit for building cross-platform software components in Rust. + +For the impatient, see [**the UniFFI user guide**](https://mozilla.github.io/uniffi-rs/) +or [**the UniFFI examples**](https://github.com/mozilla/uniffi-rs/tree/main/examples#example-uniffi-components). + +By writing your core business logic in Rust and describing its interface in an "object model", +you can use UniFFI to help you: + +* Compile your Rust code into a shared library for use on different target platforms. +* Generate bindings to load and use the library from different target languages. + +You can describe your object model in an [interface definition file](https://mozilla.github.io/uniffi-rs/udl_file_spec.html) +or [by using proc-macros](https://mozilla.github.io/uniffi-rs/proc_macro/index.html). + +UniFFI is currently used extensively by Mozilla in Firefox mobile and desktop browsers; +written once in Rust, auto-generated bindings allow that functionality to be called +from both Kotlin (for Android apps) and Swift (for iOS apps). +It also has a growing community of users shipping various cool things to many users. + +UniFFI comes with support for **Kotlin**, **Swift**, **Python** and **Ruby** with 3rd party bindings available for **C#** and **Golang**. +Additional foreign language bindings can be developed externally and we welcome contributions to list them here. +See [Third-party foreign language bindings](#third-party-foreign-language-bindings). + +## User Guide + +You can read more about using the tool in [**the UniFFI user guide**](https://mozilla.github.io/uniffi-rs/). + +We consider it ready for production use, but UniFFI is a long way from a 1.0 release with lots of internal work still going on. +We try hard to avoid breaking simple consumers, but more advanced things might break as you upgrade over time. + +### Etymology and Pronunciation + +ˈjuːnɪfaɪ. Pronounced to rhyme with "unify". + +A portmanteau word that also puns with "unify", to signify the joining of one codebase accessed from many languages. + +uni - [Latin ūni-, from ūnus, one] +FFI - [Abbreviation, Foreign Function Interface] + +## Alternative tools + +Other tools we know of which try and solve a similarly shaped problem are: + +* [Diplomat](https://github.com/rust-diplomat/diplomat/) - see our [writeup of + the different approach taken by that tool](docs/diplomat-and-macros.md) +* [Interoptopus](https://github.com/ralfbiedert/interoptopus/) + +(Please open a PR if you think other tools should be listed!) + +## Third-party foreign language bindings + +* [Kotlin Multiplatform support](https://gitlab.com/trixnity/uniffi-kotlin-multiplatform-bindings). The repository contains Kotlin Multiplatform bindings generation for UniFFI, letting you target both JVM and Native. +* [Go bindings](https://github.com/NordSecurity/uniffi-bindgen-go) +* [C# bindings](https://github.com/NordSecurity/uniffi-bindgen-cs) +* [Dart bindings](https://github.com/NiallBunting/uniffi-rs-dart) + +### External resources + +There are a few third-party resources that make it easier to work with UniFFI: + +* [Plugin support for `.udl` files](https://github.com/Lonami/uniffi-dl) for the IDEA platform ([*uniffi-dl* in the JetBrains marketplace](https://plugins.jetbrains.com/plugin/20527-uniffi-dl)). It provides syntax highlighting, code folding, code completion, reference resolution and navigation (among others features) for the [UniFFI Definition Language (UDL)](https://mozilla.github.io/uniffi-rs/). +* [cargo swift](https://github.com/antoniusnaumann/cargo-swift), a cargo plugin to build a Swift Package from Rust code. It provides an init command for setting up a UniFFI crate and a package command for building a Swift package from Rust code - without the need for additional configuration or build scripts. +* [Cargo NDK Gradle Plugin](https://github.com/willir/cargo-ndk-android-gradle) allows you to build Rust code using [`cargo-ndk`](https://github.com/bbqsrc/cargo-ndk), which generally makes Android library builds less painful. +* [`uniffi-starter`](https://github.com/ianthetechie/uniffi-starter) is a minimal project demonstrates a wide range of UniFFI in a complete project in a compact manner. It includes a full Android library build process, an XCFramework generation script, and example Swift package structure. + +(Please open a PR if you think other resources should be listed!) + +## Contributing + +If this tool sounds interesting to you, please help us develop it! You can: + +* View the [contributor guidelines](./docs/contributing.md). +* File or work on [issues](https://github.com/mozilla/uniffi-rs/issues) here in GitHub. +* Join discussions in the [#uniffi:mozilla.org](https://matrix.to/#/#uniffi:mozilla.org) + room on Matrix. + +## Code of Conduct + +This project is governed by Mozilla's [Community Participation Guidelines](./CODE_OF_CONDUCT.md). diff --git a/third_party/rust/uniffi_checksum_derive/.cargo-checksum.json b/third_party/rust/uniffi_checksum_derive/.cargo-checksum.json index 9df6eba7b4..5e211378a0 100644 --- a/third_party/rust/uniffi_checksum_derive/.cargo-checksum.json +++ b/third_party/rust/uniffi_checksum_derive/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"Cargo.toml":"028104be15a1d73a8ca192a30865b26827344808a95b21e3798e4c9406ebc4f1","src/lib.rs":"44d2e2c595b14d33d16c71dfe4ef42ad0b9e010a878ee2ec49c2e929d60275ba"},"package":"55137c122f712d9330fd985d66fa61bdc381752e89c35708c13ce63049a3002c"} \ No newline at end of file +{"files":{"Cargo.toml":"da89504b9007c2a1ea0e498a2e8ec6baeb0ff7391363cd9007e383247637792c","README.md":"37c1af00ec81a9f1bc206ab3578356e5f9ad4077dc46dd1bb623d81d804948b8","src/lib.rs":"44d2e2c595b14d33d16c71dfe4ef42ad0b9e010a878ee2ec49c2e929d60275ba"},"package":"ae7e5a6c33b1dec3f255f57ec0b6af0f0b2bb3021868be1d5eec7a38e2905ebc"} \ No newline at end of file diff --git a/third_party/rust/uniffi_checksum_derive/Cargo.toml b/third_party/rust/uniffi_checksum_derive/Cargo.toml index a3c4ed7ca3..681c8846e1 100644 --- a/third_party/rust/uniffi_checksum_derive/Cargo.toml +++ b/third_party/rust/uniffi_checksum_derive/Cargo.toml @@ -12,11 +12,12 @@ [package] edition = "2021" name = "uniffi_checksum_derive" -version = "0.25.3" +version = "0.27.1" authors = ["Firefox Sync Team "] description = "a multi-language bindings generator for rust (checksum custom derive)" homepage = "https://mozilla.github.io/uniffi-rs" documentation = "https://mozilla.github.io/uniffi-rs" +readme = "README.md" keywords = [ "ffi", "bindgen", diff --git a/third_party/rust/uniffi_checksum_derive/README.md b/third_party/rust/uniffi_checksum_derive/README.md new file mode 100644 index 0000000000..64ac3486a3 --- /dev/null +++ b/third_party/rust/uniffi_checksum_derive/README.md @@ -0,0 +1,81 @@ +# UniFFI - a multi-language bindings generator for Rust + +UniFFI is a toolkit for building cross-platform software components in Rust. + +For the impatient, see [**the UniFFI user guide**](https://mozilla.github.io/uniffi-rs/) +or [**the UniFFI examples**](https://github.com/mozilla/uniffi-rs/tree/main/examples#example-uniffi-components). + +By writing your core business logic in Rust and describing its interface in an "object model", +you can use UniFFI to help you: + +* Compile your Rust code into a shared library for use on different target platforms. +* Generate bindings to load and use the library from different target languages. + +You can describe your object model in an [interface definition file](https://mozilla.github.io/uniffi-rs/udl_file_spec.html) +or [by using proc-macros](https://mozilla.github.io/uniffi-rs/proc_macro/index.html). + +UniFFI is currently used extensively by Mozilla in Firefox mobile and desktop browsers; +written once in Rust, auto-generated bindings allow that functionality to be called +from both Kotlin (for Android apps) and Swift (for iOS apps). +It also has a growing community of users shipping various cool things to many users. + +UniFFI comes with support for **Kotlin**, **Swift**, **Python** and **Ruby** with 3rd party bindings available for **C#** and **Golang**. +Additional foreign language bindings can be developed externally and we welcome contributions to list them here. +See [Third-party foreign language bindings](#third-party-foreign-language-bindings). + +## User Guide + +You can read more about using the tool in [**the UniFFI user guide**](https://mozilla.github.io/uniffi-rs/). + +We consider it ready for production use, but UniFFI is a long way from a 1.0 release with lots of internal work still going on. +We try hard to avoid breaking simple consumers, but more advanced things might break as you upgrade over time. + +### Etymology and Pronunciation + +ˈjuːnɪfaɪ. Pronounced to rhyme with "unify". + +A portmanteau word that also puns with "unify", to signify the joining of one codebase accessed from many languages. + +uni - [Latin ūni-, from ūnus, one] +FFI - [Abbreviation, Foreign Function Interface] + +## Alternative tools + +Other tools we know of which try and solve a similarly shaped problem are: + +* [Diplomat](https://github.com/rust-diplomat/diplomat/) - see our [writeup of + the different approach taken by that tool](docs/diplomat-and-macros.md) +* [Interoptopus](https://github.com/ralfbiedert/interoptopus/) + +(Please open a PR if you think other tools should be listed!) + +## Third-party foreign language bindings + +* [Kotlin Multiplatform support](https://gitlab.com/trixnity/uniffi-kotlin-multiplatform-bindings). The repository contains Kotlin Multiplatform bindings generation for UniFFI, letting you target both JVM and Native. +* [Go bindings](https://github.com/NordSecurity/uniffi-bindgen-go) +* [C# bindings](https://github.com/NordSecurity/uniffi-bindgen-cs) +* [Dart bindings](https://github.com/NiallBunting/uniffi-rs-dart) + +### External resources + +There are a few third-party resources that make it easier to work with UniFFI: + +* [Plugin support for `.udl` files](https://github.com/Lonami/uniffi-dl) for the IDEA platform ([*uniffi-dl* in the JetBrains marketplace](https://plugins.jetbrains.com/plugin/20527-uniffi-dl)). It provides syntax highlighting, code folding, code completion, reference resolution and navigation (among others features) for the [UniFFI Definition Language (UDL)](https://mozilla.github.io/uniffi-rs/). +* [cargo swift](https://github.com/antoniusnaumann/cargo-swift), a cargo plugin to build a Swift Package from Rust code. It provides an init command for setting up a UniFFI crate and a package command for building a Swift package from Rust code - without the need for additional configuration or build scripts. +* [Cargo NDK Gradle Plugin](https://github.com/willir/cargo-ndk-android-gradle) allows you to build Rust code using [`cargo-ndk`](https://github.com/bbqsrc/cargo-ndk), which generally makes Android library builds less painful. +* [`uniffi-starter`](https://github.com/ianthetechie/uniffi-starter) is a minimal project demonstrates a wide range of UniFFI in a complete project in a compact manner. It includes a full Android library build process, an XCFramework generation script, and example Swift package structure. + +(Please open a PR if you think other resources should be listed!) + +## Contributing + +If this tool sounds interesting to you, please help us develop it! You can: + +* View the [contributor guidelines](./docs/contributing.md). +* File or work on [issues](https://github.com/mozilla/uniffi-rs/issues) here in GitHub. +* Join discussions in the [#uniffi:mozilla.org](https://matrix.to/#/#uniffi:mozilla.org) + room on Matrix. + +## Code of Conduct + +This project is governed by Mozilla's [Community Participation Guidelines](./CODE_OF_CONDUCT.md). diff --git a/third_party/rust/uniffi_core/.cargo-checksum.json b/third_party/rust/uniffi_core/.cargo-checksum.json index 59804f7c89..573f7a72c9 100644 --- a/third_party/rust/uniffi_core/.cargo-checksum.json +++ b/third_party/rust/uniffi_core/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"Cargo.toml":"b074f0db902264714faf879e99bdbc07df1550d75694f96751b499f98fecd16d","release.toml":"b150796411fc6ff90b481218cb50f8ac7c07f5845aebdb8e17877d47e55b05b9","src/ffi/callbackinterface.rs":"9e8650f0df087bf5e030a13d28f4990079e53613e656789b4b539d937a7fd288","src/ffi/ffidefault.rs":"f1ce099b92adbb12b160d513bae93342c7b6d806d7f6ebb665067db10af9a681","src/ffi/foreignbytes.rs":"d2b46e1a6317aa64801b855e0d12af6bcdef118d8036603d11c3cdaf6f35fdfe","src/ffi/foreigncallbacks.rs":"af8129a69ef23b92859e1cca0d666c95f0ed2c1fb2797f4495d824b65f774d03","src/ffi/foreignexecutor.rs":"123687921ce6dfb7f5bfa0736a630cfeff7f376b776ea03fc651da21ffd1cab8","src/ffi/mod.rs":"8117b08bbb7af3e97f66ed69c9690b60e8da0d6d8940349c7b9659a47cd8c92f","src/ffi/rustbuffer.rs":"8cc1f94b9ecba52b911da6a68155921c1b7f51b899d9874ddbc281a379941473","src/ffi/rustcalls.rs":"7caaa35ba8898c4b4983f07cefa80584ba00e753a11d496e578c80abe0cabe8b","src/ffi/rustfuture.rs":"d240426c8c8b83e3f6a2c0013e905298611287b2bb2022eb8161532209c635ca","src/ffi_converter_impls.rs":"82c1b47e02718610f2a5556997cd29ba5d8daf149d6353f470be0d9b971d968a","src/ffi_converter_traits.rs":"646c0d4aeb807d3e40db4d289f909030d0b2684087871a7d40d337680096b7d6","src/lib.rs":"4ad1a2899944a20e80a55d1c7bd01ff28395ace743a083c65847e6ea216fc5c8","src/metadata.rs":"6520ffcf2568a0d95f0f854acb6fc8aeaae26ef1f23fc576c2c50db72aa30eee","src/panichook.rs":"9f49c7994a8e5489c1105c488bb3f8c5571bc5f813e7be90441eca15da5c9851"},"package":"6121a127a3af1665cd90d12dd2b3683c2643c5103281d0fed5838324ca1fad5b"} \ No newline at end of file +{"files":{"Cargo.toml":"c8969fbc6e8f6694e260ab78c94f9b4195d61afb7836b4c130b542d3b91b9200","README.md":"37c1af00ec81a9f1bc206ab3578356e5f9ad4077dc46dd1bb623d81d804948b8","release.toml":"b150796411fc6ff90b481218cb50f8ac7c07f5845aebdb8e17877d47e55b05b9","src/ffi/callbackinterface.rs":"f0184cf76bd86abb2815d260a87f85bd7060f5373ac6ef6f71955ece2a5075af","src/ffi/ffidefault.rs":"0db83fbcbc274c4c0daf7fb27833400568839b77a3496155840734c511d801e0","src/ffi/foreignbytes.rs":"d2b46e1a6317aa64801b855e0d12af6bcdef118d8036603d11c3cdaf6f35fdfe","src/ffi/foreigncallbacks.rs":"2b820a34b78705f5debc302a25c64d515a4aa7b3bdade083f4c1cfa2803664ae","src/ffi/foreignfuture.rs":"c1d621e41ea6af0c1d3959b46af8567c3fdc4164e7a82d635fcbb1da2c0737ac","src/ffi/handle.rs":"91f91469a81cb19edebb8bba433df62658cc66f6b54d5dc8520eb5793a85abd9","src/ffi/mod.rs":"30eea545299747838bf11b0698cfb71cedd3ca04d8cfb703c53198fcc44045c1","src/ffi/rustbuffer.rs":"0e725347f916834b17156413f406d5ca6c064b2cbc7437b051fe6692ad72c2aa","src/ffi/rustcalls.rs":"51c6499871c7d5eb4f80cabc806f26dd1df3b1090a2419d0d967aa9c5299a0a6","src/ffi/rustfuture/future.rs":"426cd0ad3c8cf008a7052a7d89856b6c6d5053b94e24325f5666d0281a40ec7f","src/ffi/rustfuture/mod.rs":"44568267e591f5b37f77acfdd6e60ae55ce48ab0a17fd81af3aeb31baa3d53e6","src/ffi/rustfuture/scheduler.rs":"c6484fff14c04596df5f306f2090366435dcff92561d317fde1ea9c097a9576b","src/ffi/rustfuture/tests.rs":"211241fb484a3a103eb0418e7d295850ea021bcd583fa1488f5efc68f33d5ab8","src/ffi_converter_impls.rs":"397c813f2e765462d7a7be524e6ac75e813a91a8ffd11c7e7df05f853213f77b","src/ffi_converter_traits.rs":"24c8cf6ada9b2f63b265e62c0f9092d640e533d0d7234e9156f92c3d1902f430","src/lib.rs":"1f6a031bbb160dfe46455a8bc24596f63b1e478f45579bfff62a62f58900bee4","src/metadata.rs":"83e463c377c0f501e58ac4eb5cc47c433c1473cecd47305fa89283e736b48d96","src/panichook.rs":"9f49c7994a8e5489c1105c488bb3f8c5571bc5f813e7be90441eca15da5c9851"},"package":"0ea3eb5474d50fc149b7e4d86b9c5bd4a61dcc167f0683902bf18ae7bbb3deef"} \ No newline at end of file diff --git a/third_party/rust/uniffi_core/Cargo.toml b/third_party/rust/uniffi_core/Cargo.toml index 4d4cbe2758..ce36a2168a 100644 --- a/third_party/rust/uniffi_core/Cargo.toml +++ b/third_party/rust/uniffi_core/Cargo.toml @@ -12,11 +12,12 @@ [package] edition = "2021" name = "uniffi_core" -version = "0.25.3" +version = "0.27.1" authors = ["Firefox Sync Team "] description = "a multi-language bindings generator for rust (runtime support code)" homepage = "https://mozilla.github.io/uniffi-rs" documentation = "https://mozilla.github.io/uniffi-rs" +readme = "README.md" keywords = [ "ffi", "bindgen", @@ -44,7 +45,7 @@ version = "0.4" version = "1.10.0" [dependencies.oneshot] -version = "0.1.5" +version = "0.1.6" features = ["async"] package = "oneshot-uniffi" @@ -56,5 +57,4 @@ version = "1.1.0" [features] default = [] -extern-rustbuffer = [] tokio = ["dep:async-compat"] diff --git a/third_party/rust/uniffi_core/README.md b/third_party/rust/uniffi_core/README.md new file mode 100644 index 0000000000..64ac3486a3 --- /dev/null +++ b/third_party/rust/uniffi_core/README.md @@ -0,0 +1,81 @@ +# UniFFI - a multi-language bindings generator for Rust + +UniFFI is a toolkit for building cross-platform software components in Rust. + +For the impatient, see [**the UniFFI user guide**](https://mozilla.github.io/uniffi-rs/) +or [**the UniFFI examples**](https://github.com/mozilla/uniffi-rs/tree/main/examples#example-uniffi-components). + +By writing your core business logic in Rust and describing its interface in an "object model", +you can use UniFFI to help you: + +* Compile your Rust code into a shared library for use on different target platforms. +* Generate bindings to load and use the library from different target languages. + +You can describe your object model in an [interface definition file](https://mozilla.github.io/uniffi-rs/udl_file_spec.html) +or [by using proc-macros](https://mozilla.github.io/uniffi-rs/proc_macro/index.html). + +UniFFI is currently used extensively by Mozilla in Firefox mobile and desktop browsers; +written once in Rust, auto-generated bindings allow that functionality to be called +from both Kotlin (for Android apps) and Swift (for iOS apps). +It also has a growing community of users shipping various cool things to many users. + +UniFFI comes with support for **Kotlin**, **Swift**, **Python** and **Ruby** with 3rd party bindings available for **C#** and **Golang**. +Additional foreign language bindings can be developed externally and we welcome contributions to list them here. +See [Third-party foreign language bindings](#third-party-foreign-language-bindings). + +## User Guide + +You can read more about using the tool in [**the UniFFI user guide**](https://mozilla.github.io/uniffi-rs/). + +We consider it ready for production use, but UniFFI is a long way from a 1.0 release with lots of internal work still going on. +We try hard to avoid breaking simple consumers, but more advanced things might break as you upgrade over time. + +### Etymology and Pronunciation + +ˈjuːnɪfaɪ. Pronounced to rhyme with "unify". + +A portmanteau word that also puns with "unify", to signify the joining of one codebase accessed from many languages. + +uni - [Latin ūni-, from ūnus, one] +FFI - [Abbreviation, Foreign Function Interface] + +## Alternative tools + +Other tools we know of which try and solve a similarly shaped problem are: + +* [Diplomat](https://github.com/rust-diplomat/diplomat/) - see our [writeup of + the different approach taken by that tool](docs/diplomat-and-macros.md) +* [Interoptopus](https://github.com/ralfbiedert/interoptopus/) + +(Please open a PR if you think other tools should be listed!) + +## Third-party foreign language bindings + +* [Kotlin Multiplatform support](https://gitlab.com/trixnity/uniffi-kotlin-multiplatform-bindings). The repository contains Kotlin Multiplatform bindings generation for UniFFI, letting you target both JVM and Native. +* [Go bindings](https://github.com/NordSecurity/uniffi-bindgen-go) +* [C# bindings](https://github.com/NordSecurity/uniffi-bindgen-cs) +* [Dart bindings](https://github.com/NiallBunting/uniffi-rs-dart) + +### External resources + +There are a few third-party resources that make it easier to work with UniFFI: + +* [Plugin support for `.udl` files](https://github.com/Lonami/uniffi-dl) for the IDEA platform ([*uniffi-dl* in the JetBrains marketplace](https://plugins.jetbrains.com/plugin/20527-uniffi-dl)). It provides syntax highlighting, code folding, code completion, reference resolution and navigation (among others features) for the [UniFFI Definition Language (UDL)](https://mozilla.github.io/uniffi-rs/). +* [cargo swift](https://github.com/antoniusnaumann/cargo-swift), a cargo plugin to build a Swift Package from Rust code. It provides an init command for setting up a UniFFI crate and a package command for building a Swift package from Rust code - without the need for additional configuration or build scripts. +* [Cargo NDK Gradle Plugin](https://github.com/willir/cargo-ndk-android-gradle) allows you to build Rust code using [`cargo-ndk`](https://github.com/bbqsrc/cargo-ndk), which generally makes Android library builds less painful. +* [`uniffi-starter`](https://github.com/ianthetechie/uniffi-starter) is a minimal project demonstrates a wide range of UniFFI in a complete project in a compact manner. It includes a full Android library build process, an XCFramework generation script, and example Swift package structure. + +(Please open a PR if you think other resources should be listed!) + +## Contributing + +If this tool sounds interesting to you, please help us develop it! You can: + +* View the [contributor guidelines](./docs/contributing.md). +* File or work on [issues](https://github.com/mozilla/uniffi-rs/issues) here in GitHub. +* Join discussions in the [#uniffi:mozilla.org](https://matrix.to/#/#uniffi:mozilla.org) + room on Matrix. + +## Code of Conduct + +This project is governed by Mozilla's [Community Participation Guidelines](./CODE_OF_CONDUCT.md). diff --git a/third_party/rust/uniffi_core/src/ffi/callbackinterface.rs b/third_party/rust/uniffi_core/src/ffi/callbackinterface.rs index 7be66880bb..e7a4faab64 100644 --- a/third_party/rust/uniffi_core/src/ffi/callbackinterface.rs +++ b/third_party/rust/uniffi_core/src/ffi/callbackinterface.rs @@ -91,121 +91,20 @@ //! //! Uniffi generates a protocol or interface in client code in the foreign language must implement. //! -//! For each callback interface, a `CallbackInternals` (on the Foreign Language side) and `ForeignCallbackInternals` -//! (on Rust side) manages the process through a `ForeignCallback`. There is one `ForeignCallback` per callback interface. +//! For each callback interface, UniFFI defines a VTable. +//! This is a `repr(C)` struct where each field is a `repr(C)` callback function pointer. +//! There is one field for each method, plus an extra field for the `uniffi_free` method. +//! The foreign code registers one VTable per callback interface with Rust. //! -//! Passing a callback interface implementation from foreign language (e.g. `AndroidKeychain`) into Rust causes the -//! `KeychainCallbackInternals` to store the instance in a handlemap. -//! -//! The object handle is passed over to Rust, and used to instantiate a struct `KeychainProxy` which implements -//! the trait. This proxy implementation is generate by Uniffi. The `KeychainProxy` object is then passed to -//! client code as `Box`. -//! -//! Methods on `KeychainProxy` objects (e.g. `self.keychain.get("username".into())`) encode the arguments into a `RustBuffer`. -//! Using the `ForeignCallback`, it calls the `CallbackInternals` object on the foreign language side using the -//! object handle, and the method selector. -//! -//! The `CallbackInternals` object unpacks the arguments from the passed buffer, gets the object out from the handlemap, -//! and calls the actual implementation of the method. -//! -//! If there's a return value, it is packed up in to another `RustBuffer` and used as the return value for -//! `ForeignCallback`. The caller of `ForeignCallback`, the `KeychainProxy` unpacks the returned buffer into the correct -//! type and then returns to client code. +//! VTable methods have a similar signature to Rust scaffolding functions. +//! The one difference is that values are returned via an out pointer to work around a Python bug (https://bugs.python.org/issue5710). //! +//! The foreign object that implements the interface is represented by an opaque handle. +//! UniFFI generates a struct that implements the trait by calling VTable methods, passing the handle as the first parameter. +//! When the struct is dropped, the `uniffi_free` method is called. -use crate::{ForeignCallback, ForeignCallbackCell, Lift, LiftReturn, RustBuffer}; use std::fmt; -/// The method index used by the Drop trait to communicate to the foreign language side that Rust has finished with it, -/// and it can be deleted from the handle map. -pub const IDX_CALLBACK_FREE: u32 = 0; - -/// Result of a foreign callback invocation -#[repr(i32)] -#[derive(Debug, PartialEq, Eq)] -pub enum CallbackResult { - /// Successful call. - /// The return value is serialized to `buf_ptr`. - Success = 0, - /// Expected error. - /// This is returned when a foreign method throws an exception that corresponds to the Rust Err half of a Result. - /// The error value is serialized to `buf_ptr`. - Error = 1, - /// Unexpected error. - /// An error message string is serialized to `buf_ptr`. - UnexpectedError = 2, -} - -impl TryFrom for CallbackResult { - // On errors we return the unconverted value - type Error = i32; - - fn try_from(value: i32) -> Result { - match value { - 0 => Ok(Self::Success), - 1 => Ok(Self::Error), - 2 => Ok(Self::UnexpectedError), - n => Err(n), - } - } -} - -/// Struct to hold a foreign callback. -pub struct ForeignCallbackInternals { - callback_cell: ForeignCallbackCell, -} - -impl ForeignCallbackInternals { - pub const fn new() -> Self { - ForeignCallbackInternals { - callback_cell: ForeignCallbackCell::new(), - } - } - - pub fn set_callback(&self, callback: ForeignCallback) { - self.callback_cell.set(callback); - } - - /// Invoke a callback interface method on the foreign side and return the result - pub fn invoke_callback(&self, handle: u64, method: u32, args: RustBuffer) -> R - where - R: LiftReturn, - { - let mut ret_rbuf = RustBuffer::new(); - let callback = self.callback_cell.get(); - let raw_result = unsafe { - callback( - handle, - method, - args.data_pointer(), - args.len() as i32, - &mut ret_rbuf, - ) - }; - let result = CallbackResult::try_from(raw_result) - .unwrap_or_else(|code| panic!("Callback failed with unexpected return code: {code}")); - match result { - CallbackResult::Success => R::lift_callback_return(ret_rbuf), - CallbackResult::Error => R::lift_callback_error(ret_rbuf), - CallbackResult::UnexpectedError => { - let reason = if !ret_rbuf.is_empty() { - match >::try_lift(ret_rbuf) { - Ok(s) => s, - Err(e) => { - log::error!("{{ trait_name }} Error reading ret_buf: {e}"); - String::from("[Error reading reason]") - } - } - } else { - RustBuffer::destroy(ret_rbuf); - String::from("[Unknown Reason]") - }; - R::handle_callback_unexpected_error(UnexpectedUniFFICallbackError { reason }) - } - } - } -} - /// Used when internal/unexpected error happened when calling a foreign callback, for example when /// a unknown exception is raised /// @@ -216,8 +115,10 @@ pub struct UnexpectedUniFFICallbackError { } impl UnexpectedUniFFICallbackError { - pub fn from_reason(reason: String) -> Self { - Self { reason } + pub fn new(reason: impl fmt::Display) -> Self { + Self { + reason: reason.to_string(), + } } } diff --git a/third_party/rust/uniffi_core/src/ffi/ffidefault.rs b/third_party/rust/uniffi_core/src/ffi/ffidefault.rs index 1f86f6b13b..a992ab7384 100644 --- a/third_party/rust/uniffi_core/src/ffi/ffidefault.rs +++ b/third_party/rust/uniffi_core/src/ffi/ffidefault.rs @@ -39,6 +39,12 @@ impl FfiDefault for () { fn ffi_default() {} } +impl FfiDefault for crate::Handle { + fn ffi_default() -> Self { + Self::default() + } +} + impl FfiDefault for *const std::ffi::c_void { fn ffi_default() -> Self { std::ptr::null() @@ -51,9 +57,10 @@ impl FfiDefault for crate::RustBuffer { } } -impl FfiDefault for crate::ForeignExecutorHandle { +impl FfiDefault for crate::ForeignFuture { fn ffi_default() -> Self { - Self(std::ptr::null()) + extern "C" fn free(_handle: u64) {} + crate::ForeignFuture { handle: 0, free } } } diff --git a/third_party/rust/uniffi_core/src/ffi/foreigncallbacks.rs b/third_party/rust/uniffi_core/src/ffi/foreigncallbacks.rs index ac2463cd8e..326ff12747 100644 --- a/third_party/rust/uniffi_core/src/ffi/foreigncallbacks.rs +++ b/third_party/rust/uniffi_core/src/ffi/foreigncallbacks.rs @@ -8,96 +8,32 @@ //! code loads the exported library. For each callback type, we also define a "cell" type for //! storing the callback. -use std::sync::atomic::{AtomicUsize, Ordering}; - -use crate::{ForeignExecutorHandle, RustBuffer, RustTaskCallback}; - -/// ForeignCallback is the Rust representation of a foreign language function. -/// It is the basis for all callbacks interfaces. It is registered exactly once per callback interface, -/// at library start up time. -/// Calling this method is only done by generated objects which mirror callback interfaces objects in the foreign language. -/// -/// * The `handle` is the key into a handle map on the other side of the FFI used to look up the foreign language object -/// that implements the callback interface/trait. -/// * The `method` selector specifies the method that will be called on the object, by looking it up in a list of methods from -/// the IDL. The list is 1 indexed. Note that the list of methods is generated by UniFFI from the IDL and used in all -/// bindings, so we can rely on the method list being stable within the same run of UniFFI. -/// * `args_data` and `args_len` represents a serialized buffer of arguments to the function. The scaffolding code -/// writes the callback arguments to this buffer, in order, using `FfiConverter.write()`. The bindings code reads the -/// arguments from the buffer and passes them to the user's callback. -/// * `buf_ptr` is a pointer to where the resulting buffer will be written. UniFFI will allocate a -/// buffer to write the result into. -/// * Callbacks return one of the `CallbackResult` values -/// Note: The output buffer might still contain 0 bytes of data. -pub type ForeignCallback = unsafe extern "C" fn( - handle: u64, - method: u32, - args_data: *const u8, - args_len: i32, - buf_ptr: *mut RustBuffer, -) -> i32; - -/// Callback to schedule a Rust call with a `ForeignExecutor`. The bindings code registers exactly -/// one of these with the Rust code. -/// -/// Delay is an approximate amount of ms to wait before scheduling the call. Delay is usually 0, -/// which means schedule sometime soon. -/// -/// As a special case, when Rust drops the foreign executor, with `task=null`. The foreign -/// bindings should release the reference to the executor that was reserved for Rust. -/// -/// This callback can be invoked from any thread, including threads created by Rust. -/// -/// The callback should return one of the `ForeignExecutorCallbackResult` values. -pub type ForeignExecutorCallback = extern "C" fn( - executor: ForeignExecutorHandle, - delay: u32, - task: Option, - task_data: *const (), -) -> i8; - -/// Store a [ForeignCallback] pointer -pub(crate) struct ForeignCallbackCell(AtomicUsize); - -/// Store a [ForeignExecutorCallback] pointer -pub(crate) struct ForeignExecutorCallbackCell(AtomicUsize); - -/// Macro to define foreign callback types as well as the callback cell. -macro_rules! impl_foreign_callback_cell { - ($callback_type:ident, $cell_type:ident) => { - // Overly-paranoid sanity checking to ensure that these types are - // convertible between each-other. `transmute` actually should check this for - // us too, but this helps document the invariants we rely on in this code. - // - // Note that these are guaranteed by - // https://rust-lang.github.io/unsafe-code-guidelines/layout/function-pointers.html - // and thus this is a little paranoid. - static_assertions::assert_eq_size!(usize, $callback_type); - static_assertions::assert_eq_size!(usize, Option<$callback_type>); - - impl $cell_type { - pub const fn new() -> Self { - Self(AtomicUsize::new(0)) - } - - pub fn set(&self, callback: $callback_type) { - // Store the pointer using Ordering::Relaxed. This is sufficient since callback - // should be set at startup, before there's any chance of using them. - self.0.store(callback as usize, Ordering::Relaxed); - } - - pub fn get(&self) -> $callback_type { - let ptr_value = self.0.load(Ordering::Relaxed); - unsafe { - // SAFETY: self.0 was set in `set` from our function pointer type, so - // it's safe to transmute it back here. - ::std::mem::transmute::>(ptr_value) - .expect("Bug: callback not set. This is likely a uniffi bug.") - } - } +use std::{ + ptr::{null_mut, NonNull}, + sync::atomic::{AtomicPtr, Ordering}, +}; + +// Cell type that stores any NonNull +#[doc(hidden)] +pub struct UniffiForeignPointerCell(AtomicPtr); + +impl UniffiForeignPointerCell { + pub const fn new() -> Self { + Self(AtomicPtr::new(null_mut())) + } + + pub fn set(&self, callback: NonNull) { + self.0.store(callback.as_ptr(), Ordering::Relaxed); + } + + pub fn get(&self) -> &T { + unsafe { + NonNull::new(self.0.load(Ordering::Relaxed)) + .expect("Foreign pointer not set. This is likely a uniffi bug.") + .as_mut() } - }; + } } -impl_foreign_callback_cell!(ForeignCallback, ForeignCallbackCell); -impl_foreign_callback_cell!(ForeignExecutorCallback, ForeignExecutorCallbackCell); +unsafe impl Send for UniffiForeignPointerCell {} +unsafe impl Sync for UniffiForeignPointerCell {} diff --git a/third_party/rust/uniffi_core/src/ffi/foreignexecutor.rs b/third_party/rust/uniffi_core/src/ffi/foreignexecutor.rs deleted file mode 100644 index 7b1cb9bd80..0000000000 --- a/third_party/rust/uniffi_core/src/ffi/foreignexecutor.rs +++ /dev/null @@ -1,487 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -//! Schedule tasks using a foreign executor. - -use std::panic; - -use crate::{ForeignExecutorCallback, ForeignExecutorCallbackCell}; - -/// Opaque handle for a foreign task executor. -/// -/// Foreign code can either use an actual pointer, or use an integer value casted to it. -#[repr(transparent)] -#[derive(Clone, Copy, Debug)] -pub struct ForeignExecutorHandle(pub(crate) *const ()); - -// Implement Send + Sync for `ForeignExecutor`. The foreign bindings code is responsible for -// making the `ForeignExecutorCallback` thread-safe. -unsafe impl Send for ForeignExecutorHandle {} - -unsafe impl Sync for ForeignExecutorHandle {} - -/// Result code returned by `ForeignExecutorCallback` -#[repr(i8)] -#[derive(Debug, PartialEq, Eq)] -pub enum ForeignExecutorCallbackResult { - /// Callback was scheduled successfully - Success = 0, - /// Callback couldn't be scheduled because the foreign executor is canceled/closed. - Cancelled = 1, - /// Callback couldn't be scheduled because of some other error - Error = 2, -} - -impl ForeignExecutorCallbackResult { - /// Check the result code for the foreign executor callback - /// - /// If the result was `ForeignExecutorCallbackResult.Success`, this method returns `true`. - /// - /// If not, this method returns `false`, logging errors for any unexpected return values - pub fn check_result_code(result: i8) -> bool { - match result { - n if n == ForeignExecutorCallbackResult::Success as i8 => true, - n if n == ForeignExecutorCallbackResult::Cancelled as i8 => false, - n if n == ForeignExecutorCallbackResult::Error as i8 => { - log::error!( - "ForeignExecutorCallbackResult::Error returned by foreign executor callback" - ); - false - } - n => { - log::error!("Unknown code ({n}) returned by foreign executor callback"); - false - } - } - } -} - -// Option should use the null pointer optimization and be represented in C as a -// regular pointer. Let's check that. -static_assertions::assert_eq_size!(usize, Option); - -/// Callback for a Rust task, this is what the foreign executor invokes -/// -/// The task will be passed the `task_data` passed to `ForeignExecutorCallback` in addition to one -/// of the `RustTaskCallbackCode` values. -pub type RustTaskCallback = extern "C" fn(*const (), RustTaskCallbackCode); - -/// Passed to a `RustTaskCallback` function when the executor invokes them. -/// -/// Every `RustTaskCallback` will be invoked eventually, this code is used to distinguish the times -/// when it's invoked successfully vs times when the callback is being called because the foreign -/// executor has been cancelled / shutdown -#[repr(i8)] -#[derive(Debug, PartialEq, Eq)] -pub enum RustTaskCallbackCode { - /// Successful task callback invocation - Success = 0, - /// The `ForeignExecutor` has been cancelled. - /// - /// This signals that any progress using the executor should be halted. In particular, Futures - /// should not continue to progress. - Cancelled = 1, -} - -static FOREIGN_EXECUTOR_CALLBACK: ForeignExecutorCallbackCell = ForeignExecutorCallbackCell::new(); - -/// Set the global ForeignExecutorCallback. This is called by the foreign bindings, normally -/// during initialization. -pub fn foreign_executor_callback_set(callback: ForeignExecutorCallback) { - FOREIGN_EXECUTOR_CALLBACK.set(callback); -} - -/// Schedule Rust calls using a foreign executor -#[derive(Debug)] -pub struct ForeignExecutor { - pub(crate) handle: ForeignExecutorHandle, -} - -impl ForeignExecutor { - pub fn new(executor: ForeignExecutorHandle) -> Self { - Self { handle: executor } - } - - /// Schedule a closure to be run. - /// - /// This method can be used for "fire-and-forget" style calls, where the calling code doesn't - /// need to await the result. - /// - /// Closure requirements: - /// - Send: since the closure will likely run on a different thread - /// - 'static: since it runs at an arbitrary time, so all references need to be 'static - /// - panic::UnwindSafe: if the closure panics, it should not corrupt any data - pub fn schedule(&self, delay: u32, task: F) { - let leaked_ptr: *mut F = Box::leak(Box::new(task)); - if !schedule_raw( - self.handle, - delay, - schedule_callback::, - leaked_ptr as *const (), - ) { - // If schedule_raw() failed, drop the leaked box since `schedule_callback()` has not been - // scheduled to run. - unsafe { - drop(Box::::from_raw(leaked_ptr)); - }; - } - } - - /// Schedule a closure to be run and get a Future for the result - /// - /// Closure requirements: - /// - Send: since the closure will likely run on a different thread - /// - 'static: since it runs at an arbitrary time, so all references need to be 'static - /// - panic::UnwindSafe: if the closure panics, it should not corrupt any data - pub async fn run(&self, delay: u32, closure: F) -> T - where - F: FnOnce() -> T + Send + 'static + panic::UnwindSafe, - T: Send + 'static, - { - // Create a oneshot channel to handle the future - let (sender, receiver) = oneshot::channel(); - // We can use `AssertUnwindSafe` here because: - // - The closure is unwind safe - // - `Sender` is not marked unwind safe, maybe this is just an oversight in the oneshot - // library. However, calling `send()` and dropping the Sender should certainly be - // unwind safe. `send()` should probably not panic at all and if it does it shouldn't - // do it in a way that breaks the Receiver. - // - Calling `expect` may result in a panic, but this should should not break either the - // Sender or Receiver. - self.schedule( - delay, - panic::AssertUnwindSafe(move || { - sender.send(closure()).expect("Error sending future result") - }), - ); - receiver.await.expect("Error receiving future result") - } -} - -/// Low-level schedule interface -/// -/// When using this function, take care to ensure that the `ForeignExecutor` that holds the -/// `ForeignExecutorHandle` has not been dropped. -/// -/// Returns true if the callback was successfully scheduled -pub(crate) fn schedule_raw( - handle: ForeignExecutorHandle, - delay: u32, - callback: RustTaskCallback, - data: *const (), -) -> bool { - let result_code = (FOREIGN_EXECUTOR_CALLBACK.get())(handle, delay, Some(callback), data); - ForeignExecutorCallbackResult::check_result_code(result_code) -} - -impl Drop for ForeignExecutor { - fn drop(&mut self) { - (FOREIGN_EXECUTOR_CALLBACK.get())(self.handle, 0, None, std::ptr::null()); - } -} - -extern "C" fn schedule_callback(data: *const (), status_code: RustTaskCallbackCode) -where - F: FnOnce() + Send + 'static + panic::UnwindSafe, -{ - // No matter what, we need to call Box::from_raw() to balance the Box::leak() call. - let task = unsafe { Box::from_raw(data as *mut F) }; - // Skip running the task for the `RustTaskCallbackCode::Cancelled` code - if status_code == RustTaskCallbackCode::Success { - run_task(task); - } -} - -/// Run a scheduled task, catching any panics. -/// -/// If there are panics, then we will log a warning and return None. -fn run_task T + panic::UnwindSafe, T>(task: F) -> Option { - match panic::catch_unwind(task) { - Ok(v) => Some(v), - Err(cause) => { - let message = if let Some(s) = cause.downcast_ref::<&'static str>() { - (*s).to_string() - } else if let Some(s) = cause.downcast_ref::() { - s.clone() - } else { - "Unknown panic!".to_string() - }; - log::warn!("Error calling UniFFI callback function: {message}"); - None - } - } -} - -#[cfg(test)] -pub use test::MockEventLoop; - -#[cfg(test)] -mod test { - use super::*; - use std::{ - future::Future, - pin::Pin, - sync::{ - atomic::{AtomicU32, Ordering}, - Arc, Mutex, Once, - }, - task::{Context, Poll, Wake, Waker}, - }; - - /// Simulate an event loop / task queue / coroutine scope on the foreign side - /// - /// This simply collects scheduled calls into a Vec for testing purposes. - /// - /// Most of the MockEventLoop methods are `pub` since it's also used by the `rustfuture` tests. - pub struct MockEventLoop { - // Wrap everything in a mutex since we typically share access to MockEventLoop via an Arc - inner: Mutex, - } - - pub struct MockEventLoopInner { - // calls that have been scheduled - calls: Vec<(u32, Option, *const ())>, - // has the event loop been shutdown? - is_shutdown: bool, - } - - unsafe impl Send for MockEventLoopInner {} - - static FOREIGN_EXECUTOR_CALLBACK_INIT: Once = Once::new(); - - impl MockEventLoop { - pub fn new() -> Arc { - // Make sure we install a foreign executor callback that can deal with mock event loops - FOREIGN_EXECUTOR_CALLBACK_INIT - .call_once(|| foreign_executor_callback_set(mock_executor_callback)); - - Arc::new(Self { - inner: Mutex::new(MockEventLoopInner { - calls: vec![], - is_shutdown: false, - }), - }) - } - - /// Create a new ForeignExecutorHandle - pub fn new_handle(self: &Arc) -> ForeignExecutorHandle { - // To keep the memory management simple, we simply leak an arc reference for this. We - // only create a handful of these in the tests so there's no need for proper cleanup. - ForeignExecutorHandle(Arc::into_raw(Arc::clone(self)) as *const ()) - } - - pub fn new_executor(self: &Arc) -> ForeignExecutor { - ForeignExecutor { - handle: self.new_handle(), - } - } - - /// Get the current number of scheduled calls - pub fn call_count(&self) -> usize { - self.inner.lock().unwrap().calls.len() - } - - /// Get the last scheduled call - pub fn last_call(&self) -> (u32, Option, *const ()) { - self.inner - .lock() - .unwrap() - .calls - .last() - .cloned() - .expect("no calls scheduled") - } - - /// Run all currently scheduled calls - pub fn run_all_calls(&self) { - let mut inner = self.inner.lock().unwrap(); - let is_shutdown = inner.is_shutdown; - for (_delay, callback, data) in inner.calls.drain(..) { - if !is_shutdown { - callback.unwrap()(data, RustTaskCallbackCode::Success); - } else { - callback.unwrap()(data, RustTaskCallbackCode::Cancelled); - } - } - } - - /// Shutdown the eventloop, causing scheduled calls and future calls to be cancelled - pub fn shutdown(&self) { - self.inner.lock().unwrap().is_shutdown = true; - } - } - - // `ForeignExecutorCallback` that we install for testing - extern "C" fn mock_executor_callback( - handle: ForeignExecutorHandle, - delay: u32, - task: Option, - task_data: *const (), - ) -> i8 { - let eventloop = handle.0 as *const MockEventLoop; - let mut inner = unsafe { (*eventloop).inner.lock().unwrap() }; - if inner.is_shutdown { - ForeignExecutorCallbackResult::Cancelled as i8 - } else { - inner.calls.push((delay, task, task_data)); - ForeignExecutorCallbackResult::Success as i8 - } - } - - #[test] - fn test_schedule_raw() { - extern "C" fn callback(data: *const (), _status_code: RustTaskCallbackCode) { - unsafe { - *(data as *mut u32) += 1; - } - } - - let eventloop = MockEventLoop::new(); - - let value: u32 = 0; - assert_eq!(eventloop.call_count(), 0); - - schedule_raw( - eventloop.new_handle(), - 0, - callback, - &value as *const u32 as *const (), - ); - assert_eq!(eventloop.call_count(), 1); - assert_eq!(value, 0); - - eventloop.run_all_calls(); - assert_eq!(eventloop.call_count(), 0); - assert_eq!(value, 1); - } - - #[test] - fn test_schedule() { - let eventloop = MockEventLoop::new(); - let executor = eventloop.new_executor(); - let value = Arc::new(AtomicU32::new(0)); - assert_eq!(eventloop.call_count(), 0); - - let value2 = value.clone(); - executor.schedule(0, move || { - value2.fetch_add(1, Ordering::Relaxed); - }); - assert_eq!(eventloop.call_count(), 1); - assert_eq!(value.load(Ordering::Relaxed), 0); - - eventloop.run_all_calls(); - assert_eq!(eventloop.call_count(), 0); - assert_eq!(value.load(Ordering::Relaxed), 1); - } - - #[derive(Default)] - struct MockWaker { - wake_count: AtomicU32, - } - - impl Wake for MockWaker { - fn wake(self: Arc) { - self.wake_count.fetch_add(1, Ordering::Relaxed); - } - } - - #[test] - fn test_run() { - let eventloop = MockEventLoop::new(); - let executor = eventloop.new_executor(); - let mock_waker = Arc::new(MockWaker::default()); - let waker = Waker::from(mock_waker.clone()); - let mut context = Context::from_waker(&waker); - assert_eq!(eventloop.call_count(), 0); - - let mut future = executor.run(0, move || "test-return-value"); - unsafe { - assert_eq!( - Pin::new_unchecked(&mut future).poll(&mut context), - Poll::Pending - ); - } - assert_eq!(eventloop.call_count(), 1); - assert_eq!(mock_waker.wake_count.load(Ordering::Relaxed), 0); - - eventloop.run_all_calls(); - assert_eq!(eventloop.call_count(), 0); - assert_eq!(mock_waker.wake_count.load(Ordering::Relaxed), 1); - unsafe { - assert_eq!( - Pin::new_unchecked(&mut future).poll(&mut context), - Poll::Ready("test-return-value") - ); - } - } - - #[test] - fn test_drop() { - let eventloop = MockEventLoop::new(); - let executor = eventloop.new_executor(); - - drop(executor); - // Calling drop should schedule a call with null task data. - assert_eq!(eventloop.call_count(), 1); - assert_eq!(eventloop.last_call().1, None); - } - - // Test that cancelled calls never run - #[test] - fn test_cancelled_call() { - let eventloop = MockEventLoop::new(); - let executor = eventloop.new_executor(); - // Create a shared counter - let counter = Arc::new(AtomicU32::new(0)); - // schedule increments using both `schedule()` and run()` - let counter_clone = Arc::clone(&counter); - executor.schedule(0, move || { - counter_clone.fetch_add(1, Ordering::Relaxed); - }); - let counter_clone = Arc::clone(&counter); - let future = executor.run(0, move || { - counter_clone.fetch_add(1, Ordering::Relaxed); - }); - // shutdown the eventloop before the scheduled call gets a chance to run. - eventloop.shutdown(); - // `run_all_calls()` will cause the scheduled task callbacks to run, but will pass - // `RustTaskCallbackCode::Cancelled` to it. This drop the scheduled closure without executing - // it. - eventloop.run_all_calls(); - - assert_eq!(counter.load(Ordering::Relaxed), 0); - drop(future); - } - - // Test that when scheduled calls are cancelled, the closures are dropped properly - #[test] - fn test_cancellation_drops_closures() { - let eventloop = MockEventLoop::new(); - let executor = eventloop.new_executor(); - - // Create an Arc<> that we will move into the closures to test if they are dropped or not - let arc = Arc::new(0); - let arc_clone = Arc::clone(&arc); - executor.schedule(0, move || assert_eq!(*arc_clone, 0)); - let arc_clone = Arc::clone(&arc); - let future = executor.run(0, move || assert_eq!(*arc_clone, 0)); - - // shutdown the eventloop and run the (cancelled) scheduled calls. - eventloop.shutdown(); - eventloop.run_all_calls(); - // try to schedule some more calls now that the loop has been shutdown - let arc_clone = Arc::clone(&arc); - executor.schedule(0, move || assert_eq!(*arc_clone, 0)); - let arc_clone = Arc::clone(&arc); - let future2 = executor.run(0, move || assert_eq!(*arc_clone, 0)); - - // Drop the futures so they don't hold on to any references - drop(future); - drop(future2); - - // All of these closures should have been dropped by now, there only remaining arc - // reference should be the original - assert_eq!(Arc::strong_count(&arc), 1); - } -} diff --git a/third_party/rust/uniffi_core/src/ffi/foreignfuture.rs b/third_party/rust/uniffi_core/src/ffi/foreignfuture.rs new file mode 100644 index 0000000000..be6a214e84 --- /dev/null +++ b/third_party/rust/uniffi_core/src/ffi/foreignfuture.rs @@ -0,0 +1,241 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +//! This module defines a Rust Future that wraps an async foreign function call. +//! +//! The general idea is to create a [oneshot::Channel], hand the sender to the foreign side, and +//! await the receiver side on the Rust side. +//! +//! The foreign side should: +//! * Input a [ForeignFutureCallback] and a `u64` handle in their scaffolding function. +//! This is the sender, converted to a raw pointer, and an extern "C" function that sends the result. +//! * Return a [ForeignFuture], which represents the foreign task object corresponding to the async function. +//! * Call the [ForeignFutureCallback] when the async function completes with: +//! * The `u64` handle initially passed in +//! * The `ForeignFutureResult` for the call +//! * Wait for the [ForeignFutureHandle::free] function to be called to free the task object. +//! If this is called before the task completes, then the task will be cancelled. + +use crate::{LiftReturn, RustCallStatus, UnexpectedUniFFICallbackError}; + +/// Handle for a foreign future +pub type ForeignFutureHandle = u64; + +/// Handle for a callback data associated with a foreign future. +pub type ForeignFutureCallbackData = *mut (); + +/// Callback that's passed to a foreign async functions. +/// +/// See `LiftReturn` trait for how this is implemented. +pub type ForeignFutureCallback = + extern "C" fn(oneshot_handle: u64, ForeignFutureResult); + +/// C struct that represents the result of a foreign future +#[repr(C)] +pub struct ForeignFutureResult { + // Note: for void returns, T is `()`, which isn't directly representable with C since it's a ZST. + // Foreign code should treat that case as if there was no `return_value` field. + return_value: T, + call_status: RustCallStatus, +} + +/// Perform a call to a foreign async method + +/// C struct that represents the foreign future. +/// +/// This is what's returned by the async scaffolding functions. +#[repr(C)] +pub struct ForeignFuture { + pub handle: ForeignFutureHandle, + pub free: extern "C" fn(handle: ForeignFutureHandle), +} + +impl Drop for ForeignFuture { + fn drop(&mut self) { + (self.free)(self.handle) + } +} + +unsafe impl Send for ForeignFuture {} + +pub async fn foreign_async_call(call_scaffolding_function: F) -> T +where + F: FnOnce(ForeignFutureCallback, u64) -> ForeignFuture, + T: LiftReturn, +{ + let (sender, receiver) = oneshot::channel::>(); + // Keep the ForeignFuture around, even though we don't ever use it. + // The important thing is that the ForeignFuture will be dropped when this Future is. + let _foreign_future = + call_scaffolding_function(foreign_future_complete::, sender.into_raw() as u64); + match receiver.await { + Ok(result) => T::lift_foreign_return(result.return_value, result.call_status), + Err(e) => { + // This shouldn't happen in practice, but we can do our best to recover + T::handle_callback_unexpected_error(UnexpectedUniFFICallbackError::new(format!( + "Error awaiting foreign future: {e}" + ))) + } + } +} + +pub extern "C" fn foreign_future_complete, UT>( + oneshot_handle: u64, + result: ForeignFutureResult, +) { + let channel = unsafe { oneshot::Sender::from_raw(oneshot_handle as *mut ()) }; + // Ignore errors in send. + // + // Error means the receiver was already dropped which will happen when the future is cancelled. + let _ = channel.send(result); +} + +#[cfg(test)] +mod test { + use super::*; + use crate::{Lower, RustBuffer}; + use once_cell::sync::OnceCell; + use std::{ + future::Future, + pin::Pin, + sync::{ + atomic::{AtomicU32, Ordering}, + Arc, + }, + task::{Context, Poll, Wake}, + }; + + struct MockForeignFuture { + freed: Arc, + callback_info: Arc, u64)>>, + rust_future: Option>>>, + } + + impl MockForeignFuture { + fn new() -> Self { + let callback_info = Arc::new(OnceCell::new()); + let freed = Arc::new(AtomicU32::new(0)); + + let rust_future: Pin>> = { + let callback_info = callback_info.clone(); + let freed = freed.clone(); + Box::pin(foreign_async_call::<_, String, crate::UniFfiTag>( + move |callback, data| { + callback_info.set((callback, data)).unwrap(); + ForeignFuture { + handle: Arc::into_raw(freed) as *mut () as u64, + free: Self::free, + } + }, + )) + }; + let rust_future = Some(rust_future); + let mut mock_foreign_future = Self { + freed, + callback_info, + rust_future, + }; + // Poll the future once, to start it up. This ensures that `callback_info` is set. + let _ = mock_foreign_future.poll(); + mock_foreign_future + } + + fn poll(&mut self) -> Poll { + let waker = Arc::new(NoopWaker).into(); + let mut context = Context::from_waker(&waker); + self.rust_future + .as_mut() + .unwrap() + .as_mut() + .poll(&mut context) + } + + fn complete_success(&self, value: String) { + let (callback, data) = self.callback_info.get().unwrap(); + callback( + *data, + ForeignFutureResult { + return_value: >::lower(value), + call_status: RustCallStatus::new(), + }, + ); + } + + fn complete_error(&self, error_message: String) { + let (callback, data) = self.callback_info.get().unwrap(); + callback( + *data, + ForeignFutureResult { + return_value: RustBuffer::default(), + call_status: RustCallStatus::error(error_message), + }, + ); + } + + fn drop_future(&mut self) { + self.rust_future = None + } + + fn free_count(&self) -> u32 { + self.freed.load(Ordering::Relaxed) + } + + extern "C" fn free(handle: u64) { + let flag = unsafe { Arc::from_raw(handle as *mut AtomicU32) }; + flag.fetch_add(1, Ordering::Relaxed); + } + } + + struct NoopWaker; + + impl Wake for NoopWaker { + fn wake(self: Arc) {} + } + + #[test] + fn test_foreign_future() { + let mut mock_foreign_future = MockForeignFuture::new(); + assert_eq!(mock_foreign_future.poll(), Poll::Pending); + mock_foreign_future.complete_success("It worked!".to_owned()); + assert_eq!( + mock_foreign_future.poll(), + Poll::Ready("It worked!".to_owned()) + ); + // Since the future is complete, it should free the foreign future + assert_eq!(mock_foreign_future.free_count(), 1); + } + + #[test] + #[should_panic] + fn test_foreign_future_error() { + let mut mock_foreign_future = MockForeignFuture::new(); + assert_eq!(mock_foreign_future.poll(), Poll::Pending); + mock_foreign_future.complete_error("It Failed!".to_owned()); + let _ = mock_foreign_future.poll(); + } + + #[test] + fn test_drop_after_complete() { + let mut mock_foreign_future = MockForeignFuture::new(); + mock_foreign_future.complete_success("It worked!".to_owned()); + assert_eq!(mock_foreign_future.free_count(), 0); + assert_eq!( + mock_foreign_future.poll(), + Poll::Ready("It worked!".to_owned()) + ); + // Dropping the future after it's complete should not panic, and not cause a double-free + mock_foreign_future.drop_future(); + assert_eq!(mock_foreign_future.free_count(), 1); + } + + #[test] + fn test_drop_before_complete() { + let mut mock_foreign_future = MockForeignFuture::new(); + mock_foreign_future.complete_success("It worked!".to_owned()); + // Dropping the future before it's complete should cancel the future + assert_eq!(mock_foreign_future.free_count(), 0); + mock_foreign_future.drop_future(); + assert_eq!(mock_foreign_future.free_count(), 1); + } +} diff --git a/third_party/rust/uniffi_core/src/ffi/handle.rs b/third_party/rust/uniffi_core/src/ffi/handle.rs new file mode 100644 index 0000000000..8ee2f46c35 --- /dev/null +++ b/third_party/rust/uniffi_core/src/ffi/handle.rs @@ -0,0 +1,46 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/// Object handle +/// +/// Handles opaque `u64` values used to pass objects across the FFI, both for objects implemented in +/// Rust and ones implemented in the foreign language. +/// +/// Rust handles are generated by leaking a raw pointer +/// Foreign handles are generated with a handle map that only generates odd values. +/// For all currently supported architectures and hopefully any ones we add in the future: +/// * 0 is an invalid value. +/// * The lowest bit will always be set for foreign handles and never set for Rust ones (since the +/// leaked pointer will be aligned). +/// +/// Rust handles are mainly managed is through the [crate::HandleAlloc] trait. +#[derive(Copy, Clone, Default, Debug, PartialEq, Eq)] +#[repr(transparent)] +pub struct Handle(u64); + +impl Handle { + pub fn from_pointer(ptr: *const T) -> Self { + Self(ptr as u64) + } + + pub fn as_pointer(&self) -> *const T { + self.0 as *const T + } + + pub fn from_raw(raw: u64) -> Option { + if raw == 0 { + None + } else { + Some(Self(raw)) + } + } + + pub fn from_raw_unchecked(raw: u64) -> Self { + Self(raw) + } + + pub fn as_raw(&self) -> u64 { + self.0 + } +} diff --git a/third_party/rust/uniffi_core/src/ffi/mod.rs b/third_party/rust/uniffi_core/src/ffi/mod.rs index b606323297..acaf2b0d06 100644 --- a/third_party/rust/uniffi_core/src/ffi/mod.rs +++ b/third_party/rust/uniffi_core/src/ffi/mod.rs @@ -8,7 +8,8 @@ pub mod callbackinterface; pub mod ffidefault; pub mod foreignbytes; pub mod foreigncallbacks; -pub mod foreignexecutor; +pub mod foreignfuture; +pub mod handle; pub mod rustbuffer; pub mod rustcalls; pub mod rustfuture; @@ -17,7 +18,8 @@ pub use callbackinterface::*; pub use ffidefault::FfiDefault; pub use foreignbytes::*; pub use foreigncallbacks::*; -pub use foreignexecutor::*; +pub use foreignfuture::*; +pub use handle::*; pub use rustbuffer::*; pub use rustcalls::*; pub use rustfuture::*; diff --git a/third_party/rust/uniffi_core/src/ffi/rustbuffer.rs b/third_party/rust/uniffi_core/src/ffi/rustbuffer.rs index e09e3be89a..8b2972968c 100644 --- a/third_party/rust/uniffi_core/src/ffi/rustbuffer.rs +++ b/third_party/rust/uniffi_core/src/ffi/rustbuffer.rs @@ -52,11 +52,11 @@ use crate::ffi::{rust_call, ForeignBytes, RustCallStatus}; #[derive(Debug)] pub struct RustBuffer { /// The allocated capacity of the underlying `Vec`. - /// In Rust this is a `usize`, but we use an `i32` for compatibility with JNA. - capacity: i32, + /// In Rust this is a `usize`, but we use an `u64` to keep the foreign binding code simple. + capacity: u64, /// The occupied length of the underlying `Vec`. - /// In Rust this is a `usize`, but we use an `i32` for compatibility with JNA. - len: i32, + /// In Rust this is a `usize`, but we use an `u64` to keep the foreign binding code simple. + len: u64, /// The pointer to the allocated buffer of the `Vec`. data: *mut u8, } @@ -84,7 +84,7 @@ impl RustBuffer { /// # Safety /// /// You must ensure that the raw parts uphold the documented invariants of this class. - pub unsafe fn from_raw_parts(data: *mut u8, len: i32, capacity: i32) -> Self { + pub unsafe fn from_raw_parts(data: *mut u8, len: u64, capacity: u64) -> Self { Self { capacity, len, @@ -126,12 +126,8 @@ impl RustBuffer { /// /// Panics if the requested size is too large to fit in an `i32`, and /// hence would risk incompatibility with some foreign-language code. - pub fn new_with_size(size: usize) -> Self { - assert!( - size < i32::MAX as usize, - "RustBuffer requested size too large" - ); - Self::from_vec(vec![0u8; size]) + pub fn new_with_size(size: u64) -> Self { + Self::from_vec(vec![0u8; size as usize]) } /// Consumes a `Vec` and returns its raw parts as a `RustBuffer`. @@ -144,8 +140,8 @@ impl RustBuffer { /// Panics if the vector's length or capacity are too large to fit in an `i32`, /// and hence would risk incompatibility with some foreign-language code. pub fn from_vec(v: Vec) -> Self { - let capacity = i32::try_from(v.capacity()).expect("buffer capacity cannot fit into a i32."); - let len = i32::try_from(v.len()).expect("buffer length cannot fit into a i32."); + let capacity = u64::try_from(v.capacity()).expect("buffer capacity cannot fit into a u64."); + let len = u64::try_from(v.len()).expect("buffer length cannot fit into a u64."); let mut v = std::mem::ManuallyDrop::new(v); unsafe { Self::from_raw_parts(v.as_mut_ptr(), len, capacity) } } @@ -198,39 +194,18 @@ impl Default for RustBuffer { } } -// extern "C" functions for the RustBuffer functionality. +// Functions for the RustBuffer functionality. // -// These are used in two ways: -// 1. Code that statically links to UniFFI can use these directly to handle RustBuffer -// allocation/destruction. The plan is to use this for the Firefox desktop JS bindings. -// -// 2. The scaffolding code re-exports these functions, prefixed with the component name and UDL -// hash This creates a separate set of functions for each UniFFIed component, which is needed -// in the case where we create multiple dylib artifacts since each dylib will have its own -// allocator. +// The scaffolding code re-exports these functions, prefixed with the component name and UDL hash +// This creates a separate set of functions for each UniFFIed component, which is needed in the +// case where we create multiple dylib artifacts since each dylib will have its own allocator. /// This helper allocates a new byte buffer owned by the Rust code, and returns it /// to the foreign-language code as a `RustBuffer` struct. Callers must eventually /// free the resulting buffer, either by explicitly calling [`uniffi_rustbuffer_free`] defined /// below, or by passing ownership of the buffer back into Rust code. -#[cfg(feature = "extern-rustbuffer")] -#[no_mangle] -pub extern "C" fn uniffi_rustbuffer_alloc( - size: i32, - call_status: &mut RustCallStatus, -) -> RustBuffer { - _uniffi_rustbuffer_alloc(size, call_status) -} - -#[cfg(not(feature = "extern-rustbuffer"))] -pub fn uniffi_rustbuffer_alloc(size: i32, call_status: &mut RustCallStatus) -> RustBuffer { - _uniffi_rustbuffer_alloc(size, call_status) -} - -fn _uniffi_rustbuffer_alloc(size: i32, call_status: &mut RustCallStatus) -> RustBuffer { - rust_call(call_status, || { - Ok(RustBuffer::new_with_size(size.max(0) as usize)) - }) +pub fn uniffi_rustbuffer_alloc(size: u64, call_status: &mut RustCallStatus) -> RustBuffer { + rust_call(call_status, || Ok(RustBuffer::new_with_size(size))) } /// This helper copies bytes owned by the foreign-language code into a new byte buffer owned @@ -241,26 +216,9 @@ fn _uniffi_rustbuffer_alloc(size: i32, call_status: &mut RustCallStatus) -> Rust /// # Safety /// This function will dereference a provided pointer in order to copy bytes from it, so /// make sure the `ForeignBytes` struct contains a valid pointer and length. -#[cfg(feature = "extern-rustbuffer")] -#[no_mangle] -pub extern "C" fn uniffi_rustbuffer_from_bytes( - bytes: ForeignBytes, - call_status: &mut RustCallStatus, -) -> RustBuffer { - _uniffi_rustbuffer_from_bytes(bytes, call_status) -} - -#[cfg(not(feature = "extern-rustbuffer"))] pub fn uniffi_rustbuffer_from_bytes( bytes: ForeignBytes, call_status: &mut RustCallStatus, -) -> RustBuffer { - _uniffi_rustbuffer_from_bytes(bytes, call_status) -} - -fn _uniffi_rustbuffer_from_bytes( - bytes: ForeignBytes, - call_status: &mut RustCallStatus, ) -> RustBuffer { rust_call(call_status, || { let bytes = bytes.as_slice(); @@ -274,18 +232,7 @@ fn _uniffi_rustbuffer_from_bytes( /// The argument *must* be a uniquely-owned `RustBuffer` previously obtained from a call /// into the Rust code that returned a buffer, or you'll risk freeing unowned memory or /// corrupting the allocator state. -#[cfg(feature = "extern-rustbuffer")] -#[no_mangle] -pub extern "C" fn uniffi_rustbuffer_free(buf: RustBuffer, call_status: &mut RustCallStatus) { - _uniffi_rustbuffer_free(buf, call_status) -} - -#[cfg(not(feature = "extern-rustbuffer"))] pub fn uniffi_rustbuffer_free(buf: RustBuffer, call_status: &mut RustCallStatus) { - _uniffi_rustbuffer_free(buf, call_status) -} - -fn _uniffi_rustbuffer_free(buf: RustBuffer, call_status: &mut RustCallStatus) { rust_call(call_status, || { RustBuffer::destroy(buf); Ok(()) @@ -307,28 +254,9 @@ fn _uniffi_rustbuffer_free(buf: RustBuffer, call_status: &mut RustCallStatus) { /// The first argument *must* be a uniquely-owned `RustBuffer` previously obtained from a call /// into the Rust code that returned a buffer, or you'll risk freeing unowned memory or /// corrupting the allocator state. -#[cfg(feature = "extern-rustbuffer")] -#[no_mangle] -pub extern "C" fn uniffi_rustbuffer_reserve( - buf: RustBuffer, - additional: i32, - call_status: &mut RustCallStatus, -) -> RustBuffer { - _uniffi_rustbuffer_reserve(buf, additional, call_status) -} - -#[cfg(not(feature = "extern-rustbuffer"))] pub fn uniffi_rustbuffer_reserve( buf: RustBuffer, - additional: i32, - call_status: &mut RustCallStatus, -) -> RustBuffer { - _uniffi_rustbuffer_reserve(buf, additional, call_status) -} - -fn _uniffi_rustbuffer_reserve( - buf: RustBuffer, - additional: i32, + additional: u64, call_status: &mut RustCallStatus, ) -> RustBuffer { rust_call(call_status, || { @@ -392,24 +320,6 @@ mod test { rbuf.destroy_into_vec(); } - #[test] - #[should_panic] - fn test_rustbuffer_provided_capacity_must_be_non_negative() { - // We guard against foreign-language code providing this kind of invalid struct. - let mut v = vec![0u8, 1, 2]; - let rbuf = unsafe { RustBuffer::from_raw_parts(v.as_mut_ptr(), 3, -7) }; - rbuf.destroy_into_vec(); - } - - #[test] - #[should_panic] - fn test_rustbuffer_provided_len_must_be_non_negative() { - // We guard against foreign-language code providing this kind of invalid struct. - let mut v = vec![0u8, 1, 2]; - let rbuf = unsafe { RustBuffer::from_raw_parts(v.as_mut_ptr(), -1, 3) }; - rbuf.destroy_into_vec(); - } - #[test] #[should_panic] fn test_rustbuffer_provided_len_must_not_exceed_capacity() { diff --git a/third_party/rust/uniffi_core/src/ffi/rustcalls.rs b/third_party/rust/uniffi_core/src/ffi/rustcalls.rs index 53265393c0..16b0c76f2e 100644 --- a/third_party/rust/uniffi_core/src/ffi/rustcalls.rs +++ b/third_party/rust/uniffi_core/src/ffi/rustcalls.rs @@ -56,6 +56,13 @@ pub struct RustCallStatus { } impl RustCallStatus { + pub fn new() -> Self { + Self { + code: RustCallStatusCode::Success, + error_buf: MaybeUninit::new(RustBuffer::new()), + } + } + pub fn cancelled() -> Self { Self { code: RustCallStatusCode::Cancelled, @@ -102,7 +109,7 @@ pub enum RustCallStatusCode { /// Handle a scaffolding calls /// /// `callback` is responsible for making the actual Rust call and returning a special result type: -/// - For successfull calls, return `Ok(value)` +/// - For successful calls, return `Ok(value)` /// - For errors that should be translated into thrown exceptions in the foreign code, serialize /// the error into a `RustBuffer`, then return `Ok(buf)` /// - The success type, must implement `FfiDefault`. diff --git a/third_party/rust/uniffi_core/src/ffi/rustfuture.rs b/third_party/rust/uniffi_core/src/ffi/rustfuture.rs deleted file mode 100644 index 0c1a24174b..0000000000 --- a/third_party/rust/uniffi_core/src/ffi/rustfuture.rs +++ /dev/null @@ -1,735 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -//! [`RustFuture`] represents a [`Future`] that can be sent to the foreign code over FFI. -//! -//! This type is not instantiated directly, but via the procedural macros, such as `#[uniffi::export]`. -//! -//! # The big picture -//! -//! We implement async foreign functions using a simplified version of the Future API: -//! -//! 0. At startup, register a [RustFutureContinuationCallback] by calling -//! rust_future_continuation_callback_set. -//! 1. Call the scaffolding function to get a [RustFutureHandle] -//! 2a. In a loop: -//! - Call [rust_future_poll] -//! - Suspend the function until the [rust_future_poll] continuation function is called -//! - If the continuation was function was called with [RustFuturePoll::Ready], then break -//! otherwise continue. -//! 2b. If the async function is cancelled, then call [rust_future_cancel]. This causes the -//! continuation function to be called with [RustFuturePoll::Ready] and the [RustFuture] to -//! enter a cancelled state. -//! 3. Call [rust_future_complete] to get the result of the future. -//! 4. Call [rust_future_free] to free the future, ideally in a finally block. This: -//! - Releases any resources held by the future -//! - Calls any continuation callbacks that have not been called yet -//! -//! Note: Technically, the foreign code calls the scaffolding versions of the `rust_future_*` -//! functions. These are generated by the scaffolding macro, specially prefixed, and extern "C", -//! and manually monomorphized in the case of [rust_future_complete]. See -//! `uniffi_macros/src/setup_scaffolding.rs` for details. -//! -//! ## How does `Future` work exactly? -//! -//! A [`Future`] in Rust does nothing. When calling an async function, it just -//! returns a `Future` but nothing has happened yet. To start the computation, -//! the future must be polled. It returns [`Poll::Ready(r)`][`Poll::Ready`] if -//! the result is ready, [`Poll::Pending`] otherwise. `Poll::Pending` basically -//! means: -//! -//! > Please, try to poll me later, maybe the result will be ready! -//! -//! This model is very different than what other languages do, but it can actually -//! be translated quite easily, fortunately for us! -//! -//! But… wait a minute… who is responsible to poll the `Future` if a `Future` does -//! nothing? Well, it's _the executor_. The executor is responsible _to drive_ the -//! `Future`: that's where they are polled. -//! -//! But… wait another minute… how does the executor know when to poll a [`Future`]? -//! Does it poll them randomly in an endless loop? Well, no, actually it depends -//! on the executor! A well-designed `Future` and executor work as follows. -//! Normally, when [`Future::poll`] is called, a [`Context`] argument is -//! passed to it. It contains a [`Waker`]. The [`Waker`] is built on top of a -//! [`RawWaker`] which implements whatever is necessary. Usually, a waker will -//! signal the executor to poll a particular `Future`. A `Future` will clone -//! or pass-by-ref the waker to somewhere, as a callback, a completion, a -//! function, or anything, to the system that is responsible to notify when a -//! task is completed. So, to recap, the waker is _not_ responsible for waking the -//! `Future`, it _is_ responsible for _signaling_ the executor that a particular -//! `Future` should be polled again. That's why the documentation of -//! [`Poll::Pending`] specifies: -//! -//! > When a function returns `Pending`, the function must also ensure that the -//! > current task is scheduled to be awoken when progress can be made. -//! -//! “awakening” is done by using the `Waker`. -//! -//! [`Future`]: https://doc.rust-lang.org/std/future/trait.Future.html -//! [`Future::poll`]: https://doc.rust-lang.org/std/future/trait.Future.html#tymethod.poll -//! [`Pol::Ready`]: https://doc.rust-lang.org/std/task/enum.Poll.html#variant.Ready -//! [`Poll::Pending`]: https://doc.rust-lang.org/std/task/enum.Poll.html#variant.Pending -//! [`Context`]: https://doc.rust-lang.org/std/task/struct.Context.html -//! [`Waker`]: https://doc.rust-lang.org/std/task/struct.Waker.html -//! [`RawWaker`]: https://doc.rust-lang.org/std/task/struct.RawWaker.html - -use std::{ - future::Future, - marker::PhantomData, - mem, - ops::Deref, - panic, - pin::Pin, - sync::{Arc, Mutex}, - task::{Context, Poll, Wake}, -}; - -use crate::{rust_call_with_out_status, FfiDefault, LowerReturn, RustCallStatus}; - -/// Result code for [rust_future_poll]. This is passed to the continuation function. -#[repr(i8)] -#[derive(Debug, PartialEq, Eq)] -pub enum RustFuturePoll { - /// The future is ready and is waiting for [rust_future_complete] to be called - Ready = 0, - /// The future might be ready and [rust_future_poll] should be called again - MaybeReady = 1, -} - -/// Foreign callback that's passed to [rust_future_poll] -/// -/// The Rust side of things calls this when the foreign side should call [rust_future_poll] again -/// to continue progress on the future. -pub type RustFutureContinuationCallback = extern "C" fn(callback_data: *const (), RustFuturePoll); - -/// Opaque handle for a Rust future that's stored by the foreign language code -#[repr(transparent)] -pub struct RustFutureHandle(*const ()); - -// === Public FFI API === - -/// Create a new [RustFutureHandle] -/// -/// For each exported async function, UniFFI will create a scaffolding function that uses this to -/// create the [RustFutureHandle] to pass to the foreign code. -pub fn rust_future_new(future: F, tag: UT) -> RustFutureHandle -where - // F is the future type returned by the exported async function. It needs to be Send + `static - // since it will move between threads for an indeterminate amount of time as the foreign - // executor calls polls it and the Rust executor wakes it. It does not need to by `Sync`, - // since we synchronize all access to the values. - F: Future + Send + 'static, - // T is the output of the Future. It needs to implement [LowerReturn]. Also it must be Send + - // 'static for the same reason as F. - T: LowerReturn + Send + 'static, - // The UniFfiTag ZST. The Send + 'static bound is to keep rustc happy. - UT: Send + 'static, -{ - // Create a RustFuture and coerce to `Arc`, which is what we use to - // implement the FFI - let future_ffi = RustFuture::new(future, tag) as Arc>; - // Box the Arc, to convert the wide pointer into a normal sized pointer so that we can pass it - // to the foreign code. - let boxed_ffi = Box::new(future_ffi); - // We can now create a RustFutureHandle - RustFutureHandle(Box::into_raw(boxed_ffi) as *mut ()) -} - -/// Poll a Rust future -/// -/// When the future is ready to progress the continuation will be called with the `data` value and -/// a [RustFuturePoll] value. For each [rust_future_poll] call the continuation will be called -/// exactly once. -/// -/// # Safety -/// -/// The [RustFutureHandle] must not previously have been passed to [rust_future_free] -pub unsafe fn rust_future_poll( - handle: RustFutureHandle, - callback: RustFutureContinuationCallback, - data: *const (), -) { - let future = &*(handle.0 as *mut Arc>); - future.clone().ffi_poll(callback, data) -} - -/// Cancel a Rust future -/// -/// Any current and future continuations will be immediately called with RustFuturePoll::Ready. -/// -/// This is needed for languages like Swift, which continuation to wait for the continuation to be -/// called when tasks are cancelled. -/// -/// # Safety -/// -/// The [RustFutureHandle] must not previously have been passed to [rust_future_free] -pub unsafe fn rust_future_cancel(handle: RustFutureHandle) { - let future = &*(handle.0 as *mut Arc>); - future.clone().ffi_cancel() -} - -/// Complete a Rust future -/// -/// Note: the actually extern "C" scaffolding functions can't be generic, so we generate one for -/// each supported FFI type. -/// -/// # Safety -/// -/// - The [RustFutureHandle] must not previously have been passed to [rust_future_free] -/// - The `T` param must correctly correspond to the [rust_future_new] call. It must -/// be `>::ReturnType` -pub unsafe fn rust_future_complete( - handle: RustFutureHandle, - out_status: &mut RustCallStatus, -) -> ReturnType { - let future = &*(handle.0 as *mut Arc>); - future.ffi_complete(out_status) -} - -/// Free a Rust future, dropping the strong reference and releasing all references held by the -/// future. -/// -/// # Safety -/// -/// The [RustFutureHandle] must not previously have been passed to [rust_future_free] -pub unsafe fn rust_future_free(handle: RustFutureHandle) { - let future = Box::from_raw(handle.0 as *mut Arc>); - future.ffi_free() -} - -/// Thread-safe storage for [RustFutureContinuationCallback] data -/// -/// The basic guarantee is that all data pointers passed in are passed out exactly once to the -/// foreign continuation callback. This enables us to uphold the [rust_future_poll] guarantee. -/// -/// [ContinuationDataCell] also tracks cancellation, which is closely tied to continuation data. -#[derive(Debug)] -enum ContinuationDataCell { - /// No continuations set, neither wake() nor cancel() called. - Empty, - /// `wake()` was called when there was no continuation set. The next time `store` is called, - /// the continuation should be immediately invoked with `RustFuturePoll::MaybeReady` - Waked, - /// The future has been cancelled, any future `store` calls should immediately result in the - /// continuation being called with `RustFuturePoll::Ready`. - Cancelled, - /// Continuation set, the next time `wake()` is called is called, we should invoke it. - Set(RustFutureContinuationCallback, *const ()), -} - -impl ContinuationDataCell { - fn new() -> Self { - Self::Empty - } - - /// Store new continuation data if we are in the `Empty` state. If we are in the `Waked` or - /// `Cancelled` state, call the continuation immediately with the data. - fn store(&mut self, callback: RustFutureContinuationCallback, data: *const ()) { - match self { - Self::Empty => *self = Self::Set(callback, data), - Self::Set(old_callback, old_data) => { - log::error!( - "store: observed `Self::Set` state. Is poll() being called from multiple threads at once?" - ); - old_callback(*old_data, RustFuturePoll::Ready); - *self = Self::Set(callback, data); - } - Self::Waked => { - *self = Self::Empty; - callback(data, RustFuturePoll::MaybeReady); - } - Self::Cancelled => { - callback(data, RustFuturePoll::Ready); - } - } - } - - fn wake(&mut self) { - match self { - // If we had a continuation set, then call it and transition to the `Empty` state. - Self::Set(callback, old_data) => { - let old_data = *old_data; - let callback = *callback; - *self = Self::Empty; - callback(old_data, RustFuturePoll::MaybeReady); - } - // If we were in the `Empty` state, then transition to `Waked`. The next time `store` - // is called, we will immediately call the continuation. - Self::Empty => *self = Self::Waked, - // This is a no-op if we were in the `Cancelled` or `Waked` state. - _ => (), - } - } - - fn cancel(&mut self) { - if let Self::Set(callback, old_data) = mem::replace(self, Self::Cancelled) { - callback(old_data, RustFuturePoll::Ready); - } - } - - fn is_cancelled(&self) -> bool { - matches!(self, Self::Cancelled) - } -} - -// ContinuationDataCell is Send + Sync as long we handle the *const () pointer correctly - -unsafe impl Send for ContinuationDataCell {} -unsafe impl Sync for ContinuationDataCell {} - -/// Wraps the actual future we're polling -struct WrappedFuture -where - // See rust_future_new for an explanation of these trait bounds - F: Future + Send + 'static, - T: LowerReturn + Send + 'static, - UT: Send + 'static, -{ - // Note: this could be a single enum, but that would make it easy to mess up the future pinning - // guarantee. For example you might want to call `std::mem::take()` to try to get the result, - // but if the future happened to be stored that would move and break all internal references. - future: Option, - result: Option>, -} - -impl WrappedFuture -where - // See rust_future_new for an explanation of these trait bounds - F: Future + Send + 'static, - T: LowerReturn + Send + 'static, - UT: Send + 'static, -{ - fn new(future: F) -> Self { - Self { - future: Some(future), - result: None, - } - } - - // Poll the future and check if it's ready or not - fn poll(&mut self, context: &mut Context<'_>) -> bool { - if self.result.is_some() { - true - } else if let Some(future) = &mut self.future { - // SAFETY: We can call Pin::new_unchecked because: - // - This is the only time we get a &mut to `self.future` - // - We never poll the future after it's moved (for example by using take()) - // - We never move RustFuture, which contains us. - // - RustFuture is private to this module so no other code can move it. - let pinned = unsafe { Pin::new_unchecked(future) }; - // Run the poll and lift the result if it's ready - let mut out_status = RustCallStatus::default(); - let result: Option> = rust_call_with_out_status( - &mut out_status, - // This closure uses a `&mut F` value, which means it's not UnwindSafe by - // default. If the future panics, it may be in an invalid state. - // - // However, we can safely use `AssertUnwindSafe` since a panic will lead the `None` - // case below and we will never poll the future again. - panic::AssertUnwindSafe(|| match pinned.poll(context) { - Poll::Pending => Ok(Poll::Pending), - Poll::Ready(v) => T::lower_return(v).map(Poll::Ready), - }), - ); - match result { - Some(Poll::Pending) => false, - Some(Poll::Ready(v)) => { - self.future = None; - self.result = Some(Ok(v)); - true - } - None => { - self.future = None; - self.result = Some(Err(out_status)); - true - } - } - } else { - log::error!("poll with neither future nor result set"); - true - } - } - - fn complete(&mut self, out_status: &mut RustCallStatus) -> T::ReturnType { - let mut return_value = T::ReturnType::ffi_default(); - match self.result.take() { - Some(Ok(v)) => return_value = v, - Some(Err(call_status)) => *out_status = call_status, - None => *out_status = RustCallStatus::cancelled(), - } - self.free(); - return_value - } - - fn free(&mut self) { - self.future = None; - self.result = None; - } -} - -// If F and T are Send, then WrappedFuture is too -// -// Rust will not mark it Send by default when T::ReturnType is a raw pointer. This is promising -// that we will treat the raw pointer properly, for example by not returning it twice. -unsafe impl Send for WrappedFuture -where - // See rust_future_new for an explanation of these trait bounds - F: Future + Send + 'static, - T: LowerReturn + Send + 'static, - UT: Send + 'static, -{ -} - -/// Future that the foreign code is awaiting -struct RustFuture -where - // See rust_future_new for an explanation of these trait bounds - F: Future + Send + 'static, - T: LowerReturn + Send + 'static, - UT: Send + 'static, -{ - // This Mutex should never block if our code is working correctly, since there should not be - // multiple threads calling [Self::poll] and/or [Self::complete] at the same time. - future: Mutex>, - continuation_data: Mutex, - // UT is used as the generic parameter for [LowerReturn]. - // Let's model this with PhantomData as a function that inputs a UT value. - _phantom: PhantomData ()>, -} - -impl RustFuture -where - // See rust_future_new for an explanation of these trait bounds - F: Future + Send + 'static, - T: LowerReturn + Send + 'static, - UT: Send + 'static, -{ - fn new(future: F, _tag: UT) -> Arc { - Arc::new(Self { - future: Mutex::new(WrappedFuture::new(future)), - continuation_data: Mutex::new(ContinuationDataCell::new()), - _phantom: PhantomData, - }) - } - - fn poll(self: Arc, callback: RustFutureContinuationCallback, data: *const ()) { - let ready = self.is_cancelled() || { - let mut locked = self.future.lock().unwrap(); - let waker: std::task::Waker = Arc::clone(&self).into(); - locked.poll(&mut Context::from_waker(&waker)) - }; - if ready { - callback(data, RustFuturePoll::Ready) - } else { - self.continuation_data.lock().unwrap().store(callback, data); - } - } - - fn is_cancelled(&self) -> bool { - self.continuation_data.lock().unwrap().is_cancelled() - } - - fn wake(&self) { - self.continuation_data.lock().unwrap().wake(); - } - - fn cancel(&self) { - self.continuation_data.lock().unwrap().cancel(); - } - - fn complete(&self, call_status: &mut RustCallStatus) -> T::ReturnType { - self.future.lock().unwrap().complete(call_status) - } - - fn free(self: Arc) { - // Call cancel() to send any leftover data to the continuation callback - self.continuation_data.lock().unwrap().cancel(); - // Ensure we drop our inner future, releasing all held references - self.future.lock().unwrap().free(); - } -} - -impl Wake for RustFuture -where - // See rust_future_new for an explanation of these trait bounds - F: Future + Send + 'static, - T: LowerReturn + Send + 'static, - UT: Send + 'static, -{ - fn wake(self: Arc) { - self.deref().wake() - } - - fn wake_by_ref(self: &Arc) { - self.deref().wake() - } -} - -/// RustFuture FFI trait. This allows `Arc>` to be cast to -/// `Arc>`, which is needed to implement the public FFI API. In particular, this -/// allows you to use RustFuture functionality without knowing the concrete Future type, which is -/// unnamable. -/// -/// This is parametrized on the ReturnType rather than the `T` directly, to reduce the number of -/// scaffolding functions we need to generate. If it was parametrized on `T`, then we would need -/// to create a poll, cancel, complete, and free scaffolding function for each exported async -/// function. That would add ~1kb binary size per exported function based on a quick estimate on a -/// x86-64 machine . By parametrizing on `T::ReturnType` we can instead monomorphize by hand and -/// only create those functions for each of the 13 possible FFI return types. -#[doc(hidden)] -trait RustFutureFfi { - fn ffi_poll(self: Arc, callback: RustFutureContinuationCallback, data: *const ()); - fn ffi_cancel(&self); - fn ffi_complete(&self, call_status: &mut RustCallStatus) -> ReturnType; - fn ffi_free(self: Arc); -} - -impl RustFutureFfi for RustFuture -where - // See rust_future_new for an explanation of these trait bounds - F: Future + Send + 'static, - T: LowerReturn + Send + 'static, - UT: Send + 'static, -{ - fn ffi_poll(self: Arc, callback: RustFutureContinuationCallback, data: *const ()) { - self.poll(callback, data) - } - - fn ffi_cancel(&self) { - self.cancel() - } - - fn ffi_complete(&self, call_status: &mut RustCallStatus) -> T::ReturnType { - self.complete(call_status) - } - - fn ffi_free(self: Arc) { - self.free(); - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::{test_util::TestError, Lift, RustBuffer, RustCallStatusCode}; - use once_cell::sync::OnceCell; - use std::task::Waker; - - // Sender/Receiver pair that we use for testing - struct Channel { - result: Option>, - waker: Option, - } - - struct Sender(Arc>); - - impl Sender { - fn wake(&self) { - let inner = self.0.lock().unwrap(); - if let Some(waker) = &inner.waker { - waker.wake_by_ref(); - } - } - - fn send(&self, value: Result) { - let mut inner = self.0.lock().unwrap(); - if inner.result.replace(value).is_some() { - panic!("value already sent"); - } - if let Some(waker) = &inner.waker { - waker.wake_by_ref(); - } - } - } - - struct Receiver(Arc>); - - impl Future for Receiver { - type Output = Result; - - fn poll( - self: Pin<&mut Self>, - context: &mut Context<'_>, - ) -> Poll> { - let mut inner = self.0.lock().unwrap(); - match &inner.result { - Some(v) => Poll::Ready(v.clone()), - None => { - inner.waker = Some(context.waker().clone()); - Poll::Pending - } - } - } - } - - // Create a sender and rust future that we can use for testing - fn channel() -> (Sender, Arc>) { - let channel = Arc::new(Mutex::new(Channel { - result: None, - waker: None, - })); - let rust_future = RustFuture::new(Receiver(channel.clone()), crate::UniFfiTag); - (Sender(channel), rust_future) - } - - /// Poll a Rust future and get an OnceCell that's set when the continuation is called - fn poll(rust_future: &Arc>) -> Arc> { - let cell = Arc::new(OnceCell::new()); - let cell_ptr = Arc::into_raw(cell.clone()) as *const (); - rust_future.clone().ffi_poll(poll_continuation, cell_ptr); - cell - } - - extern "C" fn poll_continuation(data: *const (), code: RustFuturePoll) { - let cell = unsafe { Arc::from_raw(data as *const OnceCell) }; - cell.set(code).expect("Error setting OnceCell"); - } - - fn complete(rust_future: Arc>) -> (RustBuffer, RustCallStatus) { - let mut out_status_code = RustCallStatus::default(); - let return_value = rust_future.ffi_complete(&mut out_status_code); - (return_value, out_status_code) - } - - #[test] - fn test_success() { - let (sender, rust_future) = channel(); - - // Test polling the rust future before it's ready - let continuation_result = poll(&rust_future); - assert_eq!(continuation_result.get(), None); - sender.wake(); - assert_eq!(continuation_result.get(), Some(&RustFuturePoll::MaybeReady)); - - // Test polling the rust future when it's ready - let continuation_result = poll(&rust_future); - assert_eq!(continuation_result.get(), None); - sender.send(Ok("All done".into())); - assert_eq!(continuation_result.get(), Some(&RustFuturePoll::MaybeReady)); - - // Future polls should immediately return ready - let continuation_result = poll(&rust_future); - assert_eq!(continuation_result.get(), Some(&RustFuturePoll::Ready)); - - // Complete the future - let (return_buf, call_status) = complete(rust_future); - assert_eq!(call_status.code, RustCallStatusCode::Success); - assert_eq!( - >::try_lift(return_buf).unwrap(), - "All done" - ); - } - - #[test] - fn test_error() { - let (sender, rust_future) = channel(); - - let continuation_result = poll(&rust_future); - assert_eq!(continuation_result.get(), None); - sender.send(Err("Something went wrong".into())); - assert_eq!(continuation_result.get(), Some(&RustFuturePoll::MaybeReady)); - - let continuation_result = poll(&rust_future); - assert_eq!(continuation_result.get(), Some(&RustFuturePoll::Ready)); - - let (_, call_status) = complete(rust_future); - assert_eq!(call_status.code, RustCallStatusCode::Error); - unsafe { - assert_eq!( - >::try_lift_from_rust_buffer( - call_status.error_buf.assume_init() - ) - .unwrap(), - TestError::from("Something went wrong"), - ) - } - } - - // Once `complete` is called, the inner future should be released, even if wakers still hold a - // reference to the RustFuture - #[test] - fn test_cancel() { - let (_sender, rust_future) = channel(); - - let continuation_result = poll(&rust_future); - assert_eq!(continuation_result.get(), None); - rust_future.ffi_cancel(); - // Cancellation should immediately invoke the callback with RustFuturePoll::Ready - assert_eq!(continuation_result.get(), Some(&RustFuturePoll::Ready)); - - // Future polls should immediately invoke the callback with RustFuturePoll::Ready - let continuation_result = poll(&rust_future); - assert_eq!(continuation_result.get(), Some(&RustFuturePoll::Ready)); - - let (_, call_status) = complete(rust_future); - assert_eq!(call_status.code, RustCallStatusCode::Cancelled); - } - - // Once `free` is called, the inner future should be released, even if wakers still hold a - // reference to the RustFuture - #[test] - fn test_release_future() { - let (sender, rust_future) = channel(); - // Create a weak reference to the channel to use to check if rust_future has dropped its - // future. - let channel_weak = Arc::downgrade(&sender.0); - drop(sender); - // Create an extra ref to rust_future, simulating a waker that still holds a reference to - // it - let rust_future2 = rust_future.clone(); - - // Complete the rust future - rust_future.ffi_free(); - // Even though rust_future is still alive, the channel shouldn't be - assert!(Arc::strong_count(&rust_future2) > 0); - assert_eq!(channel_weak.strong_count(), 0); - assert!(channel_weak.upgrade().is_none()); - } - - // If `free` is called with a continuation still stored, we should call it them then. - // - // This shouldn't happen in practice, but it seems like good defensive programming - #[test] - fn test_complete_with_stored_continuation() { - let (_sender, rust_future) = channel(); - - let continuation_result = poll(&rust_future); - rust_future.ffi_free(); - assert_eq!(continuation_result.get(), Some(&RustFuturePoll::Ready)); - } - - // Test what happens if we see a `wake()` call while we're polling the future. This can - // happen, for example, with futures that are handled by a tokio thread pool. We should - // schedule another poll of the future in this case. - #[test] - fn test_wake_during_poll() { - let mut first_time = true; - let future = std::future::poll_fn(move |ctx| { - if first_time { - first_time = false; - // Wake the future while we are in the middle of polling it - ctx.waker().clone().wake(); - Poll::Pending - } else { - // The second time we're polled, we're ready - Poll::Ready("All done".to_owned()) - } - }); - let rust_future: Arc> = - RustFuture::new(future, crate::UniFfiTag); - let continuation_result = poll(&rust_future); - // The continuation function should called immediately - assert_eq!(continuation_result.get(), Some(&RustFuturePoll::MaybeReady)); - // A second poll should finish the future - let continuation_result = poll(&rust_future); - assert_eq!(continuation_result.get(), Some(&RustFuturePoll::Ready)); - let (return_buf, call_status) = complete(rust_future); - assert_eq!(call_status.code, RustCallStatusCode::Success); - assert_eq!( - >::try_lift(return_buf).unwrap(), - "All done" - ); - } -} diff --git a/third_party/rust/uniffi_core/src/ffi/rustfuture/future.rs b/third_party/rust/uniffi_core/src/ffi/rustfuture/future.rs new file mode 100644 index 0000000000..93c34e7543 --- /dev/null +++ b/third_party/rust/uniffi_core/src/ffi/rustfuture/future.rs @@ -0,0 +1,320 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +//! [`RustFuture`] represents a [`Future`] that can be sent to the foreign code over FFI. +//! +//! This type is not instantiated directly, but via the procedural macros, such as `#[uniffi::export]`. +//! +//! # The big picture +//! +//! We implement async foreign functions using a simplified version of the Future API: +//! +//! 0. At startup, register a [RustFutureContinuationCallback] by calling +//! rust_future_continuation_callback_set. +//! 1. Call the scaffolding function to get a [Handle] +//! 2a. In a loop: +//! - Call [rust_future_poll] +//! - Suspend the function until the [rust_future_poll] continuation function is called +//! - If the continuation was function was called with [RustFuturePoll::Ready], then break +//! otherwise continue. +//! 2b. If the async function is cancelled, then call [rust_future_cancel]. This causes the +//! continuation function to be called with [RustFuturePoll::Ready] and the [RustFuture] to +//! enter a cancelled state. +//! 3. Call [rust_future_complete] to get the result of the future. +//! 4. Call [rust_future_free] to free the future, ideally in a finally block. This: +//! - Releases any resources held by the future +//! - Calls any continuation callbacks that have not been called yet +//! +//! Note: Technically, the foreign code calls the scaffolding versions of the `rust_future_*` +//! functions. These are generated by the scaffolding macro, specially prefixed, and extern "C", +//! and manually monomorphized in the case of [rust_future_complete]. See +//! `uniffi_macros/src/setup_scaffolding.rs` for details. +//! +//! ## How does `Future` work exactly? +//! +//! A [`Future`] in Rust does nothing. When calling an async function, it just +//! returns a `Future` but nothing has happened yet. To start the computation, +//! the future must be polled. It returns [`Poll::Ready(r)`][`Poll::Ready`] if +//! the result is ready, [`Poll::Pending`] otherwise. `Poll::Pending` basically +//! means: +//! +//! > Please, try to poll me later, maybe the result will be ready! +//! +//! This model is very different than what other languages do, but it can actually +//! be translated quite easily, fortunately for us! +//! +//! But… wait a minute… who is responsible to poll the `Future` if a `Future` does +//! nothing? Well, it's _the executor_. The executor is responsible _to drive_ the +//! `Future`: that's where they are polled. +//! +//! But… wait another minute… how does the executor know when to poll a [`Future`]? +//! Does it poll them randomly in an endless loop? Well, no, actually it depends +//! on the executor! A well-designed `Future` and executor work as follows. +//! Normally, when [`Future::poll`] is called, a [`Context`] argument is +//! passed to it. It contains a [`Waker`]. The [`Waker`] is built on top of a +//! [`RawWaker`] which implements whatever is necessary. Usually, a waker will +//! signal the executor to poll a particular `Future`. A `Future` will clone +//! or pass-by-ref the waker to somewhere, as a callback, a completion, a +//! function, or anything, to the system that is responsible to notify when a +//! task is completed. So, to recap, the waker is _not_ responsible for waking the +//! `Future`, it _is_ responsible for _signaling_ the executor that a particular +//! `Future` should be polled again. That's why the documentation of +//! [`Poll::Pending`] specifies: +//! +//! > When a function returns `Pending`, the function must also ensure that the +//! > current task is scheduled to be awoken when progress can be made. +//! +//! “awakening” is done by using the `Waker`. +//! +//! [`Future`]: https://doc.rust-lang.org/std/future/trait.Future.html +//! [`Future::poll`]: https://doc.rust-lang.org/std/future/trait.Future.html#tymethod.poll +//! [`Pol::Ready`]: https://doc.rust-lang.org/std/task/enum.Poll.html#variant.Ready +//! [`Poll::Pending`]: https://doc.rust-lang.org/std/task/enum.Poll.html#variant.Pending +//! [`Context`]: https://doc.rust-lang.org/std/task/struct.Context.html +//! [`Waker`]: https://doc.rust-lang.org/std/task/struct.Waker.html +//! [`RawWaker`]: https://doc.rust-lang.org/std/task/struct.RawWaker.html + +use std::{ + future::Future, + marker::PhantomData, + ops::Deref, + panic, + pin::Pin, + sync::{Arc, Mutex}, + task::{Context, Poll, Wake}, +}; + +use super::{RustFutureContinuationCallback, RustFuturePoll, Scheduler}; +use crate::{rust_call_with_out_status, FfiDefault, LowerReturn, RustCallStatus}; + +/// Wraps the actual future we're polling +struct WrappedFuture +where + // See rust_future_new for an explanation of these trait bounds + F: Future + Send + 'static, + T: LowerReturn + Send + 'static, + UT: Send + 'static, +{ + // Note: this could be a single enum, but that would make it easy to mess up the future pinning + // guarantee. For example you might want to call `std::mem::take()` to try to get the result, + // but if the future happened to be stored that would move and break all internal references. + future: Option, + result: Option>, +} + +impl WrappedFuture +where + // See rust_future_new for an explanation of these trait bounds + F: Future + Send + 'static, + T: LowerReturn + Send + 'static, + UT: Send + 'static, +{ + fn new(future: F) -> Self { + Self { + future: Some(future), + result: None, + } + } + + // Poll the future and check if it's ready or not + fn poll(&mut self, context: &mut Context<'_>) -> bool { + if self.result.is_some() { + true + } else if let Some(future) = &mut self.future { + // SAFETY: We can call Pin::new_unchecked because: + // - This is the only time we get a &mut to `self.future` + // - We never poll the future after it's moved (for example by using take()) + // - We never move RustFuture, which contains us. + // - RustFuture is private to this module so no other code can move it. + let pinned = unsafe { Pin::new_unchecked(future) }; + // Run the poll and lift the result if it's ready + let mut out_status = RustCallStatus::default(); + let result: Option> = rust_call_with_out_status( + &mut out_status, + // This closure uses a `&mut F` value, which means it's not UnwindSafe by + // default. If the future panics, it may be in an invalid state. + // + // However, we can safely use `AssertUnwindSafe` since a panic will lead the `None` + // case below and we will never poll the future again. + panic::AssertUnwindSafe(|| match pinned.poll(context) { + Poll::Pending => Ok(Poll::Pending), + Poll::Ready(v) => T::lower_return(v).map(Poll::Ready), + }), + ); + match result { + Some(Poll::Pending) => false, + Some(Poll::Ready(v)) => { + self.future = None; + self.result = Some(Ok(v)); + true + } + None => { + self.future = None; + self.result = Some(Err(out_status)); + true + } + } + } else { + log::error!("poll with neither future nor result set"); + true + } + } + + fn complete(&mut self, out_status: &mut RustCallStatus) -> T::ReturnType { + let mut return_value = T::ReturnType::ffi_default(); + match self.result.take() { + Some(Ok(v)) => return_value = v, + Some(Err(call_status)) => *out_status = call_status, + None => *out_status = RustCallStatus::cancelled(), + } + self.free(); + return_value + } + + fn free(&mut self) { + self.future = None; + self.result = None; + } +} + +// If F and T are Send, then WrappedFuture is too +// +// Rust will not mark it Send by default when T::ReturnType is a raw pointer. This is promising +// that we will treat the raw pointer properly, for example by not returning it twice. +unsafe impl Send for WrappedFuture +where + // See rust_future_new for an explanation of these trait bounds + F: Future + Send + 'static, + T: LowerReturn + Send + 'static, + UT: Send + 'static, +{ +} + +/// Future that the foreign code is awaiting +pub(super) struct RustFuture +where + // See rust_future_new for an explanation of these trait bounds + F: Future + Send + 'static, + T: LowerReturn + Send + 'static, + UT: Send + 'static, +{ + // This Mutex should never block if our code is working correctly, since there should not be + // multiple threads calling [Self::poll] and/or [Self::complete] at the same time. + future: Mutex>, + scheduler: Mutex, + // UT is used as the generic parameter for [LowerReturn]. + // Let's model this with PhantomData as a function that inputs a UT value. + _phantom: PhantomData ()>, +} + +impl RustFuture +where + // See rust_future_new for an explanation of these trait bounds + F: Future + Send + 'static, + T: LowerReturn + Send + 'static, + UT: Send + 'static, +{ + pub(super) fn new(future: F, _tag: UT) -> Arc { + Arc::new(Self { + future: Mutex::new(WrappedFuture::new(future)), + scheduler: Mutex::new(Scheduler::new()), + _phantom: PhantomData, + }) + } + + pub(super) fn poll(self: Arc, callback: RustFutureContinuationCallback, data: u64) { + let ready = self.is_cancelled() || { + let mut locked = self.future.lock().unwrap(); + let waker: std::task::Waker = Arc::clone(&self).into(); + locked.poll(&mut Context::from_waker(&waker)) + }; + if ready { + callback(data, RustFuturePoll::Ready) + } else { + self.scheduler.lock().unwrap().store(callback, data); + } + } + + pub(super) fn is_cancelled(&self) -> bool { + self.scheduler.lock().unwrap().is_cancelled() + } + + pub(super) fn wake(&self) { + self.scheduler.lock().unwrap().wake(); + } + + pub(super) fn cancel(&self) { + self.scheduler.lock().unwrap().cancel(); + } + + pub(super) fn complete(&self, call_status: &mut RustCallStatus) -> T::ReturnType { + self.future.lock().unwrap().complete(call_status) + } + + pub(super) fn free(self: Arc) { + // Call cancel() to send any leftover data to the continuation callback + self.scheduler.lock().unwrap().cancel(); + // Ensure we drop our inner future, releasing all held references + self.future.lock().unwrap().free(); + } +} + +impl Wake for RustFuture +where + // See rust_future_new for an explanation of these trait bounds + F: Future + Send + 'static, + T: LowerReturn + Send + 'static, + UT: Send + 'static, +{ + fn wake(self: Arc) { + self.deref().wake() + } + + fn wake_by_ref(self: &Arc) { + self.deref().wake() + } +} + +/// RustFuture FFI trait. This allows `Arc>` to be cast to +/// `Arc>`, which is needed to implement the public FFI API. In particular, this +/// allows you to use RustFuture functionality without knowing the concrete Future type, which is +/// unnamable. +/// +/// This is parametrized on the ReturnType rather than the `T` directly, to reduce the number of +/// scaffolding functions we need to generate. If it was parametrized on `T`, then we would need +/// to create a poll, cancel, complete, and free scaffolding function for each exported async +/// function. That would add ~1kb binary size per exported function based on a quick estimate on a +/// x86-64 machine . By parametrizing on `T::ReturnType` we can instead monomorphize by hand and +/// only create those functions for each of the 13 possible FFI return types. +#[doc(hidden)] +pub trait RustFutureFfi: Send + Sync { + fn ffi_poll(self: Arc, callback: RustFutureContinuationCallback, data: u64); + fn ffi_cancel(&self); + fn ffi_complete(&self, call_status: &mut RustCallStatus) -> ReturnType; + fn ffi_free(self: Arc); +} + +impl RustFutureFfi for RustFuture +where + // See rust_future_new for an explanation of these trait bounds + F: Future + Send + 'static, + T: LowerReturn + Send + 'static, + UT: Send + 'static, +{ + fn ffi_poll(self: Arc, callback: RustFutureContinuationCallback, data: u64) { + self.poll(callback, data) + } + + fn ffi_cancel(&self) { + self.cancel() + } + + fn ffi_complete(&self, call_status: &mut RustCallStatus) -> T::ReturnType { + self.complete(call_status) + } + + fn ffi_free(self: Arc) { + self.free(); + } +} diff --git a/third_party/rust/uniffi_core/src/ffi/rustfuture/mod.rs b/third_party/rust/uniffi_core/src/ffi/rustfuture/mod.rs new file mode 100644 index 0000000000..3d3505e5ef --- /dev/null +++ b/third_party/rust/uniffi_core/src/ffi/rustfuture/mod.rs @@ -0,0 +1,141 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use std::{future::Future, sync::Arc}; + +mod future; +mod scheduler; +use future::*; +use scheduler::*; + +#[cfg(test)] +mod tests; + +use crate::{derive_ffi_traits, Handle, HandleAlloc, LowerReturn, RustCallStatus}; + +/// Result code for [rust_future_poll]. This is passed to the continuation function. +#[repr(i8)] +#[derive(Debug, PartialEq, Eq)] +pub enum RustFuturePoll { + /// The future is ready and is waiting for [rust_future_complete] to be called + Ready = 0, + /// The future might be ready and [rust_future_poll] should be called again + MaybeReady = 1, +} + +/// Foreign callback that's passed to [rust_future_poll] +/// +/// The Rust side of things calls this when the foreign side should call [rust_future_poll] again +/// to continue progress on the future. +pub type RustFutureContinuationCallback = extern "C" fn(callback_data: u64, RustFuturePoll); + +// === Public FFI API === + +/// Create a new [Handle] for a Rust future +/// +/// For each exported async function, UniFFI will create a scaffolding function that uses this to +/// create the [Handle] to pass to the foreign code. +pub fn rust_future_new(future: F, tag: UT) -> Handle +where + // F is the future type returned by the exported async function. It needs to be Send + `static + // since it will move between threads for an indeterminate amount of time as the foreign + // executor calls polls it and the Rust executor wakes it. It does not need to by `Sync`, + // since we synchronize all access to the values. + F: Future + Send + 'static, + // T is the output of the Future. It needs to implement [LowerReturn]. Also it must be Send + + // 'static for the same reason as F. + T: LowerReturn + Send + 'static, + // The UniFfiTag ZST. The Send + 'static bound is to keep rustc happy. + UT: Send + 'static, + // Needed to allocate a handle + dyn RustFutureFfi: HandleAlloc, +{ + as HandleAlloc>::new_handle( + RustFuture::new(future, tag) as Arc> + ) +} + +/// Poll a Rust future +/// +/// When the future is ready to progress the continuation will be called with the `data` value and +/// a [RustFuturePoll] value. For each [rust_future_poll] call the continuation will be called +/// exactly once. +/// +/// # Safety +/// +/// The [Handle] must not previously have been passed to [rust_future_free] +pub unsafe fn rust_future_poll( + handle: Handle, + callback: RustFutureContinuationCallback, + data: u64, +) where + dyn RustFutureFfi: HandleAlloc, +{ + as HandleAlloc>::get_arc(handle).ffi_poll(callback, data) +} + +/// Cancel a Rust future +/// +/// Any current and future continuations will be immediately called with RustFuturePoll::Ready. +/// +/// This is needed for languages like Swift, which continuation to wait for the continuation to be +/// called when tasks are cancelled. +/// +/// # Safety +/// +/// The [Handle] must not previously have been passed to [rust_future_free] +pub unsafe fn rust_future_cancel(handle: Handle) +where + dyn RustFutureFfi: HandleAlloc, +{ + as HandleAlloc>::get_arc(handle).ffi_cancel() +} + +/// Complete a Rust future +/// +/// Note: the actually extern "C" scaffolding functions can't be generic, so we generate one for +/// each supported FFI type. +/// +/// # Safety +/// +/// - The [Handle] must not previously have been passed to [rust_future_free] +/// - The `T` param must correctly correspond to the [rust_future_new] call. It must +/// be `>::ReturnType` +pub unsafe fn rust_future_complete( + handle: Handle, + out_status: &mut RustCallStatus, +) -> ReturnType +where + dyn RustFutureFfi: HandleAlloc, +{ + as HandleAlloc>::get_arc(handle).ffi_complete(out_status) +} + +/// Free a Rust future, dropping the strong reference and releasing all references held by the +/// future. +/// +/// # Safety +/// +/// The [Handle] must not previously have been passed to [rust_future_free] +pub unsafe fn rust_future_free(handle: Handle) +where + dyn RustFutureFfi: HandleAlloc, +{ + as HandleAlloc>::consume_handle(handle).ffi_free() +} + +// Derive HandleAlloc for dyn RustFutureFfi for all FFI return types +derive_ffi_traits!(impl HandleAlloc for dyn RustFutureFfi); +derive_ffi_traits!(impl HandleAlloc for dyn RustFutureFfi); +derive_ffi_traits!(impl HandleAlloc for dyn RustFutureFfi); +derive_ffi_traits!(impl HandleAlloc for dyn RustFutureFfi); +derive_ffi_traits!(impl HandleAlloc for dyn RustFutureFfi); +derive_ffi_traits!(impl HandleAlloc for dyn RustFutureFfi); +derive_ffi_traits!(impl HandleAlloc for dyn RustFutureFfi); +derive_ffi_traits!(impl HandleAlloc for dyn RustFutureFfi); +derive_ffi_traits!(impl HandleAlloc for dyn RustFutureFfi); +derive_ffi_traits!(impl HandleAlloc for dyn RustFutureFfi); +derive_ffi_traits!(impl HandleAlloc for dyn RustFutureFfi<*const std::ffi::c_void>); +derive_ffi_traits!(impl HandleAlloc for dyn RustFutureFfi); +derive_ffi_traits!(impl HandleAlloc for dyn RustFutureFfi<()>); diff --git a/third_party/rust/uniffi_core/src/ffi/rustfuture/scheduler.rs b/third_party/rust/uniffi_core/src/ffi/rustfuture/scheduler.rs new file mode 100644 index 0000000000..629ee0c109 --- /dev/null +++ b/third_party/rust/uniffi_core/src/ffi/rustfuture/scheduler.rs @@ -0,0 +1,96 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use std::mem; + +use super::{RustFutureContinuationCallback, RustFuturePoll}; + +/// Schedules a [crate::RustFuture] by managing the continuation data +/// +/// This struct manages the continuation callback and data that comes from the foreign side. It +/// is responsible for calling the continuation callback when the future is ready to be woken up. +/// +/// The basic guarantees are: +/// +/// * Each callback will be invoked exactly once, with its associated data. +/// * If `wake()` is called, the callback will be invoked to wake up the future -- either +/// immediately or the next time we get a callback. +/// * If `cancel()` is called, the same will happen and the schedule will stay in the cancelled +/// state, invoking any future callbacks as soon as they're stored. + +#[derive(Debug)] +pub(super) enum Scheduler { + /// No continuations set, neither wake() nor cancel() called. + Empty, + /// `wake()` was called when there was no continuation set. The next time `store` is called, + /// the continuation should be immediately invoked with `RustFuturePoll::MaybeReady` + Waked, + /// The future has been cancelled, any future `store` calls should immediately result in the + /// continuation being called with `RustFuturePoll::Ready`. + Cancelled, + /// Continuation set, the next time `wake()` is called is called, we should invoke it. + Set(RustFutureContinuationCallback, u64), +} + +impl Scheduler { + pub(super) fn new() -> Self { + Self::Empty + } + + /// Store new continuation data if we are in the `Empty` state. If we are in the `Waked` or + /// `Cancelled` state, call the continuation immediately with the data. + pub(super) fn store(&mut self, callback: RustFutureContinuationCallback, data: u64) { + match self { + Self::Empty => *self = Self::Set(callback, data), + Self::Set(old_callback, old_data) => { + log::error!( + "store: observed `Self::Set` state. Is poll() being called from multiple threads at once?" + ); + old_callback(*old_data, RustFuturePoll::Ready); + *self = Self::Set(callback, data); + } + Self::Waked => { + *self = Self::Empty; + callback(data, RustFuturePoll::MaybeReady); + } + Self::Cancelled => { + callback(data, RustFuturePoll::Ready); + } + } + } + + pub(super) fn wake(&mut self) { + match self { + // If we had a continuation set, then call it and transition to the `Empty` state. + Self::Set(callback, old_data) => { + let old_data = *old_data; + let callback = *callback; + *self = Self::Empty; + callback(old_data, RustFuturePoll::MaybeReady); + } + // If we were in the `Empty` state, then transition to `Waked`. The next time `store` + // is called, we will immediately call the continuation. + Self::Empty => *self = Self::Waked, + // This is a no-op if we were in the `Cancelled` or `Waked` state. + _ => (), + } + } + + pub(super) fn cancel(&mut self) { + if let Self::Set(callback, old_data) = mem::replace(self, Self::Cancelled) { + callback(old_data, RustFuturePoll::Ready); + } + } + + pub(super) fn is_cancelled(&self) -> bool { + matches!(self, Self::Cancelled) + } +} + +// The `*const ()` data pointer references an object on the foreign side. +// This object must be `Sync` in Rust terminology -- it must be safe for us to pass the pointer to the continuation callback from any thread. +// If the foreign side upholds their side of the contract, then `Scheduler` is Send + Sync. + +unsafe impl Send for Scheduler {} +unsafe impl Sync for Scheduler {} diff --git a/third_party/rust/uniffi_core/src/ffi/rustfuture/tests.rs b/third_party/rust/uniffi_core/src/ffi/rustfuture/tests.rs new file mode 100644 index 0000000000..886ee27c71 --- /dev/null +++ b/third_party/rust/uniffi_core/src/ffi/rustfuture/tests.rs @@ -0,0 +1,223 @@ +use once_cell::sync::OnceCell; +use std::{ + future::Future, + panic, + pin::Pin, + sync::{Arc, Mutex}, + task::{Context, Poll, Waker}, +}; + +use super::*; +use crate::{test_util::TestError, Lift, RustBuffer, RustCallStatusCode}; + +// Sender/Receiver pair that we use for testing +struct Channel { + result: Option>, + waker: Option, +} + +struct Sender(Arc>); + +impl Sender { + fn wake(&self) { + let inner = self.0.lock().unwrap(); + if let Some(waker) = &inner.waker { + waker.wake_by_ref(); + } + } + + fn send(&self, value: Result) { + let mut inner = self.0.lock().unwrap(); + if inner.result.replace(value).is_some() { + panic!("value already sent"); + } + if let Some(waker) = &inner.waker { + waker.wake_by_ref(); + } + } +} + +struct Receiver(Arc>); + +impl Future for Receiver { + type Output = Result; + + fn poll(self: Pin<&mut Self>, context: &mut Context<'_>) -> Poll> { + let mut inner = self.0.lock().unwrap(); + match &inner.result { + Some(v) => Poll::Ready(v.clone()), + None => { + inner.waker = Some(context.waker().clone()); + Poll::Pending + } + } + } +} + +// Create a sender and rust future that we can use for testing +fn channel() -> (Sender, Arc>) { + let channel = Arc::new(Mutex::new(Channel { + result: None, + waker: None, + })); + let rust_future = RustFuture::new(Receiver(channel.clone()), crate::UniFfiTag); + (Sender(channel), rust_future) +} + +/// Poll a Rust future and get an OnceCell that's set when the continuation is called +fn poll(rust_future: &Arc>) -> Arc> { + let cell = Arc::new(OnceCell::new()); + let handle = Arc::into_raw(cell.clone()) as u64; + rust_future.clone().ffi_poll(poll_continuation, handle); + cell +} + +extern "C" fn poll_continuation(data: u64, code: RustFuturePoll) { + let cell = unsafe { Arc::from_raw(data as *const OnceCell) }; + cell.set(code).expect("Error setting OnceCell"); +} + +fn complete(rust_future: Arc>) -> (RustBuffer, RustCallStatus) { + let mut out_status_code = RustCallStatus::default(); + let return_value = rust_future.ffi_complete(&mut out_status_code); + (return_value, out_status_code) +} + +#[test] +fn test_success() { + let (sender, rust_future) = channel(); + + // Test polling the rust future before it's ready + let continuation_result = poll(&rust_future); + assert_eq!(continuation_result.get(), None); + sender.wake(); + assert_eq!(continuation_result.get(), Some(&RustFuturePoll::MaybeReady)); + + // Test polling the rust future when it's ready + let continuation_result = poll(&rust_future); + assert_eq!(continuation_result.get(), None); + sender.send(Ok("All done".into())); + assert_eq!(continuation_result.get(), Some(&RustFuturePoll::MaybeReady)); + + // Future polls should immediately return ready + let continuation_result = poll(&rust_future); + assert_eq!(continuation_result.get(), Some(&RustFuturePoll::Ready)); + + // Complete the future + let (return_buf, call_status) = complete(rust_future); + assert_eq!(call_status.code, RustCallStatusCode::Success); + assert_eq!( + >::try_lift(return_buf).unwrap(), + "All done" + ); +} + +#[test] +fn test_error() { + let (sender, rust_future) = channel(); + + let continuation_result = poll(&rust_future); + assert_eq!(continuation_result.get(), None); + sender.send(Err("Something went wrong".into())); + assert_eq!(continuation_result.get(), Some(&RustFuturePoll::MaybeReady)); + + let continuation_result = poll(&rust_future); + assert_eq!(continuation_result.get(), Some(&RustFuturePoll::Ready)); + + let (_, call_status) = complete(rust_future); + assert_eq!(call_status.code, RustCallStatusCode::Error); + unsafe { + assert_eq!( + >::try_lift_from_rust_buffer( + call_status.error_buf.assume_init() + ) + .unwrap(), + TestError::from("Something went wrong"), + ) + } +} + +// Once `complete` is called, the inner future should be released, even if wakers still hold a +// reference to the RustFuture +#[test] +fn test_cancel() { + let (_sender, rust_future) = channel(); + + let continuation_result = poll(&rust_future); + assert_eq!(continuation_result.get(), None); + rust_future.ffi_cancel(); + // Cancellation should immediately invoke the callback with RustFuturePoll::Ready + assert_eq!(continuation_result.get(), Some(&RustFuturePoll::Ready)); + + // Future polls should immediately invoke the callback with RustFuturePoll::Ready + let continuation_result = poll(&rust_future); + assert_eq!(continuation_result.get(), Some(&RustFuturePoll::Ready)); + + let (_, call_status) = complete(rust_future); + assert_eq!(call_status.code, RustCallStatusCode::Cancelled); +} + +// Once `free` is called, the inner future should be released, even if wakers still hold a +// reference to the RustFuture +#[test] +fn test_release_future() { + let (sender, rust_future) = channel(); + // Create a weak reference to the channel to use to check if rust_future has dropped its + // future. + let channel_weak = Arc::downgrade(&sender.0); + drop(sender); + // Create an extra ref to rust_future, simulating a waker that still holds a reference to + // it + let rust_future2 = rust_future.clone(); + + // Complete the rust future + rust_future.ffi_free(); + // Even though rust_future is still alive, the channel shouldn't be + assert!(Arc::strong_count(&rust_future2) > 0); + assert_eq!(channel_weak.strong_count(), 0); + assert!(channel_weak.upgrade().is_none()); +} + +// If `free` is called with a continuation still stored, we should call it them then. +// +// This shouldn't happen in practice, but it seems like good defensive programming +#[test] +fn test_complete_with_stored_continuation() { + let (_sender, rust_future) = channel(); + + let continuation_result = poll(&rust_future); + rust_future.ffi_free(); + assert_eq!(continuation_result.get(), Some(&RustFuturePoll::Ready)); +} + +// Test what happens if we see a `wake()` call while we're polling the future. This can +// happen, for example, with futures that are handled by a tokio thread pool. We should +// schedule another poll of the future in this case. +#[test] +fn test_wake_during_poll() { + let mut first_time = true; + let future = std::future::poll_fn(move |ctx| { + if first_time { + first_time = false; + // Wake the future while we are in the middle of polling it + ctx.waker().clone().wake(); + Poll::Pending + } else { + // The second time we're polled, we're ready + Poll::Ready("All done".to_owned()) + } + }); + let rust_future: Arc> = RustFuture::new(future, crate::UniFfiTag); + let continuation_result = poll(&rust_future); + // The continuation function should called immediately + assert_eq!(continuation_result.get(), Some(&RustFuturePoll::MaybeReady)); + // A second poll should finish the future + let continuation_result = poll(&rust_future); + assert_eq!(continuation_result.get(), Some(&RustFuturePoll::Ready)); + let (return_buf, call_status) = complete(rust_future); + assert_eq!(call_status.code, RustCallStatusCode::Success); + assert_eq!( + >::try_lift(return_buf).unwrap(), + "All done" + ); +} diff --git a/third_party/rust/uniffi_core/src/ffi_converter_impls.rs b/third_party/rust/uniffi_core/src/ffi_converter_impls.rs index af18f3873b..aec093154a 100644 --- a/third_party/rust/uniffi_core/src/ffi_converter_impls.rs +++ b/third_party/rust/uniffi_core/src/ffi_converter_impls.rs @@ -20,11 +20,11 @@ /// /// This crate needs to implement `FFIConverter` on `UniFfiTag` instances for all UniFFI /// consumer crates. To do this, it defines blanket impls like `impl FFIConverter for u8`. -/// "UT" means an abitrary `UniFfiTag` type. +/// "UT" means an arbitrary `UniFfiTag` type. use crate::{ check_remaining, derive_ffi_traits, ffi_converter_rust_buffer_lift_and_lower, metadata, - ConvertError, FfiConverter, ForeignExecutor, Lift, LiftReturn, Lower, LowerReturn, - MetadataBuffer, Result, RustBuffer, UnexpectedUniFFICallbackError, + ConvertError, FfiConverter, Lift, LiftRef, LiftReturn, Lower, LowerReturn, MetadataBuffer, + Result, RustBuffer, UnexpectedUniFFICallbackError, }; use anyhow::bail; use bytes::buf::{Buf, BufMut}; @@ -405,47 +405,6 @@ where .concat(V::TYPE_ID_META); } -/// FFI support for [ForeignExecutor] -/// -/// These are passed over the FFI as opaque pointer-sized types representing the foreign executor. -/// The foreign bindings may use an actual pointer to the executor object, or a usized integer -/// handle. -unsafe impl FfiConverter for ForeignExecutor { - type FfiType = crate::ForeignExecutorHandle; - - // Passing these back to the foreign bindings is currently not supported - fn lower(executor: Self) -> Self::FfiType { - executor.handle - } - - fn write(executor: Self, buf: &mut Vec) { - // Use native endian when writing these values, so they can be casted to pointer values - match std::mem::size_of::() { - // Use native endian when reading these values, so they can be casted to pointer values - 4 => buf.put_u32_ne(executor.handle.0 as u32), - 8 => buf.put_u64_ne(executor.handle.0 as u64), - n => panic!("Invalid usize width: {n}"), - }; - } - - fn try_lift(executor: Self::FfiType) -> Result { - Ok(ForeignExecutor::new(executor)) - } - - fn try_read(buf: &mut &[u8]) -> Result { - let usize_val = match std::mem::size_of::() { - // Use native endian when reading these values, so they can be casted to pointer values - 4 => buf.get_u32_ne() as usize, - 8 => buf.get_u64_ne() as usize, - n => panic!("Invalid usize width: {n}"), - }; - >::try_lift(crate::ForeignExecutorHandle(usize_val as *const ())) - } - - const TYPE_ID_META: MetadataBuffer = - MetadataBuffer::from_code(metadata::codes::TYPE_FOREIGN_EXECUTOR); -} - derive_ffi_traits!(blanket u8); derive_ffi_traits!(blanket i8); derive_ffi_traits!(blanket u16); @@ -460,7 +419,6 @@ derive_ffi_traits!(blanket bool); derive_ffi_traits!(blanket String); derive_ffi_traits!(blanket Duration); derive_ffi_traits!(blanket SystemTime); -derive_ffi_traits!(blanket ForeignExecutor); // For composite types, derive LowerReturn, LiftReturn, etc, from Lift/Lower. // @@ -498,7 +456,11 @@ unsafe impl LowerReturn for () { } unsafe impl LiftReturn for () { - fn lift_callback_return(_buf: RustBuffer) -> Self {} + type ReturnType = (); + + fn try_lift_successful_return(_: ()) -> Result { + Ok(()) + } const TYPE_ID_META: MetadataBuffer = MetadataBuffer::from_code(metadata::codes::TYPE_UNIT); } @@ -535,13 +497,15 @@ where unsafe impl LiftReturn for Result where R: LiftReturn, - E: Lift + ConvertError, + E: Lift + ConvertError, { - fn lift_callback_return(buf: RustBuffer) -> Self { - Ok(R::lift_callback_return(buf)) + type ReturnType = R::ReturnType; + + fn try_lift_successful_return(v: R::ReturnType) -> Result { + R::try_lift_successful_return(v).map(Ok) } - fn lift_callback_error(buf: RustBuffer) -> Self { + fn lift_error(buf: RustBuffer) -> Self { match E::try_lift_from_rust_buffer(buf) { Ok(lifted_error) => Err(lifted_error), Err(anyhow_error) => { @@ -560,3 +524,14 @@ where .concat(R::TYPE_ID_META) .concat(E::TYPE_ID_META); } + +unsafe impl LiftRef for [T] +where + T: Lift, +{ + type LiftType = Vec; +} + +unsafe impl LiftRef for str { + type LiftType = String; +} diff --git a/third_party/rust/uniffi_core/src/ffi_converter_traits.rs b/third_party/rust/uniffi_core/src/ffi_converter_traits.rs index 3b5914e32f..4e7b9e06fa 100644 --- a/third_party/rust/uniffi_core/src/ffi_converter_traits.rs +++ b/third_party/rust/uniffi_core/src/ffi_converter_traits.rs @@ -51,7 +51,10 @@ use std::{borrow::Borrow, sync::Arc}; use anyhow::bail; use bytes::Buf; -use crate::{FfiDefault, MetadataBuffer, Result, RustBuffer, UnexpectedUniFFICallbackError}; +use crate::{ + FfiDefault, Handle, MetadataBuffer, Result, RustBuffer, RustCallStatus, RustCallStatusCode, + UnexpectedUniFFICallbackError, +}; /// Generalized FFI conversions /// @@ -302,14 +305,41 @@ pub unsafe trait LowerReturn: Sized { /// These traits should not be used directly, only in generated code, and the generated code should /// have fixture tests to test that everything works correctly together. pub unsafe trait LiftReturn: Sized { - /// Lift a Rust value for a callback interface method result - fn lift_callback_return(buf: RustBuffer) -> Self; + /// FFI return type for trait interfaces + type ReturnType; + + /// Lift a successfully returned value from a trait interface + fn try_lift_successful_return(v: Self::ReturnType) -> Result; + + /// Lift a foreign returned value from a trait interface + /// + /// When we call a foreign-implemented trait interface method, we pass a &mut RustCallStatus + /// and get [Self::ReturnType] returned. This method takes both of those and lifts `Self` from + /// it. + fn lift_foreign_return(ffi_return: Self::ReturnType, call_status: RustCallStatus) -> Self { + match call_status.code { + RustCallStatusCode::Success => Self::try_lift_successful_return(ffi_return) + .unwrap_or_else(|e| { + Self::handle_callback_unexpected_error(UnexpectedUniFFICallbackError::new(e)) + }), + RustCallStatusCode::Error => { + Self::lift_error(unsafe { call_status.error_buf.assume_init() }) + } + _ => { + let e = >::try_lift(unsafe { + call_status.error_buf.assume_init() + }) + .unwrap_or_else(|e| format!("(Error lifting message: {e}")); + Self::handle_callback_unexpected_error(UnexpectedUniFFICallbackError::new(e)) + } + } + } /// Lift a Rust value for a callback interface method error result /// /// This is called for "expected errors" -- the callback method returns a Result<> type and the /// foreign code throws an exception that corresponds to the error type. - fn lift_callback_error(_buf: RustBuffer) -> Self { + fn lift_error(_buf: RustBuffer) -> Self { panic!("Callback interface method returned unexpected error") } @@ -351,6 +381,66 @@ pub trait ConvertError: Sized { fn try_convert_unexpected_callback_error(e: UnexpectedUniFFICallbackError) -> Result; } +/// Manage handles for `Arc` instances +/// +/// Handles are used to manage objects that are passed across the FFI. They general usage is: +/// +/// * Rust creates an `Arc<>` +/// * Rust uses `new_handle` to create a handle that represents the Arc reference +/// * Rust passes the handle to the foreign code as a `u64` +/// * The foreign code passes the handle back to `Rust` to refer to the object: +/// * Handle are usually passed as borrowed values. When an FFI function inputs a handle as an +/// argument, the foreign code simply passes a copy of the `u64` to Rust, which calls `get_arc` +/// to get a new `Arc<>` clone for it. +/// * Handles are returned as owned values. When an FFI function returns a handle, the foreign +/// code either stops using the handle after returning it or calls `clone_handle` and returns +/// the clone. +/// * Eventually the foreign code may destroy their handle by passing it into a "free" FFI +/// function. This functions input an owned handle and consume it. +/// +/// The foreign code also defines their own handles. These represent foreign objects that are +/// passed to Rust. Using foreign handles is essentially the same as above, but in reverse. +/// +/// Handles must always be `Send` and the objects they reference must always be `Sync`. +/// This means that it must be safe to send handles to other threads and use them there. +/// +/// Note: this only needs to be derived for unsized types, there's a blanket impl for `T: Sized`. +/// +/// ## Safety +/// +/// All traits are unsafe (implementing it requires `unsafe impl`) because we can't guarantee +/// that it's safe to pass your type out to foreign-language code and back again. Buggy +/// implementations of this trait might violate some assumptions made by the generated code, +/// or might not match with the corresponding code in the generated foreign-language bindings. +/// These traits should not be used directly, only in generated code, and the generated code should +/// have fixture tests to test that everything works correctly together. +/// `&T` using the Arc. +pub unsafe trait HandleAlloc: Send + Sync { + /// Create a new handle for an Arc value + /// + /// Use this to lower an Arc into a handle value before passing it across the FFI. + /// The newly-created handle will have reference count = 1. + fn new_handle(value: Arc) -> Handle; + + /// Clone a handle + /// + /// This creates a new handle from an existing one. + /// It's used when the foreign code wants to pass back an owned handle and still keep a copy + /// for themselves. + fn clone_handle(handle: Handle) -> Handle; + + /// Get a clone of the `Arc<>` using a "borrowed" handle. + /// + /// Take care that the handle can not be destroyed between when it's passed and when + /// `get_arc()` is called. #1797 is a cautionary tale. + fn get_arc(handle: Handle) -> Arc { + Self::consume_handle(Self::clone_handle(handle)) + } + + /// Consume a handle, getting back the initial `Arc<>` + fn consume_handle(handle: Handle) -> Arc; +} + /// Derive FFI traits /// /// This can be used to derive: @@ -439,9 +529,10 @@ macro_rules! derive_ffi_traits { (impl $(<$($generic:ident),*>)? $(::uniffi::)? LiftReturn<$ut:path> for $ty:ty $(where $($where:tt)*)?) => { unsafe impl $(<$($generic),*>)* $crate::LiftReturn<$ut> for $ty $(where $($where)*)* { - fn lift_callback_return(buf: $crate::RustBuffer) -> Self { - >::try_lift_from_rust_buffer(buf) - .expect("Error reading callback interface result") + type ReturnType = >::FfiType; + + fn try_lift_successful_return(v: Self::ReturnType) -> $crate::Result { + >::try_lift(v) } const TYPE_ID_META: $crate::MetadataBuffer = >::TYPE_ID_META; @@ -463,4 +554,50 @@ macro_rules! derive_ffi_traits { } } }; + + (impl $(<$($generic:ident),*>)? $(::uniffi::)? HandleAlloc<$ut:path> for $ty:ty $(where $($where:tt)*)?) => { + // Derived HandleAlloc implementation. + // + // This is only needed for !Sized types like `dyn Trait`, below is a blanket implementation + // for any sized type. + unsafe impl $(<$($generic),*>)* $crate::HandleAlloc<$ut> for $ty $(where $($where)*)* + { + // To implement HandleAlloc for an unsized type, wrap it with a second Arc which + // converts the wide pointer into a normal pointer. + + fn new_handle(value: ::std::sync::Arc) -> $crate::Handle { + $crate::Handle::from_pointer(::std::sync::Arc::into_raw(::std::sync::Arc::new(value))) + } + + fn clone_handle(handle: $crate::Handle) -> $crate::Handle { + unsafe { + ::std::sync::Arc::<::std::sync::Arc>::increment_strong_count(handle.as_pointer::<::std::sync::Arc>()); + } + handle + } + + fn consume_handle(handle: $crate::Handle) -> ::std::sync::Arc { + unsafe { + ::std::sync::Arc::::clone( + &std::sync::Arc::<::std::sync::Arc::>::from_raw(handle.as_pointer::<::std::sync::Arc>()) + ) + } + } + } + }; +} + +unsafe impl HandleAlloc for T { + fn new_handle(value: Arc) -> Handle { + Handle::from_pointer(Arc::into_raw(value)) + } + + fn clone_handle(handle: Handle) -> Handle { + unsafe { Arc::increment_strong_count(handle.as_pointer::()) }; + handle + } + + fn consume_handle(handle: Handle) -> Arc { + unsafe { Arc::from_raw(handle.as_pointer()) } + } } diff --git a/third_party/rust/uniffi_core/src/lib.rs b/third_party/rust/uniffi_core/src/lib.rs index c84b403dce..1f3a2403f8 100644 --- a/third_party/rust/uniffi_core/src/lib.rs +++ b/third_party/rust/uniffi_core/src/lib.rs @@ -45,7 +45,8 @@ pub mod metadata; pub use ffi::*; pub use ffi_converter_traits::{ - ConvertError, FfiConverter, FfiConverterArc, Lift, LiftRef, LiftReturn, Lower, LowerReturn, + ConvertError, FfiConverter, FfiConverterArc, HandleAlloc, Lift, LiftRef, LiftReturn, Lower, + LowerReturn, }; pub use metadata::*; @@ -57,9 +58,8 @@ pub mod deps { pub use async_compat; pub use bytes; pub use log; + pub use oneshot; pub use static_assertions; - // Export this dependency for the 0.25 branch so that we can use it in `setup_scaffolding.rs` - pub use once_cell; } mod panichook; diff --git a/third_party/rust/uniffi_core/src/metadata.rs b/third_party/rust/uniffi_core/src/metadata.rs index 770d2b36d5..dc61a1bfcb 100644 --- a/third_party/rust/uniffi_core/src/metadata.rs +++ b/third_party/rust/uniffi_core/src/metadata.rs @@ -32,13 +32,14 @@ pub mod codes { pub const RECORD: u8 = 2; pub const ENUM: u8 = 3; pub const INTERFACE: u8 = 4; - pub const ERROR: u8 = 5; pub const NAMESPACE: u8 = 6; pub const CONSTRUCTOR: u8 = 7; pub const UDL_FILE: u8 = 8; pub const CALLBACK_INTERFACE: u8 = 9; pub const TRAIT_METHOD: u8 = 10; pub const UNIFFI_TRAIT: u8 = 11; + pub const TRAIT_INTERFACE: u8 = 12; + pub const CALLBACK_TRAIT_INTERFACE: u8 = 13; pub const UNKNOWN: u8 = 255; // Type codes @@ -66,20 +67,24 @@ pub mod codes { pub const TYPE_CALLBACK_INTERFACE: u8 = 21; pub const TYPE_CUSTOM: u8 = 22; pub const TYPE_RESULT: u8 = 23; - pub const TYPE_FUTURE: u8 = 24; - pub const TYPE_FOREIGN_EXECUTOR: u8 = 25; + pub const TYPE_TRAIT_INTERFACE: u8 = 24; + pub const TYPE_CALLBACK_TRAIT_INTERFACE: u8 = 25; pub const TYPE_UNIT: u8 = 255; - // Literal codes for LiteralMetadata - note that we don't support - // all variants in the "emit/reader" context. + // Literal codes for LiteralMetadata pub const LIT_STR: u8 = 0; pub const LIT_INT: u8 = 1; pub const LIT_FLOAT: u8 = 2; pub const LIT_BOOL: u8 = 3; - pub const LIT_NULL: u8 = 4; + pub const LIT_NONE: u8 = 4; + pub const LIT_SOME: u8 = 5; + pub const LIT_EMPTY_SEQ: u8 = 6; } -const BUF_SIZE: usize = 4096; +// For large errors (e.g. enums) a buffer size of ~4k - ~8k +// is not enough. See issues on Github: #1968 and #2041 and +// for an example see fixture/large-error +const BUF_SIZE: usize = 16384; // This struct is a kludge around the fact that Rust const generic support doesn't quite handle our // needs. @@ -168,7 +173,17 @@ impl MetadataBuffer { self.concat_value(value as u8) } - // Concatenate a string to this buffer. + // Option + pub const fn concat_option_bool(self, value: Option) -> Self { + self.concat_value(match value { + None => 0, + Some(false) => 1, + Some(true) => 2, + }) + } + + // Concatenate a string to this buffer. The maximum string length is 255 bytes. For longer strings, + // use `concat_long_str()`. // // Strings are encoded as a `u8` length, followed by the utf8 data. // @@ -189,6 +204,28 @@ impl MetadataBuffer { self } + // Concatenate a longer string to this buffer. + // + // Strings are encoded as a `u16` length, followed by the utf8 data. + // + // This consumes self, which is convenient for the proc-macro code and also allows us to avoid + // allocated an extra buffer. + pub const fn concat_long_str(mut self, string: &str) -> Self { + assert!(self.size + string.len() + 1 < BUF_SIZE); + let [lo, hi] = (string.len() as u16).to_le_bytes(); + self.bytes[self.size] = lo; + self.bytes[self.size + 1] = hi; + self.size += 2; + let bytes = string.as_bytes(); + let mut i = 0; + while i < bytes.len() { + self.bytes[self.size] = bytes[i]; + self.size += 1; + i += 1; + } + self + } + // Create an array from this MetadataBuffer // // SIZE should always be `self.size`. This is part of the kludge to hold us over until Rust diff --git a/third_party/rust/uniffi_macros/.cargo-checksum.json b/third_party/rust/uniffi_macros/.cargo-checksum.json index 96e44ac74e..9db049289c 100644 --- a/third_party/rust/uniffi_macros/.cargo-checksum.json +++ b/third_party/rust/uniffi_macros/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"Cargo.toml":"376811e12479a0cced9375a12a2e3186da053b3c7520374bf6f2b54d23282082","src/custom.rs":"36cd6c2eeb8efdc34e59dff634a22e79471ab17f49ceb0f131da5f144313f7e4","src/enum_.rs":"2d3181ef22468deb4e8f48b7b1ed9421134604c2c226d5084c453ebb2cedbfd5","src/error.rs":"0b5beb8a2c8c93c30c56f2f80538bf1f1c8e6a99b2b1c934ad12a4feb75a2fc0","src/export.rs":"6d05417f0b10a9d6df9e96a6ed771c89a5e59e6e52d1ce812025bfe68e9f717e","src/export/attributes.rs":"53a27264882ab0a802a0ee109a2ea3f3456d4c83c85eaa5c0f5912d4486ab843","src/export/callback_interface.rs":"ad2782b7ca930dc067c391394480362be1fbf331d8786be089c0a87415c85a88","src/export/item.rs":"a7b74e6400ec6c5e8fb09d8842ce718b9555d75de13fdf5fecbab2fceeec7cbf","src/export/scaffolding.rs":"8ab2b9b0c5ad5b5477963843b7a58d496344da7de1a2a4b07f30f22275c8f3c9","src/export/utrait.rs":"ce4a3d629aaf0b44b8c5ce6794c5d5b0d7f86f46f0dd6b6ecb134514be330f0d","src/fnsig.rs":"886ceec806b429c7d86fe00d0d84f7b04e21142605f7a61d182f9f616210cd2b","src/lib.rs":"501c736647eff2705c5565f80e554d2e440cceceed95044c9fe147fc309afb48","src/object.rs":"0a14d6b8ccb4faef93a1f61a97ac7d47a80b6581383f4a6e0a4789f53e66e630","src/record.rs":"fbff287bb2d0b7a9eb35ef3161fbd1abbc21f3aa08e88bd4242dd12acbfa3ee9","src/setup_scaffolding.rs":"60b48a56fae16cf01824586b90e6440da3362135d98fc07aaed624c57f806163","src/test.rs":"1673f282bb35d6b0740ad0e5f11826c2852d7a0db29604c2258f457415b537e8","src/util.rs":"217ecef0e4dabd158a7597aa3d00d94477993a90b388afbbc0fb39e14f6b013e"},"package":"11cf7a58f101fcedafa5b77ea037999b88748607f0ef3a33eaa0efc5392e92e4"} \ No newline at end of file +{"files":{"Cargo.toml":"a292239ca3c72852768fdf0e7bc2dd6386af7bf1ab0ef56dff01e1c9e781b2ca","README.md":"37c1af00ec81a9f1bc206ab3578356e5f9ad4077dc46dd1bb623d81d804948b8","src/custom.rs":"36cd6c2eeb8efdc34e59dff634a22e79471ab17f49ceb0f131da5f144313f7e4","src/default.rs":"77466ac54da69094bcdccc5927d0980b1e9dd0095647ca825830673c48847a53","src/enum_.rs":"afe0a6534d8e7f68047e3f1afad9369d5d5650f4c7555e8d4173f24126c715ba","src/error.rs":"30168378da9a23e6530ffe68647bf6618d07a0aaa236d5009137a922798a0e88","src/export.rs":"42c5e784c1dccc796c8b6ea29c2dc1811e48a531488a3ed0e2a59330778a7e41","src/export/attributes.rs":"c848f8c309c4cf7a168f038834752dc4816b5c853768d7c331ea4cd5ce0841b7","src/export/callback_interface.rs":"794b0665dc7eb02ea854c61c8bb2781e0b4ac1de646d95a8fd7791f770f2e6e3","src/export/item.rs":"4e86875692c2d2993fde12e78dbde2cbffa5675ede143577d5620126401efe05","src/export/scaffolding.rs":"b25167d2213b6d6c5ba653622f26791e8c3e74a5ecce6512ec27009fc8bf68e4","src/export/trait_interface.rs":"f07f9908ee28661de4586d89b693f3d93dae5e5cba8a089eff25f20bbf6b373b","src/export/utrait.rs":"b55533d3eef8262944d3c0d9a3a9cba0615d2d5af8608f0919abc7699989e2a8","src/fnsig.rs":"5e434a1cc87166c5245424bb14e896eb766bf680d4d50d4b8536852f91487d7c","src/lib.rs":"a28bbfd2d1dc835306ff6072f75761bb6b3a158477bba966057776c527fe6d70","src/object.rs":"5419ed64c8120aef811a77c2205f58a7a537bdf34ae04f9c92dd3aaa176eed39","src/record.rs":"29072542cc2f3e027bd7c59b45ba913458f8213d1b2b33bc70d140baa98fcdc8","src/setup_scaffolding.rs":"173fdc916967d54bd6532def16d12e5bb85467813a46a031d3338b77625756bb","src/test.rs":"1673f282bb35d6b0740ad0e5f11826c2852d7a0db29604c2258f457415b537e8","src/util.rs":"a2c3693343e78dffb2a7f7b39eeb9b7f298b66688f1766a7c08113cf9431ef4c"},"package":"18331d35003f46f0d04047fbe4227291815b83a937a8c32bc057f990962182c4"} \ No newline at end of file diff --git a/third_party/rust/uniffi_macros/Cargo.toml b/third_party/rust/uniffi_macros/Cargo.toml index 9d3908ae8d..5ae193e392 100644 --- a/third_party/rust/uniffi_macros/Cargo.toml +++ b/third_party/rust/uniffi_macros/Cargo.toml @@ -12,11 +12,12 @@ [package] edition = "2021" name = "uniffi_macros" -version = "0.25.3" +version = "0.27.1" authors = ["Firefox Sync Team "] description = "a multi-language bindings generator for rust (convenience macros)" homepage = "https://mozilla.github.io/uniffi-rs" documentation = "https://mozilla.github.io/uniffi-rs" +readme = "README.md" keywords = [ "ffi", "bindgen", @@ -47,6 +48,7 @@ version = "1.0" [dependencies.serde] version = "1.0.136" +features = ["derive"] [dependencies.syn] version = "2.0" @@ -59,11 +61,13 @@ features = [ version = "0.5.9" [dependencies.uniffi_build] -version = "=0.25.3" +version = "=0.27.1" +optional = true [dependencies.uniffi_meta] -version = "=0.25.3" +version = "=0.27.1" [features] default = [] nightly = [] +trybuild = ["dep:uniffi_build"] diff --git a/third_party/rust/uniffi_macros/README.md b/third_party/rust/uniffi_macros/README.md new file mode 100644 index 0000000000..64ac3486a3 --- /dev/null +++ b/third_party/rust/uniffi_macros/README.md @@ -0,0 +1,81 @@ +# UniFFI - a multi-language bindings generator for Rust + +UniFFI is a toolkit for building cross-platform software components in Rust. + +For the impatient, see [**the UniFFI user guide**](https://mozilla.github.io/uniffi-rs/) +or [**the UniFFI examples**](https://github.com/mozilla/uniffi-rs/tree/main/examples#example-uniffi-components). + +By writing your core business logic in Rust and describing its interface in an "object model", +you can use UniFFI to help you: + +* Compile your Rust code into a shared library for use on different target platforms. +* Generate bindings to load and use the library from different target languages. + +You can describe your object model in an [interface definition file](https://mozilla.github.io/uniffi-rs/udl_file_spec.html) +or [by using proc-macros](https://mozilla.github.io/uniffi-rs/proc_macro/index.html). + +UniFFI is currently used extensively by Mozilla in Firefox mobile and desktop browsers; +written once in Rust, auto-generated bindings allow that functionality to be called +from both Kotlin (for Android apps) and Swift (for iOS apps). +It also has a growing community of users shipping various cool things to many users. + +UniFFI comes with support for **Kotlin**, **Swift**, **Python** and **Ruby** with 3rd party bindings available for **C#** and **Golang**. +Additional foreign language bindings can be developed externally and we welcome contributions to list them here. +See [Third-party foreign language bindings](#third-party-foreign-language-bindings). + +## User Guide + +You can read more about using the tool in [**the UniFFI user guide**](https://mozilla.github.io/uniffi-rs/). + +We consider it ready for production use, but UniFFI is a long way from a 1.0 release with lots of internal work still going on. +We try hard to avoid breaking simple consumers, but more advanced things might break as you upgrade over time. + +### Etymology and Pronunciation + +ˈjuːnɪfaɪ. Pronounced to rhyme with "unify". + +A portmanteau word that also puns with "unify", to signify the joining of one codebase accessed from many languages. + +uni - [Latin ūni-, from ūnus, one] +FFI - [Abbreviation, Foreign Function Interface] + +## Alternative tools + +Other tools we know of which try and solve a similarly shaped problem are: + +* [Diplomat](https://github.com/rust-diplomat/diplomat/) - see our [writeup of + the different approach taken by that tool](docs/diplomat-and-macros.md) +* [Interoptopus](https://github.com/ralfbiedert/interoptopus/) + +(Please open a PR if you think other tools should be listed!) + +## Third-party foreign language bindings + +* [Kotlin Multiplatform support](https://gitlab.com/trixnity/uniffi-kotlin-multiplatform-bindings). The repository contains Kotlin Multiplatform bindings generation for UniFFI, letting you target both JVM and Native. +* [Go bindings](https://github.com/NordSecurity/uniffi-bindgen-go) +* [C# bindings](https://github.com/NordSecurity/uniffi-bindgen-cs) +* [Dart bindings](https://github.com/NiallBunting/uniffi-rs-dart) + +### External resources + +There are a few third-party resources that make it easier to work with UniFFI: + +* [Plugin support for `.udl` files](https://github.com/Lonami/uniffi-dl) for the IDEA platform ([*uniffi-dl* in the JetBrains marketplace](https://plugins.jetbrains.com/plugin/20527-uniffi-dl)). It provides syntax highlighting, code folding, code completion, reference resolution and navigation (among others features) for the [UniFFI Definition Language (UDL)](https://mozilla.github.io/uniffi-rs/). +* [cargo swift](https://github.com/antoniusnaumann/cargo-swift), a cargo plugin to build a Swift Package from Rust code. It provides an init command for setting up a UniFFI crate and a package command for building a Swift package from Rust code - without the need for additional configuration or build scripts. +* [Cargo NDK Gradle Plugin](https://github.com/willir/cargo-ndk-android-gradle) allows you to build Rust code using [`cargo-ndk`](https://github.com/bbqsrc/cargo-ndk), which generally makes Android library builds less painful. +* [`uniffi-starter`](https://github.com/ianthetechie/uniffi-starter) is a minimal project demonstrates a wide range of UniFFI in a complete project in a compact manner. It includes a full Android library build process, an XCFramework generation script, and example Swift package structure. + +(Please open a PR if you think other resources should be listed!) + +## Contributing + +If this tool sounds interesting to you, please help us develop it! You can: + +* View the [contributor guidelines](./docs/contributing.md). +* File or work on [issues](https://github.com/mozilla/uniffi-rs/issues) here in GitHub. +* Join discussions in the [#uniffi:mozilla.org](https://matrix.to/#/#uniffi:mozilla.org) + room on Matrix. + +## Code of Conduct + +This project is governed by Mozilla's [Community Participation Guidelines](./CODE_OF_CONDUCT.md). diff --git a/third_party/rust/uniffi_macros/src/default.rs b/third_party/rust/uniffi_macros/src/default.rs new file mode 100644 index 0000000000..000c205845 --- /dev/null +++ b/third_party/rust/uniffi_macros/src/default.rs @@ -0,0 +1,133 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use crate::util::kw; +use proc_macro2::TokenStream; +use quote::{quote, ToTokens}; +use syn::{ + bracketed, parenthesized, + parse::{Nothing, Parse, ParseStream}, + token::{Bracket, Paren}, + Lit, +}; + +/// Default value +#[derive(Clone)] +pub enum DefaultValue { + Literal(Lit), + None(kw::None), + Some { + some: kw::Some, + paren: Paren, + inner: Box, + }, + EmptySeq(Bracket), +} + +impl ToTokens for DefaultValue { + fn to_tokens(&self, tokens: &mut TokenStream) { + match self { + DefaultValue::Literal(lit) => lit.to_tokens(tokens), + DefaultValue::None(kw) => kw.to_tokens(tokens), + DefaultValue::Some { inner, .. } => tokens.extend(quote! { Some(#inner) }), + DefaultValue::EmptySeq(_) => tokens.extend(quote! { [] }), + } + } +} + +impl Parse for DefaultValue { + fn parse(input: ParseStream<'_>) -> syn::Result { + let lookahead = input.lookahead1(); + if lookahead.peek(kw::None) { + let none_kw: kw::None = input.parse()?; + Ok(Self::None(none_kw)) + } else if lookahead.peek(kw::Some) { + let some: kw::Some = input.parse()?; + let content; + let paren = parenthesized!(content in input); + Ok(Self::Some { + some, + paren, + inner: content.parse()?, + }) + } else if lookahead.peek(Bracket) { + let content; + let bracket = bracketed!(content in input); + content.parse::()?; + Ok(Self::EmptySeq(bracket)) + } else { + Ok(Self::Literal(input.parse()?)) + } + } +} + +impl DefaultValue { + fn metadata_calls(&self) -> syn::Result { + match self { + DefaultValue::Literal(Lit::Int(i)) if !i.suffix().is_empty() => Err( + syn::Error::new_spanned(i, "integer literals with suffix not supported here"), + ), + DefaultValue::Literal(Lit::Float(f)) if !f.suffix().is_empty() => Err( + syn::Error::new_spanned(f, "float literals with suffix not supported here"), + ), + + DefaultValue::Literal(Lit::Str(s)) => Ok(quote! { + .concat_value(::uniffi::metadata::codes::LIT_STR) + .concat_str(#s) + }), + DefaultValue::Literal(Lit::Int(i)) => { + let digits = i.base10_digits(); + Ok(quote! { + .concat_value(::uniffi::metadata::codes::LIT_INT) + .concat_str(#digits) + }) + } + DefaultValue::Literal(Lit::Float(f)) => { + let digits = f.base10_digits(); + Ok(quote! { + .concat_value(::uniffi::metadata::codes::LIT_FLOAT) + .concat_str(#digits) + }) + } + DefaultValue::Literal(Lit::Bool(b)) => Ok(quote! { + .concat_value(::uniffi::metadata::codes::LIT_BOOL) + .concat_bool(#b) + }), + + DefaultValue::Literal(_) => Err(syn::Error::new_spanned( + self, + "this type of literal is not currently supported as a default", + )), + + DefaultValue::EmptySeq(_) => Ok(quote! { + .concat_value(::uniffi::metadata::codes::LIT_EMPTY_SEQ) + }), + + DefaultValue::None(_) => Ok(quote! { + .concat_value(::uniffi::metadata::codes::LIT_NONE) + }), + + DefaultValue::Some { inner, .. } => { + let inner_calls = inner.metadata_calls()?; + Ok(quote! { + .concat_value(::uniffi::metadata::codes::LIT_SOME) + #inner_calls + }) + } + } + } +} + +pub fn default_value_metadata_calls(default: &Option) -> syn::Result { + Ok(match default { + Some(default) => { + let metadata_calls = default.metadata_calls()?; + quote! { + .concat_bool(true) + #metadata_calls + } + } + None => quote! { .concat_bool(false) }, + }) +} diff --git a/third_party/rust/uniffi_macros/src/enum_.rs b/third_party/rust/uniffi_macros/src/enum_.rs index 32abfa08cc..fd98da3129 100644 --- a/third_party/rust/uniffi_macros/src/enum_.rs +++ b/third_party/rust/uniffi_macros/src/enum_.rs @@ -1,13 +1,47 @@ use proc_macro2::{Ident, Span, TokenStream}; use quote::quote; -use syn::{Data, DataEnum, DeriveInput, Field, Index}; +use syn::{ + parse::{Parse, ParseStream}, + spanned::Spanned, + Attribute, Data, DataEnum, DeriveInput, Expr, Index, Lit, Variant, +}; use crate::util::{ - create_metadata_items, derive_all_ffi_traits, ident_to_string, mod_path, tagged_impl_header, - try_metadata_value_from_usize, try_read_field, + create_metadata_items, derive_all_ffi_traits, either_attribute_arg, extract_docstring, + ident_to_string, kw, mod_path, parse_comma_separated, tagged_impl_header, + try_metadata_value_from_usize, try_read_field, AttributeSliceExt, UniffiAttributeArgs, }; -pub fn expand_enum(input: DeriveInput, udl_mode: bool) -> syn::Result { +fn extract_repr(attrs: &[Attribute]) -> syn::Result> { + let mut result = None; + for attr in attrs { + if attr.path().is_ident("repr") { + attr.parse_nested_meta(|meta| { + result = match meta.path.get_ident() { + Some(i) => { + let s = i.to_string(); + match s.as_str() { + "u8" | "u16" | "u32" | "u64" | "usize" | "i8" | "i16" | "i32" + | "i64" | "isize" => Some(i.clone()), + // while the default repr for an enum is `isize` we don't apply that default here. + _ => None, + } + } + _ => None, + }; + Ok(()) + })? + } + } + Ok(result) +} + +pub fn expand_enum( + input: DeriveInput, + // Attributes from #[derive_error_for_udl()], if we are in udl mode + attr_from_udl_mode: Option, + udl_mode: bool, +) -> syn::Result { let enum_ = match input.data { Data::Enum(e) => e, _ => { @@ -18,10 +52,17 @@ pub fn expand_enum(input: DeriveInput, udl_mode: bool) -> syn::Result TokenStream { enum_or_error_ffi_converter_impl( ident, enum_, udl_mode, + attr, quote! { ::uniffi::metadata::codes::TYPE_ENUM }, ) } @@ -47,11 +90,13 @@ pub(crate) fn rich_error_ffi_converter_impl( ident: &Ident, enum_: &DataEnum, udl_mode: bool, + attr: &EnumAttr, ) -> TokenStream { enum_or_error_ffi_converter_impl( ident, enum_, udl_mode, + attr, quote! { ::uniffi::metadata::codes::TYPE_ENUM }, ) } @@ -60,6 +105,7 @@ fn enum_or_error_ffi_converter_impl( ident: &Ident, enum_: &DataEnum, udl_mode: bool, + attr: &EnumAttr, metadata_type_code: TokenStream, ) -> TokenStream { let name = ident_to_string(ident); @@ -69,19 +115,50 @@ fn enum_or_error_ffi_converter_impl( Ok(p) => p, Err(e) => return e.into_compile_error(), }; - let write_match_arms = enum_.variants.iter().enumerate().map(|(i, v)| { - let v_ident = &v.ident; - let fields = v.fields.iter().map(|f| &f.ident); - let idx = Index::from(i + 1); - let write_fields = v.fields.iter().map(write_field); + let mut write_match_arms: Vec<_> = enum_ + .variants + .iter() + .enumerate() + .map(|(i, v)| { + let v_ident = &v.ident; + let field_idents = v + .fields + .iter() + .enumerate() + .map(|(i, f)| { + f.ident + .clone() + .unwrap_or_else(|| Ident::new(&format!("e{i}"), f.span())) + }) + .collect::>(); + let idx = Index::from(i + 1); + let write_fields = + std::iter::zip(v.fields.iter(), field_idents.iter()).map(|(f, ident)| { + let ty = &f.ty; + quote! { + <#ty as ::uniffi::Lower>::write(#ident, buf); + } + }); + let is_tuple = v.fields.iter().any(|f| f.ident.is_none()); + let fields = if is_tuple { + quote! { ( #(#field_idents),* ) } + } else { + quote! { { #(#field_idents),* } } + }; - quote! { - Self::#v_ident { #(#fields),* } => { - ::uniffi::deps::bytes::BufMut::put_i32(buf, #idx); - #(#write_fields)* + quote! { + Self::#v_ident #fields => { + ::uniffi::deps::bytes::BufMut::put_i32(buf, #idx); + #(#write_fields)* + } } - } - }); + }) + .collect(); + if attr.non_exhaustive.is_some() { + write_match_arms.push(quote! { + _ => panic!("Unexpected variant in non-exhaustive enum"), + }) + } let write_impl = quote! { match obj { #(#write_match_arms)* } }; @@ -89,10 +166,17 @@ fn enum_or_error_ffi_converter_impl( let try_read_match_arms = enum_.variants.iter().enumerate().map(|(i, v)| { let idx = Index::from(i + 1); let v_ident = &v.ident; + let is_tuple = v.fields.iter().any(|f| f.ident.is_none()); let try_read_fields = v.fields.iter().map(try_read_field); - quote! { - #idx => Self::#v_ident { #(#try_read_fields)* }, + if is_tuple { + quote! { + #idx => Self::#v_ident ( #(#try_read_fields)* ), + } + } else { + quote! { + #idx => Self::#v_ident { #(#try_read_fields)* }, + } } }); let error_format_string = format!("Invalid {ident} enum value: {{}}"); @@ -127,69 +211,161 @@ fn enum_or_error_ffi_converter_impl( } } -fn write_field(f: &Field) -> TokenStream { - let ident = &f.ident; - let ty = &f.ty; - - quote! { - <#ty as ::uniffi::Lower>::write(#ident, buf); - } -} - -pub(crate) fn enum_meta_static_var(ident: &Ident, enum_: &DataEnum) -> syn::Result { +pub(crate) fn enum_meta_static_var( + ident: &Ident, + docstring: String, + discr_type: Option, + enum_: &DataEnum, + attr: &EnumAttr, +) -> syn::Result { let name = ident_to_string(ident); let module_path = mod_path()?; + let non_exhaustive = attr.non_exhaustive.is_some(); let mut metadata_expr = quote! { ::uniffi::MetadataBuffer::from_code(::uniffi::metadata::codes::ENUM) .concat_str(#module_path) .concat_str(#name) + .concat_option_bool(None) // forced_flatness }; + metadata_expr.extend(match discr_type { + None => quote! { .concat_bool(false) }, + Some(t) => quote! { .concat_bool(true).concat(<#t as ::uniffi::Lower>::TYPE_ID_META) } + }); metadata_expr.extend(variant_metadata(enum_)?); + metadata_expr.extend(quote! { + .concat_bool(#non_exhaustive) + .concat_long_str(#docstring) + }); Ok(create_metadata_items("enum", &name, metadata_expr, None)) } +fn variant_value(v: &Variant) -> syn::Result { + let Some((_, e)) = &v.discriminant else { + return Ok(quote! { .concat_bool(false) }); + }; + // Attempting to expose an enum value which we don't understand is a hard-error + // rather than silently ignoring it. If we had the ability to emit a warning that + // might make more sense. + + // We can't sanely handle most expressions other than literals, but we can handle + // negative literals. + let mut negate = false; + let lit = match e { + Expr::Lit(lit) => lit, + Expr::Unary(expr_unary) if matches!(expr_unary.op, syn::UnOp::Neg(_)) => { + negate = true; + match *expr_unary.expr { + Expr::Lit(ref lit) => lit, + _ => { + return Err(syn::Error::new_spanned( + e, + "UniFFI disciminant values must be a literal", + )); + } + } + } + _ => { + return Err(syn::Error::new_spanned( + e, + "UniFFI disciminant values must be a literal", + )); + } + }; + let Lit::Int(ref intlit) = lit.lit else { + return Err(syn::Error::new_spanned( + v, + "UniFFI disciminant values must be a literal integer", + )); + }; + if !intlit.suffix().is_empty() { + return Err(syn::Error::new_spanned( + intlit, + "integer literals with suffix not supported by UniFFI here", + )); + } + let digits = if negate { + format!("-{}", intlit.base10_digits()) + } else { + intlit.base10_digits().to_string() + }; + Ok(quote! { + .concat_bool(true) + .concat_value(::uniffi::metadata::codes::LIT_INT) + .concat_str(#digits) + }) +} + pub fn variant_metadata(enum_: &DataEnum) -> syn::Result> { let variants_len = try_metadata_value_from_usize(enum_.variants.len(), "UniFFI limits enums to 256 variants")?; std::iter::once(Ok(quote! { .concat_value(#variants_len) })) - .chain( - enum_.variants + .chain(enum_.variants.iter().map(|v| { + let fields_len = try_metadata_value_from_usize( + v.fields.len(), + "UniFFI limits enum variants to 256 fields", + )?; + + let field_names = v + .fields .iter() - .map(|v| { - let fields_len = try_metadata_value_from_usize( - v.fields.len(), - "UniFFI limits enum variants to 256 fields", - )?; - - let field_names = v.fields - .iter() - .map(|f| { - f.ident - .as_ref() - .ok_or_else(|| - syn::Error::new_spanned( - v, - "UniFFI only supports enum variants with named fields (or no fields at all)", - ) - ) - .map(ident_to_string) - }) - .collect::>>()?; - - let name = ident_to_string(&v.ident); - let field_types = v.fields.iter().map(|f| &f.ty); - Ok(quote! { - .concat_str(#name) - .concat_value(#fields_len) - #( - .concat_str(#field_names) - .concat(<#field_types as ::uniffi::Lower>::TYPE_ID_META) - // field defaults not yet supported for enums - .concat_bool(false) - )* - }) - }) - ) + .map(|f| f.ident.as_ref().map(ident_to_string).unwrap_or_default()) + .collect::>(); + + let name = ident_to_string(&v.ident); + let value_tokens = variant_value(v)?; + let docstring = extract_docstring(&v.attrs)?; + let field_types = v.fields.iter().map(|f| &f.ty); + let field_docstrings = v + .fields + .iter() + .map(|f| extract_docstring(&f.attrs)) + .collect::>>()?; + + Ok(quote! { + .concat_str(#name) + #value_tokens + .concat_value(#fields_len) + #( + .concat_str(#field_names) + .concat(<#field_types as ::uniffi::Lower>::TYPE_ID_META) + // field defaults not yet supported for enums + .concat_bool(false) + .concat_long_str(#field_docstrings) + )* + .concat_long_str(#docstring) + }) + })) .collect() } + +#[derive(Default)] +pub struct EnumAttr { + pub non_exhaustive: Option, +} + +// So ErrorAttr can be used with `parse_macro_input!` +impl Parse for EnumAttr { + fn parse(input: ParseStream<'_>) -> syn::Result { + parse_comma_separated(input) + } +} + +impl UniffiAttributeArgs for EnumAttr { + fn parse_one(input: ParseStream<'_>) -> syn::Result { + let lookahead = input.lookahead1(); + if lookahead.peek(kw::non_exhaustive) { + Ok(Self { + non_exhaustive: input.parse()?, + }) + } else { + Err(lookahead.error()) + } + } + + fn merge(self, other: Self) -> syn::Result { + Ok(Self { + non_exhaustive: either_attribute_arg(self.non_exhaustive, other.non_exhaustive)?, + }) + } +} diff --git a/third_party/rust/uniffi_macros/src/error.rs b/third_party/rust/uniffi_macros/src/error.rs index a2ee7cf603..804b438003 100644 --- a/third_party/rust/uniffi_macros/src/error.rs +++ b/third_party/rust/uniffi_macros/src/error.rs @@ -6,11 +6,11 @@ use syn::{ }; use crate::{ - enum_::{rich_error_ffi_converter_impl, variant_metadata}, + enum_::{rich_error_ffi_converter_impl, variant_metadata, EnumAttr}, util::{ - chain, create_metadata_items, derive_ffi_traits, either_attribute_arg, ident_to_string, kw, - mod_path, parse_comma_separated, tagged_impl_header, try_metadata_value_from_usize, - AttributeSliceExt, UniffiAttributeArgs, + chain, create_metadata_items, derive_ffi_traits, either_attribute_arg, extract_docstring, + ident_to_string, kw, mod_path, parse_comma_separated, tagged_impl_header, + try_metadata_value_from_usize, AttributeSliceExt, UniffiAttributeArgs, }, }; @@ -30,13 +30,14 @@ pub fn expand_error( } }; let ident = &input.ident; + let docstring = extract_docstring(&input.attrs)?; let mut attr: ErrorAttr = input.attrs.parse_uniffi_attr_args()?; if let Some(attr_from_udl_mode) = attr_from_udl_mode { attr = attr.merge(attr_from_udl_mode)?; } - let ffi_converter_impl = error_ffi_converter_impl(ident, &enum_, &attr, udl_mode); + let ffi_converter_impl = error_ffi_converter_impl(ident, &enum_, &attr, udl_mode)?; let meta_static_var = (!udl_mode).then(|| { - error_meta_static_var(ident, &enum_, attr.flat.is_some()) + error_meta_static_var(ident, docstring, &enum_, &attr) .unwrap_or_else(syn::Error::into_compile_error) }); @@ -67,23 +68,23 @@ fn error_ffi_converter_impl( enum_: &DataEnum, attr: &ErrorAttr, udl_mode: bool, -) -> TokenStream { - if attr.flat.is_some() { - flat_error_ffi_converter_impl(ident, enum_, udl_mode, attr.with_try_read.is_some()) +) -> syn::Result { + Ok(if attr.flat.is_some() { + flat_error_ffi_converter_impl(ident, enum_, udl_mode, attr) } else { - rich_error_ffi_converter_impl(ident, enum_, udl_mode) - } + rich_error_ffi_converter_impl(ident, enum_, udl_mode, &attr.clone().try_into()?) + }) } // FfiConverters for "flat errors" // -// These are errors where we only lower the to_string() value, rather than any assocated data. +// These are errors where we only lower the to_string() value, rather than any associated data. // We lower the to_string() value unconditionally, whether the enum has associated data or not. fn flat_error_ffi_converter_impl( ident: &Ident, enum_: &DataEnum, udl_mode: bool, - implement_lift: bool, + attr: &ErrorAttr, ) -> TokenStream { let name = ident_to_string(ident); let lower_impl_spec = tagged_impl_header("Lower", ident, udl_mode); @@ -95,7 +96,7 @@ fn flat_error_ffi_converter_impl( }; let lower_impl = { - let match_arms = enum_.variants.iter().enumerate().map(|(i, v)| { + let mut match_arms: Vec<_> = enum_.variants.iter().enumerate().map(|(i, v)| { let v_ident = &v.ident; let idx = Index::from(i + 1); @@ -105,7 +106,12 @@ fn flat_error_ffi_converter_impl( <::std::string::String as ::uniffi::Lower>::write(error_msg, buf); } } - }); + }).collect(); + if attr.non_exhaustive.is_some() { + match_arms.push(quote! { + _ => panic!("Unexpected variant in non-exhaustive enum"), + }) + } quote! { #[automatically_derived] @@ -128,7 +134,7 @@ fn flat_error_ffi_converter_impl( } }; - let lift_impl = if implement_lift { + let lift_impl = if attr.with_try_read.is_some() { let match_arms = enum_.variants.iter().enumerate().map(|(i, v)| { let v_ident = &v.ident; let idx = Index::from(i + 1); @@ -192,42 +198,53 @@ fn flat_error_ffi_converter_impl( pub(crate) fn error_meta_static_var( ident: &Ident, + docstring: String, enum_: &DataEnum, - flat: bool, + attr: &ErrorAttr, ) -> syn::Result { let name = ident_to_string(ident); let module_path = mod_path()?; + let flat = attr.flat.is_some(); + let non_exhaustive = attr.non_exhaustive.is_some(); let mut metadata_expr = quote! { - ::uniffi::MetadataBuffer::from_code(::uniffi::metadata::codes::ERROR) - // first our is-flat flag - .concat_bool(#flat) - // followed by an enum + ::uniffi::MetadataBuffer::from_code(::uniffi::metadata::codes::ENUM) .concat_str(#module_path) .concat_str(#name) + .concat_option_bool(Some(#flat)) + .concat_bool(false) // discr_type: None }; if flat { metadata_expr.extend(flat_error_variant_metadata(enum_)?) } else { metadata_expr.extend(variant_metadata(enum_)?); } + metadata_expr.extend(quote! { + .concat_bool(#non_exhaustive) + .concat_long_str(#docstring) + }); Ok(create_metadata_items("error", &name, metadata_expr, None)) } pub fn flat_error_variant_metadata(enum_: &DataEnum) -> syn::Result> { let variants_len = try_metadata_value_from_usize(enum_.variants.len(), "UniFFI limits enums to 256 variants")?; - Ok(std::iter::once(quote! { .concat_value(#variants_len) }) + std::iter::once(Ok(quote! { .concat_value(#variants_len) })) .chain(enum_.variants.iter().map(|v| { let name = ident_to_string(&v.ident); - quote! { .concat_str(#name) } + let docstring = extract_docstring(&v.attrs)?; + Ok(quote! { + .concat_str(#name) + .concat_long_str(#docstring) + }) })) - .collect()) + .collect() } -#[derive(Default)] +#[derive(Clone, Default)] pub struct ErrorAttr { - flat: Option, - with_try_read: Option, + pub flat: Option, + pub with_try_read: Option, + pub non_exhaustive: Option, } impl UniffiAttributeArgs for ErrorAttr { @@ -243,8 +260,13 @@ impl UniffiAttributeArgs for ErrorAttr { with_try_read: input.parse()?, ..Self::default() }) + } else if lookahead.peek(kw::non_exhaustive) { + Ok(Self { + non_exhaustive: input.parse()?, + ..Self::default() + }) } else if lookahead.peek(kw::handle_unknown_callback_error) { - // Not used anymore, but still lallowed + // Not used anymore, but still allowed Ok(Self::default()) } else { Err(lookahead.error()) @@ -255,6 +277,7 @@ impl UniffiAttributeArgs for ErrorAttr { Ok(Self { flat: either_attribute_arg(self.flat, other.flat)?, with_try_read: either_attribute_arg(self.with_try_read, other.with_try_read)?, + non_exhaustive: either_attribute_arg(self.non_exhaustive, other.non_exhaustive)?, }) } } @@ -265,3 +288,25 @@ impl Parse for ErrorAttr { parse_comma_separated(input) } } + +impl TryFrom for EnumAttr { + type Error = syn::Error; + + fn try_from(error_attr: ErrorAttr) -> Result { + if error_attr.flat.is_some() { + Err(syn::Error::new( + Span::call_site(), + "flat attribute not valid for rich enum errors", + )) + } else if error_attr.with_try_read.is_some() { + Err(syn::Error::new( + Span::call_site(), + "with_try_read attribute not valid for rich enum errors", + )) + } else { + Ok(EnumAttr { + non_exhaustive: error_attr.non_exhaustive, + }) + } + } +} diff --git a/third_party/rust/uniffi_macros/src/export.rs b/third_party/rust/uniffi_macros/src/export.rs index bbb16acf90..41657a639e 100644 --- a/third_party/rust/uniffi_macros/src/export.rs +++ b/third_party/rust/uniffi_macros/src/export.rs @@ -2,7 +2,7 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -use proc_macro2::{Ident, Span, TokenStream}; +use proc_macro2::TokenStream; use quote::{quote, quote_spanned}; use syn::{visit_mut::VisitMut, Item, Type}; @@ -10,6 +10,7 @@ mod attributes; mod callback_interface; mod item; mod scaffolding; +mod trait_interface; mod utrait; use self::{ @@ -18,20 +19,16 @@ use self::{ gen_constructor_scaffolding, gen_ffi_function, gen_fn_scaffolding, gen_method_scaffolding, }, }; -use crate::{ - object::interface_meta_static_var, - util::{ident_to_string, mod_path, tagged_impl_header}, -}; -pub use attributes::ExportAttributeArguments; +use crate::util::{ident_to_string, mod_path}; +pub use attributes::{DefaultMap, ExportFnArgs, ExportedImplFnArgs}; pub use callback_interface::ffi_converter_callback_interface_impl; -use uniffi_meta::free_fn_symbol_name; // TODO(jplatte): Ensure no generics, … // TODO(jplatte): Aggregate errors instead of short-circuiting, wherever possible pub(crate) fn expand_export( mut item: Item, - args: ExportAttributeArguments, + all_args: proc_macro::TokenStream, udl_mode: bool, ) -> syn::Result { let mod_path = mod_path()?; @@ -41,11 +38,17 @@ pub(crate) fn expand_export( // new functions outside of the `impl`). rewrite_self_type(&mut item); - let metadata = ExportItem::new(item, &args)?; + let metadata = ExportItem::new(item, all_args)?; match metadata { - ExportItem::Function { sig } => gen_fn_scaffolding(sig, &args, udl_mode), - ExportItem::Impl { items, self_ident } => { + ExportItem::Function { sig, args } => { + gen_fn_scaffolding(sig, &args.async_runtime, udl_mode) + } + ExportItem::Impl { + items, + self_ident, + args, + } => { if let Some(rt) = &args.async_runtime { if items .iter() @@ -61,8 +64,12 @@ pub(crate) fn expand_export( let item_tokens: TokenStream = items .into_iter() .map(|item| match item { - ImplItem::Constructor(sig) => gen_constructor_scaffolding(sig, &args, udl_mode), - ImplItem::Method(sig) => gen_method_scaffolding(sig, &args, udl_mode), + ImplItem::Constructor(sig) => { + gen_constructor_scaffolding(sig, &args.async_runtime, udl_mode) + } + ImplItem::Method(sig) => { + gen_method_scaffolding(sig, &args.async_runtime, udl_mode) + } }) .collect::>()?; Ok(quote_spanned! { self_ident.span() => #item_tokens }) @@ -70,111 +77,51 @@ pub(crate) fn expand_export( ExportItem::Trait { items, self_ident, - callback_interface: false, - } => { - if let Some(rt) = args.async_runtime { - return Err(syn::Error::new_spanned(rt, "not supported for traits")); - } - - let name = ident_to_string(&self_ident); - let free_fn_ident = - Ident::new(&free_fn_symbol_name(&mod_path, &name), Span::call_site()); - - let free_tokens = quote! { - #[doc(hidden)] - #[no_mangle] - pub extern "C" fn #free_fn_ident( - ptr: *const ::std::ffi::c_void, - call_status: &mut ::uniffi::RustCallStatus - ) { - uniffi::rust_call(call_status, || { - assert!(!ptr.is_null()); - drop(unsafe { ::std::boxed::Box::from_raw(ptr as *mut std::sync::Arc) }); - Ok(()) - }); - } - }; - - let impl_tokens: TokenStream = items - .into_iter() - .map(|item| match item { - ImplItem::Method(sig) => { - if sig.is_async { - return Err(syn::Error::new( - sig.span, - "async trait methods are not supported", - )); - } - gen_method_scaffolding(sig, &args, udl_mode) - } - _ => unreachable!("traits have no constructors"), - }) - .collect::>()?; - - let meta_static_var = (!udl_mode).then(|| { - interface_meta_static_var(&self_ident, true, &mod_path) - .unwrap_or_else(syn::Error::into_compile_error) - }); - let ffi_converter_tokens = ffi_converter_trait_impl(&self_ident, false); - - Ok(quote_spanned! { self_ident.span() => - #meta_static_var - #free_tokens - #ffi_converter_tokens - #impl_tokens - }) - } + with_foreign, + callback_interface_only: false, + docstring, + args, + } => trait_interface::gen_trait_scaffolding( + &mod_path, + args, + self_ident, + items, + udl_mode, + with_foreign, + docstring, + ), ExportItem::Trait { items, self_ident, - callback_interface: true, + callback_interface_only: true, + docstring, + .. } => { let trait_name = ident_to_string(&self_ident); - let trait_impl_ident = Ident::new( - &format!("UniFFICallbackHandler{trait_name}"), - Span::call_site(), - ); - let internals_ident = Ident::new( - &format!( - "UNIFFI_FOREIGN_CALLBACK_INTERNALS_{}", - trait_name.to_ascii_uppercase() - ), - Span::call_site(), - ); - - let trait_impl = callback_interface::trait_impl( - &trait_impl_ident, - &self_ident, - &internals_ident, - &items, - ) - .unwrap_or_else(|e| e.into_compile_error()); - let metadata_items = callback_interface::metadata_items(&self_ident, &items, &mod_path) - .unwrap_or_else(|e| vec![e.into_compile_error()]); - - let init_ident = Ident::new( - &uniffi_meta::init_callback_fn_symbol_name(&mod_path, &trait_name), - Span::call_site(), - ); + let trait_impl_ident = callback_interface::trait_impl_ident(&trait_name); + let trait_impl = callback_interface::trait_impl(&mod_path, &self_ident, &items) + .unwrap_or_else(|e| e.into_compile_error()); + let metadata_items = (!udl_mode).then(|| { + let items = + callback_interface::metadata_items(&self_ident, &items, &mod_path, docstring) + .unwrap_or_else(|e| vec![e.into_compile_error()]); + quote! { #(#items)* } + }); + let ffi_converter_tokens = + ffi_converter_callback_interface_impl(&self_ident, &trait_impl_ident, udl_mode); Ok(quote! { - #[doc(hidden)] - static #internals_ident: ::uniffi::ForeignCallbackInternals = ::uniffi::ForeignCallbackInternals::new(); - - #[doc(hidden)] - #[no_mangle] - pub extern "C" fn #init_ident(callback: ::uniffi::ForeignCallback, _: &mut ::uniffi::RustCallStatus) { - #internals_ident.set_callback(callback); - } - #trait_impl - #(#metadata_items)* + #ffi_converter_tokens + + #metadata_items }) } ExportItem::Struct { self_ident, uniffi_traits, + .. } => { assert!(!udl_mode); utrait::expand_uniffi_trait_export(self_ident, uniffi_traits) @@ -182,62 +129,6 @@ pub(crate) fn expand_export( } } -pub(crate) fn ffi_converter_trait_impl(trait_ident: &Ident, udl_mode: bool) -> TokenStream { - let impl_spec = tagged_impl_header("FfiConverterArc", "e! { dyn #trait_ident }, udl_mode); - let lift_ref_impl_spec = tagged_impl_header("LiftRef", "e! { dyn #trait_ident }, udl_mode); - let name = ident_to_string(trait_ident); - let mod_path = match mod_path() { - Ok(p) => p, - Err(e) => return e.into_compile_error(), - }; - - quote! { - // All traits must be `Sync + Send`. The generated scaffolding will fail to compile - // if they are not, but unfortunately it fails with an unactionably obscure error message. - // By asserting the requirement explicitly, we help Rust produce a more scrutable error message - // and thus help the user debug why the requirement isn't being met. - uniffi::deps::static_assertions::assert_impl_all!(dyn #trait_ident: Sync, Send); - - unsafe #impl_spec { - type FfiType = *const ::std::os::raw::c_void; - - fn lower(obj: ::std::sync::Arc) -> Self::FfiType { - ::std::boxed::Box::into_raw(::std::boxed::Box::new(obj)) as *const ::std::os::raw::c_void - } - - fn try_lift(v: Self::FfiType) -> ::uniffi::Result<::std::sync::Arc> { - let foreign_arc = ::std::boxed::Box::leak(unsafe { Box::from_raw(v as *mut ::std::sync::Arc) }); - // Take a clone for our own use. - Ok(::std::sync::Arc::clone(foreign_arc)) - } - - fn write(obj: ::std::sync::Arc, buf: &mut Vec) { - ::uniffi::deps::static_assertions::const_assert!(::std::mem::size_of::<*const ::std::ffi::c_void>() <= 8); - ::uniffi::deps::bytes::BufMut::put_u64( - buf, - >::lower(obj) as u64, - ); - } - - fn try_read(buf: &mut &[u8]) -> ::uniffi::Result<::std::sync::Arc> { - ::uniffi::deps::static_assertions::const_assert!(::std::mem::size_of::<*const ::std::ffi::c_void>() <= 8); - ::uniffi::check_remaining(buf, 8)?; - >::try_lift( - ::uniffi::deps::bytes::Buf::get_u64(buf) as Self::FfiType) - } - - const TYPE_ID_META: ::uniffi::MetadataBuffer = ::uniffi::MetadataBuffer::from_code(::uniffi::metadata::codes::TYPE_INTERFACE) - .concat_str(#mod_path) - .concat_str(#name) - .concat_bool(true); - } - - unsafe #lift_ref_impl_spec { - type LiftType = ::std::sync::Arc; - } - } -} - /// Rewrite Self type alias usage in an impl block to the type itself. /// /// For example, diff --git a/third_party/rust/uniffi_macros/src/export/attributes.rs b/third_party/rust/uniffi_macros/src/export/attributes.rs index c3edcd5920..be7e8902e4 100644 --- a/third_party/rust/uniffi_macros/src/export/attributes.rs +++ b/third_party/rust/uniffi_macros/src/export/attributes.rs @@ -1,31 +1,33 @@ -use crate::util::{either_attribute_arg, kw, parse_comma_separated, UniffiAttributeArgs}; +use std::collections::{HashMap, HashSet}; + +use crate::{ + default::DefaultValue, + util::{either_attribute_arg, kw, parse_comma_separated, UniffiAttributeArgs}, +}; use proc_macro2::TokenStream; use quote::ToTokens; use syn::{ + parenthesized, parse::{Parse, ParseStream}, - Attribute, LitStr, Meta, PathArguments, PathSegment, Token, + Attribute, Ident, LitStr, Meta, PathArguments, PathSegment, Token, }; +use uniffi_meta::UniffiTraitDiscriminants; #[derive(Default)] -pub struct ExportAttributeArguments { +pub struct ExportTraitArgs { pub(crate) async_runtime: Option, pub(crate) callback_interface: Option, - pub(crate) constructor: Option, - // tried to make this a vec but that got messy quickly... - pub(crate) trait_debug: Option, - pub(crate) trait_display: Option, - pub(crate) trait_hash: Option, - pub(crate) trait_eq: Option, + pub(crate) with_foreign: Option, } -impl Parse for ExportAttributeArguments { +impl Parse for ExportTraitArgs { fn parse(input: ParseStream<'_>) -> syn::Result { parse_comma_separated(input) } } -impl UniffiAttributeArgs for ExportAttributeArguments { +impl UniffiAttributeArgs for ExportTraitArgs { fn parse_one(input: ParseStream<'_>) -> syn::Result { let lookahead = input.lookahead1(); if lookahead.peek(kw::async_runtime) { @@ -40,52 +42,175 @@ impl UniffiAttributeArgs for ExportAttributeArguments { callback_interface: input.parse()?, ..Self::default() }) - } else if lookahead.peek(kw::constructor) { + } else if lookahead.peek(kw::with_foreign) { Ok(Self { - constructor: input.parse()?, + with_foreign: input.parse()?, ..Self::default() }) - } else if lookahead.peek(kw::Debug) { + } else { + Ok(Self::default()) + } + } + + fn merge(self, other: Self) -> syn::Result { + let merged = Self { + async_runtime: either_attribute_arg(self.async_runtime, other.async_runtime)?, + callback_interface: either_attribute_arg( + self.callback_interface, + other.callback_interface, + )?, + with_foreign: either_attribute_arg(self.with_foreign, other.with_foreign)?, + }; + if merged.callback_interface.is_some() && merged.with_foreign.is_some() { + return Err(syn::Error::new( + merged.callback_interface.unwrap().span, + "`callback_interface` and `with_foreign` are mutually exclusive", + )); + } + Ok(merged) + } +} + +#[derive(Clone, Default)] +pub struct ExportFnArgs { + pub(crate) async_runtime: Option, + pub(crate) name: Option, + pub(crate) defaults: DefaultMap, +} + +impl Parse for ExportFnArgs { + fn parse(input: ParseStream<'_>) -> syn::Result { + parse_comma_separated(input) + } +} + +impl UniffiAttributeArgs for ExportFnArgs { + fn parse_one(input: ParseStream<'_>) -> syn::Result { + let lookahead = input.lookahead1(); + if lookahead.peek(kw::async_runtime) { + let _: kw::async_runtime = input.parse()?; + let _: Token![=] = input.parse()?; Ok(Self { - trait_debug: input.parse()?, + async_runtime: Some(input.parse()?), ..Self::default() }) - } else if lookahead.peek(kw::Display) { + } else if lookahead.peek(kw::name) { + let _: kw::name = input.parse()?; + let _: Token![=] = input.parse()?; + let name = Some(input.parse::()?.value()); Ok(Self { - trait_display: input.parse()?, + name, ..Self::default() }) - } else if lookahead.peek(kw::Hash) { + } else if lookahead.peek(kw::default) { Ok(Self { - trait_hash: input.parse()?, + defaults: DefaultMap::parse(input)?, ..Self::default() }) - } else if lookahead.peek(kw::Eq) { + } else { + Err(syn::Error::new( + input.span(), + format!("uniffi::export attribute `{input}` is not supported here."), + )) + } + } + + fn merge(self, other: Self) -> syn::Result { + Ok(Self { + async_runtime: either_attribute_arg(self.async_runtime, other.async_runtime)?, + name: either_attribute_arg(self.name, other.name)?, + defaults: self.defaults.merge(other.defaults), + }) + } +} + +#[derive(Default)] +pub struct ExportImplArgs { + pub(crate) async_runtime: Option, +} + +impl Parse for ExportImplArgs { + fn parse(input: ParseStream<'_>) -> syn::Result { + parse_comma_separated(input) + } +} + +impl UniffiAttributeArgs for ExportImplArgs { + fn parse_one(input: ParseStream<'_>) -> syn::Result { + let lookahead = input.lookahead1(); + if lookahead.peek(kw::async_runtime) { + let _: kw::async_runtime = input.parse()?; + let _: Token![=] = input.parse()?; Ok(Self { - trait_eq: input.parse()?, - ..Self::default() + async_runtime: Some(input.parse()?), }) } else { - Ok(Self::default()) + Err(syn::Error::new( + input.span(), + format!("uniffi::export attribute `{input}` is not supported here."), + )) } } fn merge(self, other: Self) -> syn::Result { Ok(Self { async_runtime: either_attribute_arg(self.async_runtime, other.async_runtime)?, - callback_interface: either_attribute_arg( - self.callback_interface, - other.callback_interface, - )?, - constructor: either_attribute_arg(self.constructor, other.constructor)?, - trait_debug: either_attribute_arg(self.trait_debug, other.trait_debug)?, - trait_display: either_attribute_arg(self.trait_display, other.trait_display)?, - trait_hash: either_attribute_arg(self.trait_hash, other.trait_hash)?, - trait_eq: either_attribute_arg(self.trait_eq, other.trait_eq)?, }) } } +#[derive(Default)] +pub struct ExportStructArgs { + pub(crate) traits: HashSet, +} + +impl Parse for ExportStructArgs { + fn parse(input: ParseStream<'_>) -> syn::Result { + parse_comma_separated(input) + } +} + +impl UniffiAttributeArgs for ExportStructArgs { + fn parse_one(input: ParseStream<'_>) -> syn::Result { + let lookahead = input.lookahead1(); + if lookahead.peek(kw::Debug) { + input.parse::>()?; + Ok(Self { + traits: HashSet::from([UniffiTraitDiscriminants::Debug]), + }) + } else if lookahead.peek(kw::Display) { + input.parse::>()?; + Ok(Self { + traits: HashSet::from([UniffiTraitDiscriminants::Display]), + }) + } else if lookahead.peek(kw::Hash) { + input.parse::>()?; + Ok(Self { + traits: HashSet::from([UniffiTraitDiscriminants::Hash]), + }) + } else if lookahead.peek(kw::Eq) { + input.parse::>()?; + Ok(Self { + traits: HashSet::from([UniffiTraitDiscriminants::Eq]), + }) + } else { + Err(syn::Error::new( + input.span(), + format!( + "uniffi::export struct attributes must be builtin trait names; `{input}` is invalid" + ), + )) + } + } + + fn merge(self, other: Self) -> syn::Result { + let mut traits = self.traits; + traits.extend(other.traits); + Ok(Self { traits }) + } +} + +#[derive(Clone)] pub(crate) enum AsyncRuntime { Tokio(LitStr), } @@ -111,9 +236,57 @@ impl ToTokens for AsyncRuntime { } } +/// Arguments for function inside an impl block +/// +/// This stores the parsed arguments for `uniffi::constructor` and `uniffi::method` +#[derive(Clone, Default)] +pub struct ExportedImplFnArgs { + pub(crate) name: Option, + pub(crate) defaults: DefaultMap, +} + +impl Parse for ExportedImplFnArgs { + fn parse(input: ParseStream<'_>) -> syn::Result { + parse_comma_separated(input) + } +} + +impl UniffiAttributeArgs for ExportedImplFnArgs { + fn parse_one(input: ParseStream<'_>) -> syn::Result { + let lookahead = input.lookahead1(); + if lookahead.peek(kw::name) { + let _: kw::name = input.parse()?; + let _: Token![=] = input.parse()?; + let name = Some(input.parse::()?.value()); + Ok(Self { + name, + ..Self::default() + }) + } else if lookahead.peek(kw::default) { + Ok(Self { + defaults: DefaultMap::parse(input)?, + ..Self::default() + }) + } else { + Err(syn::Error::new( + input.span(), + format!("uniffi::constructor/method attribute `{input}` is not supported here."), + )) + } + } + + fn merge(self, other: Self) -> syn::Result { + Ok(Self { + name: either_attribute_arg(self.name, other.name)?, + defaults: self.defaults.merge(other.defaults), + }) + } +} + #[derive(Default)] pub(super) struct ExportedImplFnAttributes { pub constructor: bool, + pub args: ExportedImplFnArgs, } impl ExportedImplFnAttributes { @@ -130,12 +303,11 @@ impl ExportedImplFnAttributes { } ensure_no_path_args(fst)?; - if let Meta::List(_) | Meta::NameValue(_) = &attr.meta { - return Err(syn::Error::new_spanned( - &attr.meta, - "attribute arguments are not currently recognized in this position", - )); - } + let args = match &attr.meta { + Meta::List(_) => attr.parse_args::()?, + _ => Default::default(), + }; + this.args = args; if segs.len() != 2 { return Err(syn::Error::new_spanned( @@ -156,6 +328,14 @@ impl ExportedImplFnAttributes { } this.constructor = true; } + "method" => { + if this.constructor { + return Err(syn::Error::new_spanned( + attr, + "confused constructor/method attributes", + )); + } + } _ => return Err(syn::Error::new_spanned(snd, "unknown uniffi attribute")), } } @@ -171,3 +351,53 @@ fn ensure_no_path_args(seg: &PathSegment) -> syn::Result<()> { Err(syn::Error::new_spanned(&seg.arguments, "unexpected syntax")) } } + +/// Maps arguments to defaults for functions +#[derive(Clone, Default)] +pub struct DefaultMap { + map: HashMap, +} + +impl DefaultMap { + pub fn merge(self, other: Self) -> Self { + let mut map = self.map; + map.extend(other.map); + Self { map } + } + + pub fn remove(&mut self, ident: &Ident) -> Option { + self.map.remove(ident) + } + + pub fn idents(&self) -> Vec<&Ident> { + self.map.keys().collect() + } +} + +impl Parse for DefaultMap { + fn parse(input: ParseStream<'_>) -> syn::Result { + let _: kw::default = input.parse()?; + let content; + let _ = parenthesized!(content in input); + let pairs = content.parse_terminated(DefaultPair::parse, Token![,])?; + Ok(Self { + map: pairs.into_iter().map(|p| (p.name, p.value)).collect(), + }) + } +} + +pub struct DefaultPair { + pub name: Ident, + pub eq_token: Token![=], + pub value: DefaultValue, +} + +impl Parse for DefaultPair { + fn parse(input: ParseStream<'_>) -> syn::Result { + Ok(Self { + name: input.parse()?, + eq_token: input.parse()?, + value: input.parse()?, + }) + } +} diff --git a/third_party/rust/uniffi_macros/src/export/callback_interface.rs b/third_party/rust/uniffi_macros/src/export/callback_interface.rs index 2f2561bbc2..fe145384ec 100644 --- a/third_party/rust/uniffi_macros/src/export/callback_interface.rs +++ b/third_party/rust/uniffi_macros/src/export/callback_interface.rs @@ -5,69 +5,137 @@ use crate::{ export::ImplItem, fnsig::{FnKind, FnSignature, ReceiverArg}, - util::{create_metadata_items, ident_to_string, mod_path, tagged_impl_header}, + util::{ + create_metadata_items, derive_ffi_traits, ident_to_string, mod_path, tagged_impl_header, + }, }; use proc_macro2::{Span, TokenStream}; -use quote::quote; +use quote::{format_ident, quote}; use std::iter; use syn::Ident; +/// Generate a trait impl that calls foreign callbacks +/// +/// This generates: +/// * A `repr(C)` VTable struct where each field is the FFI function for the trait method. +/// * A FFI function for foreign code to set their VTable for the interface +/// * An implementation of the trait using that VTable pub(super) fn trait_impl( - ident: &Ident, + mod_path: &str, trait_ident: &Ident, - internals_ident: &Ident, items: &[ImplItem], ) -> syn::Result { - let trait_impl_methods = items + let trait_name = ident_to_string(trait_ident); + let trait_impl_ident = trait_impl_ident(&trait_name); + let vtable_type = format_ident!("UniFfiTraitVtable{trait_name}"); + let vtable_cell = format_ident!("UNIFFI_TRAIT_CELL_{}", trait_name.to_uppercase()); + let init_ident = Ident::new( + &uniffi_meta::init_callback_vtable_fn_symbol_name(mod_path, &trait_name), + Span::call_site(), + ); + let methods = items .iter() .map(|item| match item { - ImplItem::Method(sig) => gen_method_impl(sig, internals_ident), - _ => unreachable!("traits have no constructors"), + ImplItem::Constructor(sig) => Err(syn::Error::new( + sig.span, + "Constructors not allowed in trait interfaces", + )), + ImplItem::Method(sig) => Ok(sig), }) - .collect::>()?; - let ffi_converter_tokens = ffi_converter_callback_interface_impl(trait_ident, ident, false); + .collect::>>()?; + + let vtable_fields = methods.iter() + .map(|sig| { + let ident = &sig.ident; + let param_names = sig.scaffolding_param_names(); + let param_types = sig.scaffolding_param_types(); + let lift_return = sig.lift_return_impl(); + if !sig.is_async { + quote! { + #ident: extern "C" fn( + uniffi_handle: u64, + #(#param_names: #param_types,)* + uniffi_out_return: &mut #lift_return::ReturnType, + uniffi_out_call_status: &mut ::uniffi::RustCallStatus, + ), + } + } else { + quote! { + #ident: extern "C" fn( + uniffi_handle: u64, + #(#param_names: #param_types,)* + uniffi_future_callback: ::uniffi::ForeignFutureCallback<#lift_return::ReturnType>, + uniffi_callback_data: u64, + uniffi_out_return: &mut ::uniffi::ForeignFuture, + ), + } + } + }); + + let trait_impl_methods = methods + .iter() + .map(|sig| gen_method_impl(sig, &vtable_cell)) + .collect::>>()?; + let has_async_method = methods.iter().any(|m| m.is_async); + let impl_attributes = has_async_method.then(|| quote! { #[::async_trait::async_trait] }); Ok(quote! { - #[doc(hidden)] + struct #vtable_type { + #(#vtable_fields)* + uniffi_free: extern "C" fn(handle: u64), + } + + static #vtable_cell: ::uniffi::UniffiForeignPointerCell::<#vtable_type> = ::uniffi::UniffiForeignPointerCell::<#vtable_type>::new(); + + #[no_mangle] + extern "C" fn #init_ident(vtable: ::std::ptr::NonNull<#vtable_type>) { + #vtable_cell.set(vtable); + } + #[derive(Debug)] - struct #ident { + struct #trait_impl_ident { handle: u64, } - impl #ident { + impl #trait_impl_ident { fn new(handle: u64) -> Self { Self { handle } } } - impl ::std::ops::Drop for #ident { - fn drop(&mut self) { - #internals_ident.invoke_callback::<(), crate::UniFfiTag>( - self.handle, uniffi::IDX_CALLBACK_FREE, Default::default() - ) - } - } - - ::uniffi::deps::static_assertions::assert_impl_all!(#ident: Send); + ::uniffi::deps::static_assertions::assert_impl_all!(#trait_impl_ident: ::core::marker::Send); - impl #trait_ident for #ident { - #trait_impl_methods + #impl_attributes + impl #trait_ident for #trait_impl_ident { + #(#trait_impl_methods)* } - #ffi_converter_tokens + impl ::std::ops::Drop for #trait_impl_ident { + fn drop(&mut self) { + let vtable = #vtable_cell.get(); + (vtable.uniffi_free)(self.handle); + } + } }) } +pub fn trait_impl_ident(trait_name: &str) -> Ident { + Ident::new( + &format!("UniFFICallbackHandler{trait_name}"), + Span::call_site(), + ) +} + pub fn ffi_converter_callback_interface_impl( trait_ident: &Ident, trait_impl_ident: &Ident, udl_mode: bool, ) -> TokenStream { - let name = ident_to_string(trait_ident); + let trait_name = ident_to_string(trait_ident); let dyn_trait = quote! { dyn #trait_ident }; let box_dyn_trait = quote! { ::std::boxed::Box<#dyn_trait> }; let lift_impl_spec = tagged_impl_header("Lift", &box_dyn_trait, udl_mode); - let lift_ref_impl_spec = tagged_impl_header("LiftRef", &dyn_trait, udl_mode); + let derive_ffi_traits = derive_ffi_traits(&box_dyn_trait, udl_mode, &["LiftRef", "LiftReturn"]); let mod_path = match mod_path() { Ok(p) => p, Err(e) => return e.into_compile_error(), @@ -93,66 +161,85 @@ pub fn ffi_converter_callback_interface_impl( ::uniffi::metadata::codes::TYPE_CALLBACK_INTERFACE, ) .concat_str(#mod_path) - .concat_str(#name); + .concat_str(#trait_name); } - unsafe #lift_ref_impl_spec { - type LiftType = #box_dyn_trait; - } + #derive_ffi_traits } } -fn gen_method_impl(sig: &FnSignature, internals_ident: &Ident) -> syn::Result { +/// Generate a single method for [trait_impl]. This implements a trait method by invoking a +/// foreign-supplied callback. +fn gen_method_impl(sig: &FnSignature, vtable_cell: &Ident) -> syn::Result { let FnSignature { ident, + is_async, return_ty, kind, receiver, + name, + span, .. } = sig; - let index = match kind { - // Note: the callback index is 1-based, since 0 is reserved for the free function - FnKind::TraitMethod { index, .. } => index + 1, - k => { - return Err(syn::Error::new( - sig.span, - format!( - "Internal UniFFI error: Unexpected function kind for callback interface {k:?}" - ), - )); - } - }; + + if !matches!(kind, FnKind::TraitMethod { .. }) { + return Err(syn::Error::new( + *span, + format!( + "Internal UniFFI error: Unexpected function kind for callback interface {name}: {kind:?}", + ), + )); + } let self_param = match receiver { + Some(ReceiverArg::Ref) => quote! { &self }, + Some(ReceiverArg::Arc) => quote! { self: Arc }, None => { return Err(syn::Error::new( - sig.span, + *span, "callback interface methods must take &self as their first argument", )); } - Some(ReceiverArg::Ref) => quote! { &self }, - Some(ReceiverArg::Arc) => quote! { self: Arc }, }; + let params = sig.params(); - let buf_ident = Ident::new("uniffi_args_buf", Span::call_site()); - let write_exprs = sig.write_exprs(&buf_ident); + let lower_exprs = sig.args.iter().map(|a| { + let lower_impl = a.lower_impl(); + let ident = &a.ident; + quote! { #lower_impl::lower(#ident) } + }); - Ok(quote! { - fn #ident(#self_param, #(#params),*) -> #return_ty { - #[allow(unused_mut)] - let mut #buf_ident = ::std::vec::Vec::new(); - #(#write_exprs;)* - let uniffi_args_rbuf = uniffi::RustBuffer::from_vec(#buf_ident); + let lift_return = sig.lift_return_impl(); - #internals_ident.invoke_callback::<#return_ty, crate::UniFfiTag>(self.handle, #index, uniffi_args_rbuf) - } - }) + if !is_async { + Ok(quote! { + fn #ident(#self_param, #(#params),*) -> #return_ty { + let vtable = #vtable_cell.get(); + let mut uniffi_call_status = ::uniffi::RustCallStatus::new(); + let mut uniffi_return_value: #lift_return::ReturnType = ::uniffi::FfiDefault::ffi_default(); + (vtable.#ident)(self.handle, #(#lower_exprs,)* &mut uniffi_return_value, &mut uniffi_call_status); + #lift_return::lift_foreign_return(uniffi_return_value, uniffi_call_status) + } + }) + } else { + Ok(quote! { + async fn #ident(#self_param, #(#params),*) -> #return_ty { + let vtable = #vtable_cell.get(); + ::uniffi::foreign_async_call::<_, #return_ty, crate::UniFfiTag>(move |uniffi_future_callback, uniffi_future_callback_data| { + let mut uniffi_foreign_future: ::uniffi::ForeignFuture = ::uniffi::FfiDefault::ffi_default(); + (vtable.#ident)(self.handle, #(#lower_exprs,)* uniffi_future_callback, uniffi_future_callback_data, &mut uniffi_foreign_future); + uniffi_foreign_future + }).await + } + }) + } } pub(super) fn metadata_items( self_ident: &Ident, items: &[ImplItem], module_path: &str, + docstring: String, ) -> syn::Result> { let trait_name = ident_to_string(self_ident); let callback_interface_items = create_metadata_items( @@ -162,13 +249,14 @@ pub(super) fn metadata_items( ::uniffi::MetadataBuffer::from_code(::uniffi::metadata::codes::CALLBACK_INTERFACE) .concat_str(#module_path) .concat_str(#trait_name) + .concat_long_str(#docstring) }, None, ); iter::once(Ok(callback_interface_items)) .chain(items.iter().map(|item| match item { - ImplItem::Method(sig) => sig.metadata_items(), + ImplItem::Method(sig) => sig.metadata_items_for_callback_interface(), _ => unreachable!("traits have no constructors"), })) .collect() diff --git a/third_party/rust/uniffi_macros/src/export/item.rs b/third_party/rust/uniffi_macros/src/export/item.rs index 98c7d0ebe2..da3c9455c8 100644 --- a/third_party/rust/uniffi_macros/src/export/item.rs +++ b/third_party/rust/uniffi_macros/src/export/item.rs @@ -3,24 +3,34 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ use crate::fnsig::FnSignature; +use proc_macro::TokenStream; use proc_macro2::{Ident, Span}; use quote::ToTokens; -use super::attributes::{ExportAttributeArguments, ExportedImplFnAttributes}; +use super::attributes::{ + ExportFnArgs, ExportImplArgs, ExportStructArgs, ExportTraitArgs, ExportedImplFnArgs, + ExportedImplFnAttributes, +}; +use crate::util::extract_docstring; use uniffi_meta::UniffiTraitDiscriminants; pub(super) enum ExportItem { Function { sig: FnSignature, + args: ExportFnArgs, }, Impl { self_ident: Ident, items: Vec, + args: ExportImplArgs, }, Trait { self_ident: Ident, items: Vec, - callback_interface: bool, + with_foreign: bool, + callback_interface_only: bool, + docstring: String, + args: ExportTraitArgs, }, Struct { self_ident: Ident, @@ -29,15 +39,17 @@ pub(super) enum ExportItem { } impl ExportItem { - pub fn new(item: syn::Item, args: &ExportAttributeArguments) -> syn::Result { + pub fn new(item: syn::Item, attr_args: TokenStream) -> syn::Result { match item { syn::Item::Fn(item) => { - let sig = FnSignature::new_function(item.sig)?; - Ok(Self::Function { sig }) + let args: ExportFnArgs = syn::parse(attr_args)?; + let docstring = extract_docstring(&item.attrs)?; + let sig = FnSignature::new_function(item.sig, args.clone(), docstring)?; + Ok(Self::Function { sig, args }) } - syn::Item::Impl(item) => Self::from_impl(item, args.constructor.is_some()), - syn::Item::Trait(item) => Self::from_trait(item, args.callback_interface.is_some()), - syn::Item::Struct(item) => Self::from_struct(item, args), + syn::Item::Impl(item) => Self::from_impl(item, attr_args), + syn::Item::Trait(item) => Self::from_trait(item, attr_args), + syn::Item::Struct(item) => Self::from_struct(item, attr_args), // FIXME: Support const / static? _ => Err(syn::Error::new( Span::call_site(), @@ -47,7 +59,8 @@ impl ExportItem { } } - pub fn from_impl(item: syn::ItemImpl, force_constructor: bool) -> syn::Result { + pub fn from_impl(item: syn::ItemImpl, attr_args: TokenStream) -> syn::Result { + let args: ExportImplArgs = syn::parse(attr_args)?; if !item.generics.params.is_empty() || item.generics.where_clause.is_some() { return Err(syn::Error::new_spanned( &item.generics, @@ -88,14 +101,22 @@ impl ExportItem { } }; + let docstring = extract_docstring(&impl_fn.attrs)?; let attrs = ExportedImplFnAttributes::new(&impl_fn.attrs)?; - let item = if force_constructor || attrs.constructor { + let item = if attrs.constructor { ImplItem::Constructor(FnSignature::new_constructor( self_ident.clone(), impl_fn.sig, + attrs.args, + docstring, )?) } else { - ImplItem::Method(FnSignature::new_method(self_ident.clone(), impl_fn.sig)?) + ImplItem::Method(FnSignature::new_method( + self_ident.clone(), + impl_fn.sig, + attrs.args, + docstring, + )?) }; Ok(item) @@ -105,10 +126,15 @@ impl ExportItem { Ok(Self::Impl { items, self_ident: self_ident.to_owned(), + args, }) } - fn from_trait(item: syn::ItemTrait, callback_interface: bool) -> syn::Result { + fn from_trait(item: syn::ItemTrait, attr_args: TokenStream) -> syn::Result { + let args: ExportTraitArgs = syn::parse(attr_args)?; + let with_foreign = args.callback_interface.is_some() || args.with_foreign.is_some(); + let callback_interface_only = args.callback_interface.is_some(); + if !item.generics.params.is_empty() || item.generics.where_clause.is_some() { return Err(syn::Error::new_spanned( &item.generics, @@ -117,6 +143,7 @@ impl ExportItem { } let self_ident = item.ident.to_owned(); + let docstring = extract_docstring(&item.attrs)?; let items = item .items .into_iter() @@ -132,6 +159,7 @@ impl ExportItem { } }; + let docstring = extract_docstring(&tim.attrs)?; let attrs = ExportedImplFnAttributes::new(&tim.attrs)?; let item = if attrs.constructor { return Err(syn::Error::new_spanned( @@ -142,7 +170,9 @@ impl ExportItem { ImplItem::Method(FnSignature::new_trait_method( self_ident.clone(), tim.sig, + ExportedImplFnArgs::default(), i as u32, + docstring, )?) }; @@ -153,28 +183,26 @@ impl ExportItem { Ok(Self::Trait { items, self_ident, - callback_interface, + with_foreign, + callback_interface_only, + docstring, + args, }) } - fn from_struct(item: syn::ItemStruct, args: &ExportAttributeArguments) -> syn::Result { - let mut uniffi_traits = Vec::new(); - if args.trait_debug.is_some() { - uniffi_traits.push(UniffiTraitDiscriminants::Debug); - } - if args.trait_display.is_some() { - uniffi_traits.push(UniffiTraitDiscriminants::Display); - } - if args.trait_hash.is_some() { - uniffi_traits.push(UniffiTraitDiscriminants::Hash); - } - if args.trait_eq.is_some() { - uniffi_traits.push(UniffiTraitDiscriminants::Eq); + fn from_struct(item: syn::ItemStruct, attr_args: TokenStream) -> syn::Result { + let args: ExportStructArgs = syn::parse(attr_args)?; + let uniffi_traits: Vec = args.traits.into_iter().collect(); + if uniffi_traits.is_empty() { + Err(syn::Error::new(Span::call_site(), + "uniffi::export on a struct must supply a builtin trait name. Did you mean `#[derive(uniffi::Object)]`?" + )) + } else { + Ok(Self::Struct { + self_ident: item.ident, + uniffi_traits, + }) } - Ok(Self::Struct { - self_ident: item.ident, - uniffi_traits, - }) } } diff --git a/third_party/rust/uniffi_macros/src/export/scaffolding.rs b/third_party/rust/uniffi_macros/src/export/scaffolding.rs index f120ccc880..fa7b61deca 100644 --- a/third_party/rust/uniffi_macros/src/export/scaffolding.rs +++ b/third_party/rust/uniffi_macros/src/export/scaffolding.rs @@ -6,12 +6,12 @@ use proc_macro2::{Ident, TokenStream}; use quote::quote; use std::iter; -use super::attributes::{AsyncRuntime, ExportAttributeArguments}; -use crate::fnsig::{FnKind, FnSignature, NamedArg}; +use super::attributes::AsyncRuntime; +use crate::fnsig::{FnKind, FnSignature}; pub(super) fn gen_fn_scaffolding( sig: FnSignature, - arguments: &ExportAttributeArguments, + ar: &Option, udl_mode: bool, ) -> syn::Result { if sig.receiver.is_some() { @@ -21,7 +21,7 @@ pub(super) fn gen_fn_scaffolding( )); } if !sig.is_async { - if let Some(async_runtime) = &arguments.async_runtime { + if let Some(async_runtime) = ar { return Err(syn::Error::new_spanned( async_runtime, "this attribute is only allowed on async functions", @@ -32,7 +32,7 @@ pub(super) fn gen_fn_scaffolding( sig.metadata_items() .unwrap_or_else(syn::Error::into_compile_error) }); - let scaffolding_func = gen_ffi_function(&sig, arguments, udl_mode)?; + let scaffolding_func = gen_ffi_function(&sig, ar, udl_mode)?; Ok(quote! { #scaffolding_func #metadata_items @@ -41,7 +41,7 @@ pub(super) fn gen_fn_scaffolding( pub(super) fn gen_constructor_scaffolding( sig: FnSignature, - arguments: &ExportAttributeArguments, + ar: &Option, udl_mode: bool, ) -> syn::Result { if sig.receiver.is_some() { @@ -50,14 +50,11 @@ pub(super) fn gen_constructor_scaffolding( "constructors must not have a self parameter", )); } - if sig.is_async { - return Err(syn::Error::new(sig.span, "constructors can't be async")); - } let metadata_items = (!udl_mode).then(|| { sig.metadata_items() .unwrap_or_else(syn::Error::into_compile_error) }); - let scaffolding_func = gen_ffi_function(&sig, arguments, udl_mode)?; + let scaffolding_func = gen_ffi_function(&sig, ar, udl_mode)?; Ok(quote! { #scaffolding_func #metadata_items @@ -66,7 +63,7 @@ pub(super) fn gen_constructor_scaffolding( pub(super) fn gen_method_scaffolding( sig: FnSignature, - arguments: &ExportAttributeArguments, + ar: &Option, udl_mode: bool, ) -> syn::Result { let scaffolding_func = if sig.receiver.is_none() { @@ -75,7 +72,7 @@ pub(super) fn gen_method_scaffolding( "associated functions are not currently supported", )); } else { - gen_ffi_function(&sig, arguments, udl_mode)? + gen_ffi_function(&sig, ar, udl_mode)? }; let metadata_items = (!udl_mode).then(|| { @@ -90,31 +87,37 @@ pub(super) fn gen_method_scaffolding( // Pieces of code for the scaffolding function struct ScaffoldingBits { - /// Parameters for the scaffolding function - params: Vec, + /// Parameter names for the scaffolding function + param_names: Vec, + /// Parameter types for the scaffolding function + param_types: Vec, /// Lift closure. See `FnSignature::lift_closure` for an explanation of this. lift_closure: TokenStream, /// Expression to call the Rust function after a successful lift. rust_fn_call: TokenStream, + /// Convert the result of `rust_fn_call`, stored in a variable named `uniffi_result` into its final value. + /// This is used to do things like error conversion / Arc wrapping + convert_result: TokenStream, } impl ScaffoldingBits { fn new_for_function(sig: &FnSignature, udl_mode: bool) -> Self { let ident = &sig.ident; - let params: Vec<_> = sig.args.iter().map(NamedArg::scaffolding_param).collect(); let call_params = sig.rust_call_params(false); let rust_fn_call = quote! { #ident(#call_params) }; // UDL mode adds an extra conversion (#1749) - let rust_fn_call = if udl_mode && sig.looks_like_result { - quote! { #rust_fn_call.map_err(::std::convert::Into::into) } + let convert_result = if udl_mode && sig.looks_like_result { + quote! { uniffi_result.map_err(::std::convert::Into::into) } } else { - rust_fn_call + quote! { uniffi_result } }; Self { - params, + param_names: sig.scaffolding_param_names().collect(), + param_types: sig.scaffolding_param_types().collect(), lift_closure: sig.lift_closure(None), rust_fn_call, + convert_result, } } @@ -125,20 +128,32 @@ impl ScaffoldingBits { udl_mode: bool, ) -> Self { let ident = &sig.ident; - let ffi_converter = if is_trait { + let lift_impl = if is_trait { quote! { - <::std::sync::Arc as ::uniffi::FfiConverter> + <::std::sync::Arc as ::uniffi::Lift> } } else { quote! { - <::std::sync::Arc<#self_ident> as ::uniffi::FfiConverter> + <::std::sync::Arc<#self_ident> as ::uniffi::Lift> } }; - let params: Vec<_> = iter::once(quote! { uniffi_self_lowered: #ffi_converter::FfiType }) - .chain(sig.scaffolding_params()) - .collect(); + let try_lift_self = if is_trait { + // For trait interfaces we need to special case this. Trait interfaces normally lift + // foreign trait impl pointers. However, for a method call, we want to lift a Rust + // pointer. + quote! { + { + let boxed_foreign_arc = unsafe { Box::from_raw(uniffi_self_lowered as *mut ::std::sync::Arc) }; + // Take a clone for our own use. + Ok(*boxed_foreign_arc) + } + } + } else { + quote! { #lift_impl::try_lift(uniffi_self_lowered) } + }; + let lift_closure = sig.lift_closure(Some(quote! { - match #ffi_converter::try_lift(uniffi_self_lowered) { + match #try_lift_self { Ok(v) => v, Err(e) => return Err(("self", e)) } @@ -146,38 +161,45 @@ impl ScaffoldingBits { let call_params = sig.rust_call_params(true); let rust_fn_call = quote! { uniffi_args.0.#ident(#call_params) }; // UDL mode adds an extra conversion (#1749) - let rust_fn_call = if udl_mode && sig.looks_like_result { - quote! { #rust_fn_call.map_err(::std::convert::Into::into) } + let convert_result = if udl_mode && sig.looks_like_result { + quote! { uniffi_result .map_err(::std::convert::Into::into) } } else { - rust_fn_call + quote! { uniffi_result } }; Self { - params, + param_names: iter::once(quote! { uniffi_self_lowered }) + .chain(sig.scaffolding_param_names()) + .collect(), + param_types: iter::once(quote! { #lift_impl::FfiType }) + .chain(sig.scaffolding_param_types()) + .collect(), lift_closure, rust_fn_call, + convert_result, } } fn new_for_constructor(sig: &FnSignature, self_ident: &Ident, udl_mode: bool) -> Self { let ident = &sig.ident; - let params: Vec<_> = sig.args.iter().map(NamedArg::scaffolding_param).collect(); let call_params = sig.rust_call_params(false); let rust_fn_call = quote! { #self_ident::#ident(#call_params) }; // UDL mode adds extra conversions (#1749) - let rust_fn_call = match (udl_mode, sig.looks_like_result) { + let convert_result = match (udl_mode, sig.looks_like_result) { // For UDL - (true, false) => quote! { ::std::sync::Arc::new(#rust_fn_call) }, + (true, false) => quote! { ::std::sync::Arc::new(uniffi_result) }, (true, true) => { - quote! { #rust_fn_call.map(::std::sync::Arc::new).map_err(::std::convert::Into::into) } + quote! { uniffi_result.map(::std::sync::Arc::new).map_err(::std::convert::Into::into) } } - (false, _) => rust_fn_call, + (false, _) => quote! { uniffi_result }, }; Self { - params, + param_names: sig.scaffolding_param_names().collect(), + param_types: sig.scaffolding_param_types().collect(), lift_closure: sig.lift_closure(None), rust_fn_call, + convert_result, } } } @@ -188,13 +210,15 @@ impl ScaffoldingBits { /// `rust_fn` is the Rust function to call. pub(super) fn gen_ffi_function( sig: &FnSignature, - arguments: &ExportAttributeArguments, + ar: &Option, udl_mode: bool, ) -> syn::Result { let ScaffoldingBits { - params, + param_names, + param_types, lift_closure, rust_fn_call, + convert_result, } = match &sig.kind { FnKind::Function => ScaffoldingBits::new_for_function(sig, udl_mode), FnKind::Method { self_ident } => { @@ -216,14 +240,15 @@ pub(super) fn gen_ffi_function( let ffi_ident = sig.scaffolding_fn_ident()?; let name = &sig.name; - let return_impl = &sig.return_impl(); + let return_ty = &sig.return_ty; + let return_impl = &sig.lower_return_impl(); Ok(if !sig.is_async { quote! { #[doc(hidden)] #[no_mangle] #vis extern "C" fn #ffi_ident( - #(#params,)* + #(#param_names: #param_types,)* call_status: &mut ::uniffi::RustCallStatus, ) -> #return_impl::ReturnType { ::uniffi::deps::log::debug!(#name); @@ -231,7 +256,10 @@ pub(super) fn gen_ffi_function( ::uniffi::rust_call(call_status, || { #return_impl::lower_return( match uniffi_lift_args() { - Ok(uniffi_args) => #rust_fn_call, + Ok(uniffi_args) => { + let uniffi_result = #rust_fn_call; + #convert_result + } Err((arg_name, anyhow_error)) => { #return_impl::handle_failed_lift(arg_name, anyhow_error) }, @@ -242,25 +270,28 @@ pub(super) fn gen_ffi_function( } } else { let mut future_expr = rust_fn_call; - if matches!(arguments.async_runtime, Some(AsyncRuntime::Tokio(_))) { + if matches!(ar, Some(AsyncRuntime::Tokio(_))) { future_expr = quote! { ::uniffi::deps::async_compat::Compat::new(#future_expr) } } quote! { #[doc(hidden)] #[no_mangle] - pub extern "C" fn #ffi_ident(#(#params,)*) -> ::uniffi::RustFutureHandle { + pub extern "C" fn #ffi_ident(#(#param_names: #param_types,)*) -> ::uniffi::Handle { ::uniffi::deps::log::debug!(#name); let uniffi_lift_args = #lift_closure; match uniffi_lift_args() { Ok(uniffi_args) => { - ::uniffi::rust_future_new( - async move { #future_expr.await }, + ::uniffi::rust_future_new::<_, #return_ty, _>( + async move { + let uniffi_result = #future_expr.await; + #convert_result + }, crate::UniFfiTag ) }, Err((arg_name, anyhow_error)) => { - ::uniffi::rust_future_new( + ::uniffi::rust_future_new::<_, #return_ty, _>( async move { #return_impl::handle_failed_lift(arg_name, anyhow_error) }, diff --git a/third_party/rust/uniffi_macros/src/export/trait_interface.rs b/third_party/rust/uniffi_macros/src/export/trait_interface.rs new file mode 100644 index 0000000000..83587ae386 --- /dev/null +++ b/third_party/rust/uniffi_macros/src/export/trait_interface.rs @@ -0,0 +1,183 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use proc_macro2::{Ident, Span, TokenStream}; +use quote::{quote, quote_spanned}; + +use uniffi_meta::ObjectImpl; + +use crate::{ + export::{ + attributes::ExportTraitArgs, callback_interface, gen_method_scaffolding, item::ImplItem, + }, + object::interface_meta_static_var, + util::{ident_to_string, tagged_impl_header}, +}; + +pub(super) fn gen_trait_scaffolding( + mod_path: &str, + args: ExportTraitArgs, + self_ident: Ident, + items: Vec, + udl_mode: bool, + with_foreign: bool, + docstring: String, +) -> syn::Result { + if let Some(rt) = args.async_runtime { + return Err(syn::Error::new_spanned(rt, "not supported for traits")); + } + let trait_name = ident_to_string(&self_ident); + let trait_impl = with_foreign.then(|| { + callback_interface::trait_impl(mod_path, &self_ident, &items) + .unwrap_or_else(|e| e.into_compile_error()) + }); + + let clone_fn_ident = Ident::new( + &uniffi_meta::clone_fn_symbol_name(mod_path, &trait_name), + Span::call_site(), + ); + let free_fn_ident = Ident::new( + &uniffi_meta::free_fn_symbol_name(mod_path, &trait_name), + Span::call_site(), + ); + + let helper_fn_tokens = quote! { + #[doc(hidden)] + #[no_mangle] + /// Clone a pointer to this object type + /// + /// Safety: Only pass pointers returned by a UniFFI call. Do not pass pointers that were + /// passed to the free function. + pub unsafe extern "C" fn #clone_fn_ident( + ptr: *const ::std::ffi::c_void, + call_status: &mut ::uniffi::RustCallStatus + ) -> *const ::std::ffi::c_void { + uniffi::rust_call(call_status, || { + let ptr = ptr as *mut std::sync::Arc; + let arc = unsafe { ::std::sync::Arc::clone(&*ptr) }; + Ok(::std::boxed::Box::into_raw(::std::boxed::Box::new(arc)) as *const ::std::ffi::c_void) + }) + } + + #[doc(hidden)] + #[no_mangle] + /// Free a pointer to this object type + /// + /// Safety: Only pass pointers returned by a UniFFI call. Do not pass pointers that were + /// passed to the free function. + /// + /// Note: clippy doesn't complain about this being unsafe, but it definitely is since it + /// calls `Box::from_raw`. + pub unsafe extern "C" fn #free_fn_ident( + ptr: *const ::std::ffi::c_void, + call_status: &mut ::uniffi::RustCallStatus + ) { + uniffi::rust_call(call_status, || { + assert!(!ptr.is_null()); + drop(unsafe { ::std::boxed::Box::from_raw(ptr as *mut std::sync::Arc) }); + Ok(()) + }); + } + }; + + let impl_tokens: TokenStream = items + .into_iter() + .map(|item| match item { + ImplItem::Method(sig) => gen_method_scaffolding(sig, &None, udl_mode), + _ => unreachable!("traits have no constructors"), + }) + .collect::>()?; + + let meta_static_var = (!udl_mode).then(|| { + let imp = if with_foreign { + ObjectImpl::CallbackTrait + } else { + ObjectImpl::Trait + }; + interface_meta_static_var(&self_ident, imp, mod_path, docstring) + .unwrap_or_else(syn::Error::into_compile_error) + }); + let ffi_converter_tokens = ffi_converter(mod_path, &self_ident, udl_mode, with_foreign); + + Ok(quote_spanned! { self_ident.span() => + #meta_static_var + #helper_fn_tokens + #trait_impl + #impl_tokens + #ffi_converter_tokens + }) +} + +pub(crate) fn ffi_converter( + mod_path: &str, + trait_ident: &Ident, + udl_mode: bool, + with_foreign: bool, +) -> TokenStream { + let impl_spec = tagged_impl_header("FfiConverterArc", "e! { dyn #trait_ident }, udl_mode); + let lift_ref_impl_spec = tagged_impl_header("LiftRef", "e! { dyn #trait_ident }, udl_mode); + let trait_name = ident_to_string(trait_ident); + let try_lift = if with_foreign { + let trait_impl_ident = callback_interface::trait_impl_ident(&trait_name); + quote! { + fn try_lift(v: Self::FfiType) -> ::uniffi::deps::anyhow::Result<::std::sync::Arc> { + Ok(::std::sync::Arc::new(<#trait_impl_ident>::new(v as u64))) + } + } + } else { + quote! { + fn try_lift(v: Self::FfiType) -> ::uniffi::deps::anyhow::Result<::std::sync::Arc> { + unsafe { + Ok(*::std::boxed::Box::from_raw(v as *mut ::std::sync::Arc)) + } + } + } + }; + let metadata_code = if with_foreign { + quote! { ::uniffi::metadata::codes::TYPE_CALLBACK_TRAIT_INTERFACE } + } else { + quote! { ::uniffi::metadata::codes::TYPE_TRAIT_INTERFACE } + }; + + quote! { + // All traits must be `Sync + Send`. The generated scaffolding will fail to compile + // if they are not, but unfortunately it fails with an unactionably obscure error message. + // By asserting the requirement explicitly, we help Rust produce a more scrutable error message + // and thus help the user debug why the requirement isn't being met. + uniffi::deps::static_assertions::assert_impl_all!(dyn #trait_ident: ::core::marker::Sync, ::core::marker::Send); + + unsafe #impl_spec { + type FfiType = *const ::std::os::raw::c_void; + + fn lower(obj: ::std::sync::Arc) -> Self::FfiType { + ::std::boxed::Box::into_raw(::std::boxed::Box::new(obj)) as *const ::std::os::raw::c_void + } + + #try_lift + + fn write(obj: ::std::sync::Arc, buf: &mut Vec) { + ::uniffi::deps::static_assertions::const_assert!(::std::mem::size_of::<*const ::std::ffi::c_void>() <= 8); + ::uniffi::deps::bytes::BufMut::put_u64( + buf, + >::lower(obj) as u64, + ); + } + + fn try_read(buf: &mut &[u8]) -> ::uniffi::Result<::std::sync::Arc> { + ::uniffi::deps::static_assertions::const_assert!(::std::mem::size_of::<*const ::std::ffi::c_void>() <= 8); + ::uniffi::check_remaining(buf, 8)?; + >::try_lift( + ::uniffi::deps::bytes::Buf::get_u64(buf) as Self::FfiType) + } + + const TYPE_ID_META: ::uniffi::MetadataBuffer = ::uniffi::MetadataBuffer::from_code(#metadata_code) + .concat_str(#mod_path) + .concat_str(#trait_name); + } + + unsafe #lift_ref_impl_spec { + type LiftType = ::std::sync::Arc; + } + } +} diff --git a/third_party/rust/uniffi_macros/src/export/utrait.rs b/third_party/rust/uniffi_macros/src/export/utrait.rs index 3db09ea2b7..9007ae2c56 100644 --- a/third_party/rust/uniffi_macros/src/export/utrait.rs +++ b/third_party/rust/uniffi_macros/src/export/utrait.rs @@ -6,8 +6,10 @@ use proc_macro2::{Ident, TokenStream}; use quote::quote; use syn::ext::IdentExt; -use super::{attributes::ExportAttributeArguments, gen_ffi_function}; +use super::gen_ffi_function; +use crate::export::ExportedImplFnArgs; use crate::fnsig::FnSignature; +use crate::util::extract_docstring; use uniffi_meta::UniffiTraitDiscriminants; pub(crate) fn expand_uniffi_trait_export( @@ -157,12 +159,25 @@ fn process_uniffi_trait_method( unreachable!() }; + let docstring = extract_docstring(&item.attrs)?; + let ffi_func = gen_ffi_function( - &FnSignature::new_method(self_ident.clone(), item.sig.clone())?, - &ExportAttributeArguments::default(), + &FnSignature::new_method( + self_ident.clone(), + item.sig.clone(), + ExportedImplFnArgs::default(), + docstring.clone(), + )?, + &None, udl_mode, )?; // metadata for the method, which will be packed inside metadata for the trait. - let method_meta = FnSignature::new_method(self_ident.clone(), item.sig)?.metadata_expr()?; + let method_meta = FnSignature::new_method( + self_ident.clone(), + item.sig, + ExportedImplFnArgs::default(), + docstring, + )? + .metadata_expr()?; Ok((ffi_func, method_meta)) } diff --git a/third_party/rust/uniffi_macros/src/fnsig.rs b/third_party/rust/uniffi_macros/src/fnsig.rs index 69b6529d1e..9c59125207 100644 --- a/third_party/rust/uniffi_macros/src/fnsig.rs +++ b/third_party/rust/uniffi_macros/src/fnsig.rs @@ -2,8 +2,10 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -use crate::util::{ - create_metadata_items, ident_to_string, mod_path, try_metadata_value_from_usize, +use crate::{ + default::{default_value_metadata_calls, DefaultValue}, + export::{DefaultMap, ExportFnArgs, ExportedImplFnArgs}, + util::{create_metadata_items, ident_to_string, mod_path, try_metadata_value_from_usize}, }; use proc_macro2::{Span, TokenStream}; use quote::quote; @@ -13,7 +15,9 @@ pub(crate) struct FnSignature { pub kind: FnKind, pub span: Span, pub mod_path: String, + // The identifier of the Rust function. pub ident: Ident, + // The foreign name for this function, usually == ident. pub name: String, pub is_async: bool, pub receiver: Option, @@ -23,30 +27,71 @@ pub(crate) struct FnSignature { // Only use this in UDL mode. // In general, it's not reliable because it fails for type aliases. pub looks_like_result: bool, + pub docstring: String, } impl FnSignature { - pub(crate) fn new_function(sig: syn::Signature) -> syn::Result { - Self::new(FnKind::Function, sig) + pub(crate) fn new_function( + sig: syn::Signature, + args: ExportFnArgs, + docstring: String, + ) -> syn::Result { + Self::new(FnKind::Function, sig, args.name, args.defaults, docstring) } - pub(crate) fn new_method(self_ident: Ident, sig: syn::Signature) -> syn::Result { - Self::new(FnKind::Method { self_ident }, sig) + pub(crate) fn new_method( + self_ident: Ident, + sig: syn::Signature, + args: ExportedImplFnArgs, + docstring: String, + ) -> syn::Result { + Self::new( + FnKind::Method { self_ident }, + sig, + args.name, + args.defaults, + docstring, + ) } - pub(crate) fn new_constructor(self_ident: Ident, sig: syn::Signature) -> syn::Result { - Self::new(FnKind::Constructor { self_ident }, sig) + pub(crate) fn new_constructor( + self_ident: Ident, + sig: syn::Signature, + args: ExportedImplFnArgs, + docstring: String, + ) -> syn::Result { + Self::new( + FnKind::Constructor { self_ident }, + sig, + args.name, + args.defaults, + docstring, + ) } pub(crate) fn new_trait_method( self_ident: Ident, sig: syn::Signature, + args: ExportedImplFnArgs, index: u32, + docstring: String, ) -> syn::Result { - Self::new(FnKind::TraitMethod { self_ident, index }, sig) + Self::new( + FnKind::TraitMethod { self_ident, index }, + sig, + args.name, + args.defaults, + docstring, + ) } - pub(crate) fn new(kind: FnKind, sig: syn::Signature) -> syn::Result { + pub(crate) fn new( + kind: FnKind, + sig: syn::Signature, + name: Option, + mut defaults: DefaultMap, + docstring: String, + ) -> syn::Result { let span = sig.span(); let ident = sig.ident; let looks_like_result = looks_like_result(&sig.output); @@ -56,14 +101,11 @@ impl FnSignature { }; let is_async = sig.asyncness.is_some(); - if is_async && matches!(kind, FnKind::Constructor { .. }) { - return Err(syn::Error::new( - span, - "Async constructors are not supported", - )); - } - - let mut input_iter = sig.inputs.into_iter().map(Arg::try_from).peekable(); + let mut input_iter = sig + .inputs + .into_iter() + .map(|a| Arg::new(a, &mut defaults)) + .peekable(); let receiver = input_iter .next_if(|a| matches!(a, Ok(a) if a.is_receiver())) @@ -84,29 +126,43 @@ impl FnSignature { }) }) .collect::>>()?; - let mod_path = mod_path()?; + + if let Some(ident) = defaults.idents().first() { + return Err(syn::Error::new( + ident.span(), + format!("Unknown default argument: {}", ident), + )); + } Ok(Self { kind, span, - mod_path, - name: ident_to_string(&ident), + mod_path: mod_path()?, + name: name.unwrap_or_else(|| ident_to_string(&ident)), ident, is_async, receiver, args, return_ty: output, looks_like_result, + docstring, }) } - pub fn return_impl(&self) -> TokenStream { + pub fn lower_return_impl(&self) -> TokenStream { let return_ty = &self.return_ty; quote! { <#return_ty as ::uniffi::LowerReturn> } } + pub fn lift_return_impl(&self) -> TokenStream { + let return_ty = &self.return_ty; + quote! { + <#return_ty as ::uniffi::LiftReturn> + } + } + /// Generate a closure that tries to lift all arguments into a tuple. /// /// The closure moves all scaffolding arguments into itself and returns: @@ -151,14 +207,6 @@ impl FnSignature { quote! { #(#args),* } } - /// Write expressions for each of our arguments - pub fn write_exprs<'a>( - &'a self, - buf_ident: &'a Ident, - ) -> impl Iterator + 'a { - self.args.iter().map(|a| a.write_expr(buf_ident)) - } - /// Parameters expressions for each of our arguments pub fn params(&self) -> impl Iterator + '_ { self.args.iter().map(NamedArg::param) @@ -182,8 +230,18 @@ impl FnSignature { } /// Scaffolding parameters expressions for each of our arguments - pub fn scaffolding_params(&self) -> impl Iterator + '_ { - self.args.iter().map(NamedArg::scaffolding_param) + pub fn scaffolding_param_names(&self) -> impl Iterator + '_ { + self.args.iter().map(|a| { + let ident = &a.ident; + quote! { #ident } + }) + } + + pub fn scaffolding_param_types(&self) -> impl Iterator + '_ { + self.args.iter().map(|a| { + let lift_impl = a.lift_impl(); + quote! { #lift_impl::FfiType } + }) } /// Generate metadata items for this function @@ -193,6 +251,7 @@ impl FnSignature { return_ty, is_async, mod_path, + docstring, .. } = &self; let args_len = try_metadata_value_from_usize( @@ -201,7 +260,11 @@ impl FnSignature { self.args.len(), "UniFFI limits functions to 256 arguments", )?; - let arg_metadata_calls = self.args.iter().map(NamedArg::arg_metadata); + let arg_metadata_calls = self + .args + .iter() + .map(NamedArg::arg_metadata) + .collect::>>()?; match &self.kind { FnKind::Function => Ok(quote! { @@ -212,6 +275,7 @@ impl FnSignature { .concat_value(#args_len) #(#arg_metadata_calls)* .concat(<#return_ty as ::uniffi::LowerReturn>::TYPE_ID_META) + .concat_long_str(#docstring) }), FnKind::Method { self_ident } => { @@ -225,6 +289,7 @@ impl FnSignature { .concat_value(#args_len) #(#arg_metadata_calls)* .concat(<#return_ty as ::uniffi::LowerReturn>::TYPE_ID_META) + .concat_long_str(#docstring) }) } @@ -240,6 +305,7 @@ impl FnSignature { .concat_value(#args_len) #(#arg_metadata_calls)* .concat(<#return_ty as ::uniffi::LowerReturn>::TYPE_ID_META) + .concat_long_str(#docstring) }) } @@ -250,9 +316,11 @@ impl FnSignature { .concat_str(#mod_path) .concat_str(#object_name) .concat_str(#name) + .concat_bool(#is_async) .concat_value(#args_len) #(#arg_metadata_calls)* .concat(<#return_ty as ::uniffi::LowerReturn>::TYPE_ID_META) + .concat_long_str(#docstring) }) } } @@ -300,6 +368,69 @@ impl FnSignature { } } + /// Generate metadata items for callback interfaces + /// + /// Unfortunately, most of this is duplicate code from [Self::metadata_items] and + /// [Self::metadata_expr]. However, one issue with that code is that it needs to assume if the + /// arguments are being lifted vs lowered in order to get TYPE_ID_META. That code uses + /// `::TYPE_ID_META` for arguments and `::TYPE_ID_META` for + /// return types, which works for accidental/historical reasons. + /// + /// The one exception is callback interfaces (#1947), which are handled by this method. + /// + /// TODO: fix the metadata system so that this is not needed. + pub(crate) fn metadata_items_for_callback_interface(&self) -> syn::Result { + let Self { + name, + return_ty, + is_async, + mod_path, + docstring, + .. + } = &self; + match &self.kind { + FnKind::TraitMethod { + self_ident, index, .. + } => { + let object_name = ident_to_string(self_ident); + let args_len = try_metadata_value_from_usize( + // Use param_lifts to calculate this instead of sig.inputs to avoid counting any self + // params + self.args.len(), + "UniFFI limits functions to 256 arguments", + )?; + let arg_metadata_calls = self + .args + .iter() + .map(NamedArg::arg_metadata) + .collect::>>()?; + let metadata_expr = quote! { + ::uniffi::MetadataBuffer::from_code(::uniffi::metadata::codes::TRAIT_METHOD) + .concat_str(#mod_path) + .concat_str(#object_name) + .concat_u32(#index) + .concat_str(#name) + .concat_bool(#is_async) + .concat_value(#args_len) + #(#arg_metadata_calls)* + .concat(<#return_ty as ::uniffi::LiftReturn>::TYPE_ID_META) + .concat_long_str(#docstring) + }; + Ok(create_metadata_items( + "method", + &format!("{object_name}_{name}"), + metadata_expr, + Some(self.checksum_symbol_name()), + )) + } + + // This should never happen and indicates an error in the internal code + _ => panic!( + "metadata_items_for_callback_interface can only be called with `TraitMethod` sigs" + ), + } + } + pub(crate) fn checksum_symbol_name(&self) -> String { let name = &self.name; match &self.kind { @@ -331,19 +462,11 @@ pub(crate) enum ArgKind { } impl Arg { - pub(crate) fn is_receiver(&self) -> bool { - matches!(self.kind, ArgKind::Receiver(_)) - } -} - -impl TryFrom for Arg { - type Error = syn::Error; - - fn try_from(syn_arg: FnArg) -> syn::Result { + fn new(syn_arg: FnArg, defaults: &mut DefaultMap) -> syn::Result { let span = syn_arg.span(); let kind = match syn_arg { FnArg::Typed(p) => match *p.pat { - Pat::Ident(i) => Ok(ArgKind::Named(NamedArg::new(i.ident, &p.ty))), + Pat::Ident(i) => Ok(ArgKind::Named(NamedArg::new(i.ident, &p.ty, defaults)?)), _ => Err(syn::Error::new_spanned(p, "Argument name missing")), }, FnArg::Receiver(receiver) => Ok(ArgKind::Receiver(ReceiverArg::from(receiver))), @@ -351,6 +474,10 @@ impl TryFrom for Arg { Ok(Self { span, kind }) } + + pub(crate) fn is_receiver(&self) -> bool { + matches!(self.kind, ArgKind::Receiver(_)) + } } pub(crate) enum ReceiverArg { @@ -379,27 +506,30 @@ pub(crate) struct NamedArg { pub(crate) name: String, pub(crate) ty: TokenStream, pub(crate) ref_type: Option, + pub(crate) default: Option, } impl NamedArg { - pub(crate) fn new(ident: Ident, ty: &Type) -> Self { - match ty { + pub(crate) fn new(ident: Ident, ty: &Type, defaults: &mut DefaultMap) -> syn::Result { + Ok(match ty { Type::Reference(r) => { let inner = &r.elem; Self { name: ident_to_string(&ident), - ident, ty: quote! { <#inner as ::uniffi::LiftRef>::LiftType }, ref_type: Some(*inner.clone()), + default: defaults.remove(&ident), + ident, } } _ => Self { name: ident_to_string(&ident), - ident, ty: quote! { #ty }, ref_type: None, + default: defaults.remove(&ident), + ident, }, - } + }) } pub(crate) fn lift_impl(&self) -> TokenStream { @@ -419,27 +549,15 @@ impl NamedArg { quote! { #ident: #ty } } - /// Generate the scaffolding parameter for this Arg - pub(crate) fn scaffolding_param(&self) -> TokenStream { - let ident = &self.ident; - let lift_impl = self.lift_impl(); - quote! { #ident: #lift_impl::FfiType } - } - - /// Generate the expression to write the scaffolding parameter for this arg - pub(crate) fn write_expr(&self, buf_ident: &Ident) -> TokenStream { - let ident = &self.ident; - let lower_impl = self.lower_impl(); - quote! { #lower_impl::write(#ident, &mut #buf_ident) } - } - - pub(crate) fn arg_metadata(&self) -> TokenStream { + pub(crate) fn arg_metadata(&self) -> syn::Result { let name = &self.name; let lift_impl = self.lift_impl(); - quote! { + let default_calls = default_value_metadata_calls(&self.default)?; + Ok(quote! { .concat_str(#name) .concat(#lift_impl::TYPE_ID_META) - } + #default_calls + }) } } diff --git a/third_party/rust/uniffi_macros/src/lib.rs b/third_party/rust/uniffi_macros/src/lib.rs index 4cffddfa0e..929400c885 100644 --- a/third_party/rust/uniffi_macros/src/lib.rs +++ b/third_party/rust/uniffi_macros/src/lib.rs @@ -5,10 +5,8 @@ #![warn(rust_2018_idioms, unused_qualifications)] //! Macros for `uniffi`. -//! -//! Currently this is just for easily generating integration tests, but maybe -//! we'll put some other code-annotation helper macros in here at some point. +#[cfg(feature = "trybuild")] use camino::Utf8Path; use proc_macro::TokenStream; use quote::quote; @@ -18,6 +16,7 @@ use syn::{ }; mod custom; +mod default; mod enum_; mod error; mod export; @@ -33,20 +32,6 @@ use self::{ record::expand_record, }; -struct IdentPair { - lhs: Ident, - rhs: Ident, -} - -impl Parse for IdentPair { - fn parse(input: ParseStream<'_>) -> syn::Result { - let lhs = input.parse()?; - input.parse::()?; - let rhs = input.parse()?; - Ok(Self { lhs, rhs }) - } -} - struct CustomTypeInfo { ident: Ident, builtin: Path, @@ -107,9 +92,8 @@ fn do_export(attr_args: TokenStream, input: TokenStream, udl_mode: bool) -> Toke let copied_input = (!udl_mode).then(|| proc_macro2::TokenStream::from(input.clone())); let gen_output = || { - let args = syn::parse(attr_args)?; let item = syn::parse(input)?; - expand_export(item, args, udl_mode) + expand_export(item, attr_args, udl_mode) }; let output = gen_output().unwrap_or_else(syn::Error::into_compile_error); @@ -129,7 +113,7 @@ pub fn derive_record(input: TokenStream) -> TokenStream { #[proc_macro_derive(Enum)] pub fn derive_enum(input: TokenStream) -> TokenStream { - expand_enum(parse_macro_input!(input), false) + expand_enum(parse_macro_input!(input), None, false) .unwrap_or_else(syn::Error::into_compile_error) .into() } @@ -225,10 +209,14 @@ pub fn derive_record_for_udl(_attrs: TokenStream, input: TokenStream) -> TokenSt #[doc(hidden)] #[proc_macro_attribute] -pub fn derive_enum_for_udl(_attrs: TokenStream, input: TokenStream) -> TokenStream { - expand_enum(syn::parse_macro_input!(input), true) - .unwrap_or_else(syn::Error::into_compile_error) - .into() +pub fn derive_enum_for_udl(attrs: TokenStream, input: TokenStream) -> TokenStream { + expand_enum( + syn::parse_macro_input!(input), + Some(syn::parse_macro_input!(attrs)), + true, + ) + .unwrap_or_else(syn::Error::into_compile_error) + .into() } #[doc(hidden)] @@ -257,22 +245,6 @@ pub fn export_for_udl(attrs: TokenStream, input: TokenStream) -> TokenStream { do_export(attrs, input, true) } -/// Generate various support elements, including the FfiConverter implementation, -/// for a trait interface for the scaffolding code -#[doc(hidden)] -#[proc_macro] -pub fn expand_trait_interface_support(tokens: TokenStream) -> TokenStream { - export::ffi_converter_trait_impl(&syn::parse_macro_input!(tokens), true).into() -} - -/// Generate the FfiConverter implementation for an trait interface for the scaffolding code -#[doc(hidden)] -#[proc_macro] -pub fn scaffolding_ffi_converter_callback_interface(tokens: TokenStream) -> TokenStream { - let input: IdentPair = syn::parse_macro_input!(tokens); - export::ffi_converter_callback_interface_impl(&input.lhs, &input.rhs, true).into() -} - /// A helper macro to include generated component scaffolding. /// /// This is a simple convenience macro to include the UniFFI component @@ -373,6 +345,7 @@ pub fn use_udl_object(tokens: TokenStream) -> TokenStream { /// uniffi_macros::generate_and_include_scaffolding!("path/to/my/interface.udl"); /// ``` #[proc_macro] +#[cfg(feature = "trybuild")] pub fn generate_and_include_scaffolding(udl_file: TokenStream) -> TokenStream { let udl_file = syn::parse_macro_input!(udl_file as LitStr); let udl_file_string = udl_file.value(); @@ -396,15 +369,25 @@ pub fn generate_and_include_scaffolding(udl_file: TokenStream) -> TokenStream { }.into() } -/// A dummy macro that does nothing. +/// An attribute for constructors. +/// +/// Constructors are in `impl` blocks which have a `#[uniffi::export]` attribute, /// /// This exists so `#[uniffi::export]` can emit its input verbatim without -/// causing unexpected errors, plus some extra code in case everything is okay. +/// causing unexpected errors in the entire exported block. +/// This happens very often when the proc-macro is run on an incomplete +/// input by rust-analyzer while the developer is typing. /// -/// It is important for `#[uniffi::export]` to not raise unexpected errors if it -/// fails to parse the input as this happens very often when the proc-macro is -/// run on an incomplete input by rust-analyzer while the developer is typing. +/// So much better to do nothing here then let the impl block find the attribute. #[proc_macro_attribute] pub fn constructor(_attrs: TokenStream, input: TokenStream) -> TokenStream { input } + +/// An attribute for methods. +/// +/// Everything above applies here too. +#[proc_macro_attribute] +pub fn method(_attrs: TokenStream, input: TokenStream) -> TokenStream { + input +} diff --git a/third_party/rust/uniffi_macros/src/object.rs b/third_party/rust/uniffi_macros/src/object.rs index 573a1eaadd..6bcc07a14e 100644 --- a/third_party/rust/uniffi_macros/src/object.rs +++ b/third_party/rust/uniffi_macros/src/object.rs @@ -1,17 +1,27 @@ use proc_macro2::{Ident, Span, TokenStream}; use quote::quote; use syn::DeriveInput; -use uniffi_meta::free_fn_symbol_name; -use crate::util::{create_metadata_items, ident_to_string, mod_path, tagged_impl_header}; +use crate::util::{ + create_metadata_items, extract_docstring, ident_to_string, mod_path, tagged_impl_header, +}; +use uniffi_meta::ObjectImpl; pub fn expand_object(input: DeriveInput, udl_mode: bool) -> syn::Result { let module_path = mod_path()?; let ident = &input.ident; + let docstring = extract_docstring(&input.attrs)?; let name = ident_to_string(ident); - let free_fn_ident = Ident::new(&free_fn_symbol_name(&module_path, &name), Span::call_site()); + let clone_fn_ident = Ident::new( + &uniffi_meta::clone_fn_symbol_name(&module_path, &name), + Span::call_site(), + ); + let free_fn_ident = Ident::new( + &uniffi_meta::free_fn_symbol_name(&module_path, &name), + Span::call_site(), + ); let meta_static_var = (!udl_mode).then(|| { - interface_meta_static_var(ident, false, &module_path) + interface_meta_static_var(ident, ObjectImpl::Struct, &module_path, docstring) .unwrap_or_else(syn::Error::into_compile_error) }); let interface_impl = interface_impl(ident, udl_mode); @@ -19,7 +29,19 @@ pub fn expand_object(input: DeriveInput, udl_mode: bool) -> syn::Result *const ::std::ffi::c_void { + uniffi::rust_call(call_status, || { + unsafe { ::std::sync::Arc::increment_strong_count(ptr) }; + Ok(ptr) + }) + } + + #[doc(hidden)] + #[no_mangle] + pub unsafe extern "C" fn #free_fn_ident( ptr: *const ::std::ffi::c_void, call_status: &mut ::uniffi::RustCallStatus ) { @@ -41,6 +63,7 @@ pub fn expand_object(input: DeriveInput, udl_mode: bool) -> syn::Result TokenStream { let name = ident_to_string(ident); let impl_spec = tagged_impl_header("FfiConverterArc", ident, udl_mode); + let lower_return_impl_spec = tagged_impl_header("LowerReturn", ident, udl_mode); let lift_ref_impl_spec = tagged_impl_header("LiftRef", ident, udl_mode); let mod_path = match mod_path() { Ok(p) => p, @@ -52,7 +75,7 @@ pub(crate) fn interface_impl(ident: &Ident, udl_mode: bool) -> TokenStream { // if they are not, but unfortunately it fails with an unactionably obscure error message. // By asserting the requirement explicitly, we help Rust produce a more scrutable error message // and thus help the user debug why the requirement isn't being met. - uniffi::deps::static_assertions::assert_impl_all!(#ident: Sync, Send); + uniffi::deps::static_assertions::assert_impl_all!(#ident: ::core::marker::Sync, ::core::marker::Send); #[doc(hidden)] #[automatically_derived] @@ -78,17 +101,10 @@ pub(crate) fn interface_impl(ident: &Ident, udl_mode: bool) -> TokenStream { ::std::sync::Arc::into_raw(obj) as Self::FfiType } - /// When lifting, we receive a "borrow" of the `Arc` that is owned by - /// the foreign-language code, and make a clone of it for our own use. - /// - /// Safety: the provided value must be a pointer previously obtained by calling - /// the `lower()` or `write()` method of this impl. + /// When lifting, we receive an owned `Arc` that the foreign language code cloned. fn try_lift(v: Self::FfiType) -> ::uniffi::Result<::std::sync::Arc> { let v = v as *const #ident; - // We musn't drop the `Arc` that is owned by the foreign-language code. - let foreign_arc = ::std::mem::ManuallyDrop::new(unsafe { ::std::sync::Arc::::from_raw(v) }); - // Take a clone for our own use. - Ok(::std::sync::Arc::clone(&*foreign_arc)) + Ok(unsafe { ::std::sync::Arc::::from_raw(v) }) } /// When writing as a field of a complex structure, make a clone and transfer ownership @@ -117,8 +133,17 @@ pub(crate) fn interface_impl(ident: &Ident, udl_mode: bool) -> TokenStream { const TYPE_ID_META: ::uniffi::MetadataBuffer = ::uniffi::MetadataBuffer::from_code(::uniffi::metadata::codes::TYPE_INTERFACE) .concat_str(#mod_path) - .concat_str(#name) - .concat_bool(false); + .concat_str(#name); + } + + unsafe #lower_return_impl_spec { + type ReturnType = >::FfiType; + + fn lower_return(obj: Self) -> ::std::result::Result { + Ok(>::lower(::std::sync::Arc::new(obj))) + } + + const TYPE_ID_META: ::uniffi::MetadataBuffer = >::TYPE_ID_META; } unsafe #lift_ref_impl_spec { @@ -129,18 +154,25 @@ pub(crate) fn interface_impl(ident: &Ident, udl_mode: bool) -> TokenStream { pub(crate) fn interface_meta_static_var( ident: &Ident, - is_trait: bool, + imp: ObjectImpl, module_path: &str, + docstring: String, ) -> syn::Result { let name = ident_to_string(ident); + let code = match imp { + ObjectImpl::Struct => quote! { ::uniffi::metadata::codes::INTERFACE }, + ObjectImpl::Trait => quote! { ::uniffi::metadata::codes::TRAIT_INTERFACE }, + ObjectImpl::CallbackTrait => quote! { ::uniffi::metadata::codes::CALLBACK_TRAIT_INTERFACE }, + }; + Ok(create_metadata_items( "interface", &name, quote! { - ::uniffi::MetadataBuffer::from_code(::uniffi::metadata::codes::INTERFACE) - .concat_str(#module_path) - .concat_str(#name) - .concat_bool(#is_trait) + ::uniffi::MetadataBuffer::from_code(#code) + .concat_str(#module_path) + .concat_str(#name) + .concat_long_str(#docstring) }, None, )) diff --git a/third_party/rust/uniffi_macros/src/record.rs b/third_party/rust/uniffi_macros/src/record.rs index abf2743ec6..41f5d016ac 100644 --- a/third_party/rust/uniffi_macros/src/record.rs +++ b/third_party/rust/uniffi_macros/src/record.rs @@ -1,17 +1,20 @@ use proc_macro2::{Ident, Span, TokenStream}; -use quote::{quote, ToTokens}; -use syn::{ - parse::{Parse, ParseStream}, - Data, DataStruct, DeriveInput, Field, Lit, Token, -}; - -use crate::util::{ - create_metadata_items, derive_all_ffi_traits, either_attribute_arg, ident_to_string, kw, - mod_path, tagged_impl_header, try_metadata_value_from_usize, try_read_field, AttributeSliceExt, - UniffiAttributeArgs, +use quote::quote; +use syn::{parse::ParseStream, Data, DataStruct, DeriveInput, Field, Token}; + +use crate::{ + default::{default_value_metadata_calls, DefaultValue}, + util::{ + create_metadata_items, derive_all_ffi_traits, either_attribute_arg, extract_docstring, + ident_to_string, kw, mod_path, tagged_impl_header, try_metadata_value_from_usize, + try_read_field, AttributeSliceExt, UniffiAttributeArgs, + }, }; pub fn expand_record(input: DeriveInput, udl_mode: bool) -> syn::Result { + if let Some(e) = input.attrs.uniffi_attr_args_not_allowed_here() { + return Err(e); + } let record = match input.data { Data::Struct(s) => s, _ => { @@ -23,10 +26,12 @@ pub fn expand_record(input: DeriveInput, udl_mode: bool) -> syn::Result TokenStream { } } -pub enum FieldDefault { - Literal(Lit), - Null(kw::None), -} - -impl ToTokens for FieldDefault { - fn to_tokens(&self, tokens: &mut TokenStream) { - match self { - FieldDefault::Literal(lit) => lit.to_tokens(tokens), - FieldDefault::Null(kw) => kw.to_tokens(tokens), - } - } -} - -impl Parse for FieldDefault { - fn parse(input: ParseStream<'_>) -> syn::Result { - let lookahead = input.lookahead1(); - if lookahead.peek(kw::None) { - let none_kw: kw::None = input.parse()?; - Ok(Self::Null(none_kw)) - } else { - Ok(Self::Literal(input.parse()?)) - } - } -} - #[derive(Default)] pub struct FieldAttributeArguments { - pub(crate) default: Option, + pub(crate) default: Option, } impl UniffiAttributeArgs for FieldAttributeArguments { @@ -128,6 +107,7 @@ impl UniffiAttributeArgs for FieldAttributeArguments { pub(crate) fn record_meta_static_var( ident: &Ident, + docstring: String, record: &DataStruct, ) -> syn::Result { let name = ident_to_string(ident); @@ -144,17 +124,9 @@ pub(crate) fn record_meta_static_var( .parse_uniffi_attr_args::()?; let name = ident_to_string(f.ident.as_ref().unwrap()); + let docstring = extract_docstring(&f.attrs)?; let ty = &f.ty; - let default = match attrs.default { - Some(default) => { - let default_value = default_value_concat_calls(default)?; - quote! { - .concat_bool(true) - #default_value - } - } - None => quote! { .concat_bool(false) }, - }; + let default = default_value_metadata_calls(&attrs.default)?; // Note: fields need to implement both `Lower` and `Lift` to be used in a record. The // TYPE_ID_META should be the same for both traits. @@ -162,6 +134,7 @@ pub(crate) fn record_meta_static_var( .concat_str(#name) .concat(<#ty as ::uniffi::Lower>::TYPE_ID_META) #default + .concat_long_str(#docstring) }) }) .collect::>()?; @@ -175,50 +148,8 @@ pub(crate) fn record_meta_static_var( .concat_str(#name) .concat_value(#fields_len) #concat_fields + .concat_long_str(#docstring) }, None, )) } - -fn default_value_concat_calls(default: FieldDefault) -> syn::Result { - match default { - FieldDefault::Literal(Lit::Int(i)) if !i.suffix().is_empty() => Err( - syn::Error::new_spanned(i, "integer literals with suffix not supported here"), - ), - FieldDefault::Literal(Lit::Float(f)) if !f.suffix().is_empty() => Err( - syn::Error::new_spanned(f, "float literals with suffix not supported here"), - ), - - FieldDefault::Literal(Lit::Str(s)) => Ok(quote! { - .concat_value(::uniffi::metadata::codes::LIT_STR) - .concat_str(#s) - }), - FieldDefault::Literal(Lit::Int(i)) => { - let digits = i.base10_digits(); - Ok(quote! { - .concat_value(::uniffi::metadata::codes::LIT_INT) - .concat_str(#digits) - }) - } - FieldDefault::Literal(Lit::Float(f)) => { - let digits = f.base10_digits(); - Ok(quote! { - .concat_value(::uniffi::metadata::codes::LIT_FLOAT) - .concat_str(#digits) - }) - } - FieldDefault::Literal(Lit::Bool(b)) => Ok(quote! { - .concat_value(::uniffi::metadata::codes::LIT_BOOL) - .concat_bool(#b) - }), - - FieldDefault::Literal(_) => Err(syn::Error::new_spanned( - default, - "this type of literal is not currently supported as a default", - )), - - FieldDefault::Null(_) => Ok(quote! { - .concat_value(::uniffi::metadata::codes::LIT_NULL) - }), - } -} diff --git a/third_party/rust/uniffi_macros/src/setup_scaffolding.rs b/third_party/rust/uniffi_macros/src/setup_scaffolding.rs index afdb119cc4..c82e9389bb 100644 --- a/third_party/rust/uniffi_macros/src/setup_scaffolding.rs +++ b/third_party/rust/uniffi_macros/src/setup_scaffolding.rs @@ -20,15 +20,7 @@ pub fn setup_scaffolding(namespace: String) -> Result { let ffi_rustbuffer_free_ident = format_ident!("ffi_{module_path}_rustbuffer_free"); let ffi_rustbuffer_reserve_ident = format_ident!("ffi_{module_path}_rustbuffer_reserve"); let reexport_hack_ident = format_ident!("{module_path}_uniffi_reexport_hack"); - let ffi_foreign_executor_callback_set_ident = - format_ident!("ffi_{module_path}_foreign_executor_callback_set"); - let ffi_rust_future_continuation_callback_set = - format_ident!("ffi_{module_path}_rust_future_continuation_callback_set"); let ffi_rust_future_scaffolding_fns = rust_future_scaffolding_fns(&module_path); - let continuation_cell = format_ident!( - "RUST_FUTURE_CONTINUATION_CALLBACK_CELL_{}", - module_path.to_uppercase() - ); Ok(quote! { // Unit struct to parameterize the FfiConverter trait. @@ -68,7 +60,7 @@ pub fn setup_scaffolding(namespace: String) -> Result { #[allow(clippy::missing_safety_doc, missing_docs)] #[doc(hidden)] #[no_mangle] - pub extern "C" fn #ffi_rustbuffer_alloc_ident(size: i32, call_status: &mut uniffi::RustCallStatus) -> uniffi::RustBuffer { + pub extern "C" fn #ffi_rustbuffer_alloc_ident(size: u64, call_status: &mut uniffi::RustCallStatus) -> uniffi::RustBuffer { uniffi::ffi::uniffi_rustbuffer_alloc(size, call_status) } @@ -89,31 +81,10 @@ pub fn setup_scaffolding(namespace: String) -> Result { #[allow(clippy::missing_safety_doc, missing_docs)] #[doc(hidden)] #[no_mangle] - pub unsafe extern "C" fn #ffi_rustbuffer_reserve_ident(buf: uniffi::RustBuffer, additional: i32, call_status: &mut uniffi::RustCallStatus) -> uniffi::RustBuffer { + pub unsafe extern "C" fn #ffi_rustbuffer_reserve_ident(buf: uniffi::RustBuffer, additional: u64, call_status: &mut uniffi::RustCallStatus) -> uniffi::RustBuffer { uniffi::ffi::uniffi_rustbuffer_reserve(buf, additional, call_status) } - static #continuation_cell: ::uniffi::deps::once_cell::sync::OnceCell<::uniffi::RustFutureContinuationCallback> = ::uniffi::deps::once_cell::sync::OnceCell::new(); - - #[allow(clippy::missing_safety_doc, missing_docs)] - #[doc(hidden)] - #[no_mangle] - pub extern "C" fn #ffi_foreign_executor_callback_set_ident(callback: uniffi::ffi::ForeignExecutorCallback) { - uniffi::ffi::foreign_executor_callback_set(callback) - } - - #[allow(clippy::missing_safety_doc, missing_docs)] - #[doc(hidden)] - #[no_mangle] - pub unsafe extern "C" fn #ffi_rust_future_continuation_callback_set(callback: ::uniffi::RustFutureContinuationCallback) { - if let Err(existing) = #continuation_cell.set(callback) { - // Don't panic if this to be called multiple times with the same callback. - if existing != callback { - panic!("Attempt to set the RustFuture continuation callback twice"); - } - } - } - #ffi_rust_future_scaffolding_fns // Code to re-export the UniFFI scaffolding functions. @@ -158,12 +129,12 @@ pub fn setup_scaffolding(namespace: String) -> Result { /// Generates the rust_future_* functions /// -/// The foreign side uses a type-erased `RustFutureHandle` to interact with futures, which presents +/// The foreign side uses a type-erased `Handle` to interact with futures, which presents /// a problem when creating scaffolding functions. What is the `ReturnType` parameter of `RustFutureFfi`? /// /// Handle this by using some brute-force monomorphization. For each possible ffi type, we /// generate a set of scaffolding functions. The bindings code is responsible for calling the one -/// corresponds the scaffolding function that created the `RustFutureHandle`. +/// corresponds the scaffolding function that created the `Handle`. /// /// This introduces safety issues, but we do get some type checking. If the bindings code calls /// the wrong rust_future_complete function, they should get an unexpected return type, which @@ -190,41 +161,37 @@ fn rust_future_scaffolding_fns(module_path: &str) -> TokenStream { let ffi_rust_future_cancel = format_ident!("ffi_{module_path}_rust_future_cancel_{fn_suffix}"); let ffi_rust_future_complete = format_ident!("ffi_{module_path}_rust_future_complete_{fn_suffix}"); let ffi_rust_future_free = format_ident!("ffi_{module_path}_rust_future_free_{fn_suffix}"); - let continuation_cell = format_ident!("RUST_FUTURE_CONTINUATION_CALLBACK_CELL_{}", module_path.to_uppercase()); quote! { #[allow(clippy::missing_safety_doc, missing_docs)] #[doc(hidden)] #[no_mangle] - pub unsafe extern "C" fn #ffi_rust_future_poll(handle: ::uniffi::RustFutureHandle, data: *const ()) { - let callback = #continuation_cell - .get() - .expect("RustFuture continuation callback not set. This is likely a uniffi bug."); - ::uniffi::ffi::rust_future_poll::<#return_type>(handle, *callback, data); + pub unsafe extern "C" fn #ffi_rust_future_poll(handle: ::uniffi::Handle, callback: ::uniffi::RustFutureContinuationCallback, data: u64) { + ::uniffi::ffi::rust_future_poll::<#return_type, crate::UniFfiTag>(handle, callback, data); } #[allow(clippy::missing_safety_doc, missing_docs)] #[doc(hidden)] #[no_mangle] - pub unsafe extern "C" fn #ffi_rust_future_cancel(handle: ::uniffi::RustFutureHandle) { - ::uniffi::ffi::rust_future_cancel::<#return_type>(handle) + pub unsafe extern "C" fn #ffi_rust_future_cancel(handle: ::uniffi::Handle) { + ::uniffi::ffi::rust_future_cancel::<#return_type, crate::UniFfiTag>(handle) } #[allow(clippy::missing_safety_doc, missing_docs)] #[doc(hidden)] #[no_mangle] pub unsafe extern "C" fn #ffi_rust_future_complete( - handle: ::uniffi::RustFutureHandle, + handle: ::uniffi::Handle, out_status: &mut ::uniffi::RustCallStatus ) -> #return_type { - ::uniffi::ffi::rust_future_complete::<#return_type>(handle, out_status) + ::uniffi::ffi::rust_future_complete::<#return_type, crate::UniFfiTag>(handle, out_status) } #[allow(clippy::missing_safety_doc, missing_docs)] #[doc(hidden)] #[no_mangle] - pub unsafe extern "C" fn #ffi_rust_future_free(handle: ::uniffi::RustFutureHandle) { - ::uniffi::ffi::rust_future_free::<#return_type>(handle) + pub unsafe extern "C" fn #ffi_rust_future_free(handle: ::uniffi::Handle) { + ::uniffi::ffi::rust_future_free::<#return_type, crate::UniFfiTag>(handle) } } }) diff --git a/third_party/rust/uniffi_macros/src/util.rs b/third_party/rust/uniffi_macros/src/util.rs index 9f213ea1d7..97faad9c4d 100644 --- a/third_party/rust/uniffi_macros/src/util.rs +++ b/third_party/rust/uniffi_macros/src/util.rs @@ -8,7 +8,7 @@ use std::path::{Path as StdPath, PathBuf}; use syn::{ ext::IdentExt, parse::{Parse, ParseStream}, - Attribute, Token, + Attribute, Expr, Lit, Token, }; pub fn manifest_path() -> Result { @@ -79,8 +79,13 @@ pub fn try_read_field(f: &syn::Field) -> TokenStream { let ident = &f.ident; let ty = &f.ty; - quote! { - #ident: <#ty as ::uniffi::Lift>::try_read(buf)?, + match ident { + Some(ident) => quote! { + #ident: <#ty as ::uniffi::Lift>::try_read(buf)?, + }, + None => quote! { + <#ty as ::uniffi::Lift>::try_read(buf)?, + }, } } @@ -151,13 +156,7 @@ pub fn parse_comma_separated(input: ParseStream<'_>) -> } #[derive(Default)] -pub struct ArgumentNotAllowedHere; - -impl Parse for ArgumentNotAllowedHere { - fn parse(input: ParseStream<'_>) -> syn::Result { - parse_comma_separated(input) - } -} +struct ArgumentNotAllowedHere; impl UniffiAttributeArgs for ArgumentNotAllowedHere { fn parse_one(input: ParseStream<'_>) -> syn::Result { @@ -224,7 +223,11 @@ pub(crate) fn derive_all_ffi_traits(ty: &Ident, udl_mode: bool) -> TokenStream { } } -pub(crate) fn derive_ffi_traits(ty: &Ident, udl_mode: bool, trait_names: &[&str]) -> TokenStream { +pub(crate) fn derive_ffi_traits( + ty: impl ToTokens, + udl_mode: bool, + trait_names: &[&str], +) -> TokenStream { let trait_idents = trait_names .iter() .map(|name| Ident::new(name, Span::call_site())); @@ -247,11 +250,14 @@ pub(crate) fn derive_ffi_traits(ty: &Ident, udl_mode: bool, trait_names: &[&str] pub mod kw { syn::custom_keyword!(async_runtime); syn::custom_keyword!(callback_interface); - syn::custom_keyword!(constructor); + syn::custom_keyword!(with_foreign); syn::custom_keyword!(default); syn::custom_keyword!(flat_error); syn::custom_keyword!(None); + syn::custom_keyword!(Some); syn::custom_keyword!(with_try_read); + syn::custom_keyword!(name); + syn::custom_keyword!(non_exhaustive); syn::custom_keyword!(Debug); syn::custom_keyword!(Display); syn::custom_keyword!(Eq); @@ -276,3 +282,20 @@ impl Parse for ExternalTypeItem { }) } } + +pub(crate) fn extract_docstring(attrs: &[Attribute]) -> syn::Result { + return attrs + .iter() + .filter(|attr| attr.path().is_ident("doc")) + .map(|attr| { + let name_value = attr.meta.require_name_value()?; + if let Expr::Lit(expr) = &name_value.value { + if let Lit::Str(lit_str) = &expr.lit { + return Ok(lit_str.value().trim().to_owned()); + } + } + Err(syn::Error::new_spanned(attr, "Cannot parse doc attribute")) + }) + .collect::>>() + .map(|lines| lines.join("\n")); +} diff --git a/third_party/rust/uniffi_meta/.cargo-checksum.json b/third_party/rust/uniffi_meta/.cargo-checksum.json index cb02cde83f..31b45ce807 100644 --- a/third_party/rust/uniffi_meta/.cargo-checksum.json +++ b/third_party/rust/uniffi_meta/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"Cargo.toml":"cb9f8aad563572bd4f12ee234ede6773f189a79ba5bd3bfd7622d3c0ec49d6a3","src/ffi_names.rs":"422bbe9d49d5476de752a9f9b2330f59b37a79e67f19a828caceb64d1bdabff8","src/group.rs":"ae996e6b9f83d459af04eb392e36487d0fe19c7328a395823186cce76a0955ff","src/lib.rs":"a442e2271a0eb538ec1d4fc7573a3acc7e5f366e2b2ac8d0e659fd998fd7d995","src/metadata.rs":"4ae425a8eab7b8c19a6b96c914f2c02c5bee00358888fd55b936fd1fd175a93c","src/reader.rs":"57fb771584491b8e90b01c68f9d53bac7cfa3135888e11e24e14b59312185ff9","src/types.rs":"8c155ed1301e11a365863989e29c2271149048092fb7052ec145f58c948482d5"},"package":"71dc8573a7b1ac4b71643d6da34888273ebfc03440c525121f1b3634ad3417a2"} \ No newline at end of file +{"files":{"Cargo.toml":"5620cf9840477b158641547703ba353e3ad8427ec7b20b9dd5e5f5fe4df7d6d2","README.md":"37c1af00ec81a9f1bc206ab3578356e5f9ad4077dc46dd1bb623d81d804948b8","src/ffi_names.rs":"ca38b700a0a103c9faaf456ed91b67adf46d4e750aee9e9cd01ad97fb1840494","src/group.rs":"d0a43f3c528aba9403649715981ad3a8849d7a370f4ef9e2d618b88f60a3102f","src/lib.rs":"3f00d5214e2785e4b3045bc48899f6f6b1dce32ab3da6be3ebce716ee9d24c5f","src/metadata.rs":"3f236b337a1fd5082ea9cc4fee6800193a903ee88b81f1c3202843402f122a14","src/reader.rs":"579e2b87d8dd9d703b8811294abfb992621c0a46765800e4db2fad2906db2208","src/types.rs":"c2c5188da8cdf5af7f8496d4660bcfaa971b81ed73b64486c05b47256048544f"},"package":"f7224422c4cfd181c7ca9fca2154abca4d21db962f926f270f996edd38b0c4b8"} \ No newline at end of file diff --git a/third_party/rust/uniffi_meta/Cargo.toml b/third_party/rust/uniffi_meta/Cargo.toml index 34999eee18..04d8170011 100644 --- a/third_party/rust/uniffi_meta/Cargo.toml +++ b/third_party/rust/uniffi_meta/Cargo.toml @@ -12,9 +12,10 @@ [package] edition = "2021" name = "uniffi_meta" -version = "0.25.3" +version = "0.27.1" description = "uniffi_meta" homepage = "https://mozilla.github.io/uniffi-rs" +readme = "README.md" keywords = [ "ffi", "bindgen", @@ -32,4 +33,4 @@ version = "1.3" version = "0.3" [dependencies.uniffi_checksum_derive] -version = "0.25.3" +version = "0.27.1" diff --git a/third_party/rust/uniffi_meta/README.md b/third_party/rust/uniffi_meta/README.md new file mode 100644 index 0000000000..64ac3486a3 --- /dev/null +++ b/third_party/rust/uniffi_meta/README.md @@ -0,0 +1,81 @@ +# UniFFI - a multi-language bindings generator for Rust + +UniFFI is a toolkit for building cross-platform software components in Rust. + +For the impatient, see [**the UniFFI user guide**](https://mozilla.github.io/uniffi-rs/) +or [**the UniFFI examples**](https://github.com/mozilla/uniffi-rs/tree/main/examples#example-uniffi-components). + +By writing your core business logic in Rust and describing its interface in an "object model", +you can use UniFFI to help you: + +* Compile your Rust code into a shared library for use on different target platforms. +* Generate bindings to load and use the library from different target languages. + +You can describe your object model in an [interface definition file](https://mozilla.github.io/uniffi-rs/udl_file_spec.html) +or [by using proc-macros](https://mozilla.github.io/uniffi-rs/proc_macro/index.html). + +UniFFI is currently used extensively by Mozilla in Firefox mobile and desktop browsers; +written once in Rust, auto-generated bindings allow that functionality to be called +from both Kotlin (for Android apps) and Swift (for iOS apps). +It also has a growing community of users shipping various cool things to many users. + +UniFFI comes with support for **Kotlin**, **Swift**, **Python** and **Ruby** with 3rd party bindings available for **C#** and **Golang**. +Additional foreign language bindings can be developed externally and we welcome contributions to list them here. +See [Third-party foreign language bindings](#third-party-foreign-language-bindings). + +## User Guide + +You can read more about using the tool in [**the UniFFI user guide**](https://mozilla.github.io/uniffi-rs/). + +We consider it ready for production use, but UniFFI is a long way from a 1.0 release with lots of internal work still going on. +We try hard to avoid breaking simple consumers, but more advanced things might break as you upgrade over time. + +### Etymology and Pronunciation + +ˈjuːnɪfaɪ. Pronounced to rhyme with "unify". + +A portmanteau word that also puns with "unify", to signify the joining of one codebase accessed from many languages. + +uni - [Latin ūni-, from ūnus, one] +FFI - [Abbreviation, Foreign Function Interface] + +## Alternative tools + +Other tools we know of which try and solve a similarly shaped problem are: + +* [Diplomat](https://github.com/rust-diplomat/diplomat/) - see our [writeup of + the different approach taken by that tool](docs/diplomat-and-macros.md) +* [Interoptopus](https://github.com/ralfbiedert/interoptopus/) + +(Please open a PR if you think other tools should be listed!) + +## Third-party foreign language bindings + +* [Kotlin Multiplatform support](https://gitlab.com/trixnity/uniffi-kotlin-multiplatform-bindings). The repository contains Kotlin Multiplatform bindings generation for UniFFI, letting you target both JVM and Native. +* [Go bindings](https://github.com/NordSecurity/uniffi-bindgen-go) +* [C# bindings](https://github.com/NordSecurity/uniffi-bindgen-cs) +* [Dart bindings](https://github.com/NiallBunting/uniffi-rs-dart) + +### External resources + +There are a few third-party resources that make it easier to work with UniFFI: + +* [Plugin support for `.udl` files](https://github.com/Lonami/uniffi-dl) for the IDEA platform ([*uniffi-dl* in the JetBrains marketplace](https://plugins.jetbrains.com/plugin/20527-uniffi-dl)). It provides syntax highlighting, code folding, code completion, reference resolution and navigation (among others features) for the [UniFFI Definition Language (UDL)](https://mozilla.github.io/uniffi-rs/). +* [cargo swift](https://github.com/antoniusnaumann/cargo-swift), a cargo plugin to build a Swift Package from Rust code. It provides an init command for setting up a UniFFI crate and a package command for building a Swift package from Rust code - without the need for additional configuration or build scripts. +* [Cargo NDK Gradle Plugin](https://github.com/willir/cargo-ndk-android-gradle) allows you to build Rust code using [`cargo-ndk`](https://github.com/bbqsrc/cargo-ndk), which generally makes Android library builds less painful. +* [`uniffi-starter`](https://github.com/ianthetechie/uniffi-starter) is a minimal project demonstrates a wide range of UniFFI in a complete project in a compact manner. It includes a full Android library build process, an XCFramework generation script, and example Swift package structure. + +(Please open a PR if you think other resources should be listed!) + +## Contributing + +If this tool sounds interesting to you, please help us develop it! You can: + +* View the [contributor guidelines](./docs/contributing.md). +* File or work on [issues](https://github.com/mozilla/uniffi-rs/issues) here in GitHub. +* Join discussions in the [#uniffi:mozilla.org](https://matrix.to/#/#uniffi:mozilla.org) + room on Matrix. + +## Code of Conduct + +This project is governed by Mozilla's [Community Participation Guidelines](./CODE_OF_CONDUCT.md). diff --git a/third_party/rust/uniffi_meta/src/ffi_names.rs b/third_party/rust/uniffi_meta/src/ffi_names.rs index 44a5bc3e63..5c931a09e3 100644 --- a/third_party/rust/uniffi_meta/src/ffi_names.rs +++ b/third_party/rust/uniffi_meta/src/ffi_names.rs @@ -33,6 +33,12 @@ pub fn method_symbol_name(namespace: &str, object_name: &str, name: &str) -> Str format!("uniffi_{namespace}_fn_method_{object_name}_{name}") } +/// FFI symbol name for the `clone` function for an object. +pub fn clone_fn_symbol_name(namespace: &str, object_name: &str) -> String { + let object_name = object_name.to_ascii_lowercase(); + format!("uniffi_{namespace}_fn_clone_{object_name}") +} + /// FFI symbol name for the `free` function for an object. pub fn free_fn_symbol_name(namespace: &str, object_name: &str) -> String { let object_name = object_name.to_ascii_lowercase(); @@ -40,9 +46,12 @@ pub fn free_fn_symbol_name(namespace: &str, object_name: &str) -> String { } /// FFI symbol name for the `init_callback` function for a callback interface -pub fn init_callback_fn_symbol_name(namespace: &str, callback_interface_name: &str) -> String { +pub fn init_callback_vtable_fn_symbol_name( + namespace: &str, + callback_interface_name: &str, +) -> String { let callback_interface_name = callback_interface_name.to_ascii_lowercase(); - format!("uniffi_{namespace}_fn_init_callback_{callback_interface_name}") + format!("uniffi_{namespace}_fn_init_callback_vtable_{callback_interface_name}") } /// FFI checksum symbol name for a top-level function diff --git a/third_party/rust/uniffi_meta/src/group.rs b/third_party/rust/uniffi_meta/src/group.rs index f0be2e5a98..a41776bf8a 100644 --- a/third_party/rust/uniffi_meta/src/group.rs +++ b/third_party/rust/uniffi_meta/src/group.rs @@ -18,6 +18,7 @@ pub fn create_metadata_groups(items: &[Metadata]) -> MetadataGroupMap { Metadata::Namespace(namespace) => { let group = MetadataGroup { namespace: namespace.clone(), + namespace_docstring: None, items: BTreeSet::new(), }; Some((namespace.crate_name.clone(), group)) @@ -29,6 +30,7 @@ pub fn create_metadata_groups(items: &[Metadata]) -> MetadataGroupMap { }; let group = MetadataGroup { namespace, + namespace_docstring: None, items: BTreeSet::new(), }; Some((udl.module_path.clone(), group)) @@ -63,6 +65,7 @@ pub fn group_metadata(group_map: &mut MetadataGroupMap, items: Vec) -> #[derive(Debug)] pub struct MetadataGroup { pub namespace: NamespaceMetadata, + pub namespace_docstring: Option, pub items: BTreeSet, } @@ -127,12 +130,6 @@ impl<'a> ExternalTypeConverter<'a> { ..meta }), Metadata::Enum(meta) => Metadata::Enum(self.convert_enum(meta)), - Metadata::Error(meta) => Metadata::Error(match meta { - ErrorMetadata::Enum { enum_, is_flat } => ErrorMetadata::Enum { - enum_: self.convert_enum(enum_), - is_flat, - }, - }), _ => item, } } diff --git a/third_party/rust/uniffi_meta/src/lib.rs b/third_party/rust/uniffi_meta/src/lib.rs index e486d84d89..90f7b2d3cd 100644 --- a/third_party/rust/uniffi_meta/src/lib.rs +++ b/third_party/rust/uniffi_meta/src/lib.rs @@ -23,7 +23,7 @@ mod metadata; // `docs/uniffi-versioning.md` for details. // // Once we get to 1.0, then we'll need to update the scheme to something like 100 + major_version -pub const UNIFFI_CONTRACT_VERSION: u32 = 24; +pub const UNIFFI_CONTRACT_VERSION: u32 = 26; /// Similar to std::hash::Hash. /// @@ -143,6 +143,7 @@ pub struct FnMetadata { pub return_type: Option, pub throws: Option, pub checksum: Option, + pub docstring: Option, } impl FnMetadata { @@ -160,9 +161,11 @@ pub struct ConstructorMetadata { pub module_path: String, pub self_name: String, pub name: String, + pub is_async: bool, pub inputs: Vec, pub throws: Option, pub checksum: Option, + pub docstring: Option, } impl ConstructorMetadata { @@ -190,6 +193,7 @@ pub struct MethodMetadata { pub throws: Option, pub takes_self_by_arc: bool, // unused except by rust udl bindgen. pub checksum: Option, + pub docstring: Option, } impl MethodMetadata { @@ -216,6 +220,7 @@ pub struct TraitMethodMetadata { pub throws: Option, pub takes_self_by_arc: bool, // unused except by rust udl bindgen. pub checksum: Option, + pub docstring: Option, } impl TraitMethodMetadata { @@ -266,7 +271,17 @@ pub enum LiteralMetadata { Enum(String, Type), EmptySequence, EmptyMap, - Null, + None, + Some { inner: Box }, +} + +impl LiteralMetadata { + pub fn new_uint(v: u64) -> Self { + LiteralMetadata::UInt(v, Radix::Decimal, Type::UInt64) + } + pub fn new_int(v: i64) -> Self { + LiteralMetadata::Int(v, Radix::Decimal, Type::Int64) + } } // Represent the radix of integer literal values. @@ -283,6 +298,7 @@ pub struct RecordMetadata { pub module_path: String, pub name: String, pub fields: Vec, + pub docstring: Option, } #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] @@ -290,19 +306,26 @@ pub struct FieldMetadata { pub name: String, pub ty: Type, pub default: Option, + pub docstring: Option, } #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] pub struct EnumMetadata { pub module_path: String, pub name: String, + pub forced_flatness: Option, pub variants: Vec, + pub discr_type: Option, + pub non_exhaustive: bool, + pub docstring: Option, } #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] pub struct VariantMetadata { pub name: String, + pub discr: Option, pub fields: Vec, + pub docstring: Option, } #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] @@ -310,15 +333,25 @@ pub struct ObjectMetadata { pub module_path: String, pub name: String, pub imp: types::ObjectImpl, + pub docstring: Option, } #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] pub struct CallbackInterfaceMetadata { pub module_path: String, pub name: String, + pub docstring: Option, } impl ObjectMetadata { + /// FFI symbol name for the `clone` function for this object. + /// + /// This function is used to increment the reference count before lowering an object to pass + /// back to Rust. + pub fn clone_ffi_symbol_name(&self) -> String { + clone_fn_symbol_name(&self.module_path, &self.name) + } + /// FFI symbol name for the `free` function for this object. /// /// This function is used to free the memory used by this object. @@ -368,6 +401,7 @@ impl UniffiTraitMetadata { } #[repr(u8)] +#[derive(Eq, PartialEq, Hash)] pub enum UniffiTraitDiscriminants { Debug, Display, @@ -387,25 +421,6 @@ impl UniffiTraitDiscriminants { } } -#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] -pub enum ErrorMetadata { - Enum { enum_: EnumMetadata, is_flat: bool }, -} - -impl ErrorMetadata { - pub fn name(&self) -> &String { - match self { - Self::Enum { enum_, .. } => &enum_.name, - } - } - - pub fn module_path(&self) -> &String { - match self { - Self::Enum { enum_, .. } => &enum_.module_path, - } - } -} - #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] pub struct CustomTypeMetadata { pub module_path: String, @@ -433,7 +448,6 @@ pub enum Metadata { CallbackInterface(CallbackInterfaceMetadata), Record(RecordMetadata), Enum(EnumMetadata), - Error(ErrorMetadata), Constructor(ConstructorMetadata), Method(MethodMetadata), TraitMethod(TraitMethodMetadata), @@ -458,7 +472,6 @@ impl Metadata { Metadata::Object(meta) => &meta.module_path, Metadata::CallbackInterface(meta) => &meta.module_path, Metadata::TraitMethod(meta) => &meta.module_path, - Metadata::Error(meta) => meta.module_path(), Metadata::CustomType(meta) => &meta.module_path, Metadata::UniffiTrait(meta) => meta.module_path(), } @@ -507,12 +520,6 @@ impl From for Metadata { } } -impl From for Metadata { - fn from(e: ErrorMetadata) -> Self { - Self::Error(e) - } -} - impl From for Metadata { fn from(v: ObjectMetadata) -> Self { Self::Object(v) diff --git a/third_party/rust/uniffi_meta/src/metadata.rs b/third_party/rust/uniffi_meta/src/metadata.rs index 6e490a4866..9cfb77a244 100644 --- a/third_party/rust/uniffi_meta/src/metadata.rs +++ b/third_party/rust/uniffi_meta/src/metadata.rs @@ -7,7 +7,7 @@ // `uniffi_core`. // This is the easy way out of that issue and is a temporary hacky solution. -/// Metadata constants, make sure to keep this in sync with copy in `uniffi_meta::reader` +/// Metadata constants, make sure to keep this in sync with copy in `uniffi_core::metadata` pub mod codes { // Top-level metadata item codes pub const FUNC: u8 = 0; @@ -15,13 +15,14 @@ pub mod codes { pub const RECORD: u8 = 2; pub const ENUM: u8 = 3; pub const INTERFACE: u8 = 4; - pub const ERROR: u8 = 5; pub const NAMESPACE: u8 = 6; pub const CONSTRUCTOR: u8 = 7; pub const UDL_FILE: u8 = 8; pub const CALLBACK_INTERFACE: u8 = 9; pub const TRAIT_METHOD: u8 = 10; pub const UNIFFI_TRAIT: u8 = 11; + pub const TRAIT_INTERFACE: u8 = 12; + pub const CALLBACK_TRAIT_INTERFACE: u8 = 13; //pub const UNKNOWN: u8 = 255; // Type codes @@ -49,8 +50,8 @@ pub mod codes { pub const TYPE_CALLBACK_INTERFACE: u8 = 21; pub const TYPE_CUSTOM: u8 = 22; pub const TYPE_RESULT: u8 = 23; - //pub const TYPE_FUTURE: u8 = 24; - pub const TYPE_FOREIGN_EXECUTOR: u8 = 25; + pub const TYPE_TRAIT_INTERFACE: u8 = 24; + pub const TYPE_CALLBACK_TRAIT_INTERFACE: u8 = 25; pub const TYPE_UNIT: u8 = 255; // Literal codes @@ -58,7 +59,9 @@ pub mod codes { pub const LIT_INT: u8 = 1; pub const LIT_FLOAT: u8 = 2; pub const LIT_BOOL: u8 = 3; - pub const LIT_NULL: u8 = 4; + pub const LIT_NONE: u8 = 4; + pub const LIT_SOME: u8 = 5; + pub const LIT_EMPTY_SEQ: u8 = 6; } // Create a checksum for a MetadataBuffer diff --git a/third_party/rust/uniffi_meta/src/reader.rs b/third_party/rust/uniffi_meta/src/reader.rs index bf6525f2b5..6fec6cb3a3 100644 --- a/third_party/rust/uniffi_meta/src/reader.rs +++ b/third_party/rust/uniffi_meta/src/reader.rs @@ -52,9 +52,10 @@ impl<'a> MetadataReader<'a> { codes::CONSTRUCTOR => self.read_constructor()?.into(), codes::METHOD => self.read_method()?.into(), codes::RECORD => self.read_record()?.into(), - codes::ENUM => self.read_enum(false)?.into(), - codes::ERROR => self.read_error()?.into(), - codes::INTERFACE => self.read_object()?.into(), + codes::ENUM => self.read_enum()?.into(), + codes::INTERFACE => self.read_object(ObjectImpl::Struct)?.into(), + codes::TRAIT_INTERFACE => self.read_object(ObjectImpl::Trait)?.into(), + codes::CALLBACK_TRAIT_INTERFACE => self.read_object(ObjectImpl::CallbackTrait)?.into(), codes::CALLBACK_INTERFACE => self.read_callback_interface()?.into(), codes::TRAIT_METHOD => self.read_trait_method()?.into(), codes::UNIFFI_TRAIT => self.read_uniffi_trait()?.into(), @@ -80,6 +81,17 @@ impl<'a> MetadataReader<'a> { } } + fn read_u16(&mut self) -> Result { + if self.buf.len() >= 2 { + // read the value as little-endian + let value = u16::from_le_bytes([self.buf[0], self.buf[1]]); + self.buf = &self.buf[2..]; + Ok(value) + } else { + bail!("Not enough data left in buffer to read a u16 value"); + } + } + fn read_u32(&mut self) -> Result { if self.buf.len() >= 4 { // read the value as little-endian @@ -105,6 +117,17 @@ impl<'a> MetadataReader<'a> { String::from_utf8(slice.into()).context("Invalid string data") } + fn read_long_string(&mut self) -> Result { + let size = self.read_u16()? as usize; + let slice; + (slice, self.buf) = self.buf.split_at(size); + String::from_utf8(slice.into()).context("Invalid string data") + } + + fn read_optional_long_string(&mut self) -> Result> { + Ok(Some(self.read_long_string()?).filter(|str| !str.is_empty())) + } + fn read_type(&mut self) -> Result { let value = self.read_u8()?; Ok(match value { @@ -122,7 +145,6 @@ impl<'a> MetadataReader<'a> { codes::TYPE_STRING => Type::String, codes::TYPE_DURATION => Type::Duration, codes::TYPE_SYSTEM_TIME => Type::Timestamp, - codes::TYPE_FOREIGN_EXECUTOR => Type::ForeignExecutor, codes::TYPE_RECORD => Type::Record { module_path: self.read_string()?, name: self.read_string()?, @@ -134,7 +156,17 @@ impl<'a> MetadataReader<'a> { codes::TYPE_INTERFACE => Type::Object { module_path: self.read_string()?, name: self.read_string()?, - imp: ObjectImpl::from_is_trait(self.read_bool()?), + imp: ObjectImpl::Struct, + }, + codes::TYPE_TRAIT_INTERFACE => Type::Object { + module_path: self.read_string()?, + name: self.read_string()?, + imp: ObjectImpl::Trait, + }, + codes::TYPE_CALLBACK_TRAIT_INTERFACE => Type::Object { + module_path: self.read_string()?, + name: self.read_string()?, + imp: ObjectImpl::CallbackTrait, }, codes::TYPE_CALLBACK_INTERFACE => Type::CallbackInterface { module_path: self.read_string()?, @@ -198,6 +230,7 @@ impl<'a> MetadataReader<'a> { let is_async = self.read_bool()?; let inputs = self.read_inputs()?; let (return_type, throws) = self.read_return_type()?; + let docstring = self.read_optional_long_string()?; Ok(FnMetadata { module_path, name, @@ -205,6 +238,7 @@ impl<'a> MetadataReader<'a> { inputs, return_type, throws, + docstring, checksum: self.calc_checksum(), }) } @@ -213,8 +247,10 @@ impl<'a> MetadataReader<'a> { let module_path = self.read_string()?; let self_name = self.read_string()?; let name = self.read_string()?; + let is_async = self.read_bool()?; let inputs = self.read_inputs()?; let (return_type, throws) = self.read_return_type()?; + let docstring = self.read_optional_long_string()?; return_type .filter(|t| { @@ -228,10 +264,12 @@ impl<'a> MetadataReader<'a> { Ok(ConstructorMetadata { module_path, self_name, + is_async, name, inputs, throws, checksum: self.calc_checksum(), + docstring, }) } @@ -242,6 +280,7 @@ impl<'a> MetadataReader<'a> { let is_async = self.read_bool()?; let inputs = self.read_inputs()?; let (return_type, throws) = self.read_return_type()?; + let docstring = self.read_optional_long_string()?; Ok(MethodMetadata { module_path, self_name, @@ -252,6 +291,7 @@ impl<'a> MetadataReader<'a> { throws, takes_self_by_arc: false, // not emitted by macros checksum: self.calc_checksum(), + docstring, }) } @@ -260,13 +300,25 @@ impl<'a> MetadataReader<'a> { module_path: self.read_string()?, name: self.read_string()?, fields: self.read_fields()?, + docstring: self.read_optional_long_string()?, }) } - fn read_enum(&mut self, is_flat_error: bool) -> Result { + fn read_enum(&mut self) -> Result { let module_path = self.read_string()?; let name = self.read_string()?; - let variants = if is_flat_error { + let forced_flatness = match self.read_u8()? { + 0 => None, + 1 => Some(false), + 2 => Some(true), + _ => unreachable!("invalid flatness"), + }; + let discr_type = if self.read_bool()? { + Some(self.read_type()?) + } else { + None + }; + let variants = if forced_flatness == Some(true) { self.read_flat_variants()? } else { self.read_variants()? @@ -275,21 +327,20 @@ impl<'a> MetadataReader<'a> { Ok(EnumMetadata { module_path, name, + forced_flatness, + discr_type, variants, + non_exhaustive: self.read_bool()?, + docstring: self.read_optional_long_string()?, }) } - fn read_error(&mut self) -> Result { - let is_flat = self.read_bool()?; - let enum_ = self.read_enum(is_flat)?; - Ok(ErrorMetadata::Enum { enum_, is_flat }) - } - - fn read_object(&mut self) -> Result { + fn read_object(&mut self, imp: ObjectImpl) -> Result { Ok(ObjectMetadata { module_path: self.read_string()?, name: self.read_string()?, - imp: ObjectImpl::from_is_trait(self.read_bool()?), + imp, + docstring: self.read_optional_long_string()?, }) } @@ -322,6 +373,7 @@ impl<'a> MetadataReader<'a> { Ok(CallbackInterfaceMetadata { module_path: self.read_string()?, name: self.read_string()?, + docstring: self.read_optional_long_string()?, }) } @@ -333,6 +385,7 @@ impl<'a> MetadataReader<'a> { let is_async = self.read_bool()?; let inputs = self.read_inputs()?; let (return_type, throws) = self.read_return_type()?; + let docstring = self.read_optional_long_string()?; Ok(TraitMethodMetadata { module_path, trait_name, @@ -344,6 +397,7 @@ impl<'a> MetadataReader<'a> { throws, takes_self_by_arc: false, // not emitted by macros checksum: self.calc_checksum(), + docstring, }) } @@ -353,8 +407,13 @@ impl<'a> MetadataReader<'a> { .map(|_| { let name = self.read_string()?; let ty = self.read_type()?; - let default = self.read_default(&name, &ty)?; - Ok(FieldMetadata { name, ty, default }) + let default = self.read_optional_default(&name, &ty)?; + Ok(FieldMetadata { + name, + ty, + default, + docstring: self.read_optional_long_string()?, + }) }) .collect() } @@ -365,7 +424,9 @@ impl<'a> MetadataReader<'a> { .map(|_| { Ok(VariantMetadata { name: self.read_string()?, + discr: self.read_optional_default("", &Type::UInt64)?, fields: self.read_fields()?, + docstring: self.read_optional_long_string()?, }) }) .collect() @@ -377,7 +438,9 @@ impl<'a> MetadataReader<'a> { .map(|_| { Ok(VariantMetadata { name: self.read_string()?, + discr: None, fields: vec![], + docstring: self.read_optional_long_string()?, }) }) .collect() @@ -387,13 +450,16 @@ impl<'a> MetadataReader<'a> { let len = self.read_u8()?; (0..len) .map(|_| { + let name = self.read_string()?; + let ty = self.read_type()?; + let default = self.read_optional_default(&name, &ty)?; Ok(FnParamMetadata { - name: self.read_string()?, - ty: self.read_type()?, + name, + ty, + default, // not emitted by macros by_ref: false, optional: false, - default: None, }) }) .collect() @@ -405,14 +471,18 @@ impl<'a> MetadataReader<'a> { Some(checksum_metadata(metadata_buf)) } - fn read_default(&mut self, name: &str, ty: &Type) -> Result> { - let has_default = self.read_bool()?; - if !has_default { - return Ok(None); + fn read_optional_default(&mut self, name: &str, ty: &Type) -> Result> { + if self.read_bool()? { + Ok(Some(self.read_default(name, ty)?)) + } else { + Ok(None) } + } + fn read_default(&mut self, name: &str, ty: &Type) -> Result { let literal_kind = self.read_u8()?; - Ok(Some(match literal_kind { + + Ok(match literal_kind { codes::LIT_STR => { ensure!( matches!(ty, Type::String), @@ -422,12 +492,24 @@ impl<'a> MetadataReader<'a> { } codes::LIT_INT => { let base10_digits = self.read_string()?; + // procmacros emit the type for discriminant values based purely on whether the constant + // is positive or negative. + let ty = if !base10_digits.is_empty() + && base10_digits.as_bytes()[0] == b'-' + && ty == &Type::UInt64 + { + &Type::Int64 + } else { + ty + }; macro_rules! parse_int { ($ty:ident, $variant:ident) => { LiteralMetadata::$variant( base10_digits .parse::<$ty>() - .with_context(|| format!("parsing default for field {name}"))? + .with_context(|| { + format!("parsing default for field {name}: {base10_digits}") + })? .into(), Radix::Decimal, ty.to_owned(), @@ -458,8 +540,18 @@ impl<'a> MetadataReader<'a> { } }, codes::LIT_BOOL => LiteralMetadata::Boolean(self.read_bool()?), - codes::LIT_NULL => LiteralMetadata::Null, + codes::LIT_NONE => match ty { + Type::Optional { .. } => LiteralMetadata::None, + _ => bail!("field {name} of type {ty:?} can't have a default value of None"), + }, + codes::LIT_SOME => match ty { + Type::Optional { inner_type, .. } => LiteralMetadata::Some { + inner: Box::new(self.read_default(name, inner_type)?), + }, + _ => bail!("field {name} of type {ty:?} can't have a default value of None"), + }, + codes::LIT_EMPTY_SEQ => LiteralMetadata::EmptySequence, _ => bail!("Unexpected literal kind code: {literal_kind:?}"), - })) + }) } } diff --git a/third_party/rust/uniffi_meta/src/types.rs b/third_party/rust/uniffi_meta/src/types.rs index 24f8a6f2a8..51bf156b50 100644 --- a/third_party/rust/uniffi_meta/src/types.rs +++ b/third_party/rust/uniffi_meta/src/types.rs @@ -21,8 +21,12 @@ use crate::Checksum; #[derive(Debug, Copy, Clone, Eq, PartialEq, Checksum, Ord, PartialOrd)] pub enum ObjectImpl { + // A single Rust type Struct, + // A trait that's can be implemented by Rust types Trait, + // A trait + a callback interface -- can be implemented by both Rust and foreign types. + CallbackTrait, } impl ObjectImpl { @@ -31,27 +35,26 @@ impl ObjectImpl { /// Includes `r#`, traits get a leading `dyn`. If we ever supported associated types, then /// this would also include them. pub fn rust_name_for(&self, name: &str) -> String { - if self == &ObjectImpl::Trait { + if self.is_trait_interface() { format!("dyn r#{name}") } else { format!("r#{name}") } } - // uniffi_meta and procmacro support tend to carry around `is_trait` bools. This makes that - // mildly less painful - pub fn from_is_trait(is_trait: bool) -> Self { - if is_trait { - ObjectImpl::Trait - } else { - ObjectImpl::Struct - } + pub fn is_trait_interface(&self) -> bool { + matches!(self, Self::Trait | Self::CallbackTrait) + } + + pub fn has_callback_interface(&self) -> bool { + matches!(self, Self::CallbackTrait) } } #[derive(Debug, Clone, Copy, Eq, PartialEq, Checksum, Ord, PartialOrd)] pub enum ExternalKind { Interface, + Trait, // Either a record or enum DataClass, } @@ -85,7 +88,6 @@ pub enum Type { // How the object is implemented. imp: ObjectImpl, }, - ForeignExecutor, // Types defined in the component API, each of which has a string name. Record { module_path: String, diff --git a/third_party/rust/uniffi_testing/.cargo-checksum.json b/third_party/rust/uniffi_testing/.cargo-checksum.json index 0af9b557d8..47b58d8bcf 100644 --- a/third_party/rust/uniffi_testing/.cargo-checksum.json +++ b/third_party/rust/uniffi_testing/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"Cargo.toml":"300889e9b31b2f73133d69a8c5f8854c72cefd398b4f656c7fc06f4c46475dcc","README.md":"ec6aba24af9a011ef6647422aa22efabdee519cdee3da1a9f9033b07b7cbdb0d","src/lib.rs":"e19f60aed5a137401203b9054ead27894b98490bab3e2f680a23549c8ee8a13a"},"package":"118448debffcb676ddbe8c5305fb933ab7e0123753e659a71dc4a693f8d9f23c"} \ No newline at end of file +{"files":{"Cargo.toml":"f29ebbc363e01ee31c5a351aa6c19bc99343b4d293d5ea7954bca3f49e77ad54","README.md":"ec6aba24af9a011ef6647422aa22efabdee519cdee3da1a9f9033b07b7cbdb0d","src/lib.rs":"e19f60aed5a137401203b9054ead27894b98490bab3e2f680a23549c8ee8a13a"},"package":"f8ce878d0bdfc288b58797044eaaedf748526c56eef3575380bb4d4b19d69eee"} \ No newline at end of file diff --git a/third_party/rust/uniffi_testing/Cargo.toml b/third_party/rust/uniffi_testing/Cargo.toml index a4f6f0bf54..5dacf0cf31 100644 --- a/third_party/rust/uniffi_testing/Cargo.toml +++ b/third_party/rust/uniffi_testing/Cargo.toml @@ -12,7 +12,7 @@ [package] edition = "2021" name = "uniffi_testing" -version = "0.25.3" +version = "0.27.1" authors = ["Firefox Sync Team "] description = "a multi-language bindings generator for rust (testing helpers)" homepage = "https://mozilla.github.io/uniffi-rs" diff --git a/third_party/rust/uniffi_udl/.cargo-checksum.json b/third_party/rust/uniffi_udl/.cargo-checksum.json index 233a1e7795..88ed1cb4b7 100644 --- a/third_party/rust/uniffi_udl/.cargo-checksum.json +++ b/third_party/rust/uniffi_udl/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"Cargo.toml":"8bb1bcb6089114c92a0d06f884e64769edc493bd26d271eb659d6592d8d2f5fa","src/attributes.rs":"faf1c53c17ff511f587113afdb37de4890087150f32e19c9af4d878daebe7c89","src/collectors.rs":"3d80f169ccb7a64c434a6890948e49a64947062c29487b5bea58721189cb2786","src/converters/callables.rs":"c41d15cf816f5838e74f0c1d7a5806c4f1c1b2d925ca11b2b6658a978b2373fb","src/converters/enum_.rs":"1742441b812949b83ec28c2c2c4684c34fd618195cff47af2934c7ee9c895c59","src/converters/interface.rs":"a3deb1915eb4be80d21116f24bec7163736bdffbad4eb2236fb531386a97c4f8","src/converters/mod.rs":"4035ec70c56917fa7c61cced5f8a8398c99028e13959662acc1f7a48aa71ae3b","src/finder.rs":"c496aca0ac640d4ca5fd57b32131297544e5468c9e792f8b5da1644df6d22d2e","src/lib.rs":"11a5d3c288f5786164471b0870b5ca9190305757461c6a6d20777a96622457f1","src/literal.rs":"30e50d7c1d3f061bb6aaa7ad3a6eb31d75a6ea21159dfe454f7ab9ae4bdb580e","src/resolver.rs":"96212b52c4f4f7637713d0a39f5c4972e6480e4ce98070dcb8647d67c9b8d5e2"},"package":"889edb7109c6078abe0e53e9b4070cf74a6b3468d141bdf5ef1bd4d1dc24a1c3"} \ No newline at end of file +{"files":{"Cargo.toml":"e4553df91528daadbc52244f436c3ae17d9c6a4629d232643a20ed63c1311625","README.md":"37c1af00ec81a9f1bc206ab3578356e5f9ad4077dc46dd1bb623d81d804948b8","src/attributes.rs":"f1cdee01db837920dbd109d60b39dd4b0ece56f83797fe4ace1e0caf97c02483","src/collectors.rs":"00c9cd8e8f7be6949996f069b449f120f0b0694ff2f1ca92b79201721c18c594","src/converters/callables.rs":"1ad26c2782629e98272edc75c38343f58194ebf9626ae451ba84546a60d45a48","src/converters/enum_.rs":"aa0ca7a7a50521a45e296c6f53be5f1981a41872443946f72c2ca8baebe4d69b","src/converters/interface.rs":"6042d4abd5e236ed6158c5f6d4d9b81bb01fbbcb8b42bf8a5b385b978c68f6f8","src/converters/mod.rs":"c34a30e3b7a2e3c092a7074f4bab6f6c34364177c096af59a7dda13e44ffdc3c","src/finder.rs":"3586ffd3772151eabbc3d6062725933c88978e1b5feb64312bbf42cd43af30fa","src/lib.rs":"56c50ce61ba5e7266fe7fc9fa9d0022cdbfbe9801730753bd4ee66fed040221c","src/literal.rs":"4f28ad49a17246b4dc30a677cfff65b345bfac0924856e19f58e7574a74c2c40","src/resolver.rs":"c4ff362055dc4ed489e94a063ec0fd3becb8787ad35ce65cdec437a1aae518a0"},"package":"8c43c9ed40a8d20a5c3eae2d23031092db6b96dc8e571beb449ba9757484cea0"} \ No newline at end of file diff --git a/third_party/rust/uniffi_udl/Cargo.toml b/third_party/rust/uniffi_udl/Cargo.toml index 346cd71c27..fd03f3434a 100644 --- a/third_party/rust/uniffi_udl/Cargo.toml +++ b/third_party/rust/uniffi_udl/Cargo.toml @@ -12,10 +12,11 @@ [package] edition = "2021" name = "uniffi_udl" -version = "0.25.3" +version = "0.27.1" description = "udl parsing for the uniffi project" homepage = "https://mozilla.github.io/uniffi-rs" documentation = "https://mozilla.github.io/uniffi-rs" +readme = "README.md" keywords = [ "ffi", "bindgen", @@ -26,11 +27,14 @@ repository = "https://github.com/mozilla/uniffi-rs" [dependencies.anyhow] version = "1" +[dependencies.textwrap] +version = "0.16" + [dependencies.uniffi_meta] -version = "=0.25.3" +version = "=0.27.1" [dependencies.uniffi_testing] -version = "=0.25.3" +version = "=0.27.1" [dependencies.weedle2] -version = "4.0.0" +version = "5.0.0" diff --git a/third_party/rust/uniffi_udl/README.md b/third_party/rust/uniffi_udl/README.md new file mode 100644 index 0000000000..64ac3486a3 --- /dev/null +++ b/third_party/rust/uniffi_udl/README.md @@ -0,0 +1,81 @@ +# UniFFI - a multi-language bindings generator for Rust + +UniFFI is a toolkit for building cross-platform software components in Rust. + +For the impatient, see [**the UniFFI user guide**](https://mozilla.github.io/uniffi-rs/) +or [**the UniFFI examples**](https://github.com/mozilla/uniffi-rs/tree/main/examples#example-uniffi-components). + +By writing your core business logic in Rust and describing its interface in an "object model", +you can use UniFFI to help you: + +* Compile your Rust code into a shared library for use on different target platforms. +* Generate bindings to load and use the library from different target languages. + +You can describe your object model in an [interface definition file](https://mozilla.github.io/uniffi-rs/udl_file_spec.html) +or [by using proc-macros](https://mozilla.github.io/uniffi-rs/proc_macro/index.html). + +UniFFI is currently used extensively by Mozilla in Firefox mobile and desktop browsers; +written once in Rust, auto-generated bindings allow that functionality to be called +from both Kotlin (for Android apps) and Swift (for iOS apps). +It also has a growing community of users shipping various cool things to many users. + +UniFFI comes with support for **Kotlin**, **Swift**, **Python** and **Ruby** with 3rd party bindings available for **C#** and **Golang**. +Additional foreign language bindings can be developed externally and we welcome contributions to list them here. +See [Third-party foreign language bindings](#third-party-foreign-language-bindings). + +## User Guide + +You can read more about using the tool in [**the UniFFI user guide**](https://mozilla.github.io/uniffi-rs/). + +We consider it ready for production use, but UniFFI is a long way from a 1.0 release with lots of internal work still going on. +We try hard to avoid breaking simple consumers, but more advanced things might break as you upgrade over time. + +### Etymology and Pronunciation + +ˈjuːnɪfaɪ. Pronounced to rhyme with "unify". + +A portmanteau word that also puns with "unify", to signify the joining of one codebase accessed from many languages. + +uni - [Latin ūni-, from ūnus, one] +FFI - [Abbreviation, Foreign Function Interface] + +## Alternative tools + +Other tools we know of which try and solve a similarly shaped problem are: + +* [Diplomat](https://github.com/rust-diplomat/diplomat/) - see our [writeup of + the different approach taken by that tool](docs/diplomat-and-macros.md) +* [Interoptopus](https://github.com/ralfbiedert/interoptopus/) + +(Please open a PR if you think other tools should be listed!) + +## Third-party foreign language bindings + +* [Kotlin Multiplatform support](https://gitlab.com/trixnity/uniffi-kotlin-multiplatform-bindings). The repository contains Kotlin Multiplatform bindings generation for UniFFI, letting you target both JVM and Native. +* [Go bindings](https://github.com/NordSecurity/uniffi-bindgen-go) +* [C# bindings](https://github.com/NordSecurity/uniffi-bindgen-cs) +* [Dart bindings](https://github.com/NiallBunting/uniffi-rs-dart) + +### External resources + +There are a few third-party resources that make it easier to work with UniFFI: + +* [Plugin support for `.udl` files](https://github.com/Lonami/uniffi-dl) for the IDEA platform ([*uniffi-dl* in the JetBrains marketplace](https://plugins.jetbrains.com/plugin/20527-uniffi-dl)). It provides syntax highlighting, code folding, code completion, reference resolution and navigation (among others features) for the [UniFFI Definition Language (UDL)](https://mozilla.github.io/uniffi-rs/). +* [cargo swift](https://github.com/antoniusnaumann/cargo-swift), a cargo plugin to build a Swift Package from Rust code. It provides an init command for setting up a UniFFI crate and a package command for building a Swift package from Rust code - without the need for additional configuration or build scripts. +* [Cargo NDK Gradle Plugin](https://github.com/willir/cargo-ndk-android-gradle) allows you to build Rust code using [`cargo-ndk`](https://github.com/bbqsrc/cargo-ndk), which generally makes Android library builds less painful. +* [`uniffi-starter`](https://github.com/ianthetechie/uniffi-starter) is a minimal project demonstrates a wide range of UniFFI in a complete project in a compact manner. It includes a full Android library build process, an XCFramework generation script, and example Swift package structure. + +(Please open a PR if you think other resources should be listed!) + +## Contributing + +If this tool sounds interesting to you, please help us develop it! You can: + +* View the [contributor guidelines](./docs/contributing.md). +* File or work on [issues](https://github.com/mozilla/uniffi-rs/issues) here in GitHub. +* Join discussions in the [#uniffi:mozilla.org](https://matrix.to/#/#uniffi:mozilla.org) + room on Matrix. + +## Code of Conduct + +This project is governed by Mozilla's [Community Participation Guidelines](./CODE_OF_CONDUCT.md). diff --git a/third_party/rust/uniffi_udl/src/attributes.rs b/third_party/rust/uniffi_udl/src/attributes.rs index f06b4f29c1..7bb05808c4 100644 --- a/third_party/rust/uniffi_udl/src/attributes.rs +++ b/third_party/rust/uniffi_udl/src/attributes.rs @@ -37,10 +37,29 @@ pub(super) enum Attribute { kind: ExternalKind, export: bool, }, + Rust { + kind: RustKind, + }, // Custom type on the scaffolding side Custom, // The interface described is implemented as a trait. Trait, + // Modifies `Trait` to enable foreign implementations (callback interfaces) + WithForeign, + Async, + NonExhaustive, +} + +// A type defined in Rust via procmacros but which should be available +// in UDL. +#[derive(Debug, Copy, Clone, Checksum)] +pub(super) enum RustKind { + Object, + CallbackTrait, + Trait, + Record, + Enum, + CallbackInterface, } impl Attribute { @@ -67,6 +86,9 @@ impl TryFrom<&weedle::attribute::ExtendedAttribute<'_>> for Attribute { "Error" => Ok(Attribute::Error), "Custom" => Ok(Attribute::Custom), "Trait" => Ok(Attribute::Trait), + "WithForeign" => Ok(Attribute::WithForeign), + "Async" => Ok(Attribute::Async), + "NonExhaustive" => Ok(Attribute::NonExhaustive), _ => anyhow::bail!("ExtendedAttributeNoArgs not supported: {:?}", (attr.0).0), }, // Matches assignment-style attributes like ["Throws=Error"] @@ -95,6 +117,19 @@ impl TryFrom<&weedle::attribute::ExtendedAttribute<'_>> for Attribute { kind: ExternalKind::Interface, export: true, }), + "ExternalTrait" => Ok(Attribute::External { + crate_name: name_from_id_or_string(&identity.rhs), + kind: ExternalKind::Trait, + export: false, + }), + "ExternalTraitExport" => Ok(Attribute::External { + crate_name: name_from_id_or_string(&identity.rhs), + kind: ExternalKind::Trait, + export: true, + }), + "Rust" => Ok(Attribute::Rust { + kind: rust_kind_from_id_or_string(&identity.rhs)?, + }), _ => anyhow::bail!( "Attribute identity Identifier not supported: {:?}", identity.lhs_identifier.0 @@ -130,6 +165,26 @@ fn name_from_id_or_string(nm: &weedle::attribute::IdentifierOrString<'_>) -> Str } } +fn rust_kind_from_id_or_string(nm: &weedle::attribute::IdentifierOrString<'_>) -> Result { + Ok(match nm { + weedle::attribute::IdentifierOrString::String(str_lit) => match str_lit.0 { + // support names which match either procmacro or udl + "interface" => RustKind::Object, + "object" => RustKind::Object, + "record" => RustKind::Record, + "dictionary" => RustKind::Record, + "enum" => RustKind::Enum, + "trait" => RustKind::Trait, + "callback" => RustKind::CallbackInterface, + "trait_with_foreign" => RustKind::CallbackTrait, + _ => anyhow::bail!("Unknown `[Rust=]` kind {:?}", str_lit.0), + }, + weedle::attribute::IdentifierOrString::Identifier(_) => { + anyhow::bail!("Expected string attribute value, got identifier") + } + }) +} + /// Parse a weedle `ExtendedAttributeList` into a list of `Attribute`s, /// erroring out on duplicates. fn parse_attributes( @@ -161,7 +216,6 @@ where } /// Attributes that can be attached to an `enum` definition in the UDL. -/// There's only one case here: using `[Error]` to mark an enum as an error class. #[derive(Debug, Clone, Checksum, Default)] pub(super) struct EnumAttributes(Vec); @@ -169,6 +223,12 @@ impl EnumAttributes { pub fn contains_error_attr(&self) -> bool { self.0.iter().any(|attr| attr.is_error()) } + + pub fn contains_non_exhaustive_attr(&self) -> bool { + self.0 + .iter() + .any(|attr| matches!(attr, Attribute::NonExhaustive)) + } } impl TryFrom<&weedle::attribute::ExtendedAttributeList<'_>> for EnumAttributes { @@ -178,6 +238,10 @@ impl TryFrom<&weedle::attribute::ExtendedAttributeList<'_>> for EnumAttributes { ) -> Result { let attrs = parse_attributes(weedle_attributes, |attr| match attr { Attribute::Error => Ok(()), + Attribute::NonExhaustive => Ok(()), + // Allow `[Enum]`, since we may be parsing an attribute list from an interface with the + // `[Enum]` attribute. + Attribute::Enum => Ok(()), _ => bail!(format!("{attr:?} not supported for enums")), })?; Ok(Self(attrs)) @@ -196,8 +260,9 @@ impl> TryFrom> for E /// Represents UDL attributes that might appear on a function. /// -/// This supports the `[Throws=ErrorName]` attribute for functions that -/// can produce an error. +/// This supports: +/// * `[Throws=ErrorName]` attribute for functions that can produce an error. +/// * `[Async] for async functions #[derive(Debug, Clone, Checksum, Default)] pub(super) struct FunctionAttributes(Vec); @@ -210,6 +275,10 @@ impl FunctionAttributes { _ => None, }) } + + pub(super) fn is_async(&self) -> bool { + self.0.iter().any(|attr| matches!(attr, Attribute::Async)) + } } impl FromIterator for FunctionAttributes { @@ -224,7 +293,7 @@ impl TryFrom<&weedle::attribute::ExtendedAttributeList<'_>> for FunctionAttribut weedle_attributes: &weedle::attribute::ExtendedAttributeList<'_>, ) -> Result { let attrs = parse_attributes(weedle_attributes, |attr| match attr { - Attribute::Throws(_) => Ok(()), + Attribute::Throws(_) | Attribute::Async => Ok(()), _ => bail!(format!("{attr:?} not supported for functions")), })?; Ok(Self(attrs)) @@ -294,12 +363,25 @@ impl InterfaceAttributes { self.0.iter().any(|attr| attr.is_error()) } - pub fn object_impl(&self) -> ObjectImpl { - if self.0.iter().any(|attr| matches!(attr, Attribute::Trait)) { - ObjectImpl::Trait - } else { - ObjectImpl::Struct - } + pub fn contains_trait(&self) -> bool { + self.0.iter().any(|attr| matches!(attr, Attribute::Trait)) + } + + pub fn contains_with_foreign(&self) -> bool { + self.0 + .iter() + .any(|attr| matches!(attr, Attribute::WithForeign)) + } + + pub fn object_impl(&self) -> Result { + Ok( + match (self.contains_trait(), self.contains_with_foreign()) { + (true, true) => ObjectImpl::CallbackTrait, + (true, false) => ObjectImpl::Trait, + (false, false) => ObjectImpl::Struct, + (false, true) => bail!("WithForeign can't be specified without Trait"), + }, + ) } pub fn get_traits(&self) -> Vec { self.0 @@ -321,6 +403,7 @@ impl TryFrom<&weedle::attribute::ExtendedAttributeList<'_>> for InterfaceAttribu Attribute::Enum => Ok(()), Attribute::Error => Ok(()), Attribute::Trait => Ok(()), + Attribute::WithForeign => Ok(()), Attribute::Traits(_) => Ok(()), _ => bail!(format!("{attr:?} not supported for interface definition")), })?; @@ -373,6 +456,10 @@ impl ConstructorAttributes { _ => None, }) } + + pub(super) fn is_async(&self) -> bool { + self.0.iter().any(|attr| matches!(attr, Attribute::Async)) + } } impl TryFrom<&weedle::attribute::ExtendedAttributeList<'_>> for ConstructorAttributes { @@ -383,6 +470,7 @@ impl TryFrom<&weedle::attribute::ExtendedAttributeList<'_>> for ConstructorAttri let attrs = parse_attributes(weedle_attributes, |attr| match attr { Attribute::Throws(_) => Ok(()), Attribute::Name(_) => Ok(()), + Attribute::Async => Ok(()), _ => bail!(format!("{attr:?} not supported for constructors")), })?; Ok(Self(attrs)) @@ -406,6 +494,10 @@ impl MethodAttributes { }) } + pub(super) fn is_async(&self) -> bool { + self.0.iter().any(|attr| matches!(attr, Attribute::Async)) + } + pub(super) fn get_self_by_arc(&self) -> bool { self.0 .iter() @@ -425,8 +517,7 @@ impl TryFrom<&weedle::attribute::ExtendedAttributeList<'_>> for MethodAttributes weedle_attributes: &weedle::attribute::ExtendedAttributeList<'_>, ) -> Result { let attrs = parse_attributes(weedle_attributes, |attr| match attr { - Attribute::SelfType(_) => Ok(()), - Attribute::Throws(_) => Ok(()), + Attribute::SelfType(_) | Attribute::Throws(_) | Attribute::Async => Ok(()), _ => bail!(format!("{attr:?} not supported for methods")), })?; Ok(Self(attrs)) @@ -498,10 +589,18 @@ impl TypedefAttributes { }) } + pub(super) fn rust_kind(&self) -> Option { + self.0.iter().find_map(|attr| match attr { + Attribute::Rust { kind, .. } => Some(*kind), + _ => None, + }) + } + pub(super) fn external_tagged(&self) -> Option { // If it was "exported" via a proc-macro the FfiConverter was not tagged. self.0.iter().find_map(|attr| match attr { Attribute::External { export, .. } => Some(!*export), + Attribute::Rust { .. } => Some(false), _ => None, }) } @@ -513,7 +612,7 @@ impl TryFrom<&weedle::attribute::ExtendedAttributeList<'_>> for TypedefAttribute weedle_attributes: &weedle::attribute::ExtendedAttributeList<'_>, ) -> Result { let attrs = parse_attributes(weedle_attributes, |attr| match attr { - Attribute::External { .. } | Attribute::Custom => Ok(()), + Attribute::External { .. } | Attribute::Custom | Attribute::Rust { .. } => Ok(()), _ => bail!(format!("{attr:?} not supported for typedefs")), })?; Ok(Self(attrs)) @@ -641,14 +740,22 @@ mod test { } #[test] - fn test_throws_attribute() { + fn test_function_attributes() { let (_, node) = weedle::attribute::ExtendedAttributeList::parse("[Throws=Error]").unwrap(); let attrs = FunctionAttributes::try_from(&node).unwrap(); assert!(matches!(attrs.get_throws_err(), Some("Error"))); + assert!(!attrs.is_async()); let (_, node) = weedle::attribute::ExtendedAttributeList::parse("[]").unwrap(); let attrs = FunctionAttributes::try_from(&node).unwrap(); assert!(attrs.get_throws_err().is_none()); + assert!(!attrs.is_async()); + + let (_, node) = + weedle::attribute::ExtendedAttributeList::parse("[Throws=Error, Async]").unwrap(); + let attrs = FunctionAttributes::try_from(&node).unwrap(); + assert!(matches!(attrs.get_throws_err(), Some("Error"))); + assert!(attrs.is_async()); } #[test] @@ -673,22 +780,34 @@ mod test { let attrs = MethodAttributes::try_from(&node).unwrap(); assert!(!attrs.get_self_by_arc()); assert!(matches!(attrs.get_throws_err(), Some("Error"))); + assert!(!attrs.is_async()); let (_, node) = weedle::attribute::ExtendedAttributeList::parse("[]").unwrap(); let attrs = MethodAttributes::try_from(&node).unwrap(); assert!(!attrs.get_self_by_arc()); assert!(attrs.get_throws_err().is_none()); + assert!(!attrs.is_async()); let (_, node) = weedle::attribute::ExtendedAttributeList::parse("[Self=ByArc, Throws=Error]").unwrap(); let attrs = MethodAttributes::try_from(&node).unwrap(); assert!(attrs.get_self_by_arc()); assert!(attrs.get_throws_err().is_some()); + assert!(!attrs.is_async()); + + let (_, node) = + weedle::attribute::ExtendedAttributeList::parse("[Self=ByArc, Throws=Error, Async]") + .unwrap(); + let attrs = MethodAttributes::try_from(&node).unwrap(); + assert!(attrs.get_self_by_arc()); + assert!(attrs.get_throws_err().is_some()); + assert!(attrs.is_async()); let (_, node) = weedle::attribute::ExtendedAttributeList::parse("[Self=ByArc]").unwrap(); let attrs = MethodAttributes::try_from(&node).unwrap(); assert!(attrs.get_self_by_arc()); assert!(attrs.get_throws_err().is_none()); + assert!(!attrs.is_async()); } #[test] @@ -710,6 +829,11 @@ mod test { let attrs = ConstructorAttributes::try_from(&node).unwrap(); assert!(matches!(attrs.get_throws_err(), Some("Error"))); assert!(matches!(attrs.get_name(), Some("MyFactory"))); + assert!(!attrs.is_async()); + + let (_, node) = weedle::attribute::ExtendedAttributeList::parse("[Async]").unwrap(); + let attrs = ConstructorAttributes::try_from(&node).unwrap(); + assert!(attrs.is_async()); } #[test] @@ -754,15 +878,24 @@ mod test { fn test_trait_attribute() { let (_, node) = weedle::attribute::ExtendedAttributeList::parse("[Trait]").unwrap(); let attrs = InterfaceAttributes::try_from(&node).unwrap(); - assert_eq!(attrs.object_impl(), ObjectImpl::Trait); + assert_eq!(attrs.object_impl().unwrap(), ObjectImpl::Trait); + + let (_, node) = + weedle::attribute::ExtendedAttributeList::parse("[Trait, WithForeign]").unwrap(); + let attrs = InterfaceAttributes::try_from(&node).unwrap(); + assert_eq!(attrs.object_impl().unwrap(), ObjectImpl::CallbackTrait); let (_, node) = weedle::attribute::ExtendedAttributeList::parse("[]").unwrap(); let attrs = InterfaceAttributes::try_from(&node).unwrap(); - assert_eq!(attrs.object_impl(), ObjectImpl::Struct); + assert_eq!(attrs.object_impl().unwrap(), ObjectImpl::Struct); + + let (_, node) = weedle::attribute::ExtendedAttributeList::parse("[WithForeign]").unwrap(); + let attrs = InterfaceAttributes::try_from(&node).unwrap(); + assert!(attrs.object_impl().is_err()) } #[test] - fn test_enum_attribute() { + fn test_enum_attribute_on_interface() { let (_, node) = weedle::attribute::ExtendedAttributeList::parse("[Enum]").unwrap(); let attrs = InterfaceAttributes::try_from(&node).unwrap(); assert!(matches!(attrs.contains_enum_attr(), true)); @@ -783,6 +916,38 @@ mod test { ); } + // Test parsing attributes for enum definitions + #[test] + fn test_enum_attributes() { + let (_, node) = + weedle::attribute::ExtendedAttributeList::parse("[Error, NonExhaustive]").unwrap(); + let attrs = EnumAttributes::try_from(&node).unwrap(); + assert!(attrs.contains_error_attr()); + assert!(attrs.contains_non_exhaustive_attr()); + + let (_, node) = weedle::attribute::ExtendedAttributeList::parse("[Trait]").unwrap(); + let err = EnumAttributes::try_from(&node).unwrap_err(); + assert_eq!(err.to_string(), "Trait not supported for enums"); + } + + // Test parsing attributes for interface definitions with the `[Enum]` attribute + #[test] + fn test_enum_attributes_from_interface() { + let (_, node) = weedle::attribute::ExtendedAttributeList::parse("[Enum]").unwrap(); + assert!(EnumAttributes::try_from(&node).is_ok()); + + let (_, node) = + weedle::attribute::ExtendedAttributeList::parse("[Enum, Error, NonExhaustive]") + .unwrap(); + let attrs = EnumAttributes::try_from(&node).unwrap(); + assert!(attrs.contains_error_attr()); + assert!(attrs.contains_non_exhaustive_attr()); + + let (_, node) = weedle::attribute::ExtendedAttributeList::parse("[Enum, Trait]").unwrap(); + let err = EnumAttributes::try_from(&node).unwrap_err(); + assert_eq!(err.to_string(), "Trait not supported for enums"); + } + #[test] fn test_other_attributes_not_supported_for_interfaces() { let (_, node) = weedle::attribute::ExtendedAttributeList::parse("[Trait, ByRef]").unwrap(); diff --git a/third_party/rust/uniffi_udl/src/collectors.rs b/third_party/rust/uniffi_udl/src/collectors.rs index 6a91ab4a93..de5489f5f9 100644 --- a/third_party/rust/uniffi_udl/src/collectors.rs +++ b/third_party/rust/uniffi_udl/src/collectors.rs @@ -5,7 +5,7 @@ //! # Collects metadata from UDL. use crate::attributes; -use crate::converters::APIConverter; +use crate::converters::{convert_docstring, APIConverter}; use crate::finder; use crate::resolver::TypeResolver; use anyhow::{bail, Result}; @@ -138,6 +138,7 @@ impl From for uniffi_meta::MetadataGroup { crate_name: value.types.module_path(), name: value.types.namespace, }, + namespace_docstring: value.types.namespace_docstring.clone(), items: value.items, } } @@ -171,15 +172,13 @@ impl APIBuilder for weedle::Definition<'_> { match self { weedle::Definition::Namespace(d) => d.process(ci)?, weedle::Definition::Enum(d) => { + let mut e: uniffi_meta::EnumMetadata = d.convert(ci)?; // We check if the enum represents an error... let attrs = attributes::EnumAttributes::try_from(d.attributes.as_ref())?; if attrs.contains_error_attr() { - let e: uniffi_meta::ErrorMetadata = d.convert(ci)?; - ci.add_definition(e.into())?; - } else { - let e: uniffi_meta::EnumMetadata = d.convert(ci)?; - ci.add_definition(e.into())?; + e.forced_flatness = Some(true); } + ci.add_definition(e.into())?; } weedle::Definition::Dictionary(d) => { let rec = d.convert(ci)?; @@ -187,12 +186,9 @@ impl APIBuilder for weedle::Definition<'_> { } weedle::Definition::Interface(d) => { let attrs = attributes::InterfaceAttributes::try_from(d.attributes.as_ref())?; - if attrs.contains_enum_attr() { + if attrs.contains_enum_attr() || attrs.contains_error_attr() { let e: uniffi_meta::EnumMetadata = d.convert(ci)?; ci.add_definition(e.into())?; - } else if attrs.contains_error_attr() { - let e: uniffi_meta::ErrorMetadata = d.convert(ci)?; - ci.add_definition(e.into())?; } else { let obj: uniffi_meta::ObjectMetadata = d.convert(ci)?; ci.add_definition(obj.into())?; @@ -218,6 +214,7 @@ impl APIBuilder for weedle::NamespaceDefinition<'_> { if self.identifier.0 != ci.types.namespace { bail!("duplicate namespace definition"); } + ci.types.namespace_docstring = self.docstring.as_ref().map(|v| convert_docstring(&v.0)); for func in self.members.body.convert(ci)? { ci.add_definition(func.into())?; } @@ -229,6 +226,7 @@ impl APIBuilder for weedle::NamespaceDefinition<'_> { pub(crate) struct TypeCollector { /// The unique prefix that we'll use for namespacing when exposing this component's API. pub namespace: String, + pub namespace_docstring: Option, pub crate_name: String, diff --git a/third_party/rust/uniffi_udl/src/converters/callables.rs b/third_party/rust/uniffi_udl/src/converters/callables.rs index 3e15bb8e02..dda3c3a3ce 100644 --- a/third_party/rust/uniffi_udl/src/converters/callables.rs +++ b/third_party/rust/uniffi_udl/src/converters/callables.rs @@ -5,6 +5,7 @@ use super::APIConverter; use crate::attributes::ArgumentAttributes; use crate::attributes::{ConstructorAttributes, FunctionAttributes, MethodAttributes}; +use crate::converters::convert_docstring; use crate::literal::convert_default_value; use crate::InterfaceCollector; use anyhow::{bail, Result}; @@ -41,6 +42,7 @@ impl APIConverter for weedle::argument::SingleArgument<'_> { name: self.identifier.0.to_string(), ty: type_, default: None, + docstring: None, }) } } @@ -89,6 +91,7 @@ impl APIConverter for weedle::namespace::OperationNamespaceMember<'_ Some(id) => id.0.to_string(), }; let attrs = FunctionAttributes::try_from(self.attributes.as_ref())?; + let is_async = attrs.is_async(); let throws = match attrs.get_throws_err() { None => None, Some(name) => match ci.get_type(name) { @@ -99,10 +102,11 @@ impl APIConverter for weedle::namespace::OperationNamespaceMember<'_ Ok(FnMetadata { module_path: ci.module_path(), name, - is_async: false, + is_async, return_type, inputs: self.args.body.list.convert(ci)?, throws, + docstring: self.docstring.as_ref().map(|v| convert_docstring(&v.0)), checksum: None, }) } @@ -122,10 +126,12 @@ impl APIConverter for weedle::interface::ConstructorInterfa name: String::from(attributes.get_name().unwrap_or("new")), // We don't know the name of the containing `Object` at this point, fill it in later. self_name: Default::default(), + is_async: attributes.is_async(), // Also fill in checksum_fn_name later, since it depends on object_name inputs: self.args.body.list.convert(ci)?, throws, checksum: None, + docstring: self.docstring.as_ref().map(|v| convert_docstring(&v.0)), }) } } @@ -140,6 +146,7 @@ impl APIConverter for weedle::interface::OperationInterfaceMembe } let return_type = ci.resolve_return_type_expression(&self.return_type)?; let attributes = MethodAttributes::try_from(self.attributes.as_ref())?; + let is_async = attributes.is_async(); let throws = match attributes.get_throws_err() { Some(name) => match ci.get_type(name) { @@ -164,12 +171,13 @@ impl APIConverter for weedle::interface::OperationInterfaceMembe }, // We don't know the name of the containing `Object` at this point, fill it in later. self_name: Default::default(), - is_async: false, // not supported in UDL + is_async, inputs: self.args.body.list.convert(ci)?, return_type, throws, takes_self_by_arc, checksum: None, + docstring: self.docstring.as_ref().map(|v| convert_docstring(&v.0)), }) } } @@ -184,6 +192,7 @@ impl APIConverter for weedle::interface::OperationInterface } let return_type = ci.resolve_return_type_expression(&self.return_type)?; let attributes = MethodAttributes::try_from(self.attributes.as_ref())?; + let is_async = attributes.is_async(); let throws = match attributes.get_throws_err() { Some(name) => match ci.get_type(name) { @@ -208,12 +217,13 @@ impl APIConverter for weedle::interface::OperationInterface name } }, - is_async: false, // not supported in udl + is_async, inputs: self.args.body.list.convert(ci)?, return_type, throws, takes_self_by_arc, checksum: None, + docstring: self.docstring.as_ref().map(|v| convert_docstring(&v.0)), }) } } diff --git a/third_party/rust/uniffi_udl/src/converters/enum_.rs b/third_party/rust/uniffi_udl/src/converters/enum_.rs index a3e68fd23e..1615a1a7ca 100644 --- a/third_party/rust/uniffi_udl/src/converters/enum_.rs +++ b/third_party/rust/uniffi_udl/src/converters/enum_.rs @@ -3,19 +3,22 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ use super::APIConverter; -use crate::InterfaceCollector; +use crate::{attributes::EnumAttributes, converters::convert_docstring, InterfaceCollector}; use anyhow::{bail, Result}; -use uniffi_meta::{EnumMetadata, ErrorMetadata, VariantMetadata}; +use uniffi_meta::{EnumMetadata, VariantMetadata}; -// Note that we have four `APIConverter` impls here - one for the `enum` case, -// one for the `[Error] enum` case, and and one for the `[Enum] interface` case, -// and one for the `[Error] interface` case. +// Note that we have 2 `APIConverter` impls here - one for the `enum` case +// (including an enum with `[Error]`), and one for the `[Error] interface` cas +// (which is still an enum, but with different "flatness" characteristics.) impl APIConverter for weedle::EnumDefinition<'_> { fn convert(&self, ci: &mut InterfaceCollector) -> Result { + let attributes = EnumAttributes::try_from(self.attributes.as_ref())?; Ok(EnumMetadata { module_path: ci.module_path(), name: self.identifier.0.to_string(), + forced_flatness: None, + discr_type: None, variants: self .values .body @@ -23,35 +26,15 @@ impl APIConverter for weedle::EnumDefinition<'_> { .iter() .map::, _>(|v| { Ok(VariantMetadata { - name: v.0.to_string(), + name: v.value.0.to_string(), + discr: None, fields: vec![], + docstring: v.docstring.as_ref().map(|v| convert_docstring(&v.0)), }) }) .collect::>>()?, - }) - } -} - -impl APIConverter for weedle::EnumDefinition<'_> { - fn convert(&self, ci: &mut InterfaceCollector) -> Result { - Ok(ErrorMetadata::Enum { - enum_: EnumMetadata { - module_path: ci.module_path(), - name: self.identifier.0.to_string(), - variants: self - .values - .body - .list - .iter() - .map::, _>(|v| { - Ok(VariantMetadata { - name: v.0.to_string(), - fields: vec![], - }) - }) - .collect::>>()?, - }, - is_flat: true, + non_exhaustive: attributes.contains_non_exhaustive_attr(), + docstring: self.docstring.as_ref().map(|v| convert_docstring(&v.0)), }) } } @@ -61,11 +44,11 @@ impl APIConverter for weedle::InterfaceDefinition<'_> { if self.inheritance.is_some() { bail!("interface inheritance is not supported for enum interfaces"); } - // We don't need to check `self.attributes` here; if calling code has dispatched - // to this impl then we already know there was an `[Enum]` attribute. + let attributes = EnumAttributes::try_from(self.attributes.as_ref())?; Ok(EnumMetadata { module_path: ci.module_path(), name: self.identifier.0.to_string(), + forced_flatness: Some(false), variants: self .members .body @@ -78,41 +61,15 @@ impl APIConverter for weedle::InterfaceDefinition<'_> { ), }) .collect::>>()?, + discr_type: None, + non_exhaustive: attributes.contains_non_exhaustive_attr(), + docstring: self.docstring.as_ref().map(|v| convert_docstring(&v.0)), // Enums declared using the `[Enum] interface` syntax might have variants with fields. //flat: false, }) } } -impl APIConverter for weedle::InterfaceDefinition<'_> { - fn convert(&self, ci: &mut InterfaceCollector) -> Result { - if self.inheritance.is_some() { - bail!("interface inheritance is not supported for enum interfaces"); - } - // We don't need to check `self.attributes` here; callers have already checked them - // to work out which version to dispatch to. - Ok(ErrorMetadata::Enum { - enum_: EnumMetadata { - module_path: ci.module_path(), - name: self.identifier.0.to_string(), - variants: self - .members - .body - .iter() - .map::, _>(|member| match member { - weedle::interface::InterfaceMember::Operation(t) => Ok(t.convert(ci)?), - _ => bail!( - "interface member type {:?} not supported in enum interface", - member - ), - }) - .collect::>>()?, - }, - is_flat: false, - }) - } -} - #[cfg(test)] mod test { use super::*; diff --git a/third_party/rust/uniffi_udl/src/converters/interface.rs b/third_party/rust/uniffi_udl/src/converters/interface.rs index 58e6a9c8a0..ef9bdd9540 100644 --- a/third_party/rust/uniffi_udl/src/converters/interface.rs +++ b/third_party/rust/uniffi_udl/src/converters/interface.rs @@ -4,7 +4,7 @@ use super::APIConverter; use crate::attributes::InterfaceAttributes; -use crate::InterfaceCollector; +use crate::{converters::convert_docstring, InterfaceCollector}; use anyhow::{bail, Result}; use std::collections::HashSet; use uniffi_meta::{ @@ -23,7 +23,7 @@ impl APIConverter for weedle::InterfaceDefinition<'_> { }; let object_name = self.identifier.0; - let object_impl = attributes.object_impl(); + let object_impl = attributes.object_impl()?; // Convert each member into a constructor or method, guarding against duplicate names. // They get added to the ci and aren't carried in ObjectMetadata. let mut member_names = HashSet::new(); @@ -70,6 +70,7 @@ impl APIConverter for weedle::InterfaceDefinition<'_> { throws: None, takes_self_by_arc: false, checksum: None, + docstring: None, }) }; // Trait methods are in the Metadata. @@ -130,6 +131,7 @@ impl APIConverter for weedle::InterfaceDefinition<'_> { module_path: ci.module_path(), name: object_name.to_string(), imp: object_impl, + docstring: self.docstring.as_ref().map(|v| convert_docstring(&v.0)), }) } } diff --git a/third_party/rust/uniffi_udl/src/converters/mod.rs b/third_party/rust/uniffi_udl/src/converters/mod.rs index 7a2d22ac42..195d9cc0b7 100644 --- a/third_party/rust/uniffi_udl/src/converters/mod.rs +++ b/third_party/rust/uniffi_udl/src/converters/mod.rs @@ -29,6 +29,11 @@ pub(crate) trait APIConverter { fn convert(&self, ci: &mut InterfaceCollector) -> Result; } +// Convert UDL docstring into metadata docstring +pub(crate) fn convert_docstring(docstring: &str) -> String { + textwrap::dedent(docstring) +} + /// Convert a list of weedle items into a list of `InterfaceCollector` items, /// by doing a direct item-by-item mapping. impl> APIConverter> for Vec { @@ -72,6 +77,7 @@ impl APIConverter for weedle::interface::OperationInterfaceMemb }; Ok(VariantMetadata { name, + discr: None, fields: self .args .body @@ -79,6 +85,7 @@ impl APIConverter for weedle::interface::OperationInterfaceMemb .iter() .map(|arg| arg.convert(ci)) .collect::>>()?, + docstring: self.docstring.as_ref().map(|v| convert_docstring(&v.0)), }) } } @@ -95,6 +102,7 @@ impl APIConverter for weedle::DictionaryDefinition<'_> { module_path: ci.module_path(), name: self.identifier.0.to_string(), fields: self.members.body.convert(ci)?, + docstring: self.docstring.as_ref().map(|v| convert_docstring(&v.0)), }) } } @@ -113,6 +121,7 @@ impl APIConverter for weedle::dictionary::DictionaryMember<'_> { name: self.identifier.0.to_string(), ty: type_, default, + docstring: self.docstring.as_ref().map(|v| convert_docstring(&v.0)), }) } } @@ -150,6 +159,7 @@ impl APIConverter for weedle::CallbackInterfaceDefini Ok(CallbackInterfaceMetadata { module_path: ci.module_path(), name: object_name.to_string(), + docstring: self.docstring.as_ref().map(|v| convert_docstring(&v.0)), }) } } diff --git a/third_party/rust/uniffi_udl/src/finder.rs b/third_party/rust/uniffi_udl/src/finder.rs index 0c4c187dc0..259557ad07 100644 --- a/third_party/rust/uniffi_udl/src/finder.rs +++ b/third_party/rust/uniffi_udl/src/finder.rs @@ -22,8 +22,8 @@ use std::convert::TryFrom; use anyhow::{bail, Result}; use super::TypeCollector; -use crate::attributes::{InterfaceAttributes, TypedefAttributes}; -use uniffi_meta::Type; +use crate::attributes::{InterfaceAttributes, RustKind, TypedefAttributes}; +use uniffi_meta::{ObjectImpl, Type}; /// Trait to help with an early "type discovery" phase when processing the UDL. /// @@ -75,7 +75,7 @@ impl TypeFinder for weedle::InterfaceDefinition<'_> { Type::Object { name, module_path: types.module_path(), - imp: attrs.object_impl(), + imp: attrs.object_impl()?, }, ) } @@ -111,7 +111,6 @@ impl TypeFinder for weedle::EnumDefinition<'_> { impl TypeFinder for weedle::TypedefDefinition<'_> { fn add_type_definitions_to(&self, types: &mut TypeCollector) -> Result<()> { - let name = self.identifier.0; let attrs = TypedefAttributes::try_from(self.attributes.as_ref())?; // If we wanted simple `typedef`s, it would be as easy as: // > let t = types.resolve_type_expression(&self.type_)?; @@ -122,29 +121,52 @@ impl TypeFinder for weedle::TypedefDefinition<'_> { // `FfiConverter` implementation. let builtin = types.resolve_type_expression(&self.type_)?; types.add_type_definition( - name, + self.identifier.0, Type::Custom { module_path: types.module_path(), - name: name.to_string(), + name: self.identifier.0.to_string(), builtin: builtin.into(), }, ) } else { - let kind = attrs.external_kind().expect("External missing"); - let tagged = attrs.external_tagged().expect("External missing"); + let module_path = types.module_path(); + let name = self.identifier.0.to_string(); + let ty = match attrs.rust_kind() { + Some(RustKind::Object) => Type::Object { + module_path, + name, + imp: ObjectImpl::Struct, + }, + Some(RustKind::Trait) => Type::Object { + module_path, + name, + imp: ObjectImpl::Trait, + }, + Some(RustKind::CallbackTrait) => Type::Object { + module_path, + name, + imp: ObjectImpl::CallbackTrait, + }, + Some(RustKind::Record) => Type::Record { module_path, name }, + Some(RustKind::Enum) => Type::Enum { module_path, name }, + Some(RustKind::CallbackInterface) => Type::CallbackInterface { module_path, name }, + // must be external + None => { + let kind = attrs.external_kind().expect("External missing kind"); + let tagged = attrs.external_tagged().expect("External missing tagged"); + Type::External { + name, + namespace: "".to_string(), // we don't know this yet + module_path: attrs.get_crate_name(), + kind, + tagged, + } + } + }; // A crate which can supply an `FfiConverter`. // We don't reference `self._type`, so ideally we could insist on it being // the literal 'extern' but that's tricky - types.add_type_definition( - name, - Type::External { - name: name.to_string(), - namespace: "".to_string(), // we don't know this yet - module_path: attrs.get_crate_name(), - kind, - tagged, - }, - ) + types.add_type_definition(self.identifier.0, ty) } } } @@ -152,7 +174,7 @@ impl TypeFinder for weedle::TypedefDefinition<'_> { impl TypeFinder for weedle::CallbackInterfaceDefinition<'_> { fn add_type_definitions_to(&self, types: &mut TypeCollector) -> Result<()> { if self.attributes.is_some() { - bail!("no typedef attributes are currently supported"); + bail!("no callback interface attributes are currently supported"); } let name = self.identifier.0.to_string(); types.add_type_definition( diff --git a/third_party/rust/uniffi_udl/src/lib.rs b/third_party/rust/uniffi_udl/src/lib.rs index 5e6e72a7f7..a9dad5d6c5 100644 --- a/third_party/rust/uniffi_udl/src/lib.rs +++ b/third_party/rust/uniffi_udl/src/lib.rs @@ -6,7 +6,7 @@ //! //! This library is dedicated to parsing a string in a webidl syntax, as described by //! weedle and with our own custom take on the attributes etc, pushing the boundaries -//! of that syntax to describe a uniffi `MetatadataGroup`. +//! of that syntax to describe a uniffi `MetadataGroup`. //! //! The output of this module is consumed by uniffi_bindgen to generate stuff. diff --git a/third_party/rust/uniffi_udl/src/literal.rs b/third_party/rust/uniffi_udl/src/literal.rs index 78f2544254..5fbf022644 100644 --- a/third_party/rust/uniffi_udl/src/literal.rs +++ b/third_party/rust/uniffi_udl/src/literal.rs @@ -84,8 +84,10 @@ pub(super) fn convert_default_value( (weedle::literal::DefaultValue::String(s), Type::Enum { .. }) => { Literal::Enum(s.0.to_string(), type_.clone()) } - (weedle::literal::DefaultValue::Null(_), Type::Optional { .. }) => Literal::Null, - (_, Type::Optional { inner_type, .. }) => convert_default_value(default_value, inner_type)?, + (weedle::literal::DefaultValue::Null(_), Type::Optional { .. }) => Literal::None, + (_, Type::Optional { inner_type, .. }) => Literal::Some { + inner: Box::new(convert_default_value(default_value, inner_type)?), + }, // We'll ensure the type safety in the convert_* number methods. (weedle::literal::DefaultValue::Integer(i), _) => convert_integer(i, type_)?, @@ -144,7 +146,7 @@ mod test { inner_type: Box::new(Type::String) } )?, - Literal::Null + Literal::None )); Ok(()) } diff --git a/third_party/rust/uniffi_udl/src/resolver.rs b/third_party/rust/uniffi_udl/src/resolver.rs index 14a7a4c6f1..ea98cd7a99 100644 --- a/third_party/rust/uniffi_udl/src/resolver.rs +++ b/third_party/rust/uniffi_udl/src/resolver.rs @@ -209,7 +209,6 @@ pub(crate) fn resolve_builtin_type(name: &str) -> Option { "f64" => Some(Type::Float64), "timestamp" => Some(Type::Timestamp), "duration" => Some(Type::Duration), - "ForeignExecutor" => Some(Type::ForeignExecutor), _ => None, } } diff --git a/third_party/rust/wasm-encoder/.cargo-checksum.json b/third_party/rust/wasm-encoder/.cargo-checksum.json index 4c82ccdb63..b9591f5309 100644 --- a/third_party/rust/wasm-encoder/.cargo-checksum.json +++ b/third_party/rust/wasm-encoder/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"Cargo.toml":"fc84e9fbaaf9adbac39c8099ddbf6bb252a332f84e569392a7efe9a9712c1829","LICENSE":"268872b9816f90fd8e85db5a28d33f8150ebb8dd016653fb39ef1f94f2686bc5","README.md":"ac016c4843a7e1a5737255b39418732783592222dc518020730edf9dd46a1c13","src/component.rs":"700351503077106eadf612f46079418e0b437ed0a6179dc16979768205b2ecbf","src/component/aliases.rs":"be5215154b872ed5664f3bfe2aa1391f36a2575aa9d53971ab61868a1c446e9d","src/component/builder.rs":"ea35c74cb0369181e9c0ce9729ff175d25f05bd63b48dcbbf703a53dfc113c7c","src/component/canonicals.rs":"a4f9c7a2b8ac3783d7337a5579cf71df2f33b315b354965980bd2c483d3aa141","src/component/components.rs":"07b8cae3a400e1955cc39d142569910b4fef2136021053d0ddbd6404d810aa52","src/component/exports.rs":"1af747401f153ccd5c5adfcab55dc4d75e241536cbbdc62624c3a07dd7620056","src/component/imports.rs":"a6cd7f914de7706a1a93a40fb8d0031b4b140786a347da636c70d093401079cd","src/component/instances.rs":"a76c5e7ba0ef73ae4bff5569d177198b7f3a9959dbe9f5c1b4bae7d1a9c839b1","src/component/modules.rs":"9e80907e72360fae4d8057b6b0e7a6b58edd7ba6aba6e63ba17346518e169617","src/component/names.rs":"f3b6691f822d53eb743b397a4e735786501cf000451753ae9a65031ae3249835","src/component/start.rs":"4055553d5c99c99abbc01bb8abb928ecb8b909d148b647a9977fbd444bb464a3","src/component/types.rs":"f44501e9b356e5588767323f1844e768221e00843a33354ff2597a76f32def46","src/core.rs":"a00656f82a623656c59a2d7230b40a5849a5083e117bc57061746f6e3022c7bb","src/core/code.rs":"06cc521073c2cd0e33e0e417f3721b9c63b2c64ba778a927c98fb6737c8cdc34","src/core/custom.rs":"4b9f07b701dc7b6990d92fb43016c28fb971411670bb6a48553b92b1c35eb9a3","src/core/data.rs":"c9d59eab2ab811bd950da52e6766c7df2367986c7a3a0d94d7aeb47d86f203ac","src/core/dump.rs":"8feaa532e3851186277ec1f4906e7fdc82c6399b211b8c928b16e293db1205b0","src/core/elements.rs":"2df1ad85f683b5cf3ce43c8cfef3f08f74a17cec9171537e3f24205dc067b4f7","src/core/exports.rs":"9e1eb3db0e0f73e00b66d82431257923feec8c052ac8b98717c8f36ffcc03537","src/core/functions.rs":"c18b9872ac0c21048a3ce32e5e44e8e702f97a57fa1b3a07bdd98c7f6c820f09","src/core/globals.rs":"0ed5f033489318d91e974090e8bf240720baa1ba621d0b5e0956dbd68e2165c1","src/core/imports.rs":"b4b096a6b1f449c4429efff605ca597549ea31873e18a9e6e55c616b4edc76db","src/core/linking.rs":"2f1053d9c2671e91a2b6e253dd38921bfc5f1b8a1a047b10c843033fe0f492de","src/core/memories.rs":"42e11ff5e8b2634bcd87810aef3fb9057c5d0b23c68ca9862a1e99aab5ecad72","src/core/names.rs":"2624a58835b27b17260fdbd20704856dce04e1826eb2b3b6196f8f1ffcf560ef","src/core/producers.rs":"f4916c1cf61e26170cd10fe350de7dad18005461c362909b9557c16c507517bc","src/core/start.rs":"a01d4a91bcd93048977ccafc6af160357297450395bf598351f5a3e6d322e0de","src/core/tables.rs":"c8286f5624ff429d42879932dedf45975513abfd98071a4084271860225af078","src/core/tags.rs":"66251072d5108dfde8177039cef95deac21606a361cddb04f37c08b8a8969db4","src/core/types.rs":"eb61f4f53a6c283ff6788c9724ffb728d2f525a62230bd13494a797ee23fbfef","src/lib.rs":"e61325ab8de1b4c404f4ee7665eef404fca2d23322a9c8f94efec8426edc166b","src/raw.rs":"a6a72cfe8f88ea6476eccee4acf362030ba2d2e5710215bc4f13cde7de6d71ae"},"package":"d162eb64168969ae90e8668ca0593b0e47667e315aa08e717a9c9574d700d826"} \ No newline at end of file +{"files":{"Cargo.toml":"e0d334beb383d7a06a870e93a43cee25fb37c9c416b01e056a8c5589e9b5ae0c","LICENSE":"268872b9816f90fd8e85db5a28d33f8150ebb8dd016653fb39ef1f94f2686bc5","README.md":"ac016c4843a7e1a5737255b39418732783592222dc518020730edf9dd46a1c13","src/component.rs":"700351503077106eadf612f46079418e0b437ed0a6179dc16979768205b2ecbf","src/component/aliases.rs":"be5215154b872ed5664f3bfe2aa1391f36a2575aa9d53971ab61868a1c446e9d","src/component/builder.rs":"ea35c74cb0369181e9c0ce9729ff175d25f05bd63b48dcbbf703a53dfc113c7c","src/component/canonicals.rs":"a4f9c7a2b8ac3783d7337a5579cf71df2f33b315b354965980bd2c483d3aa141","src/component/components.rs":"07b8cae3a400e1955cc39d142569910b4fef2136021053d0ddbd6404d810aa52","src/component/exports.rs":"1af747401f153ccd5c5adfcab55dc4d75e241536cbbdc62624c3a07dd7620056","src/component/imports.rs":"a6cd7f914de7706a1a93a40fb8d0031b4b140786a347da636c70d093401079cd","src/component/instances.rs":"a76c5e7ba0ef73ae4bff5569d177198b7f3a9959dbe9f5c1b4bae7d1a9c839b1","src/component/modules.rs":"9e80907e72360fae4d8057b6b0e7a6b58edd7ba6aba6e63ba17346518e169617","src/component/names.rs":"ca2f46a6dbc872a9cdb297d58c958e16acaf8893318bfdba30e468841107d886","src/component/start.rs":"4055553d5c99c99abbc01bb8abb928ecb8b909d148b647a9977fbd444bb464a3","src/component/types.rs":"f44501e9b356e5588767323f1844e768221e00843a33354ff2597a76f32def46","src/core.rs":"a00656f82a623656c59a2d7230b40a5849a5083e117bc57061746f6e3022c7bb","src/core/code.rs":"23f96e90b57b1334b304bc66c9c39518d9d8baee1e2a7f5e3045f817019c62da","src/core/custom.rs":"4b9f07b701dc7b6990d92fb43016c28fb971411670bb6a48553b92b1c35eb9a3","src/core/data.rs":"c9d59eab2ab811bd950da52e6766c7df2367986c7a3a0d94d7aeb47d86f203ac","src/core/dump.rs":"8feaa532e3851186277ec1f4906e7fdc82c6399b211b8c928b16e293db1205b0","src/core/elements.rs":"2df1ad85f683b5cf3ce43c8cfef3f08f74a17cec9171537e3f24205dc067b4f7","src/core/exports.rs":"9e1eb3db0e0f73e00b66d82431257923feec8c052ac8b98717c8f36ffcc03537","src/core/functions.rs":"c18b9872ac0c21048a3ce32e5e44e8e702f97a57fa1b3a07bdd98c7f6c820f09","src/core/globals.rs":"0ed5f033489318d91e974090e8bf240720baa1ba621d0b5e0956dbd68e2165c1","src/core/imports.rs":"b4b096a6b1f449c4429efff605ca597549ea31873e18a9e6e55c616b4edc76db","src/core/linking.rs":"2f1053d9c2671e91a2b6e253dd38921bfc5f1b8a1a047b10c843033fe0f492de","src/core/memories.rs":"42e11ff5e8b2634bcd87810aef3fb9057c5d0b23c68ca9862a1e99aab5ecad72","src/core/names.rs":"2624a58835b27b17260fdbd20704856dce04e1826eb2b3b6196f8f1ffcf560ef","src/core/producers.rs":"f4916c1cf61e26170cd10fe350de7dad18005461c362909b9557c16c507517bc","src/core/start.rs":"a01d4a91bcd93048977ccafc6af160357297450395bf598351f5a3e6d322e0de","src/core/tables.rs":"c8286f5624ff429d42879932dedf45975513abfd98071a4084271860225af078","src/core/tags.rs":"66251072d5108dfde8177039cef95deac21606a361cddb04f37c08b8a8969db4","src/core/types.rs":"eb61f4f53a6c283ff6788c9724ffb728d2f525a62230bd13494a797ee23fbfef","src/lib.rs":"e61325ab8de1b4c404f4ee7665eef404fca2d23322a9c8f94efec8426edc166b","src/raw.rs":"a6a72cfe8f88ea6476eccee4acf362030ba2d2e5710215bc4f13cde7de6d71ae"},"package":"b9c7d2731df60006819b013f64ccc2019691deccf6e11a1804bc850cd6748f1a"} \ No newline at end of file diff --git a/third_party/rust/wasm-encoder/Cargo.toml b/third_party/rust/wasm-encoder/Cargo.toml index 1f9750fd1f..aa4ce5bc7c 100644 --- a/third_party/rust/wasm-encoder/Cargo.toml +++ b/third_party/rust/wasm-encoder/Cargo.toml @@ -12,7 +12,7 @@ [package] edition = "2021" name = "wasm-encoder" -version = "0.40.0" +version = "0.201.0" authors = ["Nick Fitzgerald "] description = """ A low-level WebAssembly encoder. @@ -27,7 +27,7 @@ repository = "https://github.com/bytecodealliance/wasm-tools/tree/main/crates/wa version = "0.2.4" [dependencies.wasmparser] -version = "0.120.0" +version = "0.201.0" optional = true [dev-dependencies.anyhow] @@ -35,3 +35,9 @@ version = "1.0.58" [dev-dependencies.tempfile] version = "3.2.0" + +[lints.clippy] +all = "allow" + +[lints.rust] +unsafe_code = "deny" diff --git a/third_party/rust/wasm-encoder/src/component/names.rs b/third_party/rust/wasm-encoder/src/component/names.rs index 99db10bda0..1cbb1062c6 100644 --- a/third_party/rust/wasm-encoder/src/component/names.rs +++ b/third_party/rust/wasm-encoder/src/component/names.rs @@ -1,7 +1,7 @@ use std::borrow::Cow; use super::*; -use crate::{encoding_size, CustomSection, Encode, ExportKind, NameMap, SectionId}; +use crate::{encoding_size, ExportKind, NameMap, SectionId}; /// Encoding for the `component-name` custom section which assigns /// human-readable names to items within a component. diff --git a/third_party/rust/wasm-encoder/src/core/code.rs b/third_party/rust/wasm-encoder/src/core/code.rs index cec2d4191e..76bfd7afde 100644 --- a/third_party/rust/wasm-encoder/src/core/code.rs +++ b/third_party/rust/wasm-encoder/src/core/code.rs @@ -3188,6 +3188,11 @@ impl ConstExpr { Self { bytes } } + fn with_insn(mut self, insn: Instruction) -> Self { + insn.encode(&mut self.bytes); + self + } + /// Create a constant expression containing a single `global.get` instruction. pub fn global_get(index: u32) -> Self { Self::new_insn(Instruction::GlobalGet(index)) @@ -3227,6 +3232,90 @@ impl ConstExpr { pub fn v128_const(value: i128) -> Self { Self::new_insn(Instruction::V128Const(value)) } + + /// Add a `global.get` instruction to this constant expression. + pub fn with_global_get(self, index: u32) -> Self { + self.with_insn(Instruction::GlobalGet(index)) + } + + /// Add a `ref.null` instruction to this constant expression. + pub fn with_ref_null(self, ty: HeapType) -> Self { + self.with_insn(Instruction::RefNull(ty)) + } + + /// Add a `ref.func` instruction to this constant expression. + pub fn with_ref_func(self, func: u32) -> Self { + self.with_insn(Instruction::RefFunc(func)) + } + + /// Add an `i32.const` instruction to this constant expression. + pub fn with_i32_const(self, value: i32) -> Self { + self.with_insn(Instruction::I32Const(value)) + } + + /// Add an `i64.const` instruction to this constant expression. + pub fn with_i64_const(self, value: i64) -> Self { + self.with_insn(Instruction::I64Const(value)) + } + + /// Add a `f32.const` instruction to this constant expression. + pub fn with_f32_const(self, value: f32) -> Self { + self.with_insn(Instruction::F32Const(value)) + } + + /// Add a `f64.const` instruction to this constant expression. + pub fn with_f64_const(self, value: f64) -> Self { + self.with_insn(Instruction::F64Const(value)) + } + + /// Add a `v128.const` instruction to this constant expression. + pub fn with_v128_const(self, value: i128) -> Self { + self.with_insn(Instruction::V128Const(value)) + } + + /// Add an `i32.add` instruction to this constant expression. + pub fn with_i32_add(self) -> Self { + self.with_insn(Instruction::I32Add) + } + + /// Add an `i32.sub` instruction to this constant expression. + pub fn with_i32_sub(self) -> Self { + self.with_insn(Instruction::I32Sub) + } + + /// Add an `i32.mul` instruction to this constant expression. + pub fn with_i32_mul(self) -> Self { + self.with_insn(Instruction::I32Mul) + } + + /// Add an `i64.add` instruction to this constant expression. + pub fn with_i64_add(self) -> Self { + self.with_insn(Instruction::I64Add) + } + + /// Add an `i64.sub` instruction to this constant expression. + pub fn with_i64_sub(self) -> Self { + self.with_insn(Instruction::I64Sub) + } + + /// Add an `i64.mul` instruction to this constant expression. + pub fn with_i64_mul(self) -> Self { + self.with_insn(Instruction::I64Mul) + } + + /// Returns the function, if any, referenced by this global. + pub fn get_ref_func(&self) -> Option { + let prefix = *self.bytes.get(0)?; + // 0xd2 == `ref.func` opcode, and if that's found then load the leb + // corresponding to the function index. + if prefix != 0xd2 { + return None; + } + leb128::read::unsigned(&mut &self.bytes[1..]) + .ok()? + .try_into() + .ok() + } } impl Encode for ConstExpr { diff --git a/third_party/rust/wasm-smith/.cargo-checksum.json b/third_party/rust/wasm-smith/.cargo-checksum.json index 092ef268b6..d61edb195b 100644 --- a/third_party/rust/wasm-smith/.cargo-checksum.json +++ b/third_party/rust/wasm-smith/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"Cargo.toml":"b36ff9dd80b356c2fc8e62ec88a8e6433a387d4a44b71bd4b56106da793a1558","LICENSE":"268872b9816f90fd8e85db5a28d33f8150ebb8dd016653fb39ef1f94f2686bc5","README.md":"e7b2a96cf39834aa19aed7cad8658e33445cdec7c4957658ba39e3b82cc4254b","benches/corpus.rs":"2df29556be0799f0cb1f32c8d0ae5ba0c4b9815cf4d59a8b71744d926c0693a0","src/component.rs":"f605082d1e1ed2f5a277e9f7c1a0f36842fe5bc10d9ca816fbd301a380946617","src/component/encode.rs":"5985e3f412595e07372a0787ec3db6c3018840df2bcfc638ab8798fd74b22778","src/config.rs":"9e52b815c79e1b65af21b23c8148ca9330201bfff657626352d5055e5be77abb","src/core.rs":"a5e13de2fb7c97eb12f2c1bc439f617b8afe6c509868fcb92db5e2aa9f252e65","src/core/code_builder.rs":"2e9d1eb09711f3e4ec4334d6a3952d9b5aed827778c14a7e01fb9676640ac561","src/core/code_builder/no_traps.rs":"e595dbde06551f5f8b23e03cfac0634beacab08e6243c66d6ffda95079212b24","src/core/encode.rs":"76ac812803e16cebf852f6d1265a0bd9ef16661e87f77a15f3ba997e75b351ce","src/core/terminate.rs":"d24af5206a13aee7d6a6ea900ccdf088c09d053c36026cf1607cc38c972b3ba9","src/lib.rs":"3a5521a9d4f1f38d01206339b1997e2943d6129be7d434457d59d019e2937456","tests/available_imports.rs":"3157294aa496c9e571398bdbd907539350f1015368cf47e45ecf822473df1f4e","tests/component.rs":"44684e990e832590a0b348ce33fadf39ee55dec52fdfa9353570edcbf65325ee","tests/core.rs":"1f581629a235399127b525c3aa56f1fef5da54b7887824d7c40169642cb5adb1"},"package":"51803e2c31e82f93ec1796f781f9855844ff1203f993409eb74d1d17130928a7"} \ No newline at end of file +{"files":{"Cargo.toml":"b911aa53dc1f5c0cb04a77e15b43c31cee98893c7bcb98b6d73405aeca6b447c","LICENSE":"268872b9816f90fd8e85db5a28d33f8150ebb8dd016653fb39ef1f94f2686bc5","README.md":"e7b2a96cf39834aa19aed7cad8658e33445cdec7c4957658ba39e3b82cc4254b","benches/corpus.rs":"2df29556be0799f0cb1f32c8d0ae5ba0c4b9815cf4d59a8b71744d926c0693a0","src/component.rs":"253cf8d8abf785e4b5a5c614561a85b032e7f7bdec366f83d9db0702f73d2122","src/component/encode.rs":"5985e3f412595e07372a0787ec3db6c3018840df2bcfc638ab8798fd74b22778","src/config.rs":"e75d93c0a6a5ba07ff3ec1164a9b58601fadc0a0899149e14a77e8f0ab4595e4","src/core.rs":"35d3854c8caa1f61494b2f8e2b5e150d67cf88582243feeeb198e2aa93247339","src/core/code_builder.rs":"a8b27f652c8ae4b964e05933df313016ab7ffc297ca76005faca3da2792e3afc","src/core/code_builder/no_traps.rs":"2f36c548a32a6a003e6bb22a8fe7c75517b9be00c5a299212836a7e2431a8693","src/core/encode.rs":"77e1395c78ecf1b688a3c515a58be6efe1aa58d450a20a50d69598c54f7162de","src/core/terminate.rs":"c7c5f5618d393e5d9e44ae5e9bbde4b81bfa1ca3d2111d1eea875e2490f0f64b","src/lib.rs":"d393ae9daba211ba68a48dbdee545e0570cb8298e4c62319b8c8a8b9355cd6e3","tests/available_imports.rs":"6d4c1b111f35fefce55b668474ac9ba25edf7fc3e3decb88696b644aec13d603","tests/common/mod.rs":"9d0712a0b3e30030934b04785fb8632576bdcc31921ee8f524e0b2c903dee28d","tests/component.rs":"44684e990e832590a0b348ce33fadf39ee55dec52fdfa9353570edcbf65325ee","tests/core.rs":"408a50da4438be4e5343b116aa80f5ce344ba9508fad413cc5c32cd0aa2164fb","tests/exports.rs":"dcac4e18f277cbdb821f4ad7eda0a900bd73924e0818cf1b408700c9b7bba789"},"package":"61ff53a54a853f174b0df74cdb1553f1451e7bcdc23b26b1379f664ee1913d1a"} \ No newline at end of file diff --git a/third_party/rust/wasm-smith/Cargo.toml b/third_party/rust/wasm-smith/Cargo.toml index bebb2807d5..3847409e33 100644 --- a/third_party/rust/wasm-smith/Cargo.toml +++ b/third_party/rust/wasm-smith/Cargo.toml @@ -12,7 +12,7 @@ [package] edition = "2021" name = "wasm-smith" -version = "0.15.0" +version = "0.201.0" authors = ["Nick Fitzgerald "] exclude = ["/benches/corpus"] description = "A WebAssembly test case generator" @@ -33,7 +33,6 @@ harness = false [dependencies.anyhow] version = "1.0.58" -optional = true [dependencies.arbitrary] version = "1.1.0" @@ -62,14 +61,14 @@ version = "1.0.166" optional = true [dependencies.wasm-encoder] -version = "0.40.0" +version = "0.201.0" [dependencies.wasmparser] -version = "0.120.0" +version = "0.201.0" optional = true [dependencies.wat] -version = "1.0.84" +version = "1.201.0" optional = true [dev-dependencies.criterion] @@ -80,11 +79,10 @@ version = "0.8.4" features = ["small_rng"] [dev-dependencies.wasmparser] -version = "0.120.0" +version = "0.201.0" [features] _internal_cli = [ - "anyhow", "clap", "flagset/serde", "serde", @@ -99,3 +97,9 @@ wasmparser = [ [target."cfg(not(target_family = \"wasm\"))".dev-dependencies.libfuzzer-sys] version = "0.4.0" + +[lints.clippy] +all = "allow" + +[lints.rust] +unsafe_code = "deny" diff --git a/third_party/rust/wasm-smith/src/component.rs b/third_party/rust/wasm-smith/src/component.rs index e18373b4f4..7a85b2bf10 100644 --- a/third_party/rust/wasm-smith/src/component.rs +++ b/third_party/rust/wasm-smith/src/component.rs @@ -7,12 +7,13 @@ use crate::{arbitrary_loop, Config}; use arbitrary::{Arbitrary, Result, Unstructured}; use std::collections::BTreeMap; -use std::convert::TryFrom; use std::{ collections::{HashMap, HashSet}, rc::Rc, }; -use wasm_encoder::{ComponentTypeRef, ComponentValType, PrimitiveValType, TypeBounds, ValType}; +use wasm_encoder::{ + ComponentTypeRef, ComponentValType, HeapType, PrimitiveValType, RefType, TypeBounds, ValType, +}; mod encode; @@ -540,7 +541,7 @@ impl ComponentBuilder { } let ty = match u.int_in_range::(0..=1)? { - 0 => CoreType::Func(crate::core::arbitrary_func_type( + 0 => CoreType::Func(arbitrary_func_type( u, &self.config, &self.core_valtypes, @@ -819,7 +820,7 @@ impl ComponentBuilder { // Type definition. 2 => { - let ty = crate::core::arbitrary_func_type( + let ty = arbitrary_func_type( u, &self.config, &self.core_valtypes, @@ -955,7 +956,7 @@ impl ComponentBuilder { } fn arbitrary_core_table_type(&self, u: &mut Unstructured) -> Result { - crate::core::arbitrary_table_type(u, &self.config) + crate::core::arbitrary_table_type(u, &self.config, None) } fn arbitrary_core_memory_type(&self, u: &mut Unstructured) -> Result { @@ -2176,3 +2177,45 @@ struct CoreInstanceSection {} struct CoreTypeSection { types: Vec>, } + +fn arbitrary_func_type( + u: &mut Unstructured, + config: &Config, + valtypes: &[ValType], + max_results: Option, + type_ref_limit: u32, +) -> Result> { + let mut params = vec![]; + let mut results = vec![]; + arbitrary_loop(u, 0, 20, |u| { + params.push(arbitrary_valtype(u, config, valtypes, type_ref_limit)?); + Ok(true) + })?; + arbitrary_loop(u, 0, max_results.unwrap_or(20), |u| { + results.push(arbitrary_valtype(u, config, valtypes, type_ref_limit)?); + Ok(true) + })?; + Ok(Rc::new(crate::core::FuncType { params, results })) +} + +fn arbitrary_valtype( + u: &mut Unstructured, + config: &Config, + valtypes: &[ValType], + type_ref_limit: u32, +) -> Result { + if config.gc_enabled && type_ref_limit > 0 && u.ratio(1, 20)? { + Ok(ValType::Ref(RefType { + // TODO: For now, only create allow nullable reference + // types. Eventually we should support non-nullable reference types, + // but this means that we will also need to recognize when it is + // impossible to create an instance of the reference (eg `(ref + // nofunc)` has no instances, and self-referential types that + // contain a non-null self-reference are also impossible to create). + nullable: true, + heap_type: HeapType::Concrete(u.int_in_range(0..=type_ref_limit - 1)?), + })) + } else { + Ok(*u.choose(valtypes)?) + } +} diff --git a/third_party/rust/wasm-smith/src/config.rs b/third_party/rust/wasm-smith/src/config.rs index b183530e4c..43dbecceb8 100644 --- a/third_party/rust/wasm-smith/src/config.rs +++ b/third_party/rust/wasm-smith/src/config.rs @@ -20,13 +20,13 @@ macro_rules! define_config { /// Defaults to `None` which means that any arbitrary import can be /// generated. /// - /// To only allow specific imports, override this method to return a - /// WebAssembly module which describes the imports allowed. + /// To only allow specific imports, set this field to a WebAssembly + /// module which describes the imports allowed. /// /// Note that [`Self::min_imports`] is ignored when /// `available_imports` are enabled. /// - /// The returned value must be a valid binary encoding of a + /// The provided value must be a valid binary encoding of a /// WebAssembly module. `wasm-smith` will panic if the module cannot /// be parsed. /// @@ -49,6 +49,51 @@ macro_rules! define_config { /// ``` pub available_imports: Option>, + /// If provided, the generated module will have exports with exactly + /// the same names and types as those in the provided WebAssembly + /// module. The implementation (e.g. function bodies, global + /// initializers) of each export in the generated module will be + /// random and unrelated to the implementation in the provided + /// module. Only globals and functions are supported. + /// + /// + /// Defaults to `None` which means arbitrary exports will be + /// generated. + /// + /// To specify which exports the generated modules should have, set + /// this field to a WebAssembly module which describes the desired + /// exports. To generate modules with varying exports that meet some + /// constraints, consider randomly generating the value for this + /// field. + /// + /// The provided value must be a valid binary encoding of a + /// WebAssembly module. `wasm-smith` will panic if the module cannot + /// be parsed. + /// + /// # Module Limits + /// + /// All types, functions, globals, and exports that are needed to + /// provide the required exports will be generated, even if it + /// causes the resulting module to exceed the limits defined in + /// [`Self::max_type_size`], [`Self::max_types`], + /// [`Self::max_funcs`], [`Self::max_globals`], or + /// [`Self::max_exports`]. + /// + /// # Example + /// + /// As for [`Self::available_imports`], the `wat` crate can be used + /// to provide an human-readable description of the desired exports: + /// + /// ```rust + /// Some(wat::parse_str(r#" + /// (module + /// (func (export "foo") (param i32) (result i64) unreachable) + /// (global (export "bar") f32 f32.const 0) + /// ) + /// "#)); + /// ``` + pub exports: Option>, + $( $(#[$field_attr])* pub $field: $field_ty, @@ -59,6 +104,7 @@ macro_rules! define_config { fn default() -> Config { Config { available_imports: None, + exports: None, $( $field: $default, @@ -82,12 +128,44 @@ macro_rules! define_config { /// Note that [`Self::min_imports`] is ignored when /// `available_imports` are enabled. /// - /// The returned value must be a valid binary encoding of a + /// The provided value must be a valid binary encoding of a /// WebAssembly module. `wasm-smith` will panic if the module cannot /// be parsed. #[cfg_attr(feature = "clap", clap(long))] available_imports: Option, + /// If provided, the generated module will have exports with exactly + /// the same names and types as those in the provided WebAssembly + /// module. The implementation (e.g. function bodies, global + /// initializers) of each export in the generated module will be + /// random and unrelated to the implementation in the provided + /// module. Only globals and functions are supported. + /// + /// Defaults to `None` which means arbitrary exports will be + /// generated. + /// + /// To specify which exports the generated modules should have, set + /// this field to a WebAssembly module which describes the desired + /// exports. To generate modules with varying exports that meet some + /// constraints, consider randomly generating the value for this + /// field. + /// + /// The provided value must be a valid binary encoding of a + /// WebAssembly module. `wasm-smith` will panic if the module cannot + /// be parsed. + /// + /// # Module Limits + /// + /// All types, functions, globals, and exports that are needed to + /// provide the required exports will be generated, even if it + /// causes the resulting module to exceed the limits defined in + /// [`Self::max_type_size`], [`Self::max_types`], + /// [`Self::max_funcs`], [`Self::max_globals`], or + /// [`Self::max_exports`]. + /// + #[cfg_attr(feature = "clap", clap(long))] + exports: Option, + $( $(#[$field_attr])* #[cfg_attr(feature = "clap", clap(long))] @@ -100,6 +178,7 @@ macro_rules! define_config { pub fn or(self, other: Self) -> Self { Self { available_imports: self.available_imports.or(other.available_imports), + exports: self.exports.or(other.exports), $( $field: self.$field.or(other.$field), @@ -121,6 +200,13 @@ macro_rules! define_config { } else { None }, + exports: if let Some(file) = config + .exports + .as_ref() { + Some(wat::parse_file(file)?) + } else { + None + }, $( $field: config.$field.unwrap_or(default.$field), @@ -481,6 +567,18 @@ define_config! { /// /// Defaults to `false`. pub threads_enabled: bool = false, + + /// Indicates whether wasm-smith is allowed to generate invalid function + /// bodies. + /// + /// When enabled this option will enable taking raw bytes from the input + /// byte stream and using them as a wasm function body. This means that + /// the output module is not guaranteed to be valid but can help tickle + /// various parts of validation/compilation in some circumstances as + /// well. + /// + /// Defaults to `false`. + pub allow_invalid_funcs: bool = false, } } @@ -611,12 +709,14 @@ impl<'a> Arbitrary<'a> for Config { max_type_size: 1000, canonicalize_nans: false, available_imports: None, + exports: None, threads_enabled: false, export_everything: false, disallow_traps: false, tail_call_enabled: false, gc_enabled: false, generate_custom_sections: false, + allow_invalid_funcs: false, }) } } diff --git a/third_party/rust/wasm-smith/src/core.rs b/third_party/rust/wasm-smith/src/core.rs index 6a2836ae0e..194d2101ae 100644 --- a/third_party/rust/wasm-smith/src/core.rs +++ b/third_party/rust/wasm-smith/src/core.rs @@ -9,7 +9,8 @@ use arbitrary::{Arbitrary, Result, Unstructured}; use code_builder::CodeBuilderAllocations; use flagset::{flags, FlagSet}; use std::collections::{HashMap, HashSet}; -use std::convert::TryFrom; +use std::fmt; +use std::mem; use std::ops::Range; use std::rc::Rc; use std::str::{self, FromStr}; @@ -42,7 +43,6 @@ type Instruction = wasm_encoder::Instruction<'static>; /// To configure the shape of generated module, create a /// [`Config`][crate::Config] and then call [`Module::new`][crate::Module::new] /// with it. -#[derive(Debug)] pub struct Module { config: Config, duplicate_imports_behavior: DuplicateImportsBehavior, @@ -95,9 +95,8 @@ pub struct Module { /// aliased). num_defined_funcs: usize, - /// The number of tables defined in this module (not imported or - /// aliased). - num_defined_tables: usize, + /// Initialization expressions for all defined tables in this module. + defined_tables: Vec>, /// The number of memories defined in this module (not imported or /// aliased). @@ -105,7 +104,7 @@ pub struct Module { /// The indexes and initialization expressions of globals defined in this /// module. - defined_globals: Vec<(u32, GlobalInitExpr)>, + defined_globals: Vec<(u32, ConstExpr)>, /// All tags available to this module, sorted by their index. The list /// entry is the type of each tag. @@ -140,6 +139,13 @@ pub struct Module { /// Names currently exported from this module. export_names: HashSet, + + /// Reusable buffer in `self.arbitrary_const_expr` to amortize the cost of + /// allocation. + const_expr_choices: Vec Result>>, + + /// What the maximum type index that can be referenced is. + max_type_limit: MaxTypeLimit, } impl<'a> Arbitrary<'a> for Module { @@ -148,12 +154,27 @@ impl<'a> Arbitrary<'a> for Module { } } +impl fmt::Debug for Module { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Module") + .field("config", &self.config) + .field(&"...", &"...") + .finish() + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum DuplicateImportsBehavior { Allowed, Disallowed, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum MaxTypeLimit { + ModuleTypes, + Num(u32), +} + impl Module { /// Returns a reference to the internal configuration. pub fn config(&self) -> &Config { @@ -172,7 +193,7 @@ impl Module { duplicate_imports_behavior: DuplicateImportsBehavior, ) -> Result { let mut module = Module::empty(config, duplicate_imports_behavior); - module.build(u, false)?; + module.build(u)?; Ok(module) } @@ -194,7 +215,7 @@ impl Module { num_imports: 0, num_defined_tags: 0, num_defined_funcs: 0, - num_defined_tables: 0, + defined_tables: Vec::new(), num_defined_memories: 0, defined_globals: Vec::new(), tags: Vec::new(), @@ -209,34 +230,12 @@ impl Module { data: Vec::new(), type_size: 0, export_names: HashSet::new(), + const_expr_choices: Vec::new(), + max_type_limit: MaxTypeLimit::ModuleTypes, } } } -/// Same as [`Module`], but may be invalid. -/// -/// This module generates function bodies differnetly than `Module` to try to -/// better explore wasm decoders and such. -#[derive(Debug)] -pub struct MaybeInvalidModule { - module: Module, -} - -impl MaybeInvalidModule { - /// Encode this Wasm module into bytes. - pub fn to_bytes(&self) -> Vec { - self.module.to_bytes() - } -} - -impl<'a> Arbitrary<'a> for MaybeInvalidModule { - fn arbitrary(u: &mut Unstructured<'a>) -> Result { - let mut module = Module::empty(Config::default(), DuplicateImportsBehavior::Allowed); - module.build(u, true)?; - Ok(MaybeInvalidModule { module }) - } -} - #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub(crate) struct RecGroup { pub(crate) types: Vec, @@ -357,7 +356,7 @@ enum ElementKind { #[derive(Debug)] enum Elements { Functions(Vec), - Expressions(Vec>), + Expressions(Vec), } #[derive(Debug)] @@ -391,14 +390,8 @@ pub(crate) enum Offset { Global(u32), } -#[derive(Debug)] -pub(crate) enum GlobalInitExpr { - FuncRef(u32), - ConstExpr(ConstExpr), -} - impl Module { - fn build(&mut self, u: &mut Unstructured, allow_invalid: bool) -> Result<()> { + fn build(&mut self, u: &mut Unstructured) -> Result<()> { self.valtypes = configured_valtypes(&self.config); // We attempt to figure out our available imports *before* creating the types section here, @@ -413,7 +406,6 @@ impl Module { self.arbitrary_imports(u)?; } - self.should_encode_types = !self.types.is_empty() || u.arbitrary()?; self.should_encode_imports = !self.imports.is_empty() || u.arbitrary()?; self.arbitrary_tags(u)?; @@ -421,11 +413,14 @@ impl Module { self.arbitrary_tables(u)?; self.arbitrary_memories(u)?; self.arbitrary_globals(u)?; - self.arbitrary_exports(u)?; + if !self.required_exports(u)? { + self.arbitrary_exports(u)?; + }; + self.should_encode_types = !self.types.is_empty() || u.arbitrary()?; self.arbitrary_start(u)?; self.arbitrary_elems(u)?; self.arbitrary_data(u)?; - self.arbitrary_code(u, allow_invalid)?; + self.arbitrary_code(u)?; Ok(()) } @@ -562,6 +557,8 @@ impl Module { fn arbitrary_rec_group(&mut self, u: &mut Unstructured) -> Result<()> { let rec_group_start = self.types.len(); + assert!(matches!(self.max_type_limit, MaxTypeLimit::ModuleTypes)); + if self.config.gc_enabled { // With small probability, clone an existing rec group. if self.clonable_rec_groups().next().is_some() && u.ratio(1, u8::MAX)? { @@ -572,16 +569,20 @@ impl Module { let max_rec_group_size = self.config.max_types - self.types.len(); let rec_group_size = u.int_in_range(0..=max_rec_group_size)?; let type_ref_limit = u32::try_from(self.types.len() + rec_group_size).unwrap(); + self.max_type_limit = MaxTypeLimit::Num(type_ref_limit); for _ in 0..rec_group_size { - let ty = self.arbitrary_sub_type(u, type_ref_limit)?; + let ty = self.arbitrary_sub_type(u)?; self.add_type(ty); } } else { let type_ref_limit = u32::try_from(self.types.len()).unwrap(); - let ty = self.arbitrary_sub_type(u, type_ref_limit)?; + self.max_type_limit = MaxTypeLimit::Num(type_ref_limit); + let ty = self.arbitrary_sub_type(u)?; self.add_type(ty); } + self.max_type_limit = MaxTypeLimit::ModuleTypes; + self.rec_groups.push(rec_group_start..self.types.len()); Ok(()) } @@ -615,40 +616,27 @@ impl Module { Ok(()) } - fn arbitrary_sub_type( - &mut self, - u: &mut Unstructured, - // NB: Types can be referenced up to (but not including) - // `type_ref_limit`. It is an exclusive bound to avoid an option: to - // disallow any reference to a concrete type (e.g. `(ref $type)`) you - // can simply pass `0` here. - type_ref_limit: u32, - ) -> Result { + fn arbitrary_sub_type(&mut self, u: &mut Unstructured) -> Result { if !self.config.gc_enabled { - debug_assert_eq!(type_ref_limit, u32::try_from(self.types.len()).unwrap()); return Ok(SubType { is_final: true, supertype: None, - composite_type: CompositeType::Func(self.arbitrary_func_type(u, type_ref_limit)?), + composite_type: CompositeType::Func(self.arbitrary_func_type(u)?), }); } if !self.can_subtype.is_empty() && u.ratio(1, 32_u8)? { - self.arbitrary_sub_type_of_super_type(u, type_ref_limit) + self.arbitrary_sub_type_of_super_type(u) } else { Ok(SubType { is_final: u.arbitrary()?, supertype: None, - composite_type: self.arbitrary_composite_type(u, type_ref_limit)?, + composite_type: self.arbitrary_composite_type(u)?, }) } } - fn arbitrary_sub_type_of_super_type( - &mut self, - u: &mut Unstructured, - type_ref_limit: u32, - ) -> Result { + fn arbitrary_sub_type_of_super_type(&mut self, u: &mut Unstructured) -> Result { let supertype = *u.choose(&self.can_subtype)?; let mut composite_type = self.types[usize::try_from(supertype).unwrap()] .composite_type @@ -661,7 +649,7 @@ impl Module { *f = self.arbitrary_matching_func_type(u, f)?; } CompositeType::Struct(s) => { - *s = self.arbitrary_matching_struct_type(u, s, type_ref_limit)?; + *s = self.arbitrary_matching_struct_type(u, s)?; } } Ok(SubType { @@ -675,7 +663,6 @@ impl Module { &mut self, u: &mut Unstructured, ty: &StructType, - type_ref_limit: u32, ) -> Result { let len_extra_fields = u.int_in_range(0..=5)?; let mut fields = Vec::with_capacity(ty.fields.len() + len_extra_fields); @@ -683,7 +670,7 @@ impl Module { fields.push(self.arbitrary_matching_field_type(u, *field)?); } for _ in 0..len_extra_fields { - fields.push(self.arbitrary_field_type(u, type_ref_limit)?); + fields.push(self.arbitrary_field_type(u)?); } Ok(StructType { fields: fields.into_boxed_slice(), @@ -915,71 +902,52 @@ impl Module { Ok(*u.choose(&choices)?) } - fn arbitrary_composite_type( - &mut self, - u: &mut Unstructured, - type_ref_limit: u32, - ) -> Result { + fn arbitrary_composite_type(&mut self, u: &mut Unstructured) -> Result { if !self.config.gc_enabled { - return Ok(CompositeType::Func( - self.arbitrary_func_type(u, type_ref_limit)?, - )); + return Ok(CompositeType::Func(self.arbitrary_func_type(u)?)); } match u.int_in_range(0..=2)? { 0 => Ok(CompositeType::Array(ArrayType( - self.arbitrary_field_type(u, type_ref_limit)?, + self.arbitrary_field_type(u)?, ))), - 1 => Ok(CompositeType::Func( - self.arbitrary_func_type(u, type_ref_limit)?, - )), - 2 => Ok(CompositeType::Struct( - self.arbitrary_struct_type(u, type_ref_limit)?, - )), + 1 => Ok(CompositeType::Func(self.arbitrary_func_type(u)?)), + 2 => Ok(CompositeType::Struct(self.arbitrary_struct_type(u)?)), _ => unreachable!(), } } - fn arbitrary_struct_type( - &mut self, - u: &mut Unstructured, - type_ref_limit: u32, - ) -> Result { + fn arbitrary_struct_type(&mut self, u: &mut Unstructured) -> Result { let len = u.int_in_range(0..=20)?; let mut fields = Vec::with_capacity(len); for _ in 0..len { - fields.push(self.arbitrary_field_type(u, type_ref_limit)?); + fields.push(self.arbitrary_field_type(u)?); } Ok(StructType { fields: fields.into_boxed_slice(), }) } - fn arbitrary_field_type( - &mut self, - u: &mut Unstructured, - type_ref_limit: u32, - ) -> Result { + fn arbitrary_field_type(&mut self, u: &mut Unstructured) -> Result { Ok(FieldType { - element_type: self.arbitrary_storage_type(u, type_ref_limit)?, + element_type: self.arbitrary_storage_type(u)?, mutable: u.arbitrary()?, }) } - fn arbitrary_storage_type( - &mut self, - u: &mut Unstructured, - type_ref_limit: u32, - ) -> Result { + fn arbitrary_storage_type(&mut self, u: &mut Unstructured) -> Result { match u.int_in_range(0..=2)? { 0 => Ok(StorageType::I8), 1 => Ok(StorageType::I16), - 2 => Ok(StorageType::Val(self.arbitrary_valtype(u, type_ref_limit)?)), + 2 => Ok(StorageType::Val(self.arbitrary_valtype(u)?)), _ => unreachable!(), } } fn arbitrary_ref_type(&self, u: &mut Unstructured) -> Result { + if !self.config.reference_types_enabled { + return Ok(RefType::FUNCREF); + } Ok(RefType { nullable: true, heap_type: self.arbitrary_heap_type(u)?, @@ -989,9 +957,13 @@ impl Module { fn arbitrary_heap_type(&self, u: &mut Unstructured) -> Result { assert!(self.config.reference_types_enabled); - if self.config.gc_enabled && !self.types.is_empty() && u.arbitrary()? { - let type_ref_limit = u32::try_from(self.types.len()).unwrap(); - let idx = u.int_in_range(0..=type_ref_limit)?; + let concrete_type_limit = match self.max_type_limit { + MaxTypeLimit::Num(n) => n, + MaxTypeLimit::ModuleTypes => u32::try_from(self.types.len()).unwrap(), + }; + + if self.config.gc_enabled && concrete_type_limit > 0 && u.arbitrary()? { + let idx = u.int_in_range(0..=concrete_type_limit - 1)?; return Ok(HeapType::Concrete(idx)); } @@ -1018,22 +990,24 @@ impl Module { u.choose(&choices).copied() } - fn arbitrary_func_type( - &mut self, - u: &mut Unstructured, - type_ref_limit: u32, - ) -> Result> { - arbitrary_func_type( - u, - &self.config, - &self.valtypes, - if !self.config.multi_value_enabled { - Some(1) - } else { - None - }, - type_ref_limit, - ) + fn arbitrary_func_type(&mut self, u: &mut Unstructured) -> Result> { + let mut params = vec![]; + let mut results = vec![]; + let max_params = 20; + arbitrary_loop(u, 0, max_params, |u| { + params.push(self.arbitrary_valtype(u)?); + Ok(true) + })?; + let max_results = if self.config.multi_value_enabled { + max_params + } else { + 1 + }; + arbitrary_loop(u, 0, max_results, |u| { + results.push(self.arbitrary_valtype(u)?); + Ok(true) + })?; + Ok(Rc::new(FuncType { params, results })) } fn can_add_local_or_import_tag(&self) -> bool { @@ -1097,7 +1071,7 @@ impl Module { } if self.can_add_local_or_import_table() { choices.push(|u, m| { - let ty = arbitrary_table_type(u, m.config())?; + let ty = arbitrary_table_type(u, m.config(), Some(m))?; Ok(EntityType::Table(ty)) }); } @@ -1411,13 +1385,42 @@ impl Module { .filter(move |i| self.func_type(*i).results.is_empty()) } - fn arbitrary_valtype(&self, u: &mut Unstructured, type_ref_limit: u32) -> Result { - arbitrary_valtype(u, &self.config, &self.valtypes, type_ref_limit) + fn arbitrary_valtype(&self, u: &mut Unstructured) -> Result { + #[derive(Arbitrary)] + enum ValTypeClass { + I32, + I64, + F32, + F64, + V128, + Ref, + } + + match u.arbitrary::()? { + ValTypeClass::I32 => Ok(ValType::I32), + ValTypeClass::I64 => Ok(ValType::I64), + ValTypeClass::F32 => Ok(ValType::F32), + ValTypeClass::F64 => Ok(ValType::F64), + ValTypeClass::V128 => { + if self.config.simd_enabled { + Ok(ValType::V128) + } else { + Ok(ValType::I32) + } + } + ValTypeClass::Ref => { + if self.config.reference_types_enabled { + Ok(ValType::Ref(self.arbitrary_ref_type(u)?)) + } else { + Ok(ValType::I32) + } + } + } } fn arbitrary_global_type(&self, u: &mut Unstructured) -> Result { Ok(GlobalType { - val_type: self.arbitrary_valtype(u, u32::try_from(self.types.len()).unwrap())?, + val_type: self.arbitrary_valtype(u)?, mutable: u.arbitrary()?, }) } @@ -1470,14 +1473,38 @@ impl Module { if !self.can_add_local_or_import_table() { return Ok(false); } - self.num_defined_tables += 1; - let ty = arbitrary_table_type(u, self.config())?; + let ty = arbitrary_table_type(u, self.config(), Some(self))?; + let init = self.arbitrary_table_init(u, ty.element_type)?; + self.defined_tables.push(init); self.tables.push(ty); Ok(true) }, ) } + /// Generates an arbitrary table initialization expression for a table whose + /// element type is `ty`. + /// + /// Table initialization expressions were added by the GC proposal to + /// initialize non-nullable tables. + fn arbitrary_table_init( + &mut self, + u: &mut Unstructured, + ty: RefType, + ) -> Result> { + if !self.config.gc_enabled { + assert!(ty.nullable); + return Ok(None); + } + // Even with the GC proposal an initialization expression is not + // required if the element type is nullable. + if ty.nullable && u.arbitrary()? { + return Ok(None); + } + let expr = self.arbitrary_const_expr(ValType::Ref(ty), u)?; + Ok(Some(expr)) + } + fn arbitrary_memories(&mut self, u: &mut Unstructured) -> Result<()> { arbitrary_loop( u, @@ -1494,53 +1521,260 @@ impl Module { ) } - fn arbitrary_globals(&mut self, u: &mut Unstructured) -> Result<()> { - let mut choices: Vec Result>> = - vec![]; - let num_imported_globals = self.globals.len(); + /// Add a new global of the given type and return its global index. + fn add_arbitrary_global_of_type( + &mut self, + ty: GlobalType, + u: &mut Unstructured, + ) -> Result { + let expr = self.arbitrary_const_expr(ty.val_type, u)?; + let global_idx = self.globals.len() as u32; + self.globals.push(ty); + self.defined_globals.push((global_idx, expr)); + Ok(global_idx) + } + + /// Generates an arbitrary constant expression of the type `ty`. + fn arbitrary_const_expr(&mut self, ty: ValType, u: &mut Unstructured) -> Result { + let mut choices = mem::take(&mut self.const_expr_choices); + choices.clear(); + let num_funcs = self.funcs.len() as u32; + + // MVP wasm can `global.get` any immutable imported global in a + // constant expression, and the GC proposal enables this for all + // globals, so make all matching globals a candidate. + for i in self.globals_for_const_expr(ty) { + choices.push(Box::new(move |_, _| Ok(ConstExpr::global_get(i)))); + } + + // Another option for all types is to have an actual value of each type. + // Change `ty` to any valid subtype of `ty` and then generate a matching + // type of that value. + let ty = self.arbitrary_matching_val_type(u, ty)?; + match ty { + ValType::I32 => choices.push(Box::new(|u, _| Ok(ConstExpr::i32_const(u.arbitrary()?)))), + ValType::I64 => choices.push(Box::new(|u, _| Ok(ConstExpr::i64_const(u.arbitrary()?)))), + ValType::F32 => choices.push(Box::new(|u, _| Ok(ConstExpr::f32_const(u.arbitrary()?)))), + ValType::F64 => choices.push(Box::new(|u, _| Ok(ConstExpr::f64_const(u.arbitrary()?)))), + ValType::V128 => { + choices.push(Box::new(|u, _| Ok(ConstExpr::v128_const(u.arbitrary()?)))) + } + ValType::Ref(ty) => { + if ty.nullable { + choices.push(Box::new(move |_, _| Ok(ConstExpr::ref_null(ty.heap_type)))); + } + + match ty.heap_type { + HeapType::Func if num_funcs > 0 => { + choices.push(Box::new(move |u, _| { + let func = u.int_in_range(0..=num_funcs - 1)?; + Ok(ConstExpr::ref_func(func)) + })); + } + + HeapType::Concrete(ty) => { + for (i, fty) in self.funcs.iter().map(|(t, _)| *t).enumerate() { + if ty != fty { + continue; + } + choices.push(Box::new(move |_, _| Ok(ConstExpr::ref_func(i as u32)))); + } + } + + // TODO: fill out more GC types e.g `array.new` and + // `struct.new` + _ => {} + } + } + } + + let f = u.choose(&choices)?; + let ret = f(u, ty); + self.const_expr_choices = choices; + ret + } + + fn arbitrary_globals(&mut self, u: &mut Unstructured) -> Result<()> { arbitrary_loop(u, self.config.min_globals, self.config.max_globals, |u| { if !self.can_add_local_or_import_global() { return Ok(false); } let ty = self.arbitrary_global_type(u)?; + self.add_arbitrary_global_of_type(ty, u)?; - choices.clear(); - let num_funcs = self.funcs.len() as u32; - choices.push(Box::new(move |u, ty| { - Ok(GlobalInitExpr::ConstExpr(match ty { - ValType::I32 => ConstExpr::i32_const(u.arbitrary()?), - ValType::I64 => ConstExpr::i64_const(u.arbitrary()?), - ValType::F32 => ConstExpr::f32_const(u.arbitrary()?), - ValType::F64 => ConstExpr::f64_const(u.arbitrary()?), - ValType::V128 => ConstExpr::v128_const(u.arbitrary()?), - ValType::Ref(ty) => { - assert!(ty.nullable); - if ty.heap_type == HeapType::Func && num_funcs > 0 && u.arbitrary()? { - let func = u.int_in_range(0..=num_funcs - 1)?; - return Ok(GlobalInitExpr::FuncRef(func)); + Ok(true) + }) + } + + fn required_exports(&mut self, u: &mut Unstructured) -> Result { + let example_module = if let Some(wasm) = self.config.exports.clone() { + wasm + } else { + return Ok(false); + }; + + #[cfg(feature = "wasmparser")] + { + self._required_exports(u, &example_module)?; + Ok(true) + } + #[cfg(not(feature = "wasmparser"))] + { + let _ = (example_module, u); + panic!("support for `exports` was disabled at compile time"); + } + } + + #[cfg(feature = "wasmparser")] + fn _required_exports(&mut self, u: &mut Unstructured, example_module: &[u8]) -> Result<()> { + fn convert_heap_type(ty: &wasmparser::HeapType) -> HeapType { + match ty { + wasmparser::HeapType::Concrete(_) => { + panic!("Unable to handle concrete types in exports") + } + wasmparser::HeapType::Func => HeapType::Func, + wasmparser::HeapType::Extern => HeapType::Extern, + wasmparser::HeapType::Any => HeapType::Any, + wasmparser::HeapType::None => HeapType::None, + wasmparser::HeapType::NoExtern => HeapType::NoExtern, + wasmparser::HeapType::NoFunc => HeapType::NoFunc, + wasmparser::HeapType::Eq => HeapType::Eq, + wasmparser::HeapType::Struct => HeapType::Struct, + wasmparser::HeapType::Array => HeapType::Array, + wasmparser::HeapType::I31 => HeapType::I31, + wasmparser::HeapType::Exn => HeapType::Exn, + } + } + + fn convert_val_type(ty: &wasmparser::ValType) -> ValType { + match ty { + wasmparser::ValType::I32 => ValType::I32, + wasmparser::ValType::I64 => ValType::I64, + wasmparser::ValType::F32 => ValType::F32, + wasmparser::ValType::F64 => ValType::F64, + wasmparser::ValType::V128 => ValType::V128, + wasmparser::ValType::Ref(r) => ValType::Ref(RefType { + nullable: r.is_nullable(), + heap_type: convert_heap_type(&r.heap_type()), + }), + } + } + + fn convert_export_kind(kind: &wasmparser::ExternalKind) -> ExportKind { + match kind { + wasmparser::ExternalKind::Func => ExportKind::Func, + wasmparser::ExternalKind::Table => ExportKind::Table, + wasmparser::ExternalKind::Memory => ExportKind::Memory, + wasmparser::ExternalKind::Global => ExportKind::Global, + wasmparser::ExternalKind::Tag => ExportKind::Tag, + } + } + + let mut required_exports: Vec = vec![]; + let mut validator = wasmparser::Validator::new(); + let exports_types = validator + .validate_all(&example_module) + .expect("Failed to validate `exports` Wasm"); + for payload in wasmparser::Parser::new(0).parse_all(&example_module) { + match payload.expect("Failed to read `exports` Wasm") { + wasmparser::Payload::ExportSection(export_reader) => { + required_exports = export_reader + .into_iter() + .collect::>() + .expect("Failed to read `exports` export section"); + } + _ => {} + } + } + + // For each export, add necessary prerequisites to the module. + for export in required_exports { + let new_index = match exports_types + .entity_type_from_export(&export) + .unwrap_or_else(|| { + panic!( + "Unable to get type from export {:?} in `exports` Wasm", + export, + ) + }) { + // For functions, add the type and a function with that type. + wasmparser::types::EntityType::Func(id) => { + let subtype = exports_types.get(id).unwrap_or_else(|| { + panic!( + "Unable to get subtype for function {:?} in `exports` Wasm", + id + ) + }); + match &subtype.composite_type { + wasmparser::CompositeType::Func(func_type) => { + assert!( + subtype.is_final, + "Subtype {:?} from `exports` Wasm is not final", + subtype + ); + assert!( + subtype.supertype_idx.is_none(), + "Subtype {:?} from `exports` Wasm has non-empty supertype", + subtype + ); + let new_type = Rc::new(FuncType { + params: func_type + .params() + .into_iter() + .map(convert_val_type) + .collect(), + results: func_type + .results() + .into_iter() + .map(convert_val_type) + .collect(), + }); + self.rec_groups.push(self.types.len()..self.types.len() + 1); + let type_index = self.add_type(SubType { + is_final: true, + supertype: None, + composite_type: CompositeType::Func(Rc::clone(&new_type)), + }); + let func_index = self.funcs.len() as u32; + self.funcs.push((type_index, new_type)); + self.num_defined_funcs += 1; + func_index } - ConstExpr::ref_null(ty.heap_type) + _ => panic!( + "Unable to handle type {:?} from `exports` Wasm", + subtype.composite_type + ), } - })) - })); - - for (i, g) in self.globals[..num_imported_globals].iter().enumerate() { - if !g.mutable && g.val_type == ty.val_type { - choices.push(Box::new(move |_, _| { - Ok(GlobalInitExpr::ConstExpr(ConstExpr::global_get(i as u32))) - })); } - } + // For globals, add a new global. + wasmparser::types::EntityType::Global(global_type) => self + .add_arbitrary_global_of_type( + GlobalType { + val_type: convert_val_type(&global_type.content_type), + mutable: global_type.mutable, + }, + u, + )?, + wasmparser::types::EntityType::Table(_) + | wasmparser::types::EntityType::Memory(_) + | wasmparser::types::EntityType::Tag(_) => { + panic!( + "Config `exports` has an export of type {:?} which cannot yet be handled.", + export.kind + ) + } + }; + self.exports.push(( + export.name.to_string(), + convert_export_kind(&export.kind), + new_index, + )); + self.export_names.insert(export.name.to_string()); + } - let f = u.choose(&choices)?; - let expr = f(u, ty.val_type)?; - let global_idx = self.globals.len() as u32; - self.globals.push(ty); - self.defined_globals.push((global_idx, expr)); - Ok(true) - }) + Ok(()) } fn arbitrary_exports(&mut self, u: &mut Unstructured) -> Result<()> { @@ -1644,24 +1878,17 @@ impl Module { } fn arbitrary_elems(&mut self, u: &mut Unstructured) -> Result<()> { - let func_max = self.funcs.len() as u32; - // Create a helper closure to choose an arbitrary offset. let mut offset_global_choices = vec![]; if !self.config.disallow_traps { - for (i, g) in self.globals[..self.globals.len() - self.defined_globals.len()] - .iter() - .enumerate() - { - if !g.mutable && g.val_type == ValType::I32 { - offset_global_choices.push(i as u32); - } + for i in self.globals_for_const_expr(ValType::I32) { + offset_global_choices.push(i); } } + let disallow_traps = self.config.disallow_traps; let arbitrary_active_elem = |u: &mut Unstructured, min_mem_size: u32, table: Option, - disallow_traps: bool, table_ty: &TableType| { let (offset, max_size_hint) = if !offset_global_choices.is_empty() && u.arbitrary()? { let g = u.choose(&offset_global_choices)?; @@ -1686,11 +1913,20 @@ impl Module { Ok((ElementKind::Active { table, offset }, max_size_hint)) }; + // Generate a list of candidates for "kinds" of elements segments. For + // example we can have an active segment for any existing table or + // passive/declared segments if the right wasm features are enabled. type GenElemSegment<'a> = dyn Fn(&mut Unstructured) -> Result<(ElementKind, Option)> + 'a; - let mut funcrefs: Vec> = Vec::new(); - let mut externrefs: Vec> = Vec::new(); - let disallow_traps = self.config.disallow_traps; + let mut choices: Vec> = Vec::new(); + + // Bulk memory enables passive/declared segments, and note that the + // types used are selected later. + if self.config.bulk_memory_enabled { + choices.push(Box::new(|_| Ok((ElementKind::Passive, None)))); + choices.push(Box::new(|_| Ok((ElementKind::Declared, None)))); + } + for (i, ty) in self.tables.iter().enumerate() { // If this table starts with no capacity then any non-empty element // segment placed onto it will immediately trap, which isn't too @@ -1700,93 +1936,100 @@ impl Module { continue; } - let dst = if ty.element_type == RefType::FUNCREF { - &mut funcrefs - } else { - &mut externrefs - }; let minimum = ty.minimum; // If the first table is a funcref table then it's a candidate for // the MVP encoding of element segments. + let ty = *ty; if i == 0 && ty.element_type == RefType::FUNCREF { - dst.push(Box::new(move |u| { - arbitrary_active_elem(u, minimum, None, disallow_traps, ty) + choices.push(Box::new(move |u| { + arbitrary_active_elem(u, minimum, None, &ty) })); } if self.config.bulk_memory_enabled { let idx = Some(i as u32); - dst.push(Box::new(move |u| { - arbitrary_active_elem(u, minimum, idx, disallow_traps, ty) + choices.push(Box::new(move |u| { + arbitrary_active_elem(u, minimum, idx, &ty) })); } } - // Bulk memory enables passive/declared segments for funcrefs, and - // reference types additionally enables the segments for externrefs. - if self.config.bulk_memory_enabled { - funcrefs.push(Box::new(|_| Ok((ElementKind::Passive, None)))); - funcrefs.push(Box::new(|_| Ok((ElementKind::Declared, None)))); - if self.config.reference_types_enabled { - externrefs.push(Box::new(|_| Ok((ElementKind::Passive, None)))); - externrefs.push(Box::new(|_| Ok((ElementKind::Declared, None)))); - } - } - - let mut choices = Vec::new(); - if !funcrefs.is_empty() { - choices.push((&funcrefs, RefType::FUNCREF)); - } - if !externrefs.is_empty() { - choices.push((&externrefs, RefType::EXTERNREF)); - } - if choices.is_empty() { return Ok(()); } + arbitrary_loop( u, self.config.min_element_segments, self.config.max_element_segments, |u| { - // Choose whether to generate a segment whose elements are initialized via - // expressions, or one whose elements are initialized via function indices. - let (kind_candidates, ty) = *u.choose(&choices)?; - - // Select a kind for this segment now that we know the number of - // items the segment will hold. - let (kind, max_size_hint) = u.choose(kind_candidates)?(u)?; + // Pick a kind of element segment to generate which will also + // give us a hint of the maximum size, if any. + let (kind, max_size_hint) = u.choose(&choices)?(u)?; let max = max_size_hint .map(|i| usize::try_from(i).unwrap()) .unwrap_or_else(|| self.config.max_elements); - // Pick whether we're going to use expression elements or - // indices. Note that externrefs must use expressions, - // and functions without reference types must use indices. - let items = if ty == RefType::EXTERNREF - || (self.config.reference_types_enabled && u.arbitrary()?) + // Infer, from the kind of segment, the type of the element + // segment. Passive/declared segments can be declared with any + // reference type, but active segments must match their table. + let ty = match kind { + ElementKind::Passive | ElementKind::Declared => self.arbitrary_ref_type(u)?, + ElementKind::Active { table, .. } => { + let idx = table.unwrap_or(0); + self.arbitrary_matching_ref_type(u, self.tables[idx as usize].element_type)? + } + }; + + // The `Elements::Functions` encoding is only possible when the + // element type is a `funcref` because the binary format can't + // allow encoding any other type in that form. + let can_use_function_list = ty == RefType::FUNCREF; + if !self.config.reference_types_enabled { + assert!(can_use_function_list); + } + + // If a function list is possible then build up a list of + // functions that can be selected from. + let mut func_candidates = Vec::new(); + if can_use_function_list { + match ty.heap_type { + HeapType::Func => { + func_candidates.extend(0..self.funcs.len() as u32); + } + HeapType::Concrete(ty) => { + for (i, (fty, _)) in self.funcs.iter().enumerate() { + if *fty == ty { + func_candidates.push(i as u32); + } + } + } + _ => {} + } + } + + // And finally actually generate the arbitrary elements of this + // element segment. Function indices are used if they're either + // forced or allowed, and otherwise expressions are used + // instead. + let items = if !self.config.reference_types_enabled + || (can_use_function_list && u.arbitrary()?) { let mut init = vec![]; - arbitrary_loop(u, self.config.min_elements, max, |u| { - init.push( - if ty == RefType::EXTERNREF || func_max == 0 || u.arbitrary()? { - None - } else { - Some(u.int_in_range(0..=func_max - 1)?) - }, - ); - Ok(true) - })?; - Elements::Expressions(init) - } else { - let mut init = vec![]; - if func_max > 0 { + if func_candidates.len() > 0 { arbitrary_loop(u, self.config.min_elements, max, |u| { - let func_idx = u.int_in_range(0..=func_max - 1)?; + let func_idx = *u.choose(&func_candidates)?; init.push(func_idx); Ok(true) })?; } Elements::Functions(init) + } else { + let mut init = vec![]; + arbitrary_loop(u, self.config.min_elements, max, |u| { + init.push(self.arbitrary_const_expr(ValType::Ref(ty), u)?); + Ok(true) + })?; + Elements::Expressions(init) }; self.elems.push(ElementSegment { kind, ty, items }); @@ -1795,11 +2038,11 @@ impl Module { ) } - fn arbitrary_code(&mut self, u: &mut Unstructured, allow_invalid: bool) -> Result<()> { + fn arbitrary_code(&mut self, u: &mut Unstructured) -> Result<()> { self.code.reserve(self.num_defined_funcs); - let mut allocs = CodeBuilderAllocations::new(self); + let mut allocs = CodeBuilderAllocations::new(self, self.config.exports.is_some()); for (_, ty) in self.funcs[self.funcs.len() - self.num_defined_funcs..].iter() { - let body = self.arbitrary_func_body(u, ty, &mut allocs, allow_invalid)?; + let body = self.arbitrary_func_body(u, ty, &mut allocs)?; self.code.push(body); } allocs.finish(u, self)?; @@ -1811,11 +2054,10 @@ impl Module { u: &mut Unstructured, ty: &FuncType, allocs: &mut CodeBuilderAllocations, - allow_invalid: bool, ) -> Result { let mut locals = self.arbitrary_locals(u)?; let builder = allocs.builder(ty, &mut locals); - let instructions = if allow_invalid && u.arbitrary().unwrap_or(false) { + let instructions = if self.config.allow_invalid_funcs && u.arbitrary().unwrap_or(false) { Instructions::Arbitrary(arbitrary_vec_u8(u)?) } else { Instructions::Generated(builder.arbitrary(u, self)?) @@ -1830,7 +2072,7 @@ impl Module { fn arbitrary_locals(&self, u: &mut Unstructured) -> Result> { let mut ret = Vec::new(); arbitrary_loop(u, 0, 100, |u| { - ret.push(self.arbitrary_valtype(u, u32::try_from(self.types.len()).unwrap())?); + ret.push(self.arbitrary_valtype(u)?); Ok(true) })?; Ok(ret) @@ -1865,18 +2107,11 @@ impl Module { )) })); if !self.config.disallow_traps { - for (i, g) in self.globals[..self.globals.len() - self.defined_globals.len()] - .iter() - .enumerate() - { - if g.mutable { - continue; - } - if g.val_type == ValType::I32 { - choices32.push(Box::new(move |_, _, _| Ok(Offset::Global(i as u32)))); - } else if g.val_type == ValType::I64 { - choices64.push(Box::new(move |_, _, _| Ok(Offset::Global(i as u32)))); - } + for i in self.globals_for_const_expr(ValType::I32) { + choices32.push(Box::new(move |_, _, _| Ok(Offset::Global(i)))); + } + for i in self.globals_for_const_expr(ValType::I64) { + choices64.push(Box::new(move |_, _, _| Ok(Offset::Global(i)))); } } @@ -1962,6 +2197,33 @@ impl Module { } } } + + /// Returns an iterator of all globals which can be used in constant + /// expressions for a value of type `ty` specified. + fn globals_for_const_expr(&self, ty: ValType) -> impl Iterator + '_ { + // Before the GC proposal only imported globals could be referenced, but + // the GC proposal relaxed this feature to allow any global. + let num_imported_globals = self.globals.len() - self.defined_globals.len(); + let max_global = if self.config.gc_enabled { + self.globals.len() + } else { + num_imported_globals + }; + + self.globals[..max_global] + .iter() + .enumerate() + .filter_map(move |(i, g)| { + // Mutable globals cannot participate in constant expressions, + // but otherwise so long as the global is a subtype of the + // desired type it's a candidate. + if !g.mutable && self.val_type_is_sub_type(g.val_type, ty) { + Some(i as u32) + } else { + None + } + }) + } } pub(crate) fn arbitrary_limits32( @@ -2044,49 +2306,11 @@ pub(crate) fn configured_valtypes(config: &Config) -> Vec { valtypes } -pub(crate) fn arbitrary_func_type( - u: &mut Unstructured, - config: &Config, - valtypes: &[ValType], - max_results: Option, - type_ref_limit: u32, -) -> Result> { - let mut params = vec![]; - let mut results = vec![]; - arbitrary_loop(u, 0, 20, |u| { - params.push(arbitrary_valtype(u, config, valtypes, type_ref_limit)?); - Ok(true) - })?; - arbitrary_loop(u, 0, max_results.unwrap_or(20), |u| { - results.push(arbitrary_valtype(u, config, valtypes, type_ref_limit)?); - Ok(true) - })?; - Ok(Rc::new(FuncType { params, results })) -} - -fn arbitrary_valtype( +pub(crate) fn arbitrary_table_type( u: &mut Unstructured, config: &Config, - valtypes: &[ValType], - type_ref_limit: u32, -) -> Result { - if config.gc_enabled && type_ref_limit > 0 && u.ratio(1, 20)? { - Ok(ValType::Ref(RefType { - // TODO: For now, only create allow nullable reference - // types. Eventually we should support non-nullable reference types, - // but this means that we will also need to recognize when it is - // impossible to create an instance of the reference (eg `(ref - // nofunc)` has no instances, and self-referential types that - // contain a non-null self-reference are also impossible to create). - nullable: true, - heap_type: HeapType::Concrete(u.int_in_range(0..=type_ref_limit - 1)?), - })) - } else { - Ok(*u.choose(valtypes)?) - } -} - -pub(crate) fn arbitrary_table_type(u: &mut Unstructured, config: &Config) -> Result { + module: Option<&Module>, +) -> Result { // We don't want to generate tables that are too large on average, so // keep the "inbounds" limit here a bit smaller. let max_inbounds = 10_000; @@ -2102,12 +2326,12 @@ pub(crate) fn arbitrary_table_type(u: &mut Unstructured, config: &Config) -> Res if config.disallow_traps { assert!(minimum > 0); } + let element_type = match module { + Some(module) => module.arbitrary_ref_type(u)?, + None => RefType::FUNCREF, + }; Ok(TableType { - element_type: if config.reference_types_enabled { - *u.choose(&[RefType::FUNCREF, RefType::EXTERNREF])? - } else { - RefType::FUNCREF - }, + element_type, minimum, maximum, }) diff --git a/third_party/rust/wasm-smith/src/core/code_builder.rs b/third_party/rust/wasm-smith/src/core/code_builder.rs index 1bc09008ad..a55c5aafda 100644 --- a/third_party/rust/wasm-smith/src/core/code_builder.rs +++ b/third_party/rust/wasm-smith/src/core/code_builder.rs @@ -1,11 +1,10 @@ use super::{ - CompositeType, Elements, FuncType, GlobalInitExpr, Instruction, InstructionKind::*, - InstructionKinds, Module, ValType, + CompositeType, Elements, FuncType, Instruction, InstructionKind::*, InstructionKinds, Module, + ValType, }; use crate::{unique_string, MemoryOffsetChoices}; use arbitrary::{Result, Unstructured}; use std::collections::{BTreeMap, BTreeSet}; -use std::convert::TryFrom; use std::rc::Rc; use wasm_encoder::{ ArrayType, BlockType, Catch, ConstExpr, ExportKind, FieldType, GlobalType, HeapType, MemArg, @@ -648,6 +647,10 @@ pub(crate) struct CodeBuilderAllocations { global_dropped_f32: Option, global_dropped_f64: Option, global_dropped_v128: Option, + + // Indicates that additional exports cannot be generated. This will be true + // if the `Config` specifies exactly which exports should be present. + disallow_exporting: bool, } pub(crate) struct CodeBuilder<'a> { @@ -704,7 +707,7 @@ enum Float { } impl CodeBuilderAllocations { - pub(crate) fn new(module: &Module) -> Self { + pub(crate) fn new(module: &Module, disallow_exporting: bool) -> Self { let mut mutable_globals = BTreeMap::new(); for (i, global) in module.globals.iter().enumerate() { if global.mutable { @@ -741,14 +744,14 @@ impl CodeBuilderAllocations { let mut referenced_functions = BTreeSet::new(); for (_, expr) in module.defined_globals.iter() { - if let GlobalInitExpr::FuncRef(i) = *expr { + if let Some(i) = expr.get_ref_func() { referenced_functions.insert(i); } } for g in module.elems.iter() { match &g.items { Elements::Expressions(e) => { - let iter = e.iter().filter_map(|i| *i); + let iter = e.iter().filter_map(|e| e.get_ref_func()); referenced_functions.extend(iter); } Elements::Functions(e) => { @@ -769,6 +772,42 @@ impl CodeBuilderAllocations { } } + let mut global_dropped_i32 = None; + let mut global_dropped_i64 = None; + let mut global_dropped_f32 = None; + let mut global_dropped_f64 = None; + let mut global_dropped_v128 = None; + + // If we can't export additional globals, try to use existing exported + // mutable globals for dropped values. + if disallow_exporting { + for (_, kind, index) in module.exports.iter() { + if *kind == ExportKind::Global { + let ty = module.globals[*index as usize]; + if ty.mutable { + match ty.val_type { + ValType::I32 => { + if global_dropped_i32.is_none() { + global_dropped_i32 = Some(*index) + } else { + global_dropped_f32 = Some(*index) + } + } + ValType::I64 => { + if global_dropped_i64.is_none() { + global_dropped_i64 = Some(*index) + } else { + global_dropped_f64 = Some(*index) + } + } + ValType::V128 => global_dropped_v128 = Some(*index), + _ => {} + } + } + } + } + } + CodeBuilderAllocations { controls: Vec::with_capacity(4), operands: Vec::with_capacity(16), @@ -782,13 +821,14 @@ impl CodeBuilderAllocations { memory32, memory64, - global_dropped_i32: None, - global_dropped_i64: None, - global_dropped_f32: None, - global_dropped_f64: None, - global_dropped_v128: None, + global_dropped_i32, + global_dropped_i64, + global_dropped_f32, + global_dropped_f64, + global_dropped_v128, globals_cnt: module.globals.len() as u32, new_globals: Vec::new(), + disallow_exporting, } } @@ -822,18 +862,18 @@ impl CodeBuilderAllocations { pub fn finish(self, u: &mut Unstructured<'_>, module: &mut Module) -> arbitrary::Result<()> { // Any globals injected as part of dropping operands on the stack get // injected into the module here. Each global is then exported, most of - // the time, to ensure it's part of the "image" of this module available - // for differential execution for example. + // the time (if additional exports are allowed), to ensure it's part of + // the "image" of this module available for differential execution for + // example. for (ty, init) in self.new_globals { let global_idx = module.globals.len() as u32; module.globals.push(GlobalType { val_type: ty, mutable: true, }); - let init = GlobalInitExpr::ConstExpr(init); module.defined_globals.push((global_idx, init)); - if u.ratio(1, 100).unwrap_or(false) { + if self.disallow_exporting || u.ratio(1, 100).unwrap_or(false) { continue; } @@ -922,6 +962,33 @@ impl CodeBuilder<'_> { self.allocs.operands.push(ty); } + fn pop_label_types(&mut self, module: &Module, target: u32) { + let target = usize::try_from(target).unwrap(); + let control = &self.allocs.controls[self.allocs.controls.len() - 1 - target]; + debug_assert!(self.label_types_on_stack(module, control)); + self.allocs + .operands + .truncate(self.allocs.operands.len() - control.label_types().len()); + } + + fn push_label_types(&mut self, target: u32) { + let target = usize::try_from(target).unwrap(); + let control = &self.allocs.controls[self.allocs.controls.len() - 1 - target]; + self.allocs + .operands + .extend(control.label_types().iter().copied().map(Some)); + } + + /// Pop the target label types, and then push them again. + /// + /// This is not a no-op due to subtyping: if we have a `T <: U` on the + /// stack, and the target label's type is `[U]`, then this will erase the + /// information about `T` and subsequent operations may only operate on `U`. + fn pop_push_label_types(&mut self, module: &Module, target: u32) { + self.pop_label_types(module, target); + self.push_label_types(target) + } + fn label_types_on_stack(&self, module: &Module, to_check: &Control) -> bool { self.types_on_stack(module, to_check.label_types()) } @@ -1087,12 +1154,7 @@ impl CodeBuilder<'_> { fn arbitrary_block_type(&self, u: &mut Unstructured, module: &Module) -> Result { let mut options: Vec Result>> = vec![ Box::new(|_| Ok(BlockType::Empty)), - Box::new(|u| { - Ok(BlockType::Result(module.arbitrary_valtype( - u, - u32::try_from(module.types.len()).unwrap(), - )?)) - }), + Box::new(|u| Ok(BlockType::Result(module.arbitrary_valtype(u)?))), ]; if module.config.multi_value_enabled { for (i, ty) in module.func_types() { @@ -1710,10 +1772,9 @@ fn br( .filter(|(_, l)| builder.label_types_on_stack(module, l)) .nth(i) .unwrap(); - let control = &builder.allocs.controls[builder.allocs.controls.len() - 1 - target]; - let tys = control.label_types().to_vec(); - builder.pop_operands(module, &tys); - instructions.push(Instruction::Br(target as u32)); + let target = u32::try_from(target).unwrap(); + builder.pop_label_types(module, target); + instructions.push(Instruction::Br(target)); Ok(()) } @@ -1757,7 +1818,9 @@ fn br_if( .filter(|(_, l)| builder.label_types_on_stack(module, l)) .nth(i) .unwrap(); - instructions.push(Instruction::BrIf(target as u32)); + let target = u32::try_from(target).unwrap(); + builder.pop_push_label_types(module, target); + instructions.push(Instruction::BrIf(target)); Ok(()) } @@ -2203,13 +2266,15 @@ fn br_on_null( .filter(|(_, l)| builder.label_types_on_stack(module, l)) .nth(i) .unwrap(); + let target = u32::try_from(target).unwrap(); + builder.pop_push_label_types(module, target); builder.push_operands(&[ValType::Ref(RefType { nullable: false, heap_type, })]); - instructions.push(Instruction::BrOnNull(u32::try_from(target).unwrap())); + instructions.push(Instruction::BrOnNull(target)); Ok(()) } @@ -2270,9 +2335,11 @@ fn br_on_non_null( .filter(|(_, l)| is_valid_br_on_non_null_control(module, l, builder)) .nth(i) .unwrap(); + let target = u32::try_from(target).unwrap(); + builder.pop_push_label_types(module, target); builder.pop_ref_type(); - instructions.push(Instruction::BrOnNonNull(u32::try_from(target).unwrap())); + instructions.push(Instruction::BrOnNonNull(target)); Ok(()) } @@ -2354,6 +2421,7 @@ fn br_on_cast( .unwrap(); let relative_depth = u32::try_from(relative_depth).unwrap(); + let num_label_types = control.label_types().len(); let to_ref_type = match control.label_types().last() { Some(ValType::Ref(r)) => *r, _ => unreachable!(), @@ -2363,6 +2431,15 @@ fn br_on_cast( let from_ref_type = from_ref_type.unwrap_or(to_ref_type); let from_ref_type = module.arbitrary_super_type_of_ref_type(u, from_ref_type)?; + // Do `pop_push_label_types` but without its debug assert that the types are + // on the stack, since we know that we have a `from_ref_type` but the label + // requires a `to_ref_type`. + for _ in 0..num_label_types { + builder.pop_operand(); + } + builder.push_label_types(relative_depth); + + // Replace the label's `to_ref_type` with the type difference. builder.pop_operand(); builder.push_operands(&[ValType::Ref(ref_type_difference( from_ref_type, @@ -2433,7 +2510,7 @@ fn br_on_cast_fail( debug_assert!(n > 0); let i = u.int_in_range(0..=n - 1)?; - let (target, control) = builder + let (relative_depth, control) = builder .allocs .controls .iter() @@ -2442,6 +2519,7 @@ fn br_on_cast_fail( .filter(|(_, l)| is_valid_br_on_cast_fail_control(module, builder, l, from_ref_type)) .nth(i) .unwrap(); + let relative_depth = u32::try_from(relative_depth).unwrap(); let from_ref_type = from_ref_type.unwrap_or_else(|| match control.label_types().last().unwrap() { @@ -2450,13 +2528,16 @@ fn br_on_cast_fail( }); let to_ref_type = module.arbitrary_matching_ref_type(u, from_ref_type)?; + // Pop-push the label types and then replace its last reference type with + // our `to_ref_type`. + builder.pop_push_label_types(module, relative_depth); builder.pop_operand(); builder.push_operand(Some(ValType::Ref(to_ref_type))); instructions.push(Instruction::BrOnCastFail { from_ref_type, to_ref_type, - relative_depth: u32::try_from(target).unwrap(), + relative_depth, }); Ok(()) } diff --git a/third_party/rust/wasm-smith/src/core/code_builder/no_traps.rs b/third_party/rust/wasm-smith/src/core/code_builder/no_traps.rs index a1232b6922..4f8b31ca9f 100644 --- a/third_party/rust/wasm-smith/src/core/code_builder/no_traps.rs +++ b/third_party/rust/wasm-smith/src/core/code_builder/no_traps.rs @@ -1,5 +1,5 @@ use crate::core::*; -use wasm_encoder::{BlockType, Instruction, ValType}; +use wasm_encoder::Instruction; use super::CodeBuilder; diff --git a/third_party/rust/wasm-smith/src/core/encode.rs b/third_party/rust/wasm-smith/src/core/encode.rs index 7c781dc731..5a661509a6 100644 --- a/third_party/rust/wasm-smith/src/core/encode.rs +++ b/third_party/rust/wasm-smith/src/core/encode.rs @@ -1,5 +1,4 @@ use super::*; -use std::convert::TryFrom; impl Module { /// Encode this Wasm module into bytes. @@ -122,12 +121,22 @@ impl Module { } fn encode_tables(&self, module: &mut wasm_encoder::Module) { - if self.num_defined_tables == 0 { + if self.defined_tables.is_empty() { return; } let mut tables = wasm_encoder::TableSection::new(); - for t in self.tables[self.tables.len() - self.num_defined_tables..].iter() { - tables.table(*t); + for (t, init) in self.tables[self.tables.len() - self.defined_tables.len()..] + .iter() + .zip(&self.defined_tables) + { + match init { + Some(init) => { + tables.table_with_init(*t, init); + } + None => { + tables.table(*t); + } + } } module.section(&tables); } @@ -150,10 +159,7 @@ impl Module { let mut globals = wasm_encoder::GlobalSection::new(); for (idx, expr) in &self.defined_globals { let ty = &self.globals[*idx as usize]; - match expr { - GlobalInitExpr::ConstExpr(expr) => globals.global(*ty, expr), - GlobalInitExpr::FuncRef(func) => globals.global(*ty, &ConstExpr::ref_func(*func)), - }; + globals.global(*ty, expr); } module.section(&globals); } @@ -180,24 +186,13 @@ impl Module { return; } let mut elems = wasm_encoder::ElementSection::new(); - let mut exps = vec![]; for el in &self.elems { let elements = match &el.items { - Elements::Expressions(es) => { - exps.clear(); - exps.extend(es.iter().map(|e| { - // TODO(nagisa): generate global.get of imported ref globals too. - match e { - Some(i) => match el.ty { - RefType::FUNCREF => wasm_encoder::ConstExpr::ref_func(*i), - _ => unreachable!(), - }, - None => wasm_encoder::ConstExpr::ref_null(el.ty.heap_type), - } - })); - wasm_encoder::Elements::Expressions(el.ty, &exps) + Elements::Expressions(es) => wasm_encoder::Elements::Expressions(el.ty, es), + Elements::Functions(fs) => { + assert_eq!(el.ty, RefType::FUNCREF); + wasm_encoder::Elements::Functions(fs) } - Elements::Functions(fs) => wasm_encoder::Elements::Functions(fs), }; match &el.kind { ElementKind::Active { table, offset } => { diff --git a/third_party/rust/wasm-smith/src/core/terminate.rs b/third_party/rust/wasm-smith/src/core/terminate.rs index adcfeed54f..7983c35be6 100644 --- a/third_party/rust/wasm-smith/src/core/terminate.rs +++ b/third_party/rust/wasm-smith/src/core/terminate.rs @@ -1,6 +1,5 @@ use super::*; -use std::mem; -use wasm_encoder::BlockType; +use anyhow::{bail, Result}; impl Module { /// Ensure that all of this Wasm module's functions will terminate when @@ -12,16 +11,21 @@ impl Module { /// /// The index of the fuel global is returned, so that you may control how /// much fuel the module is given. - pub fn ensure_termination(&mut self, default_fuel: u32) -> u32 { + /// + /// # Errors + /// + /// Returns an error if any function body was generated with + /// possibly-invalid bytes rather than being generated by wasm-smith. In + /// such a situation this pass does not parse the input bytes and inject + /// instructions, instead it returns an error. + pub fn ensure_termination(&mut self, default_fuel: u32) -> Result { let fuel_global = self.globals.len() as u32; self.globals.push(GlobalType { val_type: ValType::I32, mutable: true, }); - self.defined_globals.push(( - fuel_global, - GlobalInitExpr::ConstExpr(ConstExpr::i32_const(default_fuel as i32)), - )); + self.defined_globals + .push((fuel_global, ConstExpr::i32_const(default_fuel as i32))); for code in &mut self.code { let check_fuel = |insts: &mut Vec| { @@ -41,10 +45,12 @@ impl Module { let instrs = match &mut code.instructions { Instructions::Generated(list) => list, - // only present on modules contained within - // `MaybeInvalidModule`, which doesn't expose its internal - // `Module`. - Instructions::Arbitrary(_) => unreachable!(), + Instructions::Arbitrary(_) => { + bail!( + "failed to ensure that a function generated due to it \ + containing arbitrary instructions" + ) + } }; let mut new_insts = Vec::with_capacity(instrs.len() * 2); @@ -65,6 +71,6 @@ impl Module { *instrs = new_insts; } - fuel_global + Ok(fuel_global) } } diff --git a/third_party/rust/wasm-smith/src/lib.rs b/third_party/rust/wasm-smith/src/lib.rs index 8d17290473..b985791aae 100644 --- a/third_party/rust/wasm-smith/src/lib.rs +++ b/third_party/rust/wasm-smith/src/lib.rs @@ -58,7 +58,7 @@ mod component; mod config; mod core; -pub use crate::core::{InstructionKind, InstructionKinds, MaybeInvalidModule, Module}; +pub use crate::core::{InstructionKind, InstructionKinds, Module}; use arbitrary::{Result, Unstructured}; pub use component::Component; pub use config::{Config, MemoryOffsetChoices}; @@ -108,10 +108,7 @@ pub(crate) fn limited_str<'a>(max_size: usize, u: &mut Unstructured<'a>) -> Resu Err(e) => { let i = e.valid_up_to(); let valid = u.bytes(i).unwrap(); - let s = unsafe { - debug_assert!(str::from_utf8(valid).is_ok()); - str::from_utf8_unchecked(valid) - }; + let s = str::from_utf8(valid).unwrap(); Ok(s) } } diff --git a/third_party/rust/wasm-smith/tests/available_imports.rs b/third_party/rust/wasm-smith/tests/available_imports.rs index 2ccc4c1cea..27e4852da9 100644 --- a/third_party/rust/wasm-smith/tests/available_imports.rs +++ b/third_party/rust/wasm-smith/tests/available_imports.rs @@ -4,8 +4,11 @@ use arbitrary::{Arbitrary, Unstructured}; use rand::{rngs::SmallRng, RngCore, SeedableRng}; use std::collections::HashMap; use wasm_smith::{Config, Module}; +use wasmparser::Validator; use wasmparser::{Parser, TypeRef, ValType}; -use wasmparser::{Validator, WasmFeatures}; + +mod common; +use common::{parser_features_from_config, validate}; #[test] fn smoke_test_imports_config() { @@ -159,44 +162,3 @@ fn import_config( ); (config, available) } - -fn parser_features_from_config(config: &Config) -> WasmFeatures { - WasmFeatures { - mutable_global: true, - saturating_float_to_int: config.saturating_float_to_int_enabled, - sign_extension: config.sign_extension_ops_enabled, - reference_types: config.reference_types_enabled, - multi_value: config.multi_value_enabled, - bulk_memory: config.bulk_memory_enabled, - simd: config.simd_enabled, - relaxed_simd: config.relaxed_simd_enabled, - multi_memory: config.max_memories > 1, - exceptions: config.exceptions_enabled, - memory64: config.memory64_enabled, - tail_call: config.tail_call_enabled, - function_references: config.gc_enabled, - gc: config.gc_enabled, - - threads: false, - floats: true, - extended_const: false, - component_model: false, - memory_control: false, - component_model_values: false, - component_model_nested_names: false, - } -} - -fn validate(validator: &mut Validator, bytes: &[u8]) { - let err = match validator.validate_all(bytes) { - Ok(_) => return, - Err(e) => e, - }; - eprintln!("Writing Wasm to `test.wasm`"); - drop(std::fs::write("test.wasm", &bytes)); - if let Ok(text) = wasmprinter::print_bytes(bytes) { - eprintln!("Writing WAT to `test.wat`"); - drop(std::fs::write("test.wat", &text)); - } - panic!("wasm failed to validate: {}", err); -} diff --git a/third_party/rust/wasm-smith/tests/common/mod.rs b/third_party/rust/wasm-smith/tests/common/mod.rs new file mode 100644 index 0000000000..cc24eed278 --- /dev/null +++ b/third_party/rust/wasm-smith/tests/common/mod.rs @@ -0,0 +1,43 @@ +use wasm_smith::Config; +use wasmparser::{types::Types, Validator, WasmFeatures}; + +pub fn parser_features_from_config(config: &Config) -> WasmFeatures { + WasmFeatures { + mutable_global: true, + saturating_float_to_int: config.saturating_float_to_int_enabled, + sign_extension: config.sign_extension_ops_enabled, + reference_types: config.reference_types_enabled, + multi_value: config.multi_value_enabled, + bulk_memory: config.bulk_memory_enabled, + simd: config.simd_enabled, + relaxed_simd: config.relaxed_simd_enabled, + multi_memory: config.max_memories > 1, + exceptions: config.exceptions_enabled, + memory64: config.memory64_enabled, + tail_call: config.tail_call_enabled, + function_references: config.gc_enabled, + gc: config.gc_enabled, + + threads: false, + floats: true, + extended_const: false, + component_model: false, + memory_control: false, + component_model_values: false, + component_model_nested_names: false, + } +} + +pub fn validate(validator: &mut Validator, bytes: &[u8]) -> Types { + let err = match validator.validate_all(bytes) { + Ok(types) => return types, + Err(e) => e, + }; + eprintln!("Writing Wasm to `test.wasm`"); + drop(std::fs::write("test.wasm", &bytes)); + if let Ok(text) = wasmprinter::print_bytes(bytes) { + eprintln!("Writing WAT to `test.wat`"); + drop(std::fs::write("test.wat", &text)); + } + panic!("wasm failed to validate: {}", err); +} diff --git a/third_party/rust/wasm-smith/tests/core.rs b/third_party/rust/wasm-smith/tests/core.rs index 5816f8e38b..7286952dc6 100644 --- a/third_party/rust/wasm-smith/tests/core.rs +++ b/third_party/rust/wasm-smith/tests/core.rs @@ -27,7 +27,7 @@ fn smoke_test_ensure_termination() { rng.fill_bytes(&mut buf); let u = Unstructured::new(&buf); if let Ok(mut module) = Module::arbitrary_take_rest(u) { - module.ensure_termination(10); + module.ensure_termination(10).unwrap(); let wasm_bytes = module.to_bytes(); let mut validator = Validator::new_with_features(wasm_features()); @@ -128,6 +128,7 @@ fn smoke_test_wasm_gc() { let mut u = Unstructured::new(&buf); let config = Config { gc_enabled: true, + reference_types_enabled: true, ..Config::default() }; if let Ok(module) = Module::new(config, &mut u) { diff --git a/third_party/rust/wasm-smith/tests/exports.rs b/third_party/rust/wasm-smith/tests/exports.rs new file mode 100644 index 0000000000..ff1dac0cbe --- /dev/null +++ b/third_party/rust/wasm-smith/tests/exports.rs @@ -0,0 +1,147 @@ +use arbitrary::{Arbitrary, Unstructured}; +use rand::{rngs::SmallRng, RngCore, SeedableRng}; +use wasm_smith::{Config, Module}; +use wasmparser::{ + types::EntityType, CompositeType, FuncType, GlobalType, Parser, Validator, WasmFeatures, +}; + +mod common; +use common::{parser_features_from_config, validate}; + +#[derive(Debug, PartialEq)] +enum ExportType { + Func(FuncType), + Global(GlobalType), +} + +#[test] +fn smoke_test_single_export() { + let test = r#" + (module + (func (export "foo") (param i32) (result i64) + unreachable + ) + ) + "#; + smoke_test_exports(test, 11) +} + +#[test] +fn smoke_test_multiple_exports() { + let test = r#" + (module + (func (export "a") (param i32) (result i64) + unreachable + ) + (func (export "b") + unreachable + ) + (func (export "c") + unreachable + ) + ) + "#; + smoke_test_exports(test, 12) +} + +#[test] +fn smoke_test_exported_global() { + let test = r#" + (module + (func (export "a") (param i32 i32 f32 f64) (result f32) + unreachable + ) + (global (export "glob") f64 f64.const 0) + ) + "#; + smoke_test_exports(test, 20) +} + +#[test] +fn smoke_test_export_with_imports() { + let test = r#" + (module + (import "" "foo" (func (param i32))) + (import "" "bar" (global (mut f32))) + (func (param i64) unreachable) + (global i32 (i32.const 0)) + (export "a" (func 0)) + (export "b" (global 0)) + (export "c" (func 1)) + (export "d" (global 1)) + ) + "#; + smoke_test_exports(test, 21) +} + +#[test] +fn smoke_test_with_mutable_global_exports() { + let test = r#" + (module + (global (export "1i32") (mut i32) (i32.const 0)) + (global (export "2i32") (mut i32) (i32.const 0)) + (global (export "1i64") (mut i64) (i64.const 0)) + (global (export "2i64") (mut i64) (i64.const 0)) + (global (export "3i32") (mut i32) (i32.const 0)) + (global (export "3i64") (mut i64) (i64.const 0)) + (global (export "4i32") i32 (i32.const 0)) + (global (export "4i64") i64 (i64.const 0)) + )"#; + smoke_test_exports(test, 22) +} + +fn get_func_and_global_exports(features: WasmFeatures, module: &[u8]) -> Vec<(String, ExportType)> { + let mut validator = Validator::new_with_features(features); + let types = validate(&mut validator, module); + let mut exports = vec![]; + + for payload in Parser::new(0).parse_all(module) { + let payload = payload.unwrap(); + if let wasmparser::Payload::ExportSection(rdr) = payload { + for export in rdr { + let export = export.unwrap(); + match types.entity_type_from_export(&export).unwrap() { + EntityType::Func(core_id) => { + let sub_type = types.get(core_id).expect("Failed to lookup core id"); + assert!(sub_type.is_final); + assert!(sub_type.supertype_idx.is_none()); + let CompositeType::Func(func_type) = &sub_type.composite_type else { + panic!("Expected Func CompositeType, but found {:?}", sub_type); + }; + exports + .push((export.name.to_string(), ExportType::Func(func_type.clone()))); + } + EntityType::Global(global_type) => { + exports.push((export.name.to_string(), ExportType::Global(global_type))) + } + other => { + panic!("Unexpected entity type {:?}", other) + } + } + } + } + } + exports +} + +fn smoke_test_exports(exports_test_case: &str, seed: u64) { + let mut rng = SmallRng::seed_from_u64(seed); + let mut buf = vec![0; 512]; + let wasm = wat::parse_str(exports_test_case).unwrap(); + let expected_exports = get_func_and_global_exports(WasmFeatures::default(), &wasm); + + for _ in 0..1024 { + rng.fill_bytes(&mut buf); + let mut u = Unstructured::new(&buf); + + let mut config = Config::arbitrary(&mut u).expect("arbitrary config"); + config.exports = Some(wasm.clone()); + + let features = parser_features_from_config(&config); + let module = Module::new(config, &mut u).unwrap(); + let wasm_bytes = module.to_bytes(); + + let generated_exports = get_func_and_global_exports(features, &wasm_bytes); + assert_eq!(expected_exports, generated_exports); + } +} diff --git a/third_party/rust/wast/.cargo-checksum.json b/third_party/rust/wast/.cargo-checksum.json index bc41b948df..3852650553 100644 --- a/third_party/rust/wast/.cargo-checksum.json +++ b/third_party/rust/wast/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"Cargo.toml":"ed28c8d2aac09b85e36d2ab074a9a0ef823e2652c6eee337160a27d9a3e1cc77","LICENSE":"268872b9816f90fd8e85db5a28d33f8150ebb8dd016653fb39ef1f94f2686bc5","README.md":"5a0d2b894a3ac74ee2be74715a2f22c40a08520cb4ac59183f4e7356f34ac566","src/component.rs":"23a62f4f2774ccfaf60f68e9d9416e68ba203eea782ce0c39cf553ad293f1df4","src/component/alias.rs":"5ec26333e179dc3778dead489f1273815fe9c1c808ba6a7e60eff54072fad795","src/component/binary.rs":"02b07602dc755fd5cec95b6348dd919c75f368c16a81402072201cf911be5f0d","src/component/component.rs":"b23f62918bd302fd96636438c2a877fa91a5ea0c70d2c62f16528d16b1ffff9d","src/component/custom.rs":"f5b23c34b73a716a986fd999fc8d8c9e24c341e83292088fe83325cd82dab4f5","src/component/expand.rs":"71b2e23f50957b4a15d758df7f8651fdbaf5cf8f44fbfeb134b412318dbd8921","src/component/export.rs":"f51e824c839d8bb0884eca509622f376c8cce3335be324b2b25033af6216fd2a","src/component/func.rs":"4f69de6c38cc6fe77b638ed7d8000c8a170d7053a11a6585dcd5b4877a06804c","src/component/import.rs":"ffe6e4ab8f2cec68b1022c753135d675ab27ecd1315bd38517472ceaffd0610c","src/component/instance.rs":"e550a7ee9af092ae084dd41e2c0ae756b7dca8da4b91d672d90265a6a15dff83","src/component/item_ref.rs":"e9c426ccc0210dc0c37bb0448468f5f4d9e52656b72d4ff0f2dc65c89957fe60","src/component/module.rs":"d27a28199d1dea1c64294a514329a524432319288515b0a0e3091fa7d3a33f74","src/component/resolve.rs":"4bc58161f3e38499cf0d7fddcbaec875579e8c48e0f347f1c820db21b8aa2d86","src/component/types.rs":"c854a2df9613145c38720d41cda571d0d01408fc09789bd22740874628cbff99","src/component/wast.rs":"36f9440618be7755db0a2994e5ff14187392c25543cdde2069e4c69111a7f490","src/core.rs":"24b71d1ab2ad874c4e37b4dd709b6532989b60e6bc503194434f95029cc1cda7","src/core/binary.rs":"7dfa60e406206ba349ca5a78ca659e451064e184f4137b4e76fc6db527ddce06","src/core/custom.rs":"edd6044b75d79ec873c28d803fb8dc9a53724f1bba474bcdef2bc77196e0a4d2","src/core/export.rs":"1322a120d9e1dd6f3aa1485ee0bbc4294961028ae8a7584a24170af5823b73b1","src/core/expr.rs":"c3a3dd3c9c8c84007069201df8dfa75de8cc1c9681e4ce60afffa20402a3c785","src/core/func.rs":"f87239645e45b7e40ecf7f8f2b707a7cfc0620cd2632cfdaca3cb155a06da732","src/core/global.rs":"dec0abecadd2fde9298acb729259deba1ef3c4733ae5afcb4fe8cd4a633f5349","src/core/import.rs":"602a13aed2fd5fa63e2562246586546199861df57f304c2906561ab77810cadd","src/core/memory.rs":"2dc551899837d351975c6453425085ebaf4076c1f135cb1654ed8fdabb3fb695","src/core/module.rs":"0bfe00432ea192d2392d7f3f21d173781c39c353081f868632bad89df0b71405","src/core/resolve/deinline_import_export.rs":"b04b953baa68290aabcfe776ef97ef4080c42614f04671ef84ae53a626fb3d4f","src/core/resolve/mod.rs":"9b2680b00a1cafad2dbbfdac86aa1c135ea67203bbc4dee939302b27f4aff6d0","src/core/resolve/names.rs":"afa1df4d75555dd7235a117a3c7217d1ec774b062e3a7c98d7470447561c97af","src/core/resolve/types.rs":"ff34eed438539cd88ea6c2c51e49ed42eee0d8be566d40c799254bf08125cfa8","src/core/table.rs":"1be354d0040a1b29903b717baa7b87c1e17ac3bcfa16b1fe1ef386067e687a29","src/core/tag.rs":"8a3d4fcdb86dedd68cdaf25abd930ef52a461cf2a0e9393bb9cb6f425afaaf2e","src/core/types.rs":"608f59af8c41f83c0d6db41c03f187fd30d3f85a021540a5de522464b011d52a","src/core/wast.rs":"a7307f93fb20274f0c99952deeefb51c28d4afe017702ee75d96324285346442","src/encode.rs":"0b165176db54fb9136202c54180adabda843a88e5436b96c19be9d41623912a3","src/error.rs":"4526260299c7983696a49ffe30a80139beb0d081fa8f42decc5e3283b361b1cb","src/gensym.rs":"b5e02e34443085f01deb5b657e541be7dc5d6dc952304e434a0b08320ea6c185","src/lexer.rs":"bc65c4d71c6f16a55670585a31ee615c9fcfc40a4bfa165e020ed03a9a27930d","src/lib.rs":"87a394a3c8d70135ee4f79a270a733170267abdff69492f1addd3d27c8e0e937","src/names.rs":"81d49fecbff3b2abbbc323595271f32d912f03cd55a5685b7216d7cead32c420","src/parser.rs":"2a726a17de957f1bdb8abf9670946bb988c6727e1e96ba5b759b89211b78f6a7","src/token.rs":"d18c2f304ffbc017a3b0a7764daeb879a8c653c330cdb6b1eae431ae6c6dd14e","src/wast.rs":"af6e208c2a37028461a6ad9101967fbfa5c7fc343fe5eb6ecad54f948c74e15d","src/wat.rs":"b353622be42b9a1dc789b55a5a93d2de7c4b48f2d61c6a156e4a51701dc08f2d","tests/annotations.rs":"06294077550600f93a5a8c0d7e3ac38b47f00bb8933f9dc390ff31868e873afb","tests/comments.rs":"694e8a3467e9c837f723a43c729be0c6f6dfe3441ad9692759b1d55fd63055a2","tests/parse-fail.rs":"0f1d5dffd1e6145105571222096703c89c4f4a46e25c848faa730f731155ea1c","tests/parse-fail/bad-core-func-alias.wat":"b71372064c3fce9d4a616418605040fe5e1356030a709b798b4769d3619cbbfb","tests/parse-fail/bad-core-func-alias.wat.err":"bb63274c26d3a21209bad794767f48372834bdc10cfbebf568a0c65d52803c90","tests/parse-fail/bad-func-alias.wat":"237c07149e1e74afe3b991a1fee6acb63167c1ca8931341614c435000339b887","tests/parse-fail/bad-func-alias.wat.err":"4a4bfc691b06d20fdf71e1dbac04649a52c76787048415599978987d761308fa","tests/parse-fail/bad-index.wat":"d21489daeec3a35327dcc9e2ba2d0acdd05f4aeaff2272cca608fda4d2338497","tests/parse-fail/bad-index.wat.err":"dc11070de0c9160573006ea4e5fa3c4d28e71bc39b24b1938cf6ff3b03ea7154","tests/parse-fail/bad-name.wat":"e5ff5d410007779a0de6609ea4cc693f0e603d36a106b8f5098c1980dd9f8124","tests/parse-fail/bad-name.wat.err":"fb5638476c1b85d9d1919e3dbcb0f16f82d088a4a22d4a0c186d7b8ba6e1902b","tests/parse-fail/bad-name2.wat":"5a6a4d0c19e5f2e48d7cebf361aca9b9000b7ef0c652997b5bd0ffaadbd2ca8a","tests/parse-fail/bad-name2.wat.err":"129707cce45f1e3cfb3e2ca5c702182e16ca5eeb2dbb2edd0710b004a8e194a5","tests/parse-fail/bad-name3.wat":"c19133d738cc84e9174301f27d4050c216bda81c7e9918d03ac792b088f24a05","tests/parse-fail/bad-name3.wat.err":"84ea63d40a619a0782ec6e94fce63921188ab87b1c3875eacae0a371144ed83a","tests/parse-fail/block1.wat":"91e74b5c3b43be692e7a6ae74fbfa674c4b6197299eb61338c4eccf282b18f17","tests/parse-fail/block1.wat.err":"40a083ae496b41dee7002cc6a664c5db0c5e4d904ae03b815773a769c4493fca","tests/parse-fail/block2.wat":"a8c07b4c09d51f10a8ffdf19806586022552398701cd90eb6d09816d45df06e5","tests/parse-fail/block2.wat.err":"33c842ec5dd0f2fdd3a9ce8187dd98b45ceee48c12810802af809d05b9cd25e9","tests/parse-fail/block3.wat":"29739abfbabd7c55f00ddfbbb9ebd818b4a114ef2336d50514f0842f7e075905","tests/parse-fail/block3.wat.err":"fc667ae2e71a260f62a3c7393bc97272e7c0ff38b17594f4370847b8a5019060","tests/parse-fail/confusing-block-comment0.wat":"8f27c9d0d212bbb1862ea89ffd7cbeafde5dfd755d695c1ba696cd520aba1a1d","tests/parse-fail/confusing-block-comment0.wat.err":"b53cbaef7bcec3862c64e09c084b92cd61bd29b954125482b2d083db250cd9e2","tests/parse-fail/confusing-block-comment1.wat":"b1a0447c9a8eaab8938d15cd33bd4adbb8bb69c2d710209b604023991a4347cb","tests/parse-fail/confusing-block-comment1.wat.err":"2fc3b3e4f98416326e1e5ec034026301069b6a98fa24451bc7573e16b8cb3811","tests/parse-fail/confusing-block-comment2.wat":"e3f49c7a388fba81081beb25d87bbd7db0acce5dd8e3eaa04574905ed7ec420c","tests/parse-fail/confusing-block-comment2.wat.err":"2183231d6acd0b5a117f9aea747c3d5c12e758450a6cd74027bb954a3134cf19","tests/parse-fail/confusing-block-comment3.wat":"d83f89c582501eb8833e772b8462c8974984a2f7fbb80b1452dc399fac74e5ed","tests/parse-fail/confusing-block-comment3.wat.err":"8b2096a4833627905c63f49cdabe44be24336646578dcfbdc67e9bfb35cbc601","tests/parse-fail/confusing-block-comment4.wat":"b7c6c68844d918e9ef6dd5ab9c40c7de7b38f04f94fadad630eda4e596f3e0f8","tests/parse-fail/confusing-block-comment4.wat.err":"2f790cc511edfcd89a12c9207901be16039fc1a06a584d73095e77a52f861cd9","tests/parse-fail/confusing-block-comment5.wat":"a159808032638cc914fa80ac4354a68b0af4f435a09cbe3e2d577582e183eb0a","tests/parse-fail/confusing-block-comment5.wat.err":"6fe0d99894307442f83fe93beaa5da706e06c9bdaf8e39d7cbae4c4fffafcb94","tests/parse-fail/confusing-block-comment6.wat":"abe48bcba2587dca98bc80ddde4e813f94fbc8a3538704a0775ea85bca0f8466","tests/parse-fail/confusing-block-comment6.wat.err":"3c97b9bf1112bbb7335d7fe4be5befb6f91eea7bec7dd3e6b543792231003c56","tests/parse-fail/confusing-block-comment7.wat":"e125c416ea5fa0ac35a58295a83a6f345438e2d7ddc6a39bd76c8e89885b3f0e","tests/parse-fail/confusing-block-comment7.wat.err":"5c34528ff2019cd3f0b3df34fd42523c0b66120706321da2c88ec05793478d2e","tests/parse-fail/confusing-block-comment8.wat":"200cc4c0e5af21a25529d7a81633a03642cff807255d6cd72eb45cdccc605cec","tests/parse-fail/confusing-block-comment8.wat.err":"9b81237d150a784b71791eee88fb6264a8bd6412862660f7392945203809e517","tests/parse-fail/confusing-line-comment0.wat":"bcec4c5a1e52b3e392e07c6711c979aa8d7db8baaf2bcdf270ba16d1aa528d26","tests/parse-fail/confusing-line-comment0.wat.err":"41ec5a075dc6b73afe1aec6b3198c5c4ae3a1a900e1610115879058ce034d6f6","tests/parse-fail/confusing-line-comment1.wat":"a2afbcab00ec957dfd9e9bf21fa4238852247b27f0b054f4a00f6b172dddf853","tests/parse-fail/confusing-line-comment1.wat.err":"f19a645e6fb5cbd7a0dd2308732741edcf83dbae0ef62549972029856a9e7fc6","tests/parse-fail/confusing-line-comment2.wat":"7f2a68229d02aac56ec4dfccf139bf2d617a0e89430357b30444dc4239d8aa89","tests/parse-fail/confusing-line-comment2.wat.err":"08add3d33e10e1ab6b4f3ae431f5db61d6f6c0a2b7d6828482a1e51b3a2d3851","tests/parse-fail/confusing-line-comment3.wat":"61173ae54782f6de86685f9555ffb94bbe2cf20b234daf660abb69ba3326f1ff","tests/parse-fail/confusing-line-comment3.wat.err":"4a5333dc02efa3c1eeab9cafa7c707f78abe92defdb01a71d6fe20944e4785f0","tests/parse-fail/confusing-line-comment4.wat":"9ecbbbe82c750e6475af1bfb46fe7a06115e4446a437d19fc08ca3d002f2a1c9","tests/parse-fail/confusing-line-comment4.wat.err":"ddb8aee8006265253b09c313cf5eb5c2dc4da66f502b4f6d3e2e1de77b35aec9","tests/parse-fail/confusing-line-comment5.wat":"8a4c8d342111bc9d37c16dbdf67c52027e1a42632abc9f359b3e4f07a85748b5","tests/parse-fail/confusing-line-comment5.wat.err":"34e368719fc0eab2f1a43c9f8e6f1b31aa9be9f971085d72374e49bde39cbfe5","tests/parse-fail/confusing-line-comment6.wat":"15f0dcdec23736ce92db84b3a7cdfe8689c97f2a7d0b9b0bfb0dcd2675163ed1","tests/parse-fail/confusing-line-comment6.wat.err":"0570be2ede803f071925d249f3858d3a417b5a6d678c9da40fc851d788d12983","tests/parse-fail/confusing-line-comment7.wat":"c7ee59301a701dd52d56cad02df78b0ad3584460bc18efa42ee137fe0c35aef6","tests/parse-fail/confusing-line-comment7.wat.err":"feebbeee8c85d8b3b85cec89435ae18f3ade9f754ca180d747a41406b64ca07a","tests/parse-fail/confusing-line-comment8.wat":"17632a8142154624de88b3cf93516147ed3419d785200bcd7049499eca8e8f04","tests/parse-fail/confusing-line-comment8.wat.err":"9c209285f2295cd2bc999aa7a9534a654932493308ab1f102839ed15a4d04d17","tests/parse-fail/confusing-string0.wat":"497b679b32baddcd6a158f4cadd3d9a9dea3457bac2a8c2c3d4e09b7c2d80842","tests/parse-fail/confusing-string0.wat.err":"cb3d737f2319346675a038716694354cd3b272453daa8a96e32e9861a9277f7b","tests/parse-fail/confusing-string1.wat":"46654cbed1ea6aab5019aef3d20098a391e40dacafa1ad5e83bf4ec384109fce","tests/parse-fail/confusing-string1.wat.err":"de7e7da516dc6c244bd0e4f012577b69f0cacbcc10f727fadb4b50bb04e0e2b4","tests/parse-fail/confusing-string2.wat":"11938f217c14387c05312735130f00c91d9df2d3ff9df7f13395e0f2b81dad54","tests/parse-fail/confusing-string2.wat.err":"e7bd08b146a855d681fefaf9e0576a9c333a2d10044f8e268b916b22a54227c9","tests/parse-fail/confusing-string3.wat":"e0ca4903fcafb9a54a91cf99e5eac95d25c6d2eb67b076f88191ad396f839cb6","tests/parse-fail/confusing-string3.wat.err":"b88d5db9e445c798eb24f95b7661b9c0368934d27ee8208477cd1c99351b939a","tests/parse-fail/confusing-string4.wat":"3ee2aee7f77604d051519c6f1795634469c12e98ae347a98f0c8445eecf1ff3d","tests/parse-fail/confusing-string4.wat.err":"1edc65bb09d8d3eed6ff69e7d9a7a4b5941dc823fa3436fa375657510255f6f4","tests/parse-fail/confusing-string5.wat":"024e50943128840d53f17e31a9b9332ce4f0ee70a847a043015f435b1c3c6e76","tests/parse-fail/confusing-string5.wat.err":"a0f13ec40d596ea2d8b0c4292b0d28775a5116ab7e11d7de88b295d25428c661","tests/parse-fail/confusing-string6.wat":"79cf157e29319800d2652c5a7f3dc90e07ebe2145c9904a70fc12027cdee84b7","tests/parse-fail/confusing-string6.wat.err":"860555e7aa13e3de3639cc2a530d6a42b974b629c4659593e972cbb0f306abae","tests/parse-fail/confusing-string7.wat":"7d8e403766dfb4e569754160d31ed0f9a27f908ed6cff96be43ab3d37f5975d5","tests/parse-fail/confusing-string7.wat.err":"658b6a02ba6d769254485f35c20984e7135d914b4266929963d723f26a40be4a","tests/parse-fail/confusing-string8.wat":"5a9b222e578655d57ee6e9f19bc1ea8e29aa52d652975fac685213444ed6458f","tests/parse-fail/confusing-string8.wat.err":"9a4e1a510330c800a1df7966998ebc3cde931eda20b249e5360f5e9a905dce11","tests/parse-fail/inline1.wat":"4e9767d67207aace2ac5e6f63a30e7510e4aa245ba35420539509e2254470272","tests/parse-fail/inline1.wat.err":"0143017a9825e518baa6009bae2c8d63520051dedd3437705bbe36b038a57f41","tests/parse-fail/newline-in-string.wat":"5c01cf709544ade0a6cdfcc39a3836a3bc018b633dc42a6cd872b6defc763ea7","tests/parse-fail/newline-in-string.wat.err":"1504209cc37a78b2aee778f23eacf78606daf964cf7bff251f5700efcd27ffd7","tests/parse-fail/string1.wat":"620d46d585ce94b382b5fde628c1399f3e562014b7a44af46e92f7bd045ca86e","tests/parse-fail/string1.wat.err":"fc53f3a1c4a65d8f25e5af51dec7699f45cecba114ca9c7871781bc70f664320","tests/parse-fail/string10.wat":"f7409dd45e153a1b11cb23e38f4ed87da12bedde38f8f0ccfe91037b0a4d97bd","tests/parse-fail/string10.wat.err":"ce677db5e37e0ed81ca357ed6b5edb21d85c27303ee194855bea7a88457efb6a","tests/parse-fail/string11.wat":"f6e0400b8c6a2014efa1ac676c567e140d8f86b5f4d5129773e6d67af537b615","tests/parse-fail/string11.wat.err":"4c6a550d29eda38a4e1bf7a589596f11655dc779479d7b8d466cfc53f815a742","tests/parse-fail/string12.wat":"23e30070eef22271651cce096a801fc4f79f3c37343c88bb8d2fc99b32d3b8b9","tests/parse-fail/string12.wat.err":"b5ec59f2996b88b2ee157e22d1774dc3e36fc08ed5bfc621aea830d30f66f586","tests/parse-fail/string13.wat":"81a305b981159ee10e140749ea3220c9edaaff53605e63c21995de47382b5faf","tests/parse-fail/string13.wat.err":"959f26c6b54e0d367b51d11d1addd8a53b5b8ff3caf70ebdd46bbea8ccfa2418","tests/parse-fail/string14.wat":"c45c2cc9f7afbfbd4be8e513106d22f7e5e817091448576c6bdf0701b81d95dd","tests/parse-fail/string14.wat.err":"50b5bccba905ddbe275938edb7ed0b09a5ca53dcdad36a7ff736ce9bc8e7a338","tests/parse-fail/string15.wat":"b5e0d5ade40de53b2d767a132e28376bb8c7a6f6238c4d8c248ae717c41d7f1f","tests/parse-fail/string15.wat.err":"0e9fc502cc90f96d1f592a3f63369fd2a3574bc4a2345a70365dbb76804e870f","tests/parse-fail/string16.wat":"38c3688cee80a9d089d239aa06eb1d27c5364ad2bd270aca57d05997c20aa682","tests/parse-fail/string16.wat.err":"4274b3bbe4df4cf0373619b1fcd082d0c802990817d2aca26ed885168c80e489","tests/parse-fail/string2.wat":"1172964aed31537b8c466d1f045f3e756926e7b221f80b2aff4a9a6721ea0beb","tests/parse-fail/string2.wat.err":"4618d3b20a78a077337eb5d6cae14ac39d9853762f011fbd23cff8921618dbde","tests/parse-fail/string3.wat":"07e0fbcd6270c1db100917c151ee4ac3f935e4ee1b27bce3c453b22b4b74f4d6","tests/parse-fail/string3.wat.err":"08ffc6158a9e030b2e211d53bdb8aeacfd879815c7b284d6a83b030566e35928","tests/parse-fail/string4.wat":"c970da2051b0613bdd1de4664f10424e14f2ebabe604175d4fb9b763b37af577","tests/parse-fail/string4.wat.err":"406706594d305c560fabd66417ad4fc276939990b5e701bd9d13fc223d207219","tests/parse-fail/string5.wat":"386cf314bb05acdaaabdf4da1caf140167271a26bd08bf34c3a7427d4bc4431f","tests/parse-fail/string5.wat.err":"1e56b44a23a37b2b2ad05aa9dd7e1e18191b5cc22151f93bbcf9d618779a57bd","tests/parse-fail/string6.wat":"8f1fe2825ff96f2acee9130a7721f86fcc93c221baa9411bf1fb6f0870d38ccb","tests/parse-fail/string6.wat.err":"d55dfd84d94e893f167ae73b7a080aefb2bfb05cc8a1ec201c4d3066fb8549b4","tests/parse-fail/string7.wat":"b12f8c75313d7f834489d3c353422f90bc945b37139586446eda82e334a97cde","tests/parse-fail/string7.wat.err":"4cee0ca61992c249dd0faaf2529a073cf8deeb36111a3f69b43695e5682560a2","tests/parse-fail/string8.wat":"4c2e0e1f883bb4e8cba9313497ed792130e5848e62bde7716102788d7467be10","tests/parse-fail/string8.wat.err":"840c6def7c60dd7c2b7261549cab435ba78c9b3a937adf6d5d9595ff8af01c91","tests/parse-fail/string9.wat":"2b7670caed2b0688d535de6e4e416f35fa717cfbe096a6cc764a669085c8f52f","tests/parse-fail/string9.wat.err":"37b5a9c3af9631500f31f9e5e3efa821b8d96063c57d60fd01df6be6a5c323e1","tests/parse-fail/unbalanced.wat":"f664fbef53a0308f864ba496d38044eb90482636e32586512939d4930729f3fe","tests/parse-fail/unbalanced.wat.err":"aba579f7b836856e69afe05da8328aabe0643d94e369898e686aa7bb0b07e9c9","tests/recursive.rs":"ad8a2b07bf955121a7c9e326ed35f9b2bc56b440c8cc0bbde24d423a79945c1a"},"package":"f5d415036fe747a32b30c76c8bd6c73f69b7705fb7ebca5f16e852eef0c95802"} \ No newline at end of file +{"files":{"Cargo.toml":"53d3c5b19092b2144bdfe0d4cd843c2b42ce0fddfca703fd09fec319232468dc","LICENSE":"268872b9816f90fd8e85db5a28d33f8150ebb8dd016653fb39ef1f94f2686bc5","README.md":"5a0d2b894a3ac74ee2be74715a2f22c40a08520cb4ac59183f4e7356f34ac566","src/component.rs":"23a62f4f2774ccfaf60f68e9d9416e68ba203eea782ce0c39cf553ad293f1df4","src/component/alias.rs":"5ec26333e179dc3778dead489f1273815fe9c1c808ba6a7e60eff54072fad795","src/component/binary.rs":"02b07602dc755fd5cec95b6348dd919c75f368c16a81402072201cf911be5f0d","src/component/component.rs":"0c49ff1c1c4b8fe6d330eb41bce8ad176c7208c4178090b7325e9994e83c1f20","src/component/custom.rs":"f5b23c34b73a716a986fd999fc8d8c9e24c341e83292088fe83325cd82dab4f5","src/component/expand.rs":"71b2e23f50957b4a15d758df7f8651fdbaf5cf8f44fbfeb134b412318dbd8921","src/component/export.rs":"f51e824c839d8bb0884eca509622f376c8cce3335be324b2b25033af6216fd2a","src/component/func.rs":"4f69de6c38cc6fe77b638ed7d8000c8a170d7053a11a6585dcd5b4877a06804c","src/component/import.rs":"ffe6e4ab8f2cec68b1022c753135d675ab27ecd1315bd38517472ceaffd0610c","src/component/instance.rs":"e550a7ee9af092ae084dd41e2c0ae756b7dca8da4b91d672d90265a6a15dff83","src/component/item_ref.rs":"e9c426ccc0210dc0c37bb0448468f5f4d9e52656b72d4ff0f2dc65c89957fe60","src/component/module.rs":"d27a28199d1dea1c64294a514329a524432319288515b0a0e3091fa7d3a33f74","src/component/resolve.rs":"4bc58161f3e38499cf0d7fddcbaec875579e8c48e0f347f1c820db21b8aa2d86","src/component/types.rs":"c854a2df9613145c38720d41cda571d0d01408fc09789bd22740874628cbff99","src/component/wast.rs":"36f9440618be7755db0a2994e5ff14187392c25543cdde2069e4c69111a7f490","src/core.rs":"24b71d1ab2ad874c4e37b4dd709b6532989b60e6bc503194434f95029cc1cda7","src/core/binary.rs":"59462baaf8ccb7eb44a39721b71c3627c5ff2cf5e4f24c89a158c5372732f026","src/core/custom.rs":"edd6044b75d79ec873c28d803fb8dc9a53724f1bba474bcdef2bc77196e0a4d2","src/core/export.rs":"1322a120d9e1dd6f3aa1485ee0bbc4294961028ae8a7584a24170af5823b73b1","src/core/expr.rs":"c3b66daa6d2ad3f47d1d3c6a69bd1652d289e248ab3244e448f2fba7ea291b55","src/core/func.rs":"f87239645e45b7e40ecf7f8f2b707a7cfc0620cd2632cfdaca3cb155a06da732","src/core/global.rs":"dec0abecadd2fde9298acb729259deba1ef3c4733ae5afcb4fe8cd4a633f5349","src/core/import.rs":"602a13aed2fd5fa63e2562246586546199861df57f304c2906561ab77810cadd","src/core/memory.rs":"de66cc6d6a9238428ed39ea5dccfa5ff065b4f3ec96d6cfc8405cbb16742791d","src/core/module.rs":"2b608a3cfee4df1ceeefaa046863dc964172bd6f52f6678d96078091fae7657a","src/core/resolve/deinline_import_export.rs":"41226cb8c654e7ed5a22bdb774b3901c73bed52c23a4ed6f5d0302950e02c30b","src/core/resolve/mod.rs":"9b2680b00a1cafad2dbbfdac86aa1c135ea67203bbc4dee939302b27f4aff6d0","src/core/resolve/names.rs":"afa1df4d75555dd7235a117a3c7217d1ec774b062e3a7c98d7470447561c97af","src/core/resolve/types.rs":"ff34eed438539cd88ea6c2c51e49ed42eee0d8be566d40c799254bf08125cfa8","src/core/table.rs":"6b611622d7d4f83cbe8e1a82139937c0294996d5f8208ade0886d680d8ef0b5d","src/core/tag.rs":"8a3d4fcdb86dedd68cdaf25abd930ef52a461cf2a0e9393bb9cb6f425afaaf2e","src/core/types.rs":"608f59af8c41f83c0d6db41c03f187fd30d3f85a021540a5de522464b011d52a","src/core/wast.rs":"a7307f93fb20274f0c99952deeefb51c28d4afe017702ee75d96324285346442","src/encode.rs":"0b165176db54fb9136202c54180adabda843a88e5436b96c19be9d41623912a3","src/error.rs":"4526260299c7983696a49ffe30a80139beb0d081fa8f42decc5e3283b361b1cb","src/gensym.rs":"b5e02e34443085f01deb5b657e541be7dc5d6dc952304e434a0b08320ea6c185","src/lexer.rs":"bc65c4d71c6f16a55670585a31ee615c9fcfc40a4bfa165e020ed03a9a27930d","src/lib.rs":"3b1e7af5e618b9d3ac680e7f8e9b0114fedf502496fb0e62dc95d3dbc1633772","src/names.rs":"81d49fecbff3b2abbbc323595271f32d912f03cd55a5685b7216d7cead32c420","src/parser.rs":"fe6a3727c68c8075c38422de545eb189fd8f5747ca681f6704982ed5c8ea4476","src/token.rs":"d18c2f304ffbc017a3b0a7764daeb879a8c653c330cdb6b1eae431ae6c6dd14e","src/wast.rs":"af6e208c2a37028461a6ad9101967fbfa5c7fc343fe5eb6ecad54f948c74e15d","src/wat.rs":"6fd57cf40795fabe977bdae50913f0e75c3dad3a0c0ace95df2de4f92be8aaf5","tests/annotations.rs":"06294077550600f93a5a8c0d7e3ac38b47f00bb8933f9dc390ff31868e873afb","tests/comments.rs":"694e8a3467e9c837f723a43c729be0c6f6dfe3441ad9692759b1d55fd63055a2","tests/parse-fail.rs":"0f1d5dffd1e6145105571222096703c89c4f4a46e25c848faa730f731155ea1c","tests/parse-fail/bad-core-func-alias.wat":"b71372064c3fce9d4a616418605040fe5e1356030a709b798b4769d3619cbbfb","tests/parse-fail/bad-core-func-alias.wat.err":"bb63274c26d3a21209bad794767f48372834bdc10cfbebf568a0c65d52803c90","tests/parse-fail/bad-func-alias.wat":"237c07149e1e74afe3b991a1fee6acb63167c1ca8931341614c435000339b887","tests/parse-fail/bad-func-alias.wat.err":"4a4bfc691b06d20fdf71e1dbac04649a52c76787048415599978987d761308fa","tests/parse-fail/bad-index.wat":"d21489daeec3a35327dcc9e2ba2d0acdd05f4aeaff2272cca608fda4d2338497","tests/parse-fail/bad-index.wat.err":"dc11070de0c9160573006ea4e5fa3c4d28e71bc39b24b1938cf6ff3b03ea7154","tests/parse-fail/bad-name.wat":"e5ff5d410007779a0de6609ea4cc693f0e603d36a106b8f5098c1980dd9f8124","tests/parse-fail/bad-name.wat.err":"fb5638476c1b85d9d1919e3dbcb0f16f82d088a4a22d4a0c186d7b8ba6e1902b","tests/parse-fail/bad-name2.wat":"5a6a4d0c19e5f2e48d7cebf361aca9b9000b7ef0c652997b5bd0ffaadbd2ca8a","tests/parse-fail/bad-name2.wat.err":"129707cce45f1e3cfb3e2ca5c702182e16ca5eeb2dbb2edd0710b004a8e194a5","tests/parse-fail/bad-name3.wat":"c19133d738cc84e9174301f27d4050c216bda81c7e9918d03ac792b088f24a05","tests/parse-fail/bad-name3.wat.err":"84ea63d40a619a0782ec6e94fce63921188ab87b1c3875eacae0a371144ed83a","tests/parse-fail/block1.wat":"91e74b5c3b43be692e7a6ae74fbfa674c4b6197299eb61338c4eccf282b18f17","tests/parse-fail/block1.wat.err":"40a083ae496b41dee7002cc6a664c5db0c5e4d904ae03b815773a769c4493fca","tests/parse-fail/block2.wat":"a8c07b4c09d51f10a8ffdf19806586022552398701cd90eb6d09816d45df06e5","tests/parse-fail/block2.wat.err":"33c842ec5dd0f2fdd3a9ce8187dd98b45ceee48c12810802af809d05b9cd25e9","tests/parse-fail/block3.wat":"29739abfbabd7c55f00ddfbbb9ebd818b4a114ef2336d50514f0842f7e075905","tests/parse-fail/block3.wat.err":"fc667ae2e71a260f62a3c7393bc97272e7c0ff38b17594f4370847b8a5019060","tests/parse-fail/confusing-block-comment0.wat":"8f27c9d0d212bbb1862ea89ffd7cbeafde5dfd755d695c1ba696cd520aba1a1d","tests/parse-fail/confusing-block-comment0.wat.err":"b53cbaef7bcec3862c64e09c084b92cd61bd29b954125482b2d083db250cd9e2","tests/parse-fail/confusing-block-comment1.wat":"b1a0447c9a8eaab8938d15cd33bd4adbb8bb69c2d710209b604023991a4347cb","tests/parse-fail/confusing-block-comment1.wat.err":"2fc3b3e4f98416326e1e5ec034026301069b6a98fa24451bc7573e16b8cb3811","tests/parse-fail/confusing-block-comment2.wat":"e3f49c7a388fba81081beb25d87bbd7db0acce5dd8e3eaa04574905ed7ec420c","tests/parse-fail/confusing-block-comment2.wat.err":"2183231d6acd0b5a117f9aea747c3d5c12e758450a6cd74027bb954a3134cf19","tests/parse-fail/confusing-block-comment3.wat":"d83f89c582501eb8833e772b8462c8974984a2f7fbb80b1452dc399fac74e5ed","tests/parse-fail/confusing-block-comment3.wat.err":"8b2096a4833627905c63f49cdabe44be24336646578dcfbdc67e9bfb35cbc601","tests/parse-fail/confusing-block-comment4.wat":"b7c6c68844d918e9ef6dd5ab9c40c7de7b38f04f94fadad630eda4e596f3e0f8","tests/parse-fail/confusing-block-comment4.wat.err":"2f790cc511edfcd89a12c9207901be16039fc1a06a584d73095e77a52f861cd9","tests/parse-fail/confusing-block-comment5.wat":"a159808032638cc914fa80ac4354a68b0af4f435a09cbe3e2d577582e183eb0a","tests/parse-fail/confusing-block-comment5.wat.err":"6fe0d99894307442f83fe93beaa5da706e06c9bdaf8e39d7cbae4c4fffafcb94","tests/parse-fail/confusing-block-comment6.wat":"abe48bcba2587dca98bc80ddde4e813f94fbc8a3538704a0775ea85bca0f8466","tests/parse-fail/confusing-block-comment6.wat.err":"3c97b9bf1112bbb7335d7fe4be5befb6f91eea7bec7dd3e6b543792231003c56","tests/parse-fail/confusing-block-comment7.wat":"e125c416ea5fa0ac35a58295a83a6f345438e2d7ddc6a39bd76c8e89885b3f0e","tests/parse-fail/confusing-block-comment7.wat.err":"5c34528ff2019cd3f0b3df34fd42523c0b66120706321da2c88ec05793478d2e","tests/parse-fail/confusing-block-comment8.wat":"200cc4c0e5af21a25529d7a81633a03642cff807255d6cd72eb45cdccc605cec","tests/parse-fail/confusing-block-comment8.wat.err":"9b81237d150a784b71791eee88fb6264a8bd6412862660f7392945203809e517","tests/parse-fail/confusing-line-comment0.wat":"bcec4c5a1e52b3e392e07c6711c979aa8d7db8baaf2bcdf270ba16d1aa528d26","tests/parse-fail/confusing-line-comment0.wat.err":"41ec5a075dc6b73afe1aec6b3198c5c4ae3a1a900e1610115879058ce034d6f6","tests/parse-fail/confusing-line-comment1.wat":"a2afbcab00ec957dfd9e9bf21fa4238852247b27f0b054f4a00f6b172dddf853","tests/parse-fail/confusing-line-comment1.wat.err":"f19a645e6fb5cbd7a0dd2308732741edcf83dbae0ef62549972029856a9e7fc6","tests/parse-fail/confusing-line-comment2.wat":"7f2a68229d02aac56ec4dfccf139bf2d617a0e89430357b30444dc4239d8aa89","tests/parse-fail/confusing-line-comment2.wat.err":"08add3d33e10e1ab6b4f3ae431f5db61d6f6c0a2b7d6828482a1e51b3a2d3851","tests/parse-fail/confusing-line-comment3.wat":"61173ae54782f6de86685f9555ffb94bbe2cf20b234daf660abb69ba3326f1ff","tests/parse-fail/confusing-line-comment3.wat.err":"4a5333dc02efa3c1eeab9cafa7c707f78abe92defdb01a71d6fe20944e4785f0","tests/parse-fail/confusing-line-comment4.wat":"9ecbbbe82c750e6475af1bfb46fe7a06115e4446a437d19fc08ca3d002f2a1c9","tests/parse-fail/confusing-line-comment4.wat.err":"ddb8aee8006265253b09c313cf5eb5c2dc4da66f502b4f6d3e2e1de77b35aec9","tests/parse-fail/confusing-line-comment5.wat":"8a4c8d342111bc9d37c16dbdf67c52027e1a42632abc9f359b3e4f07a85748b5","tests/parse-fail/confusing-line-comment5.wat.err":"34e368719fc0eab2f1a43c9f8e6f1b31aa9be9f971085d72374e49bde39cbfe5","tests/parse-fail/confusing-line-comment6.wat":"15f0dcdec23736ce92db84b3a7cdfe8689c97f2a7d0b9b0bfb0dcd2675163ed1","tests/parse-fail/confusing-line-comment6.wat.err":"0570be2ede803f071925d249f3858d3a417b5a6d678c9da40fc851d788d12983","tests/parse-fail/confusing-line-comment7.wat":"c7ee59301a701dd52d56cad02df78b0ad3584460bc18efa42ee137fe0c35aef6","tests/parse-fail/confusing-line-comment7.wat.err":"feebbeee8c85d8b3b85cec89435ae18f3ade9f754ca180d747a41406b64ca07a","tests/parse-fail/confusing-line-comment8.wat":"17632a8142154624de88b3cf93516147ed3419d785200bcd7049499eca8e8f04","tests/parse-fail/confusing-line-comment8.wat.err":"9c209285f2295cd2bc999aa7a9534a654932493308ab1f102839ed15a4d04d17","tests/parse-fail/confusing-string0.wat":"497b679b32baddcd6a158f4cadd3d9a9dea3457bac2a8c2c3d4e09b7c2d80842","tests/parse-fail/confusing-string0.wat.err":"cb3d737f2319346675a038716694354cd3b272453daa8a96e32e9861a9277f7b","tests/parse-fail/confusing-string1.wat":"46654cbed1ea6aab5019aef3d20098a391e40dacafa1ad5e83bf4ec384109fce","tests/parse-fail/confusing-string1.wat.err":"de7e7da516dc6c244bd0e4f012577b69f0cacbcc10f727fadb4b50bb04e0e2b4","tests/parse-fail/confusing-string2.wat":"11938f217c14387c05312735130f00c91d9df2d3ff9df7f13395e0f2b81dad54","tests/parse-fail/confusing-string2.wat.err":"e7bd08b146a855d681fefaf9e0576a9c333a2d10044f8e268b916b22a54227c9","tests/parse-fail/confusing-string3.wat":"e0ca4903fcafb9a54a91cf99e5eac95d25c6d2eb67b076f88191ad396f839cb6","tests/parse-fail/confusing-string3.wat.err":"b88d5db9e445c798eb24f95b7661b9c0368934d27ee8208477cd1c99351b939a","tests/parse-fail/confusing-string4.wat":"3ee2aee7f77604d051519c6f1795634469c12e98ae347a98f0c8445eecf1ff3d","tests/parse-fail/confusing-string4.wat.err":"1edc65bb09d8d3eed6ff69e7d9a7a4b5941dc823fa3436fa375657510255f6f4","tests/parse-fail/confusing-string5.wat":"024e50943128840d53f17e31a9b9332ce4f0ee70a847a043015f435b1c3c6e76","tests/parse-fail/confusing-string5.wat.err":"a0f13ec40d596ea2d8b0c4292b0d28775a5116ab7e11d7de88b295d25428c661","tests/parse-fail/confusing-string6.wat":"79cf157e29319800d2652c5a7f3dc90e07ebe2145c9904a70fc12027cdee84b7","tests/parse-fail/confusing-string6.wat.err":"860555e7aa13e3de3639cc2a530d6a42b974b629c4659593e972cbb0f306abae","tests/parse-fail/confusing-string7.wat":"7d8e403766dfb4e569754160d31ed0f9a27f908ed6cff96be43ab3d37f5975d5","tests/parse-fail/confusing-string7.wat.err":"658b6a02ba6d769254485f35c20984e7135d914b4266929963d723f26a40be4a","tests/parse-fail/confusing-string8.wat":"5a9b222e578655d57ee6e9f19bc1ea8e29aa52d652975fac685213444ed6458f","tests/parse-fail/confusing-string8.wat.err":"9a4e1a510330c800a1df7966998ebc3cde931eda20b249e5360f5e9a905dce11","tests/parse-fail/inline1.wat":"4e9767d67207aace2ac5e6f63a30e7510e4aa245ba35420539509e2254470272","tests/parse-fail/inline1.wat.err":"0143017a9825e518baa6009bae2c8d63520051dedd3437705bbe36b038a57f41","tests/parse-fail/newline-in-string.wat":"5c01cf709544ade0a6cdfcc39a3836a3bc018b633dc42a6cd872b6defc763ea7","tests/parse-fail/newline-in-string.wat.err":"1504209cc37a78b2aee778f23eacf78606daf964cf7bff251f5700efcd27ffd7","tests/parse-fail/string1.wat":"620d46d585ce94b382b5fde628c1399f3e562014b7a44af46e92f7bd045ca86e","tests/parse-fail/string1.wat.err":"fc53f3a1c4a65d8f25e5af51dec7699f45cecba114ca9c7871781bc70f664320","tests/parse-fail/string10.wat":"f7409dd45e153a1b11cb23e38f4ed87da12bedde38f8f0ccfe91037b0a4d97bd","tests/parse-fail/string10.wat.err":"ce677db5e37e0ed81ca357ed6b5edb21d85c27303ee194855bea7a88457efb6a","tests/parse-fail/string11.wat":"f6e0400b8c6a2014efa1ac676c567e140d8f86b5f4d5129773e6d67af537b615","tests/parse-fail/string11.wat.err":"4c6a550d29eda38a4e1bf7a589596f11655dc779479d7b8d466cfc53f815a742","tests/parse-fail/string12.wat":"23e30070eef22271651cce096a801fc4f79f3c37343c88bb8d2fc99b32d3b8b9","tests/parse-fail/string12.wat.err":"b5ec59f2996b88b2ee157e22d1774dc3e36fc08ed5bfc621aea830d30f66f586","tests/parse-fail/string13.wat":"81a305b981159ee10e140749ea3220c9edaaff53605e63c21995de47382b5faf","tests/parse-fail/string13.wat.err":"959f26c6b54e0d367b51d11d1addd8a53b5b8ff3caf70ebdd46bbea8ccfa2418","tests/parse-fail/string14.wat":"c45c2cc9f7afbfbd4be8e513106d22f7e5e817091448576c6bdf0701b81d95dd","tests/parse-fail/string14.wat.err":"50b5bccba905ddbe275938edb7ed0b09a5ca53dcdad36a7ff736ce9bc8e7a338","tests/parse-fail/string15.wat":"b5e0d5ade40de53b2d767a132e28376bb8c7a6f6238c4d8c248ae717c41d7f1f","tests/parse-fail/string15.wat.err":"0e9fc502cc90f96d1f592a3f63369fd2a3574bc4a2345a70365dbb76804e870f","tests/parse-fail/string16.wat":"38c3688cee80a9d089d239aa06eb1d27c5364ad2bd270aca57d05997c20aa682","tests/parse-fail/string16.wat.err":"4274b3bbe4df4cf0373619b1fcd082d0c802990817d2aca26ed885168c80e489","tests/parse-fail/string2.wat":"1172964aed31537b8c466d1f045f3e756926e7b221f80b2aff4a9a6721ea0beb","tests/parse-fail/string2.wat.err":"4618d3b20a78a077337eb5d6cae14ac39d9853762f011fbd23cff8921618dbde","tests/parse-fail/string3.wat":"07e0fbcd6270c1db100917c151ee4ac3f935e4ee1b27bce3c453b22b4b74f4d6","tests/parse-fail/string3.wat.err":"08ffc6158a9e030b2e211d53bdb8aeacfd879815c7b284d6a83b030566e35928","tests/parse-fail/string4.wat":"c970da2051b0613bdd1de4664f10424e14f2ebabe604175d4fb9b763b37af577","tests/parse-fail/string4.wat.err":"406706594d305c560fabd66417ad4fc276939990b5e701bd9d13fc223d207219","tests/parse-fail/string5.wat":"386cf314bb05acdaaabdf4da1caf140167271a26bd08bf34c3a7427d4bc4431f","tests/parse-fail/string5.wat.err":"1e56b44a23a37b2b2ad05aa9dd7e1e18191b5cc22151f93bbcf9d618779a57bd","tests/parse-fail/string6.wat":"8f1fe2825ff96f2acee9130a7721f86fcc93c221baa9411bf1fb6f0870d38ccb","tests/parse-fail/string6.wat.err":"d55dfd84d94e893f167ae73b7a080aefb2bfb05cc8a1ec201c4d3066fb8549b4","tests/parse-fail/string7.wat":"b12f8c75313d7f834489d3c353422f90bc945b37139586446eda82e334a97cde","tests/parse-fail/string7.wat.err":"4cee0ca61992c249dd0faaf2529a073cf8deeb36111a3f69b43695e5682560a2","tests/parse-fail/string8.wat":"4c2e0e1f883bb4e8cba9313497ed792130e5848e62bde7716102788d7467be10","tests/parse-fail/string8.wat.err":"840c6def7c60dd7c2b7261549cab435ba78c9b3a937adf6d5d9595ff8af01c91","tests/parse-fail/string9.wat":"2b7670caed2b0688d535de6e4e416f35fa717cfbe096a6cc764a669085c8f52f","tests/parse-fail/string9.wat.err":"37b5a9c3af9631500f31f9e5e3efa821b8d96063c57d60fd01df6be6a5c323e1","tests/parse-fail/unbalanced.wat":"f664fbef53a0308f864ba496d38044eb90482636e32586512939d4930729f3fe","tests/parse-fail/unbalanced.wat.err":"aba579f7b836856e69afe05da8328aabe0643d94e369898e686aa7bb0b07e9c9","tests/recursive.rs":"ad8a2b07bf955121a7c9e326ed35f9b2bc56b440c8cc0bbde24d423a79945c1a"},"package":"1ef6e1ef34d7da3e2b374fd2b1a9c0227aff6cad596e1b24df9b58d0f6222faa"} \ No newline at end of file diff --git a/third_party/rust/wast/Cargo.toml b/third_party/rust/wast/Cargo.toml index e2ac927aa5..7f2d159a8b 100644 --- a/third_party/rust/wast/Cargo.toml +++ b/third_party/rust/wast/Cargo.toml @@ -12,7 +12,7 @@ [package] edition = "2021" name = "wast" -version = "70.0.1" +version = "201.0.0" authors = ["Alex Crichton "] description = """ Customizable Rust parsers for the WebAssembly Text formats WAT and WAST @@ -27,6 +27,9 @@ repository = "https://github.com/bytecodealliance/wasm-tools/tree/main/crates/wa name = "parse-fail" harness = false +[dependencies.bumpalo] +version = "3.14.0" + [dependencies.leb128] version = "0.2.4" @@ -37,7 +40,7 @@ version = "2.4.1" version = "0.1.9" [dependencies.wasm-encoder] -version = "0.40.0" +version = "0.201.0" [dev-dependencies.anyhow] version = "1.0.58" @@ -48,3 +51,9 @@ version = "1.3" [features] default = ["wasm-module"] wasm-module = [] + +[lints.clippy] +all = "allow" + +[lints.rust] +unsafe_code = "deny" diff --git a/third_party/rust/wast/src/component/component.rs b/third_party/rust/wast/src/component/component.rs index f954935d87..cc9171b505 100644 --- a/third_party/rust/wast/src/component/component.rs +++ b/third_party/rust/wast/src/component/component.rs @@ -111,6 +111,7 @@ impl<'a> Parse<'a> for Component<'a> { let _r = parser.register_annotation("custom"); let _r = parser.register_annotation("producers"); let _r = parser.register_annotation("name"); + let _r = parser.register_annotation("metadata.code.branch_hint"); let span = parser.parse::()?.0; let id = parser.parse()?; diff --git a/third_party/rust/wast/src/core/binary.rs b/third_party/rust/wast/src/core/binary.rs index 68facd6431..da94da0241 100644 --- a/third_party/rust/wast/src/core/binary.rs +++ b/third_party/rust/wast/src/core/binary.rs @@ -69,7 +69,7 @@ pub fn encode( if needs_data_count(&funcs) { e.section(12, &data.len()); } - e.section_list(10, Code, &funcs); + e.code_section(&funcs, &imports); e.section_list(11, Data, &data); let names = find_names(module_id, module_name, fields); @@ -121,6 +121,47 @@ impl Encoder<'_> { } self.custom_sections(CustomPlace::After(anchor)); } + + /// Encodes the code section of a wasm module module while additionally + /// handling the branch hinting proposal. + /// + /// The branch hinting proposal requires to encode the offsets of the + /// instructions relative from the beginning of the function. Here we encode + /// each instruction and we save its offset. If needed, we use this + /// information to build the branch hint section and insert it before the + /// code section. + fn code_section<'a>(&'a mut self, list: &[&'a Func<'_>], imports: &[&Import<'_>]) { + self.custom_sections(CustomPlace::Before(CustomPlaceAnchor::Code)); + + if !list.is_empty() { + let mut branch_hints = Vec::new(); + let mut code_section = Vec::new(); + + list.len().encode(&mut code_section); + let mut func_index = imports + .iter() + .filter(|i| matches!(i.item.kind, ItemKind::Func(..))) + .count() as u32; + for func in list.iter() { + let hints = func.encode(&mut code_section); + if !hints.is_empty() { + branch_hints.push(FunctionBranchHints { func_index, hints }); + } + func_index += 1; + } + + // Branch hints section has to be inserted before the Code section + // Insert the section only if we have some hints + if !branch_hints.is_empty() { + self.section(0, &("metadata.code.branch_hint", branch_hints)); + } + + // Finally, insert the Code section from the tmp buffer + self.wasm.push(10); + code_section.encode(&mut self.wasm); + } + self.custom_sections(CustomPlace::After(CustomPlaceAnchor::Code)); + } } impl Encode for FunctionType<'_> { @@ -475,7 +516,7 @@ impl Encode for Table<'_> { e.push(0x40); e.push(0x00); ty.encode(e); - init_expr.encode(e); + init_expr.encode(e, 0); } _ => panic!("TableKind should be normal during encoding"), } @@ -497,7 +538,9 @@ impl Encode for Global<'_> { assert!(self.exports.names.is_empty()); self.ty.encode(e); match &self.kind { - GlobalKind::Inline(expr) => expr.encode(e), + GlobalKind::Inline(expr) => { + let _hints = expr.encode(e, 0); + } _ => panic!("GlobalKind should be inline during encoding"), } } @@ -534,7 +577,7 @@ impl Encode for Elem<'_> { ElemPayload::Indices(_), ) => { e.push(0x00); - offset.encode(e); + offset.encode(e, 0); } (ElemKind::Passive, ElemPayload::Indices(_)) => { e.push(0x01); // flags @@ -543,7 +586,7 @@ impl Encode for Elem<'_> { (ElemKind::Active { table, offset }, ElemPayload::Indices(_)) => { e.push(0x02); // flags table.encode(e); - offset.encode(e); + offset.encode(e, 0); e.push(0x00); // extern_kind } (ElemKind::Declared, ElemPayload::Indices(_)) => { @@ -565,7 +608,7 @@ impl Encode for Elem<'_> { }, ) => { e.push(0x04); - offset.encode(e); + offset.encode(e, 0); } (ElemKind::Passive, ElemPayload::Exprs { ty, .. }) => { e.push(0x05); @@ -574,7 +617,7 @@ impl Encode for Elem<'_> { (ElemKind::Active { table, offset }, ElemPayload::Exprs { ty, .. }) => { e.push(0x06); table.encode(e); - offset.encode(e); + offset.encode(e, 0); ty.encode(e); } (ElemKind::Declared, ElemPayload::Exprs { ty, .. }) => { @@ -594,7 +637,7 @@ impl Encode for ElemPayload<'_> { ElemPayload::Exprs { exprs, ty: _ } => { exprs.len().encode(e); for expr in exprs { - expr.encode(e); + expr.encode(e, 0); } } } @@ -610,12 +653,12 @@ impl Encode for Data<'_> { offset, } => { e.push(0x00); - offset.encode(e); + offset.encode(e, 0); } DataKind::Active { memory, offset } => { e.push(0x02); memory.encode(e); - offset.encode(e); + offset.encode(e, 0); } } self.data.iter().map(|l| l.len()).sum::().encode(e); @@ -625,20 +668,25 @@ impl Encode for Data<'_> { } } -impl Encode for Func<'_> { - fn encode(&self, e: &mut Vec) { +impl Func<'_> { + /// Encodes the function into `e` while returning all branch hints with + /// known relative offsets after encoding. + fn encode(&self, e: &mut Vec) -> Vec { assert!(self.exports.names.is_empty()); - let mut tmp = Vec::new(); let (expr, locals) = match &self.kind { FuncKind::Inline { expression, locals } => (expression, locals), _ => panic!("should only have inline functions in emission"), }; + // Encode the function into a temporary vector because functions are + // prefixed with their length. The temporary vector, when encoded, + // encodes its length first then the body. + let mut tmp = Vec::new(); locals.encode(&mut tmp); - expr.encode(&mut tmp); + let branch_hints = expr.encode(&mut tmp, 0); + tmp.encode(e); - tmp.len().encode(e); - e.extend_from_slice(&tmp); + branch_hints } } @@ -658,12 +706,25 @@ impl Encode for Box<[Local<'_>]> { } } -impl Encode for Expression<'_> { - fn encode(&self, e: &mut Vec) { - for instr in self.instrs.iter() { +// Encode the expression and store the offset from the beginning +// for each instruction. +impl Expression<'_> { + fn encode(&self, e: &mut Vec, relative_start: usize) -> Vec { + let mut hints = Vec::with_capacity(self.branch_hints.len()); + let mut next_hint = self.branch_hints.iter().peekable(); + + for (i, instr) in self.instrs.iter().enumerate() { + if let Some(hint) = next_hint.next_if(|h| h.instr_index == i) { + hints.push(BranchHint { + branch_func_offset: u32::try_from(e.len() - relative_start).unwrap(), + branch_hint_value: hint.value, + }); + } instr.encode(e); } e.push(0x0b); + + hints } } @@ -1146,6 +1207,31 @@ impl Encode for Dylink0Subsection<'_> { } } +struct FunctionBranchHints { + func_index: u32, + hints: Vec, +} + +struct BranchHint { + branch_func_offset: u32, + branch_hint_value: u32, +} + +impl Encode for FunctionBranchHints { + fn encode(&self, e: &mut Vec) { + self.func_index.encode(e); + self.hints.encode(e); + } +} + +impl Encode for BranchHint { + fn encode(&self, e: &mut Vec) { + self.branch_func_offset.encode(e); + 1u32.encode(e); + self.branch_hint_value.encode(e); + } +} + impl Encode for Tag<'_> { fn encode(&self, e: &mut Vec) { self.ty.encode(e); diff --git a/third_party/rust/wast/src/core/expr.rs b/third_party/rust/wast/src/core/expr.rs index 489ac205af..b45950b896 100644 --- a/third_party/rust/wast/src/core/expr.rs +++ b/third_party/rust/wast/src/core/expr.rs @@ -1,3 +1,4 @@ +use crate::annotation; use crate::core::*; use crate::encode::Encode; use crate::kw; @@ -14,6 +15,20 @@ use std::mem; #[allow(missing_docs)] pub struct Expression<'a> { pub instrs: Box<[Instruction<'a>]>, + pub branch_hints: Vec, +} + +/// A `@metadata.code.branch_hint` in the code, associated with a If or BrIf +/// This instruction is a placeholder and won't produce anything. Its purpose +/// is to store the offset of the following instruction and check that +/// it's followed by `br_if` or `if`. +#[derive(Debug)] +pub struct BranchHint { + /// Index of instructions in `instrs` field of `Expression` that this hint + /// appplies to. + pub instr_index: usize, + /// The value of this branch hint + pub value: u32, } impl<'a> Parse<'a> for Expression<'a> { @@ -22,6 +37,7 @@ impl<'a> Parse<'a> for Expression<'a> { exprs.parse(parser)?; Ok(Expression { instrs: exprs.instrs.into(), + branch_hints: exprs.branch_hints, }) } } @@ -47,6 +63,7 @@ impl<'a> Expression<'a> { exprs.parse_folded_instruction(parser)?; Ok(Expression { instrs: exprs.instrs.into(), + branch_hints: exprs.branch_hints, }) } } @@ -66,6 +83,11 @@ struct ExpressionParser<'a> { /// Descriptor of all our nested s-expr blocks. This only happens when /// instructions themselves are nested. stack: Vec>, + + /// Related to the branch hints proposal. + /// Will be used later to collect the offsets in the final binary. + /// <(index of branch instructions, BranchHintAnnotation)> + branch_hints: Vec, } enum Paren { @@ -89,6 +111,9 @@ enum Level<'a> { /// which don't correspond to terminating instructions, we're just in a /// nested block. IfArm, + + /// This means we are finishing the parsing of a branch hint annotation. + BranchHint, } /// Possible states of "what is currently being parsed?" in an `if` expression. @@ -145,6 +170,14 @@ impl<'a> ExpressionParser<'a> { if self.handle_if_lparen(parser)? { continue; } + + // Handle the case of a branch hint annotation + if parser.peek::()? { + self.parse_branch_hint(parser)?; + self.stack.push(Level::BranchHint); + continue; + } + match parser.parse()? { // If block/loop show up then we just need to be sure to // push an `end` instruction whenever the `)` token is @@ -177,6 +210,7 @@ impl<'a> ExpressionParser<'a> { Paren::Right => match self.stack.pop().unwrap() { Level::EndWith(i) => self.instrs.push(i), Level::IfArm => {} + Level::BranchHint => {} // If an `if` statement hasn't parsed the clause or `then` // block, then that's an error because there weren't enough @@ -191,7 +225,6 @@ impl<'a> ExpressionParser<'a> { }, } } - Ok(()) } @@ -287,6 +320,24 @@ impl<'a> ExpressionParser<'a> { If::Else => Err(parser.error("unexpected token: too many payloads inside of `(if)`")), } } + + fn parse_branch_hint(&mut self, parser: Parser<'a>) -> Result<()> { + parser.parse::()?; + + let hint = parser.parse::()?; + + let value = match hint.as_bytes() { + [0] => 0, + [1] => 1, + _ => return Err(parser.error("invalid value for branch hint")), + }; + + self.branch_hints.push(BranchHint { + instr_index: self.instrs.len(), + value, + }); + Ok(()) + } } // TODO: document this obscenity diff --git a/third_party/rust/wast/src/core/memory.rs b/third_party/rust/wast/src/core/memory.rs index 3bc7345ef2..eb1baa1a95 100644 --- a/third_party/rust/wast/src/core/memory.rs +++ b/third_party/rust/wast/src/core/memory.rs @@ -165,6 +165,7 @@ impl<'a> Parse<'a> for Data<'a> { if parser.is_empty() { return Ok(Expression { instrs: [insn].into(), + branch_hints: Vec::new(), }); } @@ -184,6 +185,7 @@ impl<'a> Parse<'a> for Data<'a> { instrs.push(insn); Ok(Expression { instrs: instrs.into(), + branch_hints: Vec::new(), }) } })?; diff --git a/third_party/rust/wast/src/core/module.rs b/third_party/rust/wast/src/core/module.rs index 569a8884d4..f74ce6b619 100644 --- a/third_party/rust/wast/src/core/module.rs +++ b/third_party/rust/wast/src/core/module.rs @@ -114,6 +114,7 @@ impl<'a> Parse<'a> for Module<'a> { let _r = parser.register_annotation("producers"); let _r = parser.register_annotation("name"); let _r = parser.register_annotation("dylink.0"); + let _r = parser.register_annotation("metadata.code.branch_hint"); let span = parser.parse::()?.0; let id = parser.parse()?; diff --git a/third_party/rust/wast/src/core/resolve/deinline_import_export.rs b/third_party/rust/wast/src/core/resolve/deinline_import_export.rs index c338407182..98e680b58a 100644 --- a/third_party/rust/wast/src/core/resolve/deinline_import_export.rs +++ b/third_party/rust/wast/src/core/resolve/deinline_import_export.rs @@ -85,6 +85,7 @@ pub fn run(fields: &mut Vec) { } else { Instruction::I64Const(0) }]), + branch_hints: Vec::new(), }, }, data, @@ -143,6 +144,7 @@ pub fn run(fields: &mut Vec) { table: Index::Id(id), offset: Expression { instrs: Box::new([Instruction::I32Const(0)]), + branch_hints: Vec::new(), }, }, payload, diff --git a/third_party/rust/wast/src/core/table.rs b/third_party/rust/wast/src/core/table.rs index 280244498f..e7f0b0f974 100644 --- a/third_party/rust/wast/src/core/table.rs +++ b/third_party/rust/wast/src/core/table.rs @@ -253,6 +253,7 @@ impl<'a> ElemPayload<'a> { ElemPayload::Exprs { exprs, .. } => { let expr = Expression { instrs: [Instruction::RefFunc(func)].into(), + branch_hints: Vec::new(), }; exprs.push(expr); } diff --git a/third_party/rust/wast/src/lib.rs b/third_party/rust/wast/src/lib.rs index 7923a343b5..bb16574177 100644 --- a/third_party/rust/wast/src/lib.rs +++ b/third_party/rust/wast/src/lib.rs @@ -538,4 +538,5 @@ pub mod annotation { annotation!(name); annotation!(producers); annotation!(dylink_0 = "dylink.0"); + annotation!(metadata_code_branch_hint = "metadata.code.branch_hint"); } diff --git a/third_party/rust/wast/src/parser.rs b/third_party/rust/wast/src/parser.rs index 0c85923f83..7a20ebe255 100644 --- a/third_party/rust/wast/src/parser.rs +++ b/third_party/rust/wast/src/parser.rs @@ -65,6 +65,7 @@ use crate::lexer::{Float, Integer, Lexer, Token, TokenKind}; use crate::token::Span; use crate::Error; +use bumpalo::Bump; use std::borrow::Cow; use std::cell::{Cell, RefCell}; use std::collections::HashMap; @@ -303,7 +304,7 @@ pub struct ParseBuffer<'a> { cur: Cell, known_annotations: RefCell>, depth: Cell, - strings: RefCell>>, + strings: Bump, } /// The current position within a `Lexer` that we're at. This simultaneously @@ -396,14 +397,7 @@ impl ParseBuffer<'_> { /// This will return a reference to `s`, but one that's safely rooted in the /// `Parser`. fn push_str(&self, s: Vec) -> &[u8] { - let s = Box::from(s); - let ret = &*s as *const [u8]; - self.strings.borrow_mut().push(s); - // This should be safe in that the address of `ret` isn't changing as - // it's on the heap itself. Additionally the lifetime of this return - // value is tied to the lifetime of `self` (nothing is deallocated - // early), so it should be safe to say the two have the same lifetime. - unsafe { &*ret } + self.strings.alloc_slice_copy(&s) } /// Lexes the next "significant" token from the `pos` specified. diff --git a/third_party/rust/wast/src/wat.rs b/third_party/rust/wast/src/wat.rs index f74121187d..6d9a233359 100644 --- a/third_party/rust/wast/src/wat.rs +++ b/third_party/rust/wast/src/wat.rs @@ -43,6 +43,7 @@ impl<'a> Parse<'a> for Wat<'a> { let _r = parser.register_annotation("custom"); let _r = parser.register_annotation("producers"); let _r = parser.register_annotation("name"); + let _r = parser.register_annotation("metadata.code.branch_hint"); let wat = if parser.peek2::()? { Wat::Module(parser.parens(|parser| parser.parse())?) } else if parser.peek2::()? { diff --git a/third_party/rust/webext-storage/.cargo-checksum.json b/third_party/rust/webext-storage/.cargo-checksum.json index 98fa4d6a22..25f2af17bf 100644 --- a/third_party/rust/webext-storage/.cargo-checksum.json +++ b/third_party/rust/webext-storage/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"Cargo.toml":"a11f7fbc29c375034289e7bdd11da4aadac9cb2d939a4f2e5dc61aaea35cf465","README.md":"1fd617294339930ee1ad5172377648b268cce0216fc3971facbfe7c6839e9ab1","build.rs":"92f7d380f3d8fab1e6d80276915af57192e276321d132a5f800ea4520e9cb469","sql/create_schema.sql":"a17311a407ec10e033886b7125da4c8b84bc6d761f6b28edc9594de430e1d964","sql/create_sync_temp_tables.sql":"860ede362c94feb47d85522553fa2852f9bdb9f9b025d6438dd5dee3d4acd527","sql/tests/create_schema_v1.sql":"77cf0c90eaac3e1aea626537147e1b8ec349b68d6076c92fa7ae402aac613050","src/api.rs":"6fe362e4f437def2ad2249de385cca8f0d1d5d67679240351e9f57523fefe5e7","src/db.rs":"b95024c1d8f36a76a6f3098acea5a82bc49de144a24cdc280ed38e9bcc8e772b","src/error.rs":"6437e9a0edefac2707af85eef13bdbfcd53a84d7aa7859599155d10451d42361","src/ffi.rs":"f66a81393bebe7a4b7e7960cb426df106ff1f02bfebcaa6e335b4b8b56c5c936","src/lib.rs":"ab25e7c6ea67fb905fe6dad866c0d2c462b1e93bcff283db947513aeabbb2d73","src/migration.rs":"8d92f82b2ba38e1039fd054c8c75078a6b896a0d3cdc1a52571456b25a32c9c3","src/schema.rs":"d8dd8f66cad71e3e369722734e0d5d16fd9423d5f6a5abba1854a27e1e814724","src/store.rs":"d208689c46fb97cd2c60a0c610ba1998a7132fb50fffa2eefa1d6b169b7c34f0","src/sync/bridge.rs":"996de05beb2904f84b3cbfc9ef85c4844078fdb4867d9068390d496156bee614","src/sync/incoming.rs":"dd77c64e2ade4f39cba258decab6d3db8ad0b5f513aa018efbd56b9869a021d9","src/sync/mod.rs":"bd1bc5c428dfda6aee7efe53b6e74b8015da5129a303638a21ca8d63516e4061","src/sync/outgoing.rs":"dacb77b956f2546fd60a89367927a199d9b662b17201d0781145f7405b61fdce","src/sync/sync_tests.rs":"f3846ca7e463315ba9788826613b987ddcff7b21672ff257a98769ee94f4191a","src/webext-storage.udl":"0341d431ba837cf64ea210ef6157010c6664a0b5a194e89acb0414938636b391","uniffi.toml":"beeec89c2f877eb89be0090dc304dbc7c74e787385e7459bad78c6165bb66791"},"package":null} \ No newline at end of file +{"files":{"Cargo.toml":"b20c30e7d9ae0460d04b744d5a9274985ded6e5b9ed0262b914553ca71425821","README.md":"1fd617294339930ee1ad5172377648b268cce0216fc3971facbfe7c6839e9ab1","build.rs":"92f7d380f3d8fab1e6d80276915af57192e276321d132a5f800ea4520e9cb469","sql/create_schema.sql":"a17311a407ec10e033886b7125da4c8b84bc6d761f6b28edc9594de430e1d964","sql/create_sync_temp_tables.sql":"860ede362c94feb47d85522553fa2852f9bdb9f9b025d6438dd5dee3d4acd527","sql/tests/create_schema_v1.sql":"77cf0c90eaac3e1aea626537147e1b8ec349b68d6076c92fa7ae402aac613050","src/api.rs":"6fe362e4f437def2ad2249de385cca8f0d1d5d67679240351e9f57523fefe5e7","src/db.rs":"b95024c1d8f36a76a6f3098acea5a82bc49de144a24cdc280ed38e9bcc8e772b","src/error.rs":"6437e9a0edefac2707af85eef13bdbfcd53a84d7aa7859599155d10451d42361","src/ffi.rs":"f66a81393bebe7a4b7e7960cb426df106ff1f02bfebcaa6e335b4b8b56c5c936","src/lib.rs":"ab25e7c6ea67fb905fe6dad866c0d2c462b1e93bcff283db947513aeabbb2d73","src/migration.rs":"8d92f82b2ba38e1039fd054c8c75078a6b896a0d3cdc1a52571456b25a32c9c3","src/schema.rs":"d8dd8f66cad71e3e369722734e0d5d16fd9423d5f6a5abba1854a27e1e814724","src/store.rs":"d208689c46fb97cd2c60a0c610ba1998a7132fb50fffa2eefa1d6b169b7c34f0","src/sync/bridge.rs":"996de05beb2904f84b3cbfc9ef85c4844078fdb4867d9068390d496156bee614","src/sync/incoming.rs":"dd77c64e2ade4f39cba258decab6d3db8ad0b5f513aa018efbd56b9869a021d9","src/sync/mod.rs":"bd1bc5c428dfda6aee7efe53b6e74b8015da5129a303638a21ca8d63516e4061","src/sync/outgoing.rs":"dacb77b956f2546fd60a89367927a199d9b662b17201d0781145f7405b61fdce","src/sync/sync_tests.rs":"f3846ca7e463315ba9788826613b987ddcff7b21672ff257a98769ee94f4191a","src/webext-storage.udl":"0341d431ba837cf64ea210ef6157010c6664a0b5a194e89acb0414938636b391","uniffi.toml":"beeec89c2f877eb89be0090dc304dbc7c74e787385e7459bad78c6165bb66791"},"package":null} \ No newline at end of file diff --git a/third_party/rust/webext-storage/Cargo.toml b/third_party/rust/webext-storage/Cargo.toml index c3f9b170df..8306a25dd2 100644 --- a/third_party/rust/webext-storage/Cargo.toml +++ b/third_party/rust/webext-storage/Cargo.toml @@ -28,7 +28,7 @@ serde = "1" serde_derive = "1" serde_json = "1" thiserror = "1.0" -uniffi = "0.25.2" +uniffi = "0.27.1" [dependencies.error-support] path = "../support/error" @@ -78,7 +78,7 @@ path = "../support/sql" path = "../support/rc_crypto/nss/nss_build_common" [build-dependencies.uniffi] -version = "0.25.2" +version = "0.27.1" features = ["build"] [features] diff --git a/third_party/rust/weedle2/.cargo-checksum.json b/third_party/rust/weedle2/.cargo-checksum.json index 9eb63aa3c1..13a9d357d5 100644 --- a/third_party/rust/weedle2/.cargo-checksum.json +++ b/third_party/rust/weedle2/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"Cargo.toml":"1c9da38b083da061c42208494a1161d95938a524054f95b39097248da32ebfd0","LICENSE.md":"467e2e44913e850ca4fb6760fa3b78ffefda2a21688a6b1e26a8b706963669eb","README.md":"32092d702b93e9cc8d95fbcad1e30e315882a55496ad90a855ea575dc47aa7d4","release.toml":"fce2a51533478f7e7d663b11ac69676323856780adf8af158ce213c9dbc86c75","src/argument.rs":"aa7a119364de0a1f91b00db6cffb0a7b2ee73e0fb79a764211165ccebed56dc8","src/attribute.rs":"b3577059be9c1262d3360b4b4cf4cc6cb052251e5b9d64906fb40fc5e925da69","src/common.rs":"9d781dacc2582ac375f2c3b6b22bcb01add4a63db525d18ebb3af6fe633a311e","src/dictionary.rs":"be209d70b0db33acf753ca63998a63dd37a9d4b193583a1490782781360c2d01","src/interface.rs":"bd2f32bfbcab2167f056fabbf92da9852300141ffc80d9542c6e1bbea9d521ac","src/lib.rs":"42466a14426416dc0d3500842c86fe00471c2d73be6df823c3e05ba0e4572d20","src/literal.rs":"16d44ae36f893c3b2d8d807921b31b992b7e3ce7d1729913b4f392f52db44f9e","src/macros.rs":"2f24c285806c863fd9dc65534479d36961d9666ca5e11c9137f2ce2659a712b6","src/mixin.rs":"d65ff3a49615dabdf3f7845723f01ae730a83c8ef5bdd9a945d35149a9ef2858","src/namespace.rs":"fe6b406c2ab8bd904d0dff8a12321c77f4082ab364559e578e53ab415c8541fa","src/term.rs":"049514b2b44f1bcb5c23d225d3bb0a6135892a5660049c694681c5b99d47d8e7","src/types.rs":"0fe236336e71079c88ae4848624af29c87dd16266ba36c30a0d47e7dc4d33587","src/whitespace.rs":"552ebc98857859e8714e06bb25681ceab6052d009e00f55cceda0d4fb3c4f059"},"package":"2e79c5206e1f43a2306fd64bdb95025ee4228960f2e6c5a8b173f3caaf807741"} \ No newline at end of file +{"files":{"Cargo.toml":"cd75893e67b7ed5e1ee200efddc874a7d235a7275b3d9bbadd728d47a5d5b8c6","LICENSE.md":"467e2e44913e850ca4fb6760fa3b78ffefda2a21688a6b1e26a8b706963669eb","README.md":"b18a5a8af93ebf01f28b3ec637e64fb0bd21d556e356984974d0031f6eeda47a","release.toml":"fce2a51533478f7e7d663b11ac69676323856780adf8af158ce213c9dbc86c75","src/argument.rs":"aa7a119364de0a1f91b00db6cffb0a7b2ee73e0fb79a764211165ccebed56dc8","src/attribute.rs":"b3577059be9c1262d3360b4b4cf4cc6cb052251e5b9d64906fb40fc5e925da69","src/common.rs":"c0ae3236856bbe9917326ac9769e2fe15283cd4d1a0216aafcea3bc03ad724cc","src/dictionary.rs":"d0c9a40e694eb563d7d8307d36a3f9a3be1766846bdfa809d614a6e1e9e149d1","src/interface.rs":"352f7773e9e3870fa61efdb2b737d777a34def1eb8301ec66f7c416b31d5e88b","src/lib.rs":"95c6b27082959165f3899b4a5a520b10fbcab80dcf94ecf030aba74df0748203","src/literal.rs":"16d44ae36f893c3b2d8d807921b31b992b7e3ce7d1729913b4f392f52db44f9e","src/macros.rs":"2f24c285806c863fd9dc65534479d36961d9666ca5e11c9137f2ce2659a712b6","src/mixin.rs":"d65ff3a49615dabdf3f7845723f01ae730a83c8ef5bdd9a945d35149a9ef2858","src/namespace.rs":"417a9a6cb26efc8f84bd84e59c6ae7378502f920a76738b355e8c905e62efed8","src/term.rs":"049514b2b44f1bcb5c23d225d3bb0a6135892a5660049c694681c5b99d47d8e7","src/types.rs":"0fe236336e71079c88ae4848624af29c87dd16266ba36c30a0d47e7dc4d33587","src/whitespace.rs":"9b58cfc0a4b8667a26ff3d4dcd424a3d00bf23abab4dd0d286318cb18bd77a13"},"package":"998d2c24ec099a87daf9467808859f9d82b61f1d9c9701251aea037f514eae0e"} \ No newline at end of file diff --git a/third_party/rust/weedle2/Cargo.toml b/third_party/rust/weedle2/Cargo.toml index 072d3f9a2a..66703cf607 100644 --- a/third_party/rust/weedle2/Cargo.toml +++ b/third_party/rust/weedle2/Cargo.toml @@ -12,7 +12,7 @@ [package] edition = "2021" name = "weedle2" -version = "4.0.0" +version = "5.0.0" authors = [ "Sharad Chand ", "Jan-Erik Rediger ", @@ -21,10 +21,9 @@ exclude = ["tests"] description = "A WebIDL Parser" homepage = "https://github.com/mozilla/uniffi-rs/tree/main/weedle2" documentation = "https://docs.rs/weedle2" -readme = "./README.md" +readme = "README.md" license = "MIT" repository = "https://github.com/mozilla/uniffi-rs" -resolver = "2" [lib] name = "weedle" diff --git a/third_party/rust/weedle2/README.md b/third_party/rust/weedle2/README.md index 889dc0da19..88e4cec017 100644 --- a/third_party/rust/weedle2/README.md +++ b/third_party/rust/weedle2/README.md @@ -28,7 +28,7 @@ Parses valid WebIDL definitions & produces a data structure starting from ```toml [dependencies] -weedle2 = "4.0.0" +weedle2 = "5.0.0" ``` ### `src/main.rs` diff --git a/third_party/rust/weedle2/src/common.rs b/third_party/rust/weedle2/src/common.rs index fadf89ba8b..d36f6e5438 100644 --- a/third_party/rust/weedle2/src/common.rs +++ b/third_party/rust/weedle2/src/common.rs @@ -33,6 +33,18 @@ impl<'a, T: Parse<'a>, U: Parse<'a>, V: Parse<'a>> Parse<'a> for (T, U, V) { parser!(nom::sequence::tuple((T::parse, U::parse, V::parse))); } +pub(crate) fn docstring(input: &str) -> IResult<&str, String> { + nom::multi::many1(nom::sequence::preceded( + nom::character::complete::multispace0, + nom::sequence::delimited( + nom::bytes::complete::tag("///"), + nom::bytes::complete::take_until("\n"), + nom::bytes::complete::tag("\n"), + ), + ))(input) + .map(|io| (io.0, io.1.join("\n"))) +} + ast_types! { /// Parses `( body )` #[derive(Copy, Default)] @@ -103,6 +115,11 @@ ast_types! { assign: term!(=), value: DefaultValue<'a>, } + + /// Represents consecutive comment lines starting with `///`, joined by `\n`. + struct Docstring( + String = docstring, + ) } #[cfg(test)] @@ -211,4 +228,32 @@ mod test { Identifier; 0 == "hello"; }); + + test!(should_parse_docstring { "///hello world\n" => + ""; + Docstring; + 0 == "hello world"; + }); + + test!(should_parse_multiline_docstring { "///hello\n///world\n" => + ""; + Docstring; + 0 == "hello\nworld"; + }); + + test!(should_parse_multiline_indented_docstring { "///hello\n ///world\n" => + ""; + Docstring; + 0 == "hello\nworld"; + }); + + test!(should_not_parse_docstring_with_comments { "///hello\n//comment1\n///world\n" => + "//comment1\n///world\n"; + Docstring; + 0 == "hello"; + }); + + test!(err should_not_parse_not_docstring { "" => + Docstring + }); } diff --git a/third_party/rust/weedle2/src/dictionary.rs b/third_party/rust/weedle2/src/dictionary.rs index 3c9b23cac5..b775d526dc 100644 --- a/third_party/rust/weedle2/src/dictionary.rs +++ b/third_party/rust/weedle2/src/dictionary.rs @@ -1,5 +1,5 @@ use crate::attribute::ExtendedAttributeList; -use crate::common::{Default, Identifier}; +use crate::common::{Default, Docstring, Identifier}; use crate::types::Type; /// Parses dictionary members @@ -8,6 +8,7 @@ pub type DictionaryMembers<'a> = Vec>; ast_types! { /// Parses dictionary member `[attributes]? required? type identifier ( = default )?;` struct DictionaryMember<'a> { + docstring: Option, attributes: Option>, required: Option, type_: Type<'a>, diff --git a/third_party/rust/weedle2/src/interface.rs b/third_party/rust/weedle2/src/interface.rs index 5e30909c38..ab3c10d3c3 100644 --- a/third_party/rust/weedle2/src/interface.rs +++ b/third_party/rust/weedle2/src/interface.rs @@ -1,6 +1,6 @@ use crate::argument::ArgumentList; use crate::attribute::ExtendedAttributeList; -use crate::common::{Generics, Identifier, Parenthesized}; +use crate::common::{Docstring, Generics, Identifier, Parenthesized}; use crate::literal::ConstValue; use crate::types::{AttributedType, ConstType, ReturnType}; @@ -41,6 +41,7 @@ ast_types! { /// /// (( )) means ( ) chars Constructor(struct ConstructorInterfaceMember<'a> { + docstring: Option, attributes: Option>, constructor: term!(constructor), args: Parenthesized>, @@ -50,6 +51,7 @@ ast_types! { /// /// (( )) means ( ) chars Operation(struct OperationInterfaceMember<'a> { + docstring: Option, attributes: Option>, modifier: Option, special: Option, diff --git a/third_party/rust/weedle2/src/lib.rs b/third_party/rust/weedle2/src/lib.rs index 610a34fa14..71ab7c33b5 100644 --- a/third_party/rust/weedle2/src/lib.rs +++ b/third_party/rust/weedle2/src/lib.rs @@ -23,7 +23,7 @@ use self::argument::ArgumentList; use self::attribute::ExtendedAttributeList; -use self::common::{Braced, Identifier, Parenthesized, PunctuatedNonEmpty}; +use self::common::{Braced, Docstring, Identifier, Parenthesized, PunctuatedNonEmpty}; use self::dictionary::DictionaryMembers; use self::interface::{Inheritance, InterfaceMembers}; use self::literal::StringLit; @@ -109,6 +109,7 @@ ast_types! { }), /// Parses `[attributes]? callback interface identifier ( : inheritance )? { members };` CallbackInterface(struct CallbackInterfaceDefinition<'a> { + docstring: Option, attributes: Option>, callback: term!(callback), interface: term!(interface), @@ -119,6 +120,7 @@ ast_types! { }), /// Parses `[attributes]? interface identifier ( : inheritance )? { members };` Interface(struct InterfaceDefinition<'a> { + docstring: Option, attributes: Option>, interface: term!(interface), identifier: Identifier<'a>, @@ -137,6 +139,7 @@ ast_types! { }), /// Parses `[attributes]? namespace identifier { members };` Namespace(struct NamespaceDefinition<'a> { + docstring: Option, attributes: Option>, namespace: term!(namespace), identifier: Identifier<'a>, @@ -145,6 +148,7 @@ ast_types! { }), /// Parses `[attributes]? dictionary identifier ( : inheritance )? { members };` Dictionary(struct DictionaryDefinition<'a> { + docstring: Option, attributes: Option>, dictionary: term!(dictionary), identifier: Identifier<'a>, @@ -191,6 +195,7 @@ ast_types! { }), /// Parses `[attributes]? enum identifier { values };` Enum(struct EnumDefinition<'a> { + docstring: Option, attributes: Option>, enum_: term!(enum), identifier: Identifier<'a>, @@ -224,8 +229,15 @@ ast_types! { } } +ast_types! { + struct EnumVariant<'a> { + docstring: Option, + value: StringLit<'a>, + } +} + /// Parses a non-empty enum value list -pub type EnumValueList<'a> = PunctuatedNonEmpty, term!(,)>; +pub type EnumValueList<'a> = PunctuatedNonEmpty, term!(,)>; #[cfg(test)] mod test { diff --git a/third_party/rust/weedle2/src/namespace.rs b/third_party/rust/weedle2/src/namespace.rs index ed28573218..60673cdcec 100644 --- a/third_party/rust/weedle2/src/namespace.rs +++ b/third_party/rust/weedle2/src/namespace.rs @@ -1,6 +1,6 @@ use crate::argument::ArgumentList; use crate::attribute::ExtendedAttributeList; -use crate::common::{Identifier, Parenthesized}; +use crate::common::{Docstring, Identifier, Parenthesized}; use crate::types::{AttributedType, ReturnType}; /// Parses namespace members declaration @@ -13,6 +13,7 @@ ast_types! { /// /// (( )) means ( ) chars Operation(struct OperationNamespaceMember<'a> { + docstring: Option, attributes: Option>, return_type: ReturnType<'a>, identifier: Option>, @@ -21,6 +22,7 @@ ast_types! { }), /// Parses `[attribute]? readonly attributetype type identifier;` Attribute(struct AttributeNamespaceMember<'a> { + docstring: Option, attributes: Option>, readonly: term!(readonly), attribute: term!(attribute), diff --git a/third_party/rust/weedle2/src/whitespace.rs b/third_party/rust/weedle2/src/whitespace.rs index 336e4784e1..4be3ca43e8 100644 --- a/third_party/rust/weedle2/src/whitespace.rs +++ b/third_party/rust/weedle2/src/whitespace.rs @@ -7,6 +7,7 @@ pub(crate) fn sp(input: &str) -> IResult<&str, &str> { (), nom::sequence::tuple(( nom::bytes::complete::tag("//"), + nom::combinator::not(nom::bytes::complete::tag("/")), nom::bytes::complete::take_until("\n"), nom::bytes::complete::tag("\n"), )), diff --git a/third_party/rust/wgpu-core/.cargo-checksum.json b/third_party/rust/wgpu-core/.cargo-checksum.json index d22e0914d7..f025f2ccaa 100644 --- a/third_party/rust/wgpu-core/.cargo-checksum.json +++ b/third_party/rust/wgpu-core/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"Cargo.toml":"4880d66b004519ca6e424fc9e2e6ac065536d36334a2e327b90422e97f2a2a35","LICENSE.APACHE":"a6cba85bc92e0cff7a450b1d873c0eaa2e9fc96bf472df0247a26bec77bf3ff9","LICENSE.MIT":"c7fea58d1cfe49634cd92e54fc10a9d871f4b275321a4cd8c09e449122caaeb4","build.rs":"a99478d7f63fb41429e3834f4d0e5cd333f94ba1834c68295f929170e16987de","src/any_surface.rs":"1c032bc1894a222a47f0116b976f1543c1140c0534678502ee1172d4f77fc515","src/binding_model.rs":"bb4aefad17957e770a5f70f00bf5853dc13da1d9f836493c9aa9adbbe7bb8147","src/command/bind.rs":"a37f042484b65d9fdea4cdab3667381623ee9a8943a6d32683d410b92736d306","src/command/bundle.rs":"fea00382acdf204bcb58522953335dd8f0092565693fa65d0c008e2698e39445","src/command/clear.rs":"03cfc0d4c689d56010391440ab279e615ef1d3235eb1f9f9df0323682d275109","src/command/compute.rs":"2b6beed328ed351ad6fe7088cfa1824c1bf4be50deaeab971cdcb09914d791de","src/command/draw.rs":"15f9ad857504d8098279f9c789317feba321c9b6b8f0de20b8ba98f358c99d89","src/command/memory_init.rs":"6ec93b9e2eb21edaa534e60770b4ba95735e9de61e74d827bc492df8e3639449","src/command/mod.rs":"1d347e1746194f7a07d1f75bd3a9d3cbe121fbaa479c25ba6b8c16e9d699e06b","src/command/query.rs":"43b78a163eb0eb5f1427b7a57b6d39a2748c25f880ba024c91e2f71e2a6a817d","src/command/render.rs":"808dc8106811b32877637851e63baeba7c7438748dec67cbb17ea93c58dc61bd","src/command/transfer.rs":"bf1077d1a99a258bad46087ae7234703627e7f4d30b38e6142d016c02deaad3a","src/conv.rs":"7e3ffe33b47a6fd3617aabf9f11cc68f1ccbee2c7343b8dbbcd0e8f3447e1ad8","src/device/any_device.rs":"65f47b58939b60f88f47861e65d5d45209492df8e73e7c1b60b3b459f510c09e","src/device/bgl.rs":"ec8bdd6e9b4cd50c25bed317275863d0c16bb6619f62ed85bf0464948010dfc1","src/device/global.rs":"ff90a9e3b261bedbec37ab1aed0bf23f1e50c5418da72184e2b175057ed18fce","src/device/life.rs":"3cacaaa74df04bb1285a36d70395b35cfa17059f8d6289b41e665ecbc64cb66a","src/device/mod.rs":"fff41f92e1a9f6660e18dc30452d9911ca827701bb8303af2ae06f1c1e1a795f","src/device/queue.rs":"da0aeebfd1d1c6e155dc89cebf75dfdb6ec18062f9512044ed7e0fef0bda2f74","src/device/resource.rs":"74d3180c12602133bee46925d3788ac510d2ad5ea141a2b46f6904f38549053b","src/device/trace.rs":"9deb1b083165e07253b4928ac2f564aba06f9089c3aca1c0a1d438d87d981542","src/error.rs":"e3b6b7a69877437f4e46af7f0e8ca1db1822beae7c8448db41c2bae0f64b2bb4","src/global.rs":"0966475959706650fd036a18d51441a8e14c3ef10107db617f597614ca47e50a","src/hal_api.rs":"1cd9c3fe1c9d8c3a24e3e7f963a2ef26e056a2b26d529b840dbc969090aaf201","src/hash_utils.rs":"e8d484027c7ce81978e4679a5e20af9416ab7d2fa595f1ca95992b29d625b0ca","src/hub.rs":"352a1b75d4535f24b06d16134421db98f910e6e719f50f863a204df6768e3369","src/id.rs":"9f67dbef5d7a416eb740281ecf8a94673f624da16f21ec33c425c11d9ed01e90","src/identity.rs":"12b820eb4b8bd7b226e15eec97d0f100a695f6b9be7acd79ad2421f2d0fe1985","src/init_tracker/buffer.rs":"61eb9cfaa312135b7a937ff6a3117f531b5b7323fae6553a41d6de9bc106d7e0","src/init_tracker/mod.rs":"a0f64730cc025113b656b4690f9dcb0ec18b8770bc7ef24c7b4ad8bebae03d24","src/init_tracker/texture.rs":"030fd594bf9948fad391390d85c5e1fec7eaf67b6e812c60f2dd59bc4fda8fd5","src/instance.rs":"b6de2a371ef3b43d3217102fe87e423dd1eb12da86b65f54b902d9eaa38b6b9f","src/lib.rs":"4ad9979442cf88557fb3b9f8d3b26c7b929a710c60cabcd1f51788917c95aecb","src/pipeline.rs":"89d88de4b8b8e1dd2bc834d101a1bdf34816ebcaa616dc795f604e9183a21cd0","src/pool.rs":"778ea1c23fcfaaa5001606e686f712f606826039d60dd5a3cd26e7de91ac057a","src/present.rs":"f69580ee0baf181162f9dd82b159596c738558d8abb60db93047effbe1436b2f","src/registry.rs":"913e651dc585ff12fe7659443c38d635a2904881e56cb7159c5ca72d45ae5800","src/resource.rs":"59731bc9a207d87b07b6db9c897e20d64be27c144bb8eb8ab2505807163acfc4","src/snatch.rs":"29a1135ee09c06883eac4df6f45b7220c2ba8f89f34232ea1d270d6e7b05c7a8","src/storage.rs":"f0c41461b8f9cdc862dbd3de04c8e720ee416c7c57310696f6f4fd22183fcc85","src/track/buffer.rs":"83a0cbb8026dbd651d32ea5a47f332f691afed1c5e6f14e78a4fe8aa25e2ad12","src/track/metadata.rs":"655985fdfdd1c7fe8220af98abadf33de7e8920b485e3dd27c28688c3dd2e47d","src/track/mod.rs":"52470a48de6b5dce55385e23ba7a3cbf512cc10cdf431a35aa42190e2fc4306d","src/track/range.rs":"2a15794e79b0470d5ba6b3267173a42f34312878e1cb288f198d2854a7888e53","src/track/stateless.rs":"305e0a493fb1cd0a325274c0757e99c19f9d14deaa8ca11ada41c1399a4ae5c4","src/track/texture.rs":"ba3e3814b341b5242548b55d77bef1d1d9e7d52d63784be98c51e342da7fefff","src/validation.rs":"026168ac4f23bc6a58a90c78fd3eb73485b3c1aad630ef43755604d1babade79"},"package":null} \ No newline at end of file +{"files":{"Cargo.toml":"3f5fa464854359b0150d4fe82cf5b0c17874dbb3c3c708b2a9cc24ebc1a61349","LICENSE.APACHE":"a6cba85bc92e0cff7a450b1d873c0eaa2e9fc96bf472df0247a26bec77bf3ff9","LICENSE.MIT":"c7fea58d1cfe49634cd92e54fc10a9d871f4b275321a4cd8c09e449122caaeb4","build.rs":"a99478d7f63fb41429e3834f4d0e5cd333f94ba1834c68295f929170e16987de","src/any_surface.rs":"1c032bc1894a222a47f0116b976f1543c1140c0534678502ee1172d4f77fc515","src/binding_model.rs":"bb4aefad17957e770a5f70f00bf5853dc13da1d9f836493c9aa9adbbe7bb8147","src/command/bind.rs":"a37f042484b65d9fdea4cdab3667381623ee9a8943a6d32683d410b92736d306","src/command/bundle.rs":"3435ea21daba6f5a26f9a4158b342d0b9a1839da39b82fa8c23d84ee144b9da0","src/command/clear.rs":"2eb9c323c3ae3644563cab1bb1d26e4a17a54f5a120698c7f09ef9f9e9077780","src/command/compute.rs":"1ad1d8b83774422bd73c7c67c153205e451fd71602b37f34162b59e5f7dbbc75","src/command/draw.rs":"15f9ad857504d8098279f9c789317feba321c9b6b8f0de20b8ba98f358c99d89","src/command/memory_init.rs":"71550dabbf7cc3c3ff6aa4ccd31af080bb5e1cb1e21422daea63fee30294476f","src/command/mod.rs":"1d347e1746194f7a07d1f75bd3a9d3cbe121fbaa479c25ba6b8c16e9d699e06b","src/command/query.rs":"43b78a163eb0eb5f1427b7a57b6d39a2748c25f880ba024c91e2f71e2a6a817d","src/command/render.rs":"875545efa83e56face9a857c5d8c814bc46f4b0538a152933265533176ad1e84","src/command/transfer.rs":"63042151145825c5cac6459a5a254a82142c86d299895f72bef2682f71de7ad1","src/conv.rs":"7e3ffe33b47a6fd3617aabf9f11cc68f1ccbee2c7343b8dbbcd0e8f3447e1ad8","src/device/any_device.rs":"65f47b58939b60f88f47861e65d5d45209492df8e73e7c1b60b3b459f510c09e","src/device/bgl.rs":"ec8bdd6e9b4cd50c25bed317275863d0c16bb6619f62ed85bf0464948010dfc1","src/device/global.rs":"6a08dcc25059f2194999c4f5e3a57a1a96b17031873b1a871095dc58115470fb","src/device/life.rs":"efba75c8632e0bd9c655dbff1f8ff027900f388b6eeb357a9c02ddad24474b23","src/device/mod.rs":"e578d03253a9af314e1e81168d45ea8cc0a8f56df79e440dc1ade945d75277ec","src/device/queue.rs":"4405a6b8ea29093239f1556e5352f7c777f4f6b783fd5dea458fde22556741ca","src/device/resource.rs":"65f84150c467aa50615a097d15f6b4dba3514b0adf2de1e9b9837e5084619a80","src/device/trace.rs":"9deb1b083165e07253b4928ac2f564aba06f9089c3aca1c0a1d438d87d981542","src/error.rs":"e3b6b7a69877437f4e46af7f0e8ca1db1822beae7c8448db41c2bae0f64b2bb4","src/global.rs":"0966475959706650fd036a18d51441a8e14c3ef10107db617f597614ca47e50a","src/hal_api.rs":"1cd9c3fe1c9d8c3a24e3e7f963a2ef26e056a2b26d529b840dbc969090aaf201","src/hash_utils.rs":"e8d484027c7ce81978e4679a5e20af9416ab7d2fa595f1ca95992b29d625b0ca","src/hub.rs":"352a1b75d4535f24b06d16134421db98f910e6e719f50f863a204df6768e3369","src/id.rs":"9f67dbef5d7a416eb740281ecf8a94673f624da16f21ec33c425c11d9ed01e90","src/identity.rs":"12b820eb4b8bd7b226e15eec97d0f100a695f6b9be7acd79ad2421f2d0fe1985","src/init_tracker/buffer.rs":"61eb9cfaa312135b7a937ff6a3117f531b5b7323fae6553a41d6de9bc106d7e0","src/init_tracker/mod.rs":"a0f64730cc025113b656b4690f9dcb0ec18b8770bc7ef24c7b4ad8bebae03d24","src/init_tracker/texture.rs":"030fd594bf9948fad391390d85c5e1fec7eaf67b6e812c60f2dd59bc4fda8fd5","src/instance.rs":"b6de2a371ef3b43d3217102fe87e423dd1eb12da86b65f54b902d9eaa38b6b9f","src/lib.rs":"4ad9979442cf88557fb3b9f8d3b26c7b929a710c60cabcd1f51788917c95aecb","src/pipeline.rs":"89d88de4b8b8e1dd2bc834d101a1bdf34816ebcaa616dc795f604e9183a21cd0","src/pool.rs":"778ea1c23fcfaaa5001606e686f712f606826039d60dd5a3cd26e7de91ac057a","src/present.rs":"f69580ee0baf181162f9dd82b159596c738558d8abb60db93047effbe1436b2f","src/registry.rs":"913e651dc585ff12fe7659443c38d635a2904881e56cb7159c5ca72d45ae5800","src/resource.rs":"59731bc9a207d87b07b6db9c897e20d64be27c144bb8eb8ab2505807163acfc4","src/snatch.rs":"5cf0e4e4611083e8256092696ecf0caa9e855dd680c18fc7dbb227d071aa2548","src/storage.rs":"f0c41461b8f9cdc862dbd3de04c8e720ee416c7c57310696f6f4fd22183fcc85","src/track/buffer.rs":"d1b4c85dfaf1bcc93013d7f51ba677f97c36245c9de224fb328595aa15109cf6","src/track/metadata.rs":"7016911136bc5b97053f54241b01d344f5d65c4337a292c8aa32bb142a26be0c","src/track/mod.rs":"5365c532a531d224a958bd8217769ef59d67d293b4f95972c10db71f6bff1a80","src/track/range.rs":"2a15794e79b0470d5ba6b3267173a42f34312878e1cb288f198d2854a7888e53","src/track/stateless.rs":"305e0a493fb1cd0a325274c0757e99c19f9d14deaa8ca11ada41c1399a4ae5c4","src/track/texture.rs":"da83403d5990021c5db78f1ffdf69198eadb99cc77a6bc481648337a5403295c","src/validation.rs":"026168ac4f23bc6a58a90c78fd3eb73485b3c1aad630ef43755604d1babade79"},"package":null} \ No newline at end of file diff --git a/third_party/rust/wgpu-core/Cargo.toml b/third_party/rust/wgpu-core/Cargo.toml index 3d3b4dc80c..aef86b3bbf 100644 --- a/third_party/rust/wgpu-core/Cargo.toml +++ b/third_party/rust/wgpu-core/Cargo.toml @@ -11,7 +11,7 @@ [package] edition = "2021" -rust-version = "1.70" +rust-version = "1.74" name = "wgpu-core" version = "0.19.0" authors = ["gfx-rs developers"] @@ -127,7 +127,7 @@ vulkan = ["hal/vulkan"] wgsl = ["naga/wgsl-in"] [target."cfg(all(target_arch = \"wasm32\", not(target_os = \"emscripten\")))".dependencies.web-sys] -version = "0.3.67" +version = "0.3.69" features = [ "HtmlCanvasElement", "OffscreenCanvas", diff --git a/third_party/rust/wgpu-core/src/command/bundle.rs b/third_party/rust/wgpu-core/src/command/bundle.rs index ab2d18bc59..47beda8ec6 100644 --- a/third_party/rust/wgpu-core/src/command/bundle.rs +++ b/third_party/rust/wgpu-core/src/command/bundle.rs @@ -99,6 +99,7 @@ use crate::{ pipeline::{PipelineFlags, RenderPipeline, VertexStep}, resource::{Buffer, Resource, ResourceInfo, ResourceType}, resource_log, + snatch::SnatchGuard, track::RenderBundleScope, validation::check_buffer_usage, Label, LabelHelpers, @@ -165,7 +166,7 @@ fn validate_indexed_draw( ) -> Result<(), DrawError> { let last_index = first_index as u64 + index_count as u64; let index_limit = index_state.limit(); - if last_index <= index_limit { + if last_index > index_limit { return Err(DrawError::IndexBeyondLimit { last_index, index_limit, @@ -894,7 +895,11 @@ impl RenderBundle { /// Note that the function isn't expected to fail, generally. /// All the validation has already been done by this point. /// The only failure condition is if some of the used buffers are destroyed. - pub(super) unsafe fn execute(&self, raw: &mut A::CommandEncoder) -> Result<(), ExecutionError> { + pub(super) unsafe fn execute( + &self, + raw: &mut A::CommandEncoder, + snatch_guard: &SnatchGuard, + ) -> Result<(), ExecutionError> { let mut offsets = self.base.dynamic_offsets.as_slice(); let mut pipeline_layout = None::>>; if !self.discard_hal_labels { @@ -903,8 +908,6 @@ impl RenderBundle { } } - let snatch_guard = self.device.snatchable_lock.read(); - use ArcRenderCommand as Cmd; for command in self.base.commands.iter() { match command { @@ -914,7 +917,7 @@ impl RenderBundle { bind_group, } => { let raw_bg = bind_group - .raw(&snatch_guard) + .raw(snatch_guard) .ok_or(ExecutionError::InvalidBindGroup(bind_group.info.id()))?; unsafe { raw.set_bind_group( @@ -938,7 +941,7 @@ impl RenderBundle { size, } => { let buffer: &A::Buffer = buffer - .raw(&snatch_guard) + .raw(snatch_guard) .ok_or(ExecutionError::DestroyedBuffer(buffer.info.id()))?; let bb = hal::BufferBinding { buffer, @@ -954,7 +957,7 @@ impl RenderBundle { size, } => { let buffer = buffer - .raw(&snatch_guard) + .raw(snatch_guard) .ok_or(ExecutionError::DestroyedBuffer(buffer.info.id()))?; let bb = hal::BufferBinding { buffer, @@ -1041,7 +1044,7 @@ impl RenderBundle { indexed: false, } => { let buffer = buffer - .raw(&snatch_guard) + .raw(snatch_guard) .ok_or(ExecutionError::DestroyedBuffer(buffer.info.id()))?; unsafe { raw.draw_indirect(buffer, *offset, 1) }; } @@ -1052,7 +1055,7 @@ impl RenderBundle { indexed: true, } => { let buffer = buffer - .raw(&snatch_guard) + .raw(snatch_guard) .ok_or(ExecutionError::DestroyedBuffer(buffer.info.id()))?; unsafe { raw.draw_indexed_indirect(buffer, *offset, 1) }; } diff --git a/third_party/rust/wgpu-core/src/command/clear.rs b/third_party/rust/wgpu-core/src/command/clear.rs index e404fabb14..72c923f82e 100644 --- a/third_party/rust/wgpu-core/src/command/clear.rs +++ b/third_party/rust/wgpu-core/src/command/clear.rs @@ -12,6 +12,7 @@ use crate::{ id::{BufferId, CommandEncoderId, DeviceId, TextureId}, init_tracker::{MemoryInitKind, TextureInitRange}, resource::{Resource, Texture, TextureClearMode}, + snatch::SnatchGuard, track::{TextureSelector, TextureTracker}, }; @@ -239,6 +240,7 @@ impl Global { } let (encoder, tracker) = cmd_buf_data.open_encoder_and_tracker()?; + let snatch_guard = device.snatchable_lock.read(); clear_texture( &dst_texture, TextureInitRange { @@ -249,6 +251,7 @@ impl Global { &mut tracker.textures, &device.alignments, device.zero_buffer.as_ref().unwrap(), + &snatch_guard, ) } } @@ -260,10 +263,10 @@ pub(crate) fn clear_texture( texture_tracker: &mut TextureTracker, alignments: &hal::Alignments, zero_buffer: &A::Buffer, + snatch_guard: &SnatchGuard<'_>, ) -> Result<(), ClearError> { - let snatch_guard = dst_texture.device.snatchable_lock.read(); let dst_raw = dst_texture - .raw(&snatch_guard) + .raw(snatch_guard) .ok_or_else(|| ClearError::InvalidTexture(dst_texture.as_info().id()))?; // Issue the right barrier. diff --git a/third_party/rust/wgpu-core/src/command/compute.rs b/third_party/rust/wgpu-core/src/command/compute.rs index c2fd3ab397..b38324984c 100644 --- a/third_party/rust/wgpu-core/src/command/compute.rs +++ b/third_party/rust/wgpu-core/src/command/compute.rs @@ -272,14 +272,14 @@ where } } -struct State { +struct State<'a, A: HalApi> { binder: Binder, pipeline: Option, - scope: UsageScope, + scope: UsageScope<'a, A>, debug_scope_depth: u32, } -impl State { +impl<'a, A: HalApi> State<'a, A> { fn is_ready(&self) -> Result<(), DispatchError> { let bind_mask = self.binder.invalid_mask(); if bind_mask != 0 { @@ -407,7 +407,7 @@ impl Global { let mut state = State { binder: Binder::new(), pipeline: None, - scope: UsageScope::new(&device.tracker_indices), + scope: device.new_usage_scope(), debug_scope_depth: 0, }; let mut temp_offsets = Vec::new(); @@ -868,6 +868,7 @@ impl Global { transit, &mut tracker.textures, device, + &snatch_guard, ); CommandBuffer::insert_barriers_from_tracker( transit, diff --git a/third_party/rust/wgpu-core/src/command/memory_init.rs b/third_party/rust/wgpu-core/src/command/memory_init.rs index 3bfc71f4f7..54bdedb792 100644 --- a/third_party/rust/wgpu-core/src/command/memory_init.rs +++ b/third_party/rust/wgpu-core/src/command/memory_init.rs @@ -7,6 +7,7 @@ use crate::{ hal_api::HalApi, init_tracker::*, resource::{Resource, Texture}, + snatch::SnatchGuard, track::{TextureTracker, Tracker}, FastHashMap, }; @@ -144,6 +145,7 @@ pub(crate) fn fixup_discarded_surfaces< encoder: &mut A::CommandEncoder, texture_tracker: &mut TextureTracker, device: &Device, + snatch_guard: &SnatchGuard<'_>, ) { for init in inits { clear_texture( @@ -156,6 +158,7 @@ pub(crate) fn fixup_discarded_surfaces< texture_tracker, &device.alignments, device.zero_buffer.as_ref().unwrap(), + snatch_guard, ) .unwrap(); } @@ -167,6 +170,7 @@ impl BakedCommands { pub(crate) fn initialize_buffer_memory( &mut self, device_tracker: &mut Tracker, + snatch_guard: &SnatchGuard<'_>, ) -> Result<(), DestroyedBufferError> { // Gather init ranges for each buffer so we can collapse them. // It is not possible to do this at an earlier point since previously @@ -225,16 +229,15 @@ impl BakedCommands { .unwrap() .1; - let snatch_guard = buffer.device.snatchable_lock.read(); let raw_buf = buffer .raw - .get(&snatch_guard) + .get(snatch_guard) .ok_or(DestroyedBufferError(buffer_id))?; unsafe { self.encoder.transition_buffers( transition - .map(|pending| pending.into_hal(&buffer, &snatch_guard)) + .map(|pending| pending.into_hal(&buffer, snatch_guard)) .into_iter(), ); } @@ -271,6 +274,7 @@ impl BakedCommands { &mut self, device_tracker: &mut Tracker, device: &Device, + snatch_guard: &SnatchGuard<'_>, ) -> Result<(), DestroyedTextureError> { let mut ranges: Vec = Vec::new(); for texture_use in self.texture_memory_actions.drain_init_actions() { @@ -310,6 +314,7 @@ impl BakedCommands { &mut device_tracker.textures, &device.alignments, device.zero_buffer.as_ref().unwrap(), + snatch_guard, ); // A Texture can be destroyed between the command recording diff --git a/third_party/rust/wgpu-core/src/command/render.rs b/third_party/rust/wgpu-core/src/command/render.rs index 9141ddb021..7e859e3cc8 100644 --- a/third_party/rust/wgpu-core/src/command/render.rs +++ b/third_party/rust/wgpu-core/src/command/render.rs @@ -739,9 +739,9 @@ impl TextureView { const MAX_TOTAL_ATTACHMENTS: usize = hal::MAX_COLOR_ATTACHMENTS + hal::MAX_COLOR_ATTACHMENTS + 1; type AttachmentDataVec = ArrayVec; -struct RenderPassInfo<'a, A: HalApi> { +struct RenderPassInfo<'a, 'd, A: HalApi> { context: RenderPassContext, - usage_scope: UsageScope, + usage_scope: UsageScope<'d, A>, /// All render attachments, including depth/stencil render_attachments: AttachmentDataVec>, is_depth_read_only: bool, @@ -754,7 +754,7 @@ struct RenderPassInfo<'a, A: HalApi> { multiview: Option, } -impl<'a, A: HalApi> RenderPassInfo<'a, A> { +impl<'a, 'd, A: HalApi> RenderPassInfo<'a, 'd, A> { fn add_pass_texture_init_actions( channel: &PassChannel, texture_memory_actions: &mut CommandBufferTextureMemoryActions, @@ -790,7 +790,7 @@ impl<'a, A: HalApi> RenderPassInfo<'a, A> { } fn start( - device: &Device, + device: &'d Device, label: Option<&str>, color_attachments: &[Option], depth_stencil_attachment: Option<&RenderPassDepthStencilAttachment>, @@ -1214,7 +1214,7 @@ impl<'a, A: HalApi> RenderPassInfo<'a, A> { Ok(Self { context, - usage_scope: UsageScope::new(&device.tracker_indices), + usage_scope: device.new_usage_scope(), render_attachments, is_depth_read_only, is_stencil_read_only, @@ -1230,7 +1230,7 @@ impl<'a, A: HalApi> RenderPassInfo<'a, A> { mut self, raw: &mut A::CommandEncoder, snatch_guard: &SnatchGuard, - ) -> Result<(UsageScope, SurfacesInDiscardState), RenderPassErrorInner> { + ) -> Result<(UsageScope<'d, A>, SurfacesInDiscardState), RenderPassErrorInner> { profiling::scope!("RenderPassInfo::finish"); unsafe { raw.end_render_pass(); @@ -2374,7 +2374,7 @@ impl Global { .extend(texture_memory_actions.register_init_action(action)); } - unsafe { bundle.execute(raw) } + unsafe { bundle.execute(raw, &snatch_guard) } .map_err(|e| match e { ExecutionError::DestroyedBuffer(id) => { RenderCommandError::DestroyedBuffer(id) @@ -2427,6 +2427,7 @@ impl Global { transit, &mut tracker.textures, &cmd_buf.device, + &snatch_guard, ); cmd_buf_data diff --git a/third_party/rust/wgpu-core/src/command/transfer.rs b/third_party/rust/wgpu-core/src/command/transfer.rs index 0a952dfc84..8e98a4c9b9 100644 --- a/third_party/rust/wgpu-core/src/command/transfer.rs +++ b/third_party/rust/wgpu-core/src/command/transfer.rs @@ -14,6 +14,7 @@ use crate::{ TextureInitTrackerAction, }, resource::{Resource, Texture, TextureErrorDimension}, + snatch::SnatchGuard, track::{TextureSelector, Tracker}, }; @@ -452,6 +453,7 @@ fn handle_texture_init( copy_texture: &ImageCopyTexture, copy_size: &Extent3d, texture: &Arc>, + snatch_guard: &SnatchGuard<'_>, ) -> Result<(), ClearError> { let init_action = TextureInitTrackerAction { texture: texture.clone(), @@ -480,6 +482,7 @@ fn handle_texture_init( &mut trackers.textures, &device.alignments, device.zero_buffer.as_ref().unwrap(), + snatch_guard, )?; } } @@ -499,6 +502,7 @@ fn handle_src_texture_init( source: &ImageCopyTexture, copy_size: &Extent3d, texture: &Arc>, + snatch_guard: &SnatchGuard<'_>, ) -> Result<(), TransferError> { handle_texture_init( MemoryInitKind::NeedsInitializedMemory, @@ -509,6 +513,7 @@ fn handle_src_texture_init( source, copy_size, texture, + snatch_guard, )?; Ok(()) } @@ -525,6 +530,7 @@ fn handle_dst_texture_init( destination: &ImageCopyTexture, copy_size: &Extent3d, texture: &Arc>, + snatch_guard: &SnatchGuard<'_>, ) -> Result<(), TransferError> { // Attention: If we don't write full texture subresources, we need to a full // clear first since we don't track subrects. This means that in rare cases @@ -549,6 +555,7 @@ fn handle_dst_texture_init( destination, copy_size, texture, + snatch_guard, )?; Ok(()) } @@ -779,6 +786,8 @@ impl Global { let (dst_range, dst_base) = extract_texture_selector(destination, copy_size, &dst_texture)?; + let snatch_guard = device.snatchable_lock.read(); + // Handle texture init *before* dealing with barrier transitions so we // have an easier time inserting "immediate-inits" that may be required // by prior discards in rare cases. @@ -790,10 +799,9 @@ impl Global { destination, copy_size, &dst_texture, + &snatch_guard, )?; - let snatch_guard = device.snatchable_lock.read(); - let (src_buffer, src_pending) = { let buffer_guard = hub.buffers.read(); let src_buffer = buffer_guard @@ -935,6 +943,8 @@ impl Global { let (src_range, src_base) = extract_texture_selector(source, copy_size, &src_texture)?; + let snatch_guard = device.snatchable_lock.read(); + // Handle texture init *before* dealing with barrier transitions so we // have an easier time inserting "immediate-inits" that may be required // by prior discards in rare cases. @@ -946,10 +956,9 @@ impl Global { source, copy_size, &src_texture, + &snatch_guard, )?; - let snatch_guard = device.snatchable_lock.read(); - let src_pending = tracker .textures .set_single(&src_texture, src_range, hal::TextureUses::COPY_SRC) @@ -1152,6 +1161,7 @@ impl Global { source, copy_size, &src_texture, + &snatch_guard, )?; handle_dst_texture_init( encoder, @@ -1161,6 +1171,7 @@ impl Global { destination, copy_size, &dst_texture, + &snatch_guard, )?; let src_pending = cmd_buf_data diff --git a/third_party/rust/wgpu-core/src/device/global.rs b/third_party/rust/wgpu-core/src/device/global.rs index 539b92e0f3..0c97e1b504 100644 --- a/third_party/rust/wgpu-core/src/device/global.rs +++ b/third_party/rust/wgpu-core/src/device/global.rs @@ -192,7 +192,15 @@ impl Global { let ptr = if map_size == 0 { std::ptr::NonNull::dangling() } else { - match map_buffer(device.raw(), &buffer, 0, map_size, HostMap::Write) { + let snatch_guard = device.snatchable_lock.read(); + match map_buffer( + device.raw(), + &buffer, + 0, + map_size, + HostMap::Write, + &snatch_guard, + ) { Ok(ptr) => ptr, Err(e) => { to_destroy.push(buffer); @@ -2008,9 +2016,10 @@ impl Global { } // Wait for all work to finish before configuring the surface. + let snatch_guard = device.snatchable_lock.read(); let fence = device.fence.read(); let fence = fence.as_ref().unwrap(); - match device.maintain(fence, wgt::Maintain::Wait) { + match device.maintain(fence, wgt::Maintain::Wait, snatch_guard) { Ok((closures, _)) => { user_callbacks = closures; } @@ -2120,9 +2129,10 @@ impl Global { device: &crate::device::Device, maintain: wgt::Maintain, ) -> Result { + let snatch_guard = device.snatchable_lock.read(); let fence = device.fence.read(); let fence = fence.as_ref().unwrap(); - let (closures, queue_empty) = device.maintain(fence, maintain)?; + let (closures, queue_empty) = device.maintain(fence, maintain, snatch_guard)?; // Some deferred destroys are scheduled in maintain so run this right after // to avoid holding on to them until the next device poll. @@ -2240,6 +2250,15 @@ impl Global { } } + // This is a test-only function to force the device into an + // invalid state by inserting an error value in its place in + // the registry. + pub fn device_make_invalid(&self, device_id: DeviceId) { + let hub = A::hub(self); + hub.devices + .force_replace_with_error(device_id, "Made invalid."); + } + pub fn device_drop(&self, device_id: DeviceId) { profiling::scope!("Device::drop"); api_log!("Device::drop {device_id:?}"); @@ -2275,7 +2294,7 @@ impl Global { ) { let hub = A::hub(self); - if let Ok(device) = hub.devices.get(device_id) { + if let Ok(Some(device)) = hub.devices.try_get(device_id) { let mut life_tracker = device.lock_life(); if let Some(existing_closure) = life_tracker.device_lost_closure.take() { // It's important to not hold the lock while calling the closure. @@ -2284,6 +2303,12 @@ impl Global { life_tracker = device.lock_life(); } life_tracker.device_lost_closure = Some(device_lost_closure); + } else { + // No device? Okay. Just like we have to call any existing closure + // before we drop it, we need to call this closure before we exit + // this function, because there's no device that is ever going to + // call it. + device_lost_closure.call(DeviceLostReason::DeviceInvalid, "".to_string()); } } diff --git a/third_party/rust/wgpu-core/src/device/life.rs b/third_party/rust/wgpu-core/src/device/life.rs index 7b06a4a30b..af345015df 100644 --- a/third_party/rust/wgpu-core/src/device/life.rs +++ b/third_party/rust/wgpu-core/src/device/life.rs @@ -12,6 +12,7 @@ use crate::{ self, Buffer, DestroyedBuffer, DestroyedTexture, QuerySet, Resource, Sampler, StagingBuffer, Texture, TextureView, }, + snatch::SnatchGuard, track::{ResourceTracker, Tracker, TrackerIndex}, FastHashMap, SubmissionIndex, }; @@ -309,12 +310,12 @@ impl LifetimeTracker { } pub fn post_submit(&mut self) { - for v in self.future_suspected_buffers.drain(..).take(1) { + for v in self.future_suspected_buffers.drain(..) { self.suspected_resources .buffers .insert(v.as_info().tracker_index(), v); } - for v in self.future_suspected_textures.drain(..).take(1) { + for v in self.future_suspected_textures.drain(..) { self.suspected_resources .textures .insert(v.as_info().tracker_index(), v); @@ -780,6 +781,7 @@ impl LifetimeTracker { &mut self, raw: &A::Device, trackers: &Mutex>, + snatch_guard: &SnatchGuard, ) -> Vec { if self.ready_to_map.is_empty() { return Vec::new(); @@ -816,7 +818,14 @@ impl LifetimeTracker { log::debug!("Buffer {tracker_index:?} map state -> Active"); let host = mapping.op.host; let size = mapping.range.end - mapping.range.start; - match super::map_buffer(raw, &buffer, mapping.range.start, size, host) { + match super::map_buffer( + raw, + &buffer, + mapping.range.start, + size, + host, + snatch_guard, + ) { Ok(ptr) => { *buffer.map_state.lock() = resource::BufferMapState::Active { ptr, diff --git a/third_party/rust/wgpu-core/src/device/mod.rs b/third_party/rust/wgpu-core/src/device/mod.rs index 7ecda830a3..e2ab6c2690 100644 --- a/third_party/rust/wgpu-core/src/device/mod.rs +++ b/third_party/rust/wgpu-core/src/device/mod.rs @@ -3,9 +3,10 @@ use crate::{ hal_api::HalApi, hub::Hub, id::{BindGroupLayoutId, PipelineLayoutId}, - resource::{Buffer, BufferAccessResult}, - resource::{BufferAccessError, BufferMapOperation}, - resource_log, Label, DOWNLEVEL_ERROR_MESSAGE, + resource::{Buffer, BufferAccessError, BufferAccessResult, BufferMapOperation}, + resource_log, + snatch::SnatchGuard, + Label, DOWNLEVEL_ERROR_MESSAGE, }; use arrayvec::ArrayVec; @@ -317,10 +318,10 @@ fn map_buffer( offset: BufferAddress, size: BufferAddress, kind: HostMap, + snatch_guard: &SnatchGuard, ) -> Result, BufferAccessError> { - let snatch_guard = buffer.device.snatchable_lock.read(); let raw_buffer = buffer - .raw(&snatch_guard) + .raw(snatch_guard) .ok_or(BufferAccessError::Destroyed)?; let mapping = unsafe { raw.map_buffer(raw_buffer, offset..offset + size) diff --git a/third_party/rust/wgpu-core/src/device/queue.rs b/third_party/rust/wgpu-core/src/device/queue.rs index 6ebb9eb09b..3cb5f695a7 100644 --- a/third_party/rust/wgpu-core/src/device/queue.rs +++ b/third_party/rust/wgpu-core/src/device/queue.rs @@ -815,6 +815,7 @@ impl Global { &mut trackers.textures, &device.alignments, device.zero_buffer.as_ref().unwrap(), + &device.snatchable_lock.read(), ) .map_err(QueueWriteError::from)?; } @@ -1084,6 +1085,7 @@ impl Global { &mut trackers.textures, &device.alignments, device.zero_buffer.as_ref().unwrap(), + &device.snatchable_lock.read(), ) .map_err(QueueWriteError::from)?; } @@ -1147,6 +1149,9 @@ impl Global { let device = queue.device.as_ref().unwrap(); + let snatch_guard = device.snatchable_lock.read(); + + // Fence lock must be acquired after the snatch lock everywhere to avoid deadlocks. let mut fence = device.fence.write(); let fence = fence.as_mut().unwrap(); let submit_index = device @@ -1155,9 +1160,7 @@ impl Global { + 1; let mut active_executions = Vec::new(); - let mut used_surface_textures = track::TextureUsageScope::new(); - - let snatch_guard = device.snatchable_lock.read(); + let mut used_surface_textures = track::TextureUsageScope::default(); let mut submit_surface_textures_owned = SmallVec::<[_; 2]>::new(); @@ -1391,10 +1394,10 @@ impl Global { //Note: locking the trackers has to be done after the storages let mut trackers = device.trackers.lock(); baked - .initialize_buffer_memory(&mut *trackers) + .initialize_buffer_memory(&mut *trackers, &snatch_guard) .map_err(|err| QueueSubmitError::DestroyedBuffer(err.0))?; baked - .initialize_texture_memory(&mut *trackers, device) + .initialize_texture_memory(&mut *trackers, device, &snatch_guard) .map_err(|err| QueueSubmitError::DestroyedTexture(err.0))?; //Note: stateless trackers are not merged: // device already knows these resources exist. @@ -1435,7 +1438,7 @@ impl Global { baked.encoder.end_encoding().unwrap() }; baked.list.push(present); - used_surface_textures = track::TextureUsageScope::new(); + used_surface_textures = track::TextureUsageScope::default(); } // done @@ -1542,7 +1545,7 @@ impl Global { // This will schedule destruction of all resources that are no longer needed // by the user but used in the command stream, among other things. - let (closures, _) = match device.maintain(fence, wgt::Maintain::Poll) { + let (closures, _) = match device.maintain(fence, wgt::Maintain::Poll, snatch_guard) { Ok(closures) => closures, Err(WaitIdleError::Device(err)) => return Err(QueueSubmitError::Queue(err)), Err(WaitIdleError::StuckGpu) => return Err(QueueSubmitError::StuckGpu), diff --git a/third_party/rust/wgpu-core/src/device/resource.rs b/third_party/rust/wgpu-core/src/device/resource.rs index 28ba0eafb1..4892aecb75 100644 --- a/third_party/rust/wgpu-core/src/device/resource.rs +++ b/third_party/rust/wgpu-core/src/device/resource.rs @@ -28,7 +28,10 @@ use crate::{ resource_log, snatch::{SnatchGuard, SnatchLock, Snatchable}, storage::Storage, - track::{BindGroupStates, TextureSelector, Tracker, TrackerIndexAllocators}, + track::{ + BindGroupStates, TextureSelector, Tracker, TrackerIndexAllocators, UsageScope, + UsageScopePool, + }, validation::{ self, check_buffer_usage, check_texture_usage, validate_color_attachment_bytes_per_sample, }, @@ -97,6 +100,8 @@ pub struct Device { pub(crate) command_allocator: Mutex>>, //Note: The submission index here corresponds to the last submission that is done. pub(crate) active_submission_index: AtomicU64, //SubmissionIndex, + // NOTE: if both are needed, the `snatchable_lock` must be consistently acquired before the + // `fence` lock to avoid deadlocks. pub(crate) fence: RwLock>, pub(crate) snatchable_lock: SnatchLock, @@ -135,6 +140,7 @@ pub struct Device { pub(crate) deferred_destroy: Mutex>>, #[cfg(feature = "trace")] pub(crate) trace: Mutex>, + pub(crate) usage_scopes: UsageScopePool, } pub(crate) enum DeferredDestroy { @@ -296,6 +302,7 @@ impl Device { instance_flags, pending_writes: Mutex::new(Some(pending_writes)), deferred_destroy: Mutex::new(Vec::new()), + usage_scopes: Default::default(), }) } @@ -387,6 +394,7 @@ impl Device { &'this self, fence: &A::Fence, maintain: wgt::Maintain, + snatch_guard: SnatchGuard, ) -> Result<(UserClosures, bool), WaitIdleError> { profiling::scope!("Device::maintain"); let last_done_index = if maintain.is_wait() { @@ -440,7 +448,8 @@ impl Device { life_tracker.triage_mapped(); } - let mapping_closures = life_tracker.handle_mapping(self.raw(), &self.trackers); + let mapping_closures = + life_tracker.handle_mapping(self.raw(), &self.trackers, &snatch_guard); let queue_empty = life_tracker.queue_empty(); @@ -467,8 +476,9 @@ impl Device { } } - // Don't hold the lock while calling release_gpu_resources. + // Don't hold the locks while calling release_gpu_resources. drop(life_tracker); + drop(snatch_guard); if should_release_gpu_resource { self.release_gpu_resources(); @@ -3568,6 +3578,10 @@ impl Device { let _ = texture.destroy(); } } + + pub(crate) fn new_usage_scope(&self) -> UsageScope<'_, A> { + UsageScope::new_pooled(&self.usage_scopes, &self.tracker_indices) + } } impl Device { diff --git a/third_party/rust/wgpu-core/src/snatch.rs b/third_party/rust/wgpu-core/src/snatch.rs index 2324d33574..d5cd1a3d37 100644 --- a/third_party/rust/wgpu-core/src/snatch.rs +++ b/third_party/rust/wgpu-core/src/snatch.rs @@ -1,7 +1,12 @@ #![allow(unused)] use parking_lot::{RwLock, RwLockReadGuard, RwLockWriteGuard}; -use std::cell::UnsafeCell; +use std::{ + backtrace::Backtrace, + cell::{Cell, RefCell, UnsafeCell}, + panic::{self, Location}, + thread, +}; /// A guard that provides read access to snatchable data. pub struct SnatchGuard<'a>(RwLockReadGuard<'a, ()>); @@ -59,6 +64,10 @@ impl std::fmt::Debug for Snatchable { unsafe impl Sync for Snatchable {} +thread_local! { + static READ_LOCK_LOCATION: Cell, Backtrace)>> = const { Cell::new(None) }; +} + /// A Device-global lock for all snatchable data. pub struct SnatchLock { lock: RwLock<()>, @@ -76,7 +85,24 @@ impl SnatchLock { } /// Request read access to snatchable resources. + #[track_caller] pub fn read(&self) -> SnatchGuard { + if cfg!(debug_assertions) { + let caller = Location::caller(); + let backtrace = Backtrace::capture(); + if let Some((prev, bt)) = READ_LOCK_LOCATION.take() { + let current = thread::current(); + let name = current.name().unwrap_or(""); + panic!( + "thread '{name}' attempted to acquire a snatch read lock recursively.\n + - {prev}\n{bt}\n + - {caller}\n{backtrace}" + ); + } else { + READ_LOCK_LOCATION.set(Some((caller, backtrace))); + } + } + SnatchGuard(self.lock.read()) } @@ -89,3 +115,10 @@ impl SnatchLock { ExclusiveSnatchGuard(self.lock.write()) } } + +impl Drop for SnatchGuard<'_> { + fn drop(&mut self) { + #[cfg(debug_assertions)] + READ_LOCK_LOCATION.take(); + } +} diff --git a/third_party/rust/wgpu-core/src/track/buffer.rs b/third_party/rust/wgpu-core/src/track/buffer.rs index a30ac2a225..6cf1fdda6f 100644 --- a/third_party/rust/wgpu-core/src/track/buffer.rs +++ b/third_party/rust/wgpu-core/src/track/buffer.rs @@ -108,23 +108,27 @@ impl BufferBindGroupState { #[derive(Debug)] pub(crate) struct BufferUsageScope { state: Vec, - metadata: ResourceMetadata>, } -impl BufferUsageScope { - pub fn new() -> Self { +impl Default for BufferUsageScope { + fn default() -> Self { Self { state: Vec::new(), - metadata: ResourceMetadata::new(), } } +} +impl BufferUsageScope { fn tracker_assert_in_bounds(&self, index: usize) { strict_assert!(index < self.state.len()); self.metadata.tracker_assert_in_bounds(index); } + pub fn clear(&mut self) { + self.state.clear(); + self.metadata.clear(); + } /// Sets the size of all the vectors inside the tracker. /// diff --git a/third_party/rust/wgpu-core/src/track/metadata.rs b/third_party/rust/wgpu-core/src/track/metadata.rs index 744783a7fa..3e71e0e084 100644 --- a/third_party/rust/wgpu-core/src/track/metadata.rs +++ b/third_party/rust/wgpu-core/src/track/metadata.rs @@ -39,6 +39,11 @@ impl ResourceMetadata { resize_bitvec(&mut self.owned, size); } + pub(super) fn clear(&mut self) { + self.resources.clear(); + self.owned.clear(); + } + /// Ensures a given index is in bounds for all arrays and does /// sanity checks of the presence of a refcount. /// diff --git a/third_party/rust/wgpu-core/src/track/mod.rs b/third_party/rust/wgpu-core/src/track/mod.rs index 9ca37ebadc..374dfe7493 100644 --- a/third_party/rust/wgpu-core/src/track/mod.rs +++ b/third_party/rust/wgpu-core/src/track/mod.rs @@ -480,8 +480,8 @@ impl RenderBundleScope { /// Create the render bundle scope and pull the maximum IDs from the hubs. pub fn new() -> Self { Self { - buffers: RwLock::new(BufferUsageScope::new()), - textures: RwLock::new(TextureUsageScope::new()), + buffers: RwLock::new(BufferUsageScope::default()), + textures: RwLock::new(TextureUsageScope::default()), bind_groups: RwLock::new(StatelessTracker::new()), render_pipelines: RwLock::new(StatelessTracker::new()), query_sets: RwLock::new(StatelessTracker::new()), @@ -512,28 +512,52 @@ impl RenderBundleScope { } } +/// A pool for storing the memory used by [`UsageScope`]s. We take and store this memory when the +/// scope is dropped to avoid reallocating. The memory required only grows and allocation cost is +/// significant when a large number of resources have been used. +pub(crate) type UsageScopePool = Mutex, TextureUsageScope)>>; + /// A usage scope tracker. Only needs to store stateful resources as stateless /// resources cannot possibly have a usage conflict. #[derive(Debug)] -pub(crate) struct UsageScope { +pub(crate) struct UsageScope<'a, A: HalApi> { + pub pool: &'a UsageScopePool, pub buffers: BufferUsageScope, pub textures: TextureUsageScope, } -impl UsageScope { - /// Create the render bundle scope and pull the maximum IDs from the hubs. - pub fn new(tracker_indices: &TrackerIndexAllocators) -> Self { - let mut value = Self { - buffers: BufferUsageScope::new(), - textures: TextureUsageScope::new(), - }; +impl<'a, A: HalApi> Drop for UsageScope<'a, A> { + fn drop(&mut self) { + // clear vecs and push into pool + self.buffers.clear(); + self.textures.clear(); + self.pool.lock().push(( + std::mem::take(&mut self.buffers), + std::mem::take(&mut self.textures), + )); + } +} - value.buffers.set_size(tracker_indices.buffers.size()); - value.textures.set_size(tracker_indices.textures.size()); +impl UsageScope<'static, A> { + pub fn new_pooled<'d>( + pool: &'d UsageScopePool, + tracker_indices: &TrackerIndexAllocators, + ) -> UsageScope<'d, A> { + let pooled = pool.lock().pop().unwrap_or_default(); + + let mut scope = UsageScope::<'d, A> { + pool, + buffers: pooled.0, + textures: pooled.1, + }; - value + scope.buffers.set_size(tracker_indices.buffers.size()); + scope.textures.set_size(tracker_indices.textures.size()); + scope } +} +impl<'a, A: HalApi> UsageScope<'a, A> { /// Merge the inner contents of a bind group into the usage scope. /// /// Only stateful things are merged in here, all other resources are owned diff --git a/third_party/rust/wgpu-core/src/track/texture.rs b/third_party/rust/wgpu-core/src/track/texture.rs index e7c4707c93..3cf95ff38a 100644 --- a/third_party/rust/wgpu-core/src/track/texture.rs +++ b/third_party/rust/wgpu-core/src/track/texture.rs @@ -210,6 +210,7 @@ pub(crate) struct TextureStateSet { simple: Vec, complex: FastHashMap, } + impl TextureStateSet { fn new() -> Self { Self { @@ -235,15 +236,16 @@ pub(crate) struct TextureUsageScope { metadata: ResourceMetadata>, } -impl TextureUsageScope { - pub fn new() -> Self { +impl Default for TextureUsageScope { + fn default() -> Self { Self { set: TextureStateSet::new(), - metadata: ResourceMetadata::new(), } } +} +impl TextureUsageScope { fn tracker_assert_in_bounds(&self, index: usize) { self.metadata.tracker_assert_in_bounds(index); @@ -258,6 +260,11 @@ impl TextureUsageScope { }); } + pub fn clear(&mut self) { + self.set.clear(); + self.metadata.clear(); + } + /// Sets the size of all the vectors inside the tracker. /// /// Must be called with the highest possible Texture ID before diff --git a/third_party/rust/wgpu-hal/.cargo-checksum.json b/third_party/rust/wgpu-hal/.cargo-checksum.json index de9bc38719..2cba88cfdf 100644 --- a/third_party/rust/wgpu-hal/.cargo-checksum.json +++ b/third_party/rust/wgpu-hal/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"Cargo.toml":"eaa7b2b51fbe98c0721dc52d94c64b48d2d6e351bf36da3e756378a8d8ebc1de","LICENSE.APACHE":"a6cba85bc92e0cff7a450b1d873c0eaa2e9fc96bf472df0247a26bec77bf3ff9","LICENSE.MIT":"c7fea58d1cfe49634cd92e54fc10a9d871f4b275321a4cd8c09e449122caaeb4","README.md":"099ee611a911dc19330a61bffcde13663929a51b25ac528ee33ea796d695491e","build.rs":"c80bdc0152a00471eec6ed0dd0f7d55d0b975498a00ba05e94100c84ad639a49","examples/halmark/main.rs":"4604737f714943383c57feac2b8468ecf15e9e60c54a5303455e9953ec5c79fb","examples/halmark/shader.wgsl":"26c256ec36d6f0e9a1647431ca772766bee4382d64eaa718ba7b488dcfb6bcca","examples/raw-gles.em.html":"70fbe68394a1a4522192de1dcfaf7d399f60d7bdf5de70b708f9bb0417427546","examples/raw-gles.rs":"095113a1ba0851652a77aabfc8fa6ea7edcc2d09e91fd1e5009ead87d5998ea9","examples/ray-traced-triangle/main.rs":"955c2b8700c3b2daf14e9ef963ff499ed185b6f349dbc63caa422b2cf4942a1f","examples/ray-traced-triangle/shader.wgsl":"cc10caf92746724a71f6dd0dbc3a71e57b37c7d1d83278556805a535c0728a9d","src/auxil/dxgi/conv.rs":"760cd4eaa79b530368a30140b96bf73ac4fbdb4025eb95f0bed581638c8bb1cb","src/auxil/dxgi/exception.rs":"f0cfb5a0adcdc3b6db909601fee51ad51368f5da269bcd46e4dbea45a3bec4b1","src/auxil/dxgi/factory.rs":"5f861fbfe2f4cce08722a95283549b8f62b96f24a306d080d9f1730ae53501d8","src/auxil/dxgi/mod.rs":"a202564d9ac97530b16a234b87d180cd345aae705e082a9b1177dcde813645f9","src/auxil/dxgi/result.rs":"79fe5aa17a2b21a7f06b1b604200c3c3e73fca31e8193aab80b5b15e7e9818a0","src/auxil/dxgi/time.rs":"b6f966b250e9424d5d7e4065f2108cba87197c1e30baae6d87083055d1bc5a4b","src/auxil/mod.rs":"720ef2aae258733322a3274fd858f91effb8951dabaf7bbfd8a9a0be2d2dba97","src/auxil/renderdoc.rs":"c2f849f70f576b0c9b0d32dd155b6a6353f74dff59cbeeaa994a12789d047c0f","src/dx12/adapter.rs":"7d647c9a1211e564fb1220c65df26fe2c519e5eddfa89291eaea45be4b60746a","src/dx12/command.rs":"6fe77b8b27c6428128ed0c3bcf7517e511c3c1eec8491a08936a696d5cb30751","src/dx12/conv.rs":"94d35f117ae003b07049f3a0bc6c45a0ffda9fb8053233d39c173cfb1b644403","src/dx12/descriptor.rs":"e06eb08bee4c805fa76b6ab791893b5b563ee60de9c8f8d8e0e21ab97ade5664","src/dx12/device.rs":"f7ca4a30085fdaecc321a01344f9d8cd907b7ba5a1b92f13a3bd9faad1934ed8","src/dx12/instance.rs":"351a4e0d526de8eafc74bf5f01a41da48efa39e0c66704a85da72e1140b159d4","src/dx12/mod.rs":"4b9d5e2414d628ed537f32f46604eeb95912ad9d5ee61cf4ce11c8dd6a88c8ab","src/dx12/shader_compilation.rs":"5087adb8576e2d7751619dfdf8b37c573bb4e494290c594077ca3208cce1e746","src/dx12/suballocation.rs":"6939fc36223a15cc070c744d0418f9ac6fa2829d794af17cdea7c61eb5f8d2c0","src/dx12/types.rs":"9573736baaa0ef607367c3b72144556d24faf677a26bb8df49a4372a1348e06b","src/dx12/view.rs":"792772e9c87840dcd045b7381a03162eb4a501492a95ca586e77e81aed621c67","src/empty.rs":"5c3a5e39d45b4522ff3496fe6ec3b4a7afd906b6095dff1cad113c826aa9ea62","src/gles/adapter.rs":"3175c86212b6c8caa099a3e34750c18251107461314c02f77c984e5b8301051a","src/gles/command.rs":"9f9ef3d97fcb2bc521b85141dee1ca9e8fe06b08d861766c3b3e9a2f3a53b494","src/gles/conv.rs":"5d15d3a33032d32ff99bc338fba0689fa54c76d0714e335fe48523d841df386f","src/gles/device.rs":"7ccd7aa3b878159190092bf279158289d754cc695bd27b9ec7177cd9b86b37c5","src/gles/egl.rs":"ad9b0ddc66877ae4088511283b8c860dd09b0b4d2c1fc51246c6935aa16703eb","src/gles/emscripten.rs":"19bb73a9d140645f3f32cd48b002151711a9b8456e213eab5f3a2be79239e147","src/gles/mod.rs":"b8999f76ad45e07312b291457100f12699ba6a2635c1f1913b0648e9a9394015","src/gles/queue.rs":"3ead252c54c673da6736a0c0c6b63c848791bc78042def3f3ffff8ffce2c6e64","src/gles/shaders/clear.frag":"9133ed8ed97d3641fbb6b5f5ea894a3554c629ccc1b80a5fc9221d7293aa1954","src/gles/shaders/clear.vert":"a543768725f4121ff2e9e1fb5b00644931e9d6f2f946c0ef01968afb5a135abd","src/gles/shaders/srgb_present.frag":"dd9a43c339a2fa4ccf7f6a1854c6f400cabf271a7d5e9230768e9f39d47f3ff5","src/gles/shaders/srgb_present.vert":"6e85d489403d80b81cc94790730bb53b309dfc5eeede8f1ea3412a660f31d357","src/gles/web.rs":"d263695d45736d3c6ec3528c8c33fe6cf3767d3429a13a92d88b4fdc7b6340fb","src/gles/wgl.rs":"06e947912c357c5275090b12b7e31e596ff264fd460e2449b6db4b79284eb74d","src/lib.rs":"c8b8a95f5bfd58eaada0af2cd0abc80f888aeea85969a1363f4061cc9b542ca4","src/metal/adapter.rs":"bb5d0ca1cecbd914cbb29487303be4ed69035469a8bc137784d5bbb6ab36cec7","src/metal/command.rs":"661b38a75d4f4cd1b0d6957f1f09db0743ec3a13bbafba9baa931894ee193f48","src/metal/conv.rs":"0bce6a8d0ccef16783475803d70d35e03ab7938c19374e22c9d253abe1f8b111","src/metal/device.rs":"c5deeecf475e0aa4b2027c656ea19207716f84b56cfa7c9132dca504d1abebfb","src/metal/mod.rs":"f6d12246a6c7e6d998db796a009702f289b5f56bd35f01c0a619f5345fb363c9","src/metal/surface.rs":"f2b9b65d4117db2b16c04469c573358eb65de104d5a72aa02da8483ee243cbd3","src/metal/time.rs":"c32d69f30e846dfcc0e39e01097fb80df63b2bebb6586143bb62494999850246","src/vulkan/adapter.rs":"ed980734c8239bad7f3371e0e778ec63ecea5fe971f04c3dcdd3fe55c359f63b","src/vulkan/command.rs":"e5a88eab59b3864cdf44ba2231270e16045505dc549b8b90251031de452ba826","src/vulkan/conv.rs":"7e6266e3a0b7d0b8d5d51362a0386a84bc047350eeac663b6352a94d5e5c0a87","src/vulkan/device.rs":"9824d597dbb51030bd337e80bb0f1eab6fdb6935fc87dfd8beae2c1f1048fbcf","src/vulkan/instance.rs":"cd4aa3a8ed343076446117bae21fc438fe8761054489ec7d1ed7c31512c2e5ec","src/vulkan/mod.rs":"0c6bfb321b693930bcae3e61d06ff7b71965a64761ce39d757fc609d4b46a03e"},"package":null} \ No newline at end of file +{"files":{"Cargo.toml":"332863a1b354637fc62ff5729e7c2f5089fa65db05d330ec268a7aab04bb3d42","LICENSE.APACHE":"a6cba85bc92e0cff7a450b1d873c0eaa2e9fc96bf472df0247a26bec77bf3ff9","LICENSE.MIT":"c7fea58d1cfe49634cd92e54fc10a9d871f4b275321a4cd8c09e449122caaeb4","README.md":"099ee611a911dc19330a61bffcde13663929a51b25ac528ee33ea796d695491e","build.rs":"c80bdc0152a00471eec6ed0dd0f7d55d0b975498a00ba05e94100c84ad639a49","examples/halmark/main.rs":"4604737f714943383c57feac2b8468ecf15e9e60c54a5303455e9953ec5c79fb","examples/halmark/shader.wgsl":"26c256ec36d6f0e9a1647431ca772766bee4382d64eaa718ba7b488dcfb6bcca","examples/raw-gles.em.html":"70fbe68394a1a4522192de1dcfaf7d399f60d7bdf5de70b708f9bb0417427546","examples/raw-gles.rs":"095113a1ba0851652a77aabfc8fa6ea7edcc2d09e91fd1e5009ead87d5998ea9","examples/ray-traced-triangle/main.rs":"955c2b8700c3b2daf14e9ef963ff499ed185b6f349dbc63caa422b2cf4942a1f","examples/ray-traced-triangle/shader.wgsl":"cc10caf92746724a71f6dd0dbc3a71e57b37c7d1d83278556805a535c0728a9d","src/auxil/dxgi/conv.rs":"760cd4eaa79b530368a30140b96bf73ac4fbdb4025eb95f0bed581638c8bb1cb","src/auxil/dxgi/exception.rs":"f0cfb5a0adcdc3b6db909601fee51ad51368f5da269bcd46e4dbea45a3bec4b1","src/auxil/dxgi/factory.rs":"5f861fbfe2f4cce08722a95283549b8f62b96f24a306d080d9f1730ae53501d8","src/auxil/dxgi/mod.rs":"a202564d9ac97530b16a234b87d180cd345aae705e082a9b1177dcde813645f9","src/auxil/dxgi/result.rs":"79fe5aa17a2b21a7f06b1b604200c3c3e73fca31e8193aab80b5b15e7e9818a0","src/auxil/dxgi/time.rs":"b6f966b250e9424d5d7e4065f2108cba87197c1e30baae6d87083055d1bc5a4b","src/auxil/mod.rs":"720ef2aae258733322a3274fd858f91effb8951dabaf7bbfd8a9a0be2d2dba97","src/auxil/renderdoc.rs":"c2f849f70f576b0c9b0d32dd155b6a6353f74dff59cbeeaa994a12789d047c0f","src/dx12/adapter.rs":"2a90a4222702e50fabfc64339ac2aa667e1ee2d2bf801c3e2e59a91e261c7a04","src/dx12/command.rs":"e0675560784214a18e078062cbd0965c21a35c99eecf0e697d1badb9c692db35","src/dx12/conv.rs":"94d35f117ae003b07049f3a0bc6c45a0ffda9fb8053233d39c173cfb1b644403","src/dx12/descriptor.rs":"e06eb08bee4c805fa76b6ab791893b5b563ee60de9c8f8d8e0e21ab97ade5664","src/dx12/device.rs":"6a38dabd4db9d7ca79bfcfbeb3483320ad88f2db6e93a7210051bbf81acb1bbd","src/dx12/instance.rs":"030a86ac810d5ebf151328bbc06b9b5a89788721aa963cb4e0f3ff943b2c8633","src/dx12/mod.rs":"87e7012a113554c976abdbecd10b2fe6aab3ba695c88dc77d6beb3880d96ba54","src/dx12/shader_compilation.rs":"5087adb8576e2d7751619dfdf8b37c573bb4e494290c594077ca3208cce1e746","src/dx12/suballocation.rs":"6939fc36223a15cc070c744d0418f9ac6fa2829d794af17cdea7c61eb5f8d2c0","src/dx12/types.rs":"9573736baaa0ef607367c3b72144556d24faf677a26bb8df49a4372a1348e06b","src/dx12/view.rs":"792772e9c87840dcd045b7381a03162eb4a501492a95ca586e77e81aed621c67","src/empty.rs":"0849e0b7210d33145b3fb1368ed08b13fbc2144b1ccc0a1af913896e916bbe46","src/gles/adapter.rs":"22fe404177d4abb077f35a5be54295f81eacb021d448ad01aa96450ade8b089a","src/gles/command.rs":"788505168ae1af312781c86d31d5bc289fcc4e2b3cb047fb41deb62542db1a95","src/gles/conv.rs":"5d15d3a33032d32ff99bc338fba0689fa54c76d0714e335fe48523d841df386f","src/gles/device.rs":"e126042653b0651a12684504e19c05db3fe333bde3ac408063675eef6c27754a","src/gles/egl.rs":"72feefbcd399e686b5c210a67151ef040e45e123ad371539274d1d52b3dd162d","src/gles/emscripten.rs":"19bb73a9d140645f3f32cd48b002151711a9b8456e213eab5f3a2be79239e147","src/gles/mod.rs":"b8999f76ad45e07312b291457100f12699ba6a2635c1f1913b0648e9a9394015","src/gles/queue.rs":"7f62e9c1cb670f66ac8bb0d62cd531b9c11b2a29f8ed28736c40e21071fbe25d","src/gles/shaders/clear.frag":"9133ed8ed97d3641fbb6b5f5ea894a3554c629ccc1b80a5fc9221d7293aa1954","src/gles/shaders/clear.vert":"a543768725f4121ff2e9e1fb5b00644931e9d6f2f946c0ef01968afb5a135abd","src/gles/shaders/srgb_present.frag":"dd9a43c339a2fa4ccf7f6a1854c6f400cabf271a7d5e9230768e9f39d47f3ff5","src/gles/shaders/srgb_present.vert":"6e85d489403d80b81cc94790730bb53b309dfc5eeede8f1ea3412a660f31d357","src/gles/web.rs":"000ed39eae4a6442ca16bdca68e1e61459e6815a5db8eeec53e0ee0f5f14ae66","src/gles/wgl.rs":"78bfe5bcdacfde8d8df0d6f3987048f8ad8a2c2c93df6d607cf5a16795d3bb8e","src/lib.rs":"b80b7d14f524f25875b1d7e7ee67cd6057bbe9944ea92b59d5d4d08d7b17580d","src/metal/adapter.rs":"e473a1857817030ee5c5358e8387daff66d1aca1f5133224cc844ea4b17d3e49","src/metal/command.rs":"61640f2cb7269f44487df2d911771fbe1f78484d44a5363a6ddfce7d77143b02","src/metal/conv.rs":"0bce6a8d0ccef16783475803d70d35e03ab7938c19374e22c9d253abe1f8b111","src/metal/device.rs":"1343ad60a423b81649a6eaad69de743a2247234a75ec48c0698973af67592e48","src/metal/mod.rs":"3e3f69ac864cb6a96c5b9df551a934bbc4bf08503430b9507561eef973daf57f","src/metal/surface.rs":"397dab6726eead96f0be9ecb6686874e691079da94c072921a6422e9b1284fb5","src/metal/time.rs":"c32d69f30e846dfcc0e39e01097fb80df63b2bebb6586143bb62494999850246","src/vulkan/adapter.rs":"ac4525114d37d6ffa31f2c240b2bb9e1d97d19cd6d400598c2a1153d9144bd0f","src/vulkan/command.rs":"82060e8040eea27b717ec676525139a9c7238074ad5ce6902f5174ac5c7aa048","src/vulkan/conv.rs":"7e6266e3a0b7d0b8d5d51362a0386a84bc047350eeac663b6352a94d5e5c0a87","src/vulkan/device.rs":"24abe83a2cbe8fa3377d0fb8d64fcaac3a3e6b0521ec28143fa8b7dee0c1afee","src/vulkan/instance.rs":"60742c3c1feee63eca0953165dae57c86b0d58e496ad374743ff83ae50a4e8f1","src/vulkan/mod.rs":"dfd8e079b7b783de3dfdfbf2349d7283c1b9108242d1ffbb4e072376901c7bd1"},"package":null} \ No newline at end of file diff --git a/third_party/rust/wgpu-hal/Cargo.toml b/third_party/rust/wgpu-hal/Cargo.toml index 47195f996d..f601949231 100644 --- a/third_party/rust/wgpu-hal/Cargo.toml +++ b/third_party/rust/wgpu-hal/Cargo.toml @@ -11,7 +11,7 @@ [package] edition = "2021" -rust-version = "1.70" +rust-version = "1.74" name = "wgpu-hal" version = "0.19.0" authors = ["gfx-rs developers"] @@ -78,7 +78,7 @@ package = "wgpu-types" [dev-dependencies] cfg-if = "1" -env_logger = "0.10" +env_logger = "0.11" glam = "0.25.0" [dev-dependencies.naga] @@ -87,7 +87,7 @@ path = "../naga" features = ["wgsl-in"] [dev-dependencies.winit] -version = "0.29.10" +version = "0.29.14" features = ["android-native-activity"] [build-dependencies] @@ -142,11 +142,11 @@ vulkan = [ windows_rs = ["gpu-allocator"] [target."cfg(all(target_arch = \"wasm32\", not(target_os = \"emscripten\")))".dependencies] -js-sys = "0.3.67" +js-sys = "0.3.69" wasm-bindgen = "0.2.87" [target."cfg(all(target_arch = \"wasm32\", not(target_os = \"emscripten\")))".dependencies.web-sys] -version = "0.3.67" +version = "0.3.69" features = [ "Window", "HtmlCanvasElement", @@ -185,7 +185,7 @@ version = ">=0.7, <0.9" optional = true [target."cfg(not(target_arch = \"wasm32\"))".dependencies.renderdoc-sys] -version = "1.0.0" +version = "1.1.0" optional = true [target."cfg(not(target_arch = \"wasm32\"))".dependencies.smallvec] diff --git a/third_party/rust/wgpu-hal/src/dx12/adapter.rs b/third_party/rust/wgpu-hal/src/dx12/adapter.rs index 960e1790a9..b417a88a6f 100644 --- a/third_party/rust/wgpu-hal/src/dx12/adapter.rs +++ b/third_party/rust/wgpu-hal/src/dx12/adapter.rs @@ -432,7 +432,9 @@ impl super::Adapter { } } -impl crate::Adapter for super::Adapter { +impl crate::Adapter for super::Adapter { + type A = super::Api; + unsafe fn open( &self, _features: wgt::Features, diff --git a/third_party/rust/wgpu-hal/src/dx12/command.rs b/third_party/rust/wgpu-hal/src/dx12/command.rs index 9d96d29cae..3c535b2234 100644 --- a/third_party/rust/wgpu-hal/src/dx12/command.rs +++ b/third_party/rust/wgpu-hal/src/dx12/command.rs @@ -249,7 +249,9 @@ impl super::CommandEncoder { } } -impl crate::CommandEncoder for super::CommandEncoder { +impl crate::CommandEncoder for super::CommandEncoder { + type A = super::Api; + unsafe fn begin_encoding(&mut self, label: crate::Label) -> Result<(), crate::DeviceError> { let list = loop { if let Some(list) = self.free_lists.pop() { diff --git a/third_party/rust/wgpu-hal/src/dx12/device.rs b/third_party/rust/wgpu-hal/src/dx12/device.rs index 3603b033b8..23bd409dc4 100644 --- a/third_party/rust/wgpu-hal/src/dx12/device.rs +++ b/third_party/rust/wgpu-hal/src/dx12/device.rs @@ -323,7 +323,9 @@ impl super::Device { } } -impl crate::Device for super::Device { +impl crate::Device for super::Device { + type A = super::Api; + unsafe fn exit(mut self, _queue: super::Queue) { self.rtv_pool.lock().free_handle(self.null_rtv_handle); self.mem_allocator = None; @@ -1098,7 +1100,16 @@ impl crate::Device for super::Device { } let mut dynamic_buffers = Vec::new(); - for (layout, entry) in desc.layout.entries.iter().zip(desc.entries.iter()) { + let layout_and_entry_iter = desc.entries.iter().map(|entry| { + let layout = desc + .layout + .entries + .iter() + .find(|layout_entry| layout_entry.binding == entry.binding) + .expect("internal error: no layout entry found with binding slot"); + (layout, entry) + }); + for (layout, entry) in layout_and_entry_iter { match layout.ty { wgt::BindingType::Buffer { has_dynamic_offset: true, diff --git a/third_party/rust/wgpu-hal/src/dx12/instance.rs b/third_party/rust/wgpu-hal/src/dx12/instance.rs index 020809328e..1dba7101df 100644 --- a/third_party/rust/wgpu-hal/src/dx12/instance.rs +++ b/third_party/rust/wgpu-hal/src/dx12/instance.rs @@ -13,7 +13,9 @@ impl Drop for super::Instance { } } -impl crate::Instance for super::Instance { +impl crate::Instance for super::Instance { + type A = super::Api; + unsafe fn init(desc: &crate::InstanceDescriptor) -> Result { profiling::scope!("Init DX12 Backend"); let lib_main = d3d12::D3D12Lib::new().map_err(|e| { diff --git a/third_party/rust/wgpu-hal/src/dx12/mod.rs b/third_party/rust/wgpu-hal/src/dx12/mod.rs index 13b43f8aca..4f958943ca 100644 --- a/third_party/rust/wgpu-hal/src/dx12/mod.rs +++ b/third_party/rust/wgpu-hal/src/dx12/mod.rs @@ -639,7 +639,9 @@ impl SwapChain { } } -impl crate::Surface for Surface { +impl crate::Surface for Surface { + type A = Api; + unsafe fn configure( &self, device: &Device, @@ -884,7 +886,9 @@ impl crate::Surface for Surface { } } -impl crate::Queue for Queue { +impl crate::Queue for Queue { + type A = Api; + unsafe fn submit( &self, command_buffers: &[&CommandBuffer], diff --git a/third_party/rust/wgpu-hal/src/empty.rs b/third_party/rust/wgpu-hal/src/empty.rs index d58e779b96..ad00da1b7f 100644 --- a/third_party/rust/wgpu-hal/src/empty.rs +++ b/third_party/rust/wgpu-hal/src/empty.rs @@ -39,7 +39,9 @@ impl crate::Api for Api { type ComputePipeline = Resource; } -impl crate::Instance for Context { +impl crate::Instance for Context { + type A = Api; + unsafe fn init(desc: &crate::InstanceDescriptor) -> Result { Ok(Context) } @@ -56,7 +58,9 @@ impl crate::Instance for Context { } } -impl crate::Surface for Context { +impl crate::Surface for Context { + type A = Api; + unsafe fn configure( &self, device: &Context, @@ -76,7 +80,9 @@ impl crate::Surface for Context { unsafe fn discard_texture(&self, texture: Resource) {} } -impl crate::Adapter for Context { +impl crate::Adapter for Context { + type A = Api; + unsafe fn open( &self, features: wgt::Features, @@ -100,7 +106,9 @@ impl crate::Adapter for Context { } } -impl crate::Queue for Context { +impl crate::Queue for Context { + type A = Api; + unsafe fn submit( &self, command_buffers: &[&Resource], @@ -122,7 +130,9 @@ impl crate::Queue for Context { } } -impl crate::Device for Context { +impl crate::Device for Context { + type A = Api; + unsafe fn exit(self, queue: Context) {} unsafe fn create_buffer(&self, desc: &crate::BufferDescriptor) -> DeviceResult { Ok(Resource) @@ -259,7 +269,9 @@ impl crate::Device for Context { unsafe fn destroy_acceleration_structure(&self, _acceleration_structure: Resource) {} } -impl crate::CommandEncoder for Encoder { +impl crate::CommandEncoder for Encoder { + type A = Api; + unsafe fn begin_encoding(&mut self, label: crate::Label) -> DeviceResult<()> { Ok(()) } diff --git a/third_party/rust/wgpu-hal/src/gles/adapter.rs b/third_party/rust/wgpu-hal/src/gles/adapter.rs index c09725e85f..b9d044337c 100644 --- a/third_party/rust/wgpu-hal/src/gles/adapter.rs +++ b/third_party/rust/wgpu-hal/src/gles/adapter.rs @@ -922,7 +922,9 @@ impl super::Adapter { } } -impl crate::Adapter for super::Adapter { +impl crate::Adapter for super::Adapter { + type A = super::Api; + unsafe fn open( &self, features: wgt::Features, diff --git a/third_party/rust/wgpu-hal/src/gles/command.rs b/third_party/rust/wgpu-hal/src/gles/command.rs index 4385e2a31e..258dee76e5 100644 --- a/third_party/rust/wgpu-hal/src/gles/command.rs +++ b/third_party/rust/wgpu-hal/src/gles/command.rs @@ -250,7 +250,9 @@ impl super::CommandEncoder { } } -impl crate::CommandEncoder for super::CommandEncoder { +impl crate::CommandEncoder for super::CommandEncoder { + type A = super::Api; + unsafe fn begin_encoding(&mut self, label: crate::Label) -> Result<(), crate::DeviceError> { self.state = State::default(); self.cmd_buffer.label = label.map(str::to_string); diff --git a/third_party/rust/wgpu-hal/src/gles/device.rs b/third_party/rust/wgpu-hal/src/gles/device.rs index 2678488cf8..50c07f3ff0 100644 --- a/third_party/rust/wgpu-hal/src/gles/device.rs +++ b/third_party/rust/wgpu-hal/src/gles/device.rs @@ -483,7 +483,9 @@ impl super::Device { } } -impl crate::Device for super::Device { +impl crate::Device for super::Device { + type A = super::Api; + unsafe fn exit(self, queue: super::Queue) { let gl = &self.shared.context.lock(); unsafe { gl.delete_vertex_array(self.main_vao) }; @@ -1123,8 +1125,10 @@ impl crate::Device for super::Device { !0; bg_layout .entries - .last() - .map_or(0, |b| b.binding as usize + 1) + .iter() + .map(|b| b.binding) + .max() + .map_or(0, |idx| idx as usize + 1) ] .into_boxed_slice(); @@ -1177,7 +1181,16 @@ impl crate::Device for super::Device { ) -> Result { let mut contents = Vec::new(); - for (entry, layout) in desc.entries.iter().zip(desc.layout.entries.iter()) { + let layout_and_entry_iter = desc.entries.iter().map(|entry| { + let layout = desc + .layout + .entries + .iter() + .find(|layout_entry| layout_entry.binding == entry.binding) + .expect("internal error: no layout entry found with binding slot"); + (entry, layout) + }); + for (entry, layout) in layout_and_entry_iter { let binding = match layout.ty { wgt::BindingType::Buffer { .. } => { let bb = &desc.buffers[entry.resource_index as usize]; diff --git a/third_party/rust/wgpu-hal/src/gles/egl.rs b/third_party/rust/wgpu-hal/src/gles/egl.rs index f4bfcf5487..b166f4f102 100644 --- a/third_party/rust/wgpu-hal/src/gles/egl.rs +++ b/third_party/rust/wgpu-hal/src/gles/egl.rs @@ -703,7 +703,9 @@ impl Instance { unsafe impl Send for Instance {} unsafe impl Sync for Instance {} -impl crate::Instance for Instance { +impl crate::Instance for Instance { + type A = super::Api; + unsafe fn init(desc: &crate::InstanceDescriptor) -> Result { profiling::scope!("Init OpenGL (EGL) Backend"); #[cfg(Emscripten)] @@ -1165,7 +1167,9 @@ impl Surface { } } -impl crate::Surface for Surface { +impl crate::Surface for Surface { + type A = super::Api; + unsafe fn configure( &self, device: &super::Device, diff --git a/third_party/rust/wgpu-hal/src/gles/queue.rs b/third_party/rust/wgpu-hal/src/gles/queue.rs index 5db5af9a16..29dfb79d04 100644 --- a/third_party/rust/wgpu-hal/src/gles/queue.rs +++ b/third_party/rust/wgpu-hal/src/gles/queue.rs @@ -1748,7 +1748,9 @@ impl super::Queue { } } -impl crate::Queue for super::Queue { +impl crate::Queue for super::Queue { + type A = super::Api; + unsafe fn submit( &self, command_buffers: &[&super::CommandBuffer], diff --git a/third_party/rust/wgpu-hal/src/gles/web.rs b/third_party/rust/wgpu-hal/src/gles/web.rs index 797d6f91d7..ab2ccef8b6 100644 --- a/third_party/rust/wgpu-hal/src/gles/web.rs +++ b/third_party/rust/wgpu-hal/src/gles/web.rs @@ -116,7 +116,9 @@ unsafe impl Sync for Instance {} #[cfg(send_sync)] unsafe impl Send for Instance {} -impl crate::Instance for Instance { +impl crate::Instance for Instance { + type A = super::Api; + unsafe fn init(_desc: &crate::InstanceDescriptor) -> Result { profiling::scope!("Init OpenGL (WebGL) Backend"); Ok(Instance { @@ -309,7 +311,9 @@ impl Surface { } } -impl crate::Surface for Surface { +impl crate::Surface for Surface { + type A = super::Api; + unsafe fn configure( &self, device: &super::Device, diff --git a/third_party/rust/wgpu-hal/src/gles/wgl.rs b/third_party/rust/wgpu-hal/src/gles/wgl.rs index c9039090b7..2564892969 100644 --- a/third_party/rust/wgpu-hal/src/gles/wgl.rs +++ b/third_party/rust/wgpu-hal/src/gles/wgl.rs @@ -422,7 +422,9 @@ fn create_instance_device() -> Result { Ok(InstanceDevice { dc, _tx: drop_tx }) } -impl crate::Instance for Instance { +impl crate::Instance for Instance { + type A = super::Api; + unsafe fn init(desc: &crate::InstanceDescriptor) -> Result { profiling::scope!("Init OpenGL (WGL) Backend"); let opengl_module = unsafe { LoadLibraryA("opengl32.dll\0".as_ptr() as *const _) }; @@ -676,7 +678,9 @@ impl Surface { } } -impl crate::Surface for Surface { +impl crate::Surface for Surface { + type A = super::Api; + unsafe fn configure( &self, device: &super::Device, diff --git a/third_party/rust/wgpu-hal/src/lib.rs b/third_party/rust/wgpu-hal/src/lib.rs index f1794a4a89..79bd54e66e 100644 --- a/third_party/rust/wgpu-hal/src/lib.rs +++ b/third_party/rust/wgpu-hal/src/lib.rs @@ -191,13 +191,13 @@ impl InstanceError { } pub trait Api: Clone + fmt::Debug + Sized { - type Instance: Instance; - type Surface: Surface; - type Adapter: Adapter; - type Device: Device; + type Instance: Instance; + type Surface: Surface; + type Adapter: Adapter; + type Device: Device; - type Queue: Queue; - type CommandEncoder: CommandEncoder; + type Queue: Queue; + type CommandEncoder: CommandEncoder; type CommandBuffer: WasmNotSendSync + fmt::Debug; type Buffer: fmt::Debug + WasmNotSendSync + 'static; @@ -218,18 +218,22 @@ pub trait Api: Clone + fmt::Debug + Sized { type AccelerationStructure: fmt::Debug + WasmNotSendSync + 'static; } -pub trait Instance: Sized + WasmNotSendSync { +pub trait Instance: Sized + WasmNotSendSync { + type A: Api; + unsafe fn init(desc: &InstanceDescriptor) -> Result; unsafe fn create_surface( &self, display_handle: raw_window_handle::RawDisplayHandle, window_handle: raw_window_handle::RawWindowHandle, - ) -> Result; - unsafe fn destroy_surface(&self, surface: A::Surface); - unsafe fn enumerate_adapters(&self) -> Vec>; + ) -> Result<::Surface, InstanceError>; + unsafe fn destroy_surface(&self, surface: ::Surface); + unsafe fn enumerate_adapters(&self) -> Vec>; } -pub trait Surface: WasmNotSendSync { +pub trait Surface: WasmNotSendSync { + type A: Api; + /// Configures the surface to use the given device. /// /// # Safety @@ -240,7 +244,7 @@ pub trait Surface: WasmNotSendSync { /// - All surfaces created using other devices must have been unconfigured before this call. unsafe fn configure( &self, - device: &A::Device, + device: &::Device, config: &SurfaceConfiguration, ) -> Result<(), SurfaceError>; @@ -252,7 +256,7 @@ pub trait Surface: WasmNotSendSync { /// - All [`AcquiredSurfaceTexture`]s must have been destroyed. /// - All [`Api::TextureView`]s derived from the [`AcquiredSurfaceTexture`]s must have been destroyed. /// - The surface must have been configured on the given device. - unsafe fn unconfigure(&self, device: &A::Device); + unsafe fn unconfigure(&self, device: &::Device); /// Returns the next texture to be presented by the swapchain for drawing /// @@ -267,16 +271,18 @@ pub trait Surface: WasmNotSendSync { unsafe fn acquire_texture( &self, timeout: Option, - ) -> Result>, SurfaceError>; - unsafe fn discard_texture(&self, texture: A::SurfaceTexture); + ) -> Result>, SurfaceError>; + unsafe fn discard_texture(&self, texture: ::SurfaceTexture); } -pub trait Adapter: WasmNotSendSync { +pub trait Adapter: WasmNotSendSync { + type A: Api; + unsafe fn open( &self, features: wgt::Features, limits: &wgt::Limits, - ) -> Result, DeviceError>; + ) -> Result, DeviceError>; /// Return the set of supported capabilities for a texture format. unsafe fn texture_format_capabilities( @@ -287,7 +293,10 @@ pub trait Adapter: WasmNotSendSync { /// Returns the capabilities of working with a specified surface. /// /// `None` means presentation is not supported for it. - unsafe fn surface_capabilities(&self, surface: &A::Surface) -> Option; + unsafe fn surface_capabilities( + &self, + surface: &::Surface, + ) -> Option; /// Creates a [`PresentationTimestamp`] using the adapter's WSI. /// @@ -295,97 +304,111 @@ pub trait Adapter: WasmNotSendSync { unsafe fn get_presentation_timestamp(&self) -> wgt::PresentationTimestamp; } -pub trait Device: WasmNotSendSync { +pub trait Device: WasmNotSendSync { + type A: Api; + /// Exit connection to this logical device. - unsafe fn exit(self, queue: A::Queue); + unsafe fn exit(self, queue: ::Queue); /// Creates a new buffer. /// /// The initial usage is `BufferUses::empty()`. - unsafe fn create_buffer(&self, desc: &BufferDescriptor) -> Result; - unsafe fn destroy_buffer(&self, buffer: A::Buffer); + unsafe fn create_buffer( + &self, + desc: &BufferDescriptor, + ) -> Result<::Buffer, DeviceError>; + unsafe fn destroy_buffer(&self, buffer: ::Buffer); //TODO: clarify if zero-sized mapping is allowed unsafe fn map_buffer( &self, - buffer: &A::Buffer, + buffer: &::Buffer, range: MemoryRange, ) -> Result; - unsafe fn unmap_buffer(&self, buffer: &A::Buffer) -> Result<(), DeviceError>; - unsafe fn flush_mapped_ranges(&self, buffer: &A::Buffer, ranges: I) + unsafe fn unmap_buffer(&self, buffer: &::Buffer) -> Result<(), DeviceError>; + unsafe fn flush_mapped_ranges(&self, buffer: &::Buffer, ranges: I) where I: Iterator; - unsafe fn invalidate_mapped_ranges(&self, buffer: &A::Buffer, ranges: I) + unsafe fn invalidate_mapped_ranges(&self, buffer: &::Buffer, ranges: I) where I: Iterator; /// Creates a new texture. /// /// The initial usage for all subresources is `TextureUses::UNINITIALIZED`. - unsafe fn create_texture(&self, desc: &TextureDescriptor) -> Result; - unsafe fn destroy_texture(&self, texture: A::Texture); + unsafe fn create_texture( + &self, + desc: &TextureDescriptor, + ) -> Result<::Texture, DeviceError>; + unsafe fn destroy_texture(&self, texture: ::Texture); unsafe fn create_texture_view( &self, - texture: &A::Texture, + texture: &::Texture, desc: &TextureViewDescriptor, - ) -> Result; - unsafe fn destroy_texture_view(&self, view: A::TextureView); - unsafe fn create_sampler(&self, desc: &SamplerDescriptor) -> Result; - unsafe fn destroy_sampler(&self, sampler: A::Sampler); + ) -> Result<::TextureView, DeviceError>; + unsafe fn destroy_texture_view(&self, view: ::TextureView); + unsafe fn create_sampler( + &self, + desc: &SamplerDescriptor, + ) -> Result<::Sampler, DeviceError>; + unsafe fn destroy_sampler(&self, sampler: ::Sampler); /// Create a fresh [`CommandEncoder`]. /// /// The new `CommandEncoder` is in the "closed" state. unsafe fn create_command_encoder( &self, - desc: &CommandEncoderDescriptor, - ) -> Result; - unsafe fn destroy_command_encoder(&self, pool: A::CommandEncoder); + desc: &CommandEncoderDescriptor, + ) -> Result<::CommandEncoder, DeviceError>; + unsafe fn destroy_command_encoder(&self, pool: ::CommandEncoder); /// Creates a bind group layout. unsafe fn create_bind_group_layout( &self, desc: &BindGroupLayoutDescriptor, - ) -> Result; - unsafe fn destroy_bind_group_layout(&self, bg_layout: A::BindGroupLayout); + ) -> Result<::BindGroupLayout, DeviceError>; + unsafe fn destroy_bind_group_layout(&self, bg_layout: ::BindGroupLayout); unsafe fn create_pipeline_layout( &self, - desc: &PipelineLayoutDescriptor, - ) -> Result; - unsafe fn destroy_pipeline_layout(&self, pipeline_layout: A::PipelineLayout); + desc: &PipelineLayoutDescriptor, + ) -> Result<::PipelineLayout, DeviceError>; + unsafe fn destroy_pipeline_layout(&self, pipeline_layout: ::PipelineLayout); unsafe fn create_bind_group( &self, - desc: &BindGroupDescriptor, - ) -> Result; - unsafe fn destroy_bind_group(&self, group: A::BindGroup); + desc: &BindGroupDescriptor, + ) -> Result<::BindGroup, DeviceError>; + unsafe fn destroy_bind_group(&self, group: ::BindGroup); unsafe fn create_shader_module( &self, desc: &ShaderModuleDescriptor, shader: ShaderInput, - ) -> Result; - unsafe fn destroy_shader_module(&self, module: A::ShaderModule); + ) -> Result<::ShaderModule, ShaderError>; + unsafe fn destroy_shader_module(&self, module: ::ShaderModule); unsafe fn create_render_pipeline( &self, - desc: &RenderPipelineDescriptor, - ) -> Result; - unsafe fn destroy_render_pipeline(&self, pipeline: A::RenderPipeline); + desc: &RenderPipelineDescriptor, + ) -> Result<::RenderPipeline, PipelineError>; + unsafe fn destroy_render_pipeline(&self, pipeline: ::RenderPipeline); unsafe fn create_compute_pipeline( &self, - desc: &ComputePipelineDescriptor, - ) -> Result; - unsafe fn destroy_compute_pipeline(&self, pipeline: A::ComputePipeline); + desc: &ComputePipelineDescriptor, + ) -> Result<::ComputePipeline, PipelineError>; + unsafe fn destroy_compute_pipeline(&self, pipeline: ::ComputePipeline); unsafe fn create_query_set( &self, desc: &wgt::QuerySetDescriptor, + desc: &GetAccelerationStructureBuildSizesDescriptor, ) -> AccelerationStructureBuildSizes; unsafe fn get_acceleration_structure_device_address( &self, - acceleration_structure: &A::AccelerationStructure, + acceleration_structure: &::AccelerationStructure, ) -> wgt::BufferAddress; unsafe fn destroy_acceleration_structure( &self, - acceleration_structure: A::AccelerationStructure, + acceleration_structure: ::AccelerationStructure, ); } -pub trait Queue: WasmNotSendSync { +pub trait Queue: WasmNotSendSync { + type A: Api; + /// Submits the command buffers for execution on GPU. /// /// Valid usage: @@ -422,14 +447,14 @@ pub trait Queue: WasmNotSendSync { /// passed to the surface_textures argument. unsafe fn submit( &self, - command_buffers: &[&A::CommandBuffer], - surface_textures: &[&A::SurfaceTexture], - signal_fence: Option<(&mut A::Fence, FenceValue)>, + command_buffers: &[&::CommandBuffer], + surface_textures: &[&::SurfaceTexture], + signal_fence: Option<(&mut ::Fence, FenceValue)>, ) -> Result<(), DeviceError>; unsafe fn present( &self, - surface: &A::Surface, - texture: A::SurfaceTexture, + surface: &::Surface, + texture: ::SurfaceTexture, ) -> Result<(), SurfaceError>; unsafe fn get_timestamp_period(&self) -> f32; } @@ -472,7 +497,9 @@ pub trait Queue: WasmNotSendSync { /// built it. /// /// - A `CommandEncoder` must not outlive its `Device`. -pub trait CommandEncoder: WasmNotSendSync + fmt::Debug { +pub trait CommandEncoder: WasmNotSendSync + fmt::Debug { + type A: Api; + /// Begin encoding a new command buffer. /// /// This puts this `CommandEncoder` in the "recording" state. @@ -510,7 +537,7 @@ pub trait CommandEncoder: WasmNotSendSync + fmt::Debug { /// /// [`CommandBuffer`]: Api::CommandBuffer /// [`begin_encoding`]: CommandEncoder::begin_encoding - unsafe fn end_encoding(&mut self) -> Result; + unsafe fn end_encoding(&mut self) -> Result<::CommandBuffer, DeviceError>; /// Reclaim all resources belonging to this `CommandEncoder`. /// @@ -525,22 +552,26 @@ pub trait CommandEncoder: WasmNotSendSync + fmt::Debug { /// [`CommandBuffer`]: Api::CommandBuffer unsafe fn reset_all(&mut self, command_buffers: I) where - I: Iterator; + I: Iterator::CommandBuffer>; unsafe fn transition_buffers<'a, T>(&mut self, barriers: T) where - T: Iterator>; + T: Iterator>; unsafe fn transition_textures<'a, T>(&mut self, barriers: T) where - T: Iterator>; + T: Iterator>; // copy operations - unsafe fn clear_buffer(&mut self, buffer: &A::Buffer, range: MemoryRange); + unsafe fn clear_buffer(&mut self, buffer: &::Buffer, range: MemoryRange); - unsafe fn copy_buffer_to_buffer(&mut self, src: &A::Buffer, dst: &A::Buffer, regions: T) - where + unsafe fn copy_buffer_to_buffer( + &mut self, + src: &::Buffer, + dst: &::Buffer, + regions: T, + ) where T: Iterator; /// Copy from an external image to an internal texture. @@ -551,7 +582,7 @@ pub trait CommandEncoder: WasmNotSendSync + fmt::Debug { unsafe fn copy_external_image_to_texture( &mut self, src: &wgt::ImageCopyExternalImage, - dst: &A::Texture, + dst: &::Texture, dst_premultiplication: bool, regions: T, ) where @@ -563,9 +594,9 @@ pub trait CommandEncoder: WasmNotSendSync + fmt::Debug { /// Note: the copy extent is in physical size (rounded to the block size) unsafe fn copy_texture_to_texture( &mut self, - src: &A::Texture, + src: &::Texture, src_usage: TextureUses, - dst: &A::Texture, + dst: &::Texture, regions: T, ) where T: Iterator; @@ -574,8 +605,12 @@ pub trait CommandEncoder: WasmNotSendSync + fmt::Debug { /// Works with a single array layer. /// Note: `dst` current usage has to be `TextureUses::COPY_DST`. /// Note: the copy extent is in physical size (rounded to the block size) - unsafe fn copy_buffer_to_texture(&mut self, src: &A::Buffer, dst: &A::Texture, regions: T) - where + unsafe fn copy_buffer_to_texture( + &mut self, + src: &::Buffer, + dst: &::Texture, + regions: T, + ) where T: Iterator; /// Copy from texture to buffer. @@ -583,9 +618,9 @@ pub trait CommandEncoder: WasmNotSendSync + fmt::Debug { /// Note: the copy extent is in physical size (rounded to the block size) unsafe fn copy_texture_to_buffer( &mut self, - src: &A::Texture, + src: &::Texture, src_usage: TextureUses, - dst: &A::Buffer, + dst: &::Buffer, regions: T, ) where T: Iterator; @@ -596,9 +631,9 @@ pub trait CommandEncoder: WasmNotSendSync + fmt::Debug { /// of all the preceding groups to be taken from `layout`. unsafe fn set_bind_group( &mut self, - layout: &A::PipelineLayout, + layout: &::PipelineLayout, index: u32, - group: &A::BindGroup, + group: &::BindGroup, dynamic_offsets: &[wgt::DynamicOffset], ); @@ -612,7 +647,7 @@ pub trait CommandEncoder: WasmNotSendSync + fmt::Debug { /// - The range of push constants written must be valid for the pipeline layout at draw time. unsafe fn set_push_constants( &mut self, - layout: &A::PipelineLayout, + layout: &::PipelineLayout, stages: wgt::ShaderStages, offset_bytes: u32, data: &[u32], @@ -627,18 +662,18 @@ pub trait CommandEncoder: WasmNotSendSync + fmt::Debug { /// # Safety: /// /// - If `set` is an occlusion query set, it must be the same one as used in the [`RenderPassDescriptor::occlusion_query_set`] parameter. - unsafe fn begin_query(&mut self, set: &A::QuerySet, index: u32); + unsafe fn begin_query(&mut self, set: &::QuerySet, index: u32); /// # Safety: /// /// - If `set` is an occlusion query set, it must be the same one as used in the [`RenderPassDescriptor::occlusion_query_set`] parameter. - unsafe fn end_query(&mut self, set: &A::QuerySet, index: u32); - unsafe fn write_timestamp(&mut self, set: &A::QuerySet, index: u32); - unsafe fn reset_queries(&mut self, set: &A::QuerySet, range: Range); + unsafe fn end_query(&mut self, set: &::QuerySet, index: u32); + unsafe fn write_timestamp(&mut self, set: &::QuerySet, index: u32); + unsafe fn reset_queries(&mut self, set: &::QuerySet, range: Range); unsafe fn copy_query_results( &mut self, - set: &A::QuerySet, + set: &::QuerySet, range: Range, - buffer: &A::Buffer, + buffer: &::Buffer, offset: wgt::BufferAddress, stride: wgt::BufferSize, ); @@ -646,17 +681,17 @@ pub trait CommandEncoder: WasmNotSendSync + fmt::Debug { // render passes // Begins a render pass, clears all active bindings. - unsafe fn begin_render_pass(&mut self, desc: &RenderPassDescriptor); + unsafe fn begin_render_pass(&mut self, desc: &RenderPassDescriptor); unsafe fn end_render_pass(&mut self); - unsafe fn set_render_pipeline(&mut self, pipeline: &A::RenderPipeline); + unsafe fn set_render_pipeline(&mut self, pipeline: &::RenderPipeline); unsafe fn set_index_buffer<'a>( &mut self, - binding: BufferBinding<'a, A>, + binding: BufferBinding<'a, Self::A>, format: wgt::IndexFormat, ); - unsafe fn set_vertex_buffer<'a>(&mut self, index: u32, binding: BufferBinding<'a, A>); + unsafe fn set_vertex_buffer<'a>(&mut self, index: u32, binding: BufferBinding<'a, Self::A>); unsafe fn set_viewport(&mut self, rect: &Rect, depth_range: Range); unsafe fn set_scissor_rect(&mut self, rect: &Rect); unsafe fn set_stencil_reference(&mut self, value: u32); @@ -679,29 +714,29 @@ pub trait CommandEncoder: WasmNotSendSync + fmt::Debug { ); unsafe fn draw_indirect( &mut self, - buffer: &A::Buffer, + buffer: &::Buffer, offset: wgt::BufferAddress, draw_count: u32, ); unsafe fn draw_indexed_indirect( &mut self, - buffer: &A::Buffer, + buffer: &::Buffer, offset: wgt::BufferAddress, draw_count: u32, ); unsafe fn draw_indirect_count( &mut self, - buffer: &A::Buffer, + buffer: &::Buffer, offset: wgt::BufferAddress, - count_buffer: &A::Buffer, + count_buffer: &::Buffer, count_offset: wgt::BufferAddress, max_count: u32, ); unsafe fn draw_indexed_indirect_count( &mut self, - buffer: &A::Buffer, + buffer: &::Buffer, offset: wgt::BufferAddress, - count_buffer: &A::Buffer, + count_buffer: &::Buffer, count_offset: wgt::BufferAddress, max_count: u32, ); @@ -709,13 +744,17 @@ pub trait CommandEncoder: WasmNotSendSync + fmt::Debug { // compute passes // Begins a compute pass, clears all active bindings. - unsafe fn begin_compute_pass(&mut self, desc: &ComputePassDescriptor); + unsafe fn begin_compute_pass(&mut self, desc: &ComputePassDescriptor); unsafe fn end_compute_pass(&mut self); - unsafe fn set_compute_pipeline(&mut self, pipeline: &A::ComputePipeline); + unsafe fn set_compute_pipeline(&mut self, pipeline: &::ComputePipeline); unsafe fn dispatch(&mut self, count: [u32; 3]); - unsafe fn dispatch_indirect(&mut self, buffer: &A::Buffer, offset: wgt::BufferAddress); + unsafe fn dispatch_indirect( + &mut self, + buffer: &::Buffer, + offset: wgt::BufferAddress, + ); /// To get the required sizes for the buffer allocations use `get_acceleration_structure_build_sizes` per descriptor /// All buffers must be synchronized externally @@ -729,8 +768,8 @@ pub trait CommandEncoder: WasmNotSendSync + fmt::Debug { descriptor_count: u32, descriptors: T, ) where - A: 'a, - T: IntoIterator>; + Self::A: 'a, + T: IntoIterator>; unsafe fn place_acceleration_structure_barrier( &mut self, diff --git a/third_party/rust/wgpu-hal/src/metal/adapter.rs b/third_party/rust/wgpu-hal/src/metal/adapter.rs index 9ec777b0f0..6211896838 100644 --- a/third_party/rust/wgpu-hal/src/metal/adapter.rs +++ b/third_party/rust/wgpu-hal/src/metal/adapter.rs @@ -18,7 +18,9 @@ impl super::Adapter { } } -impl crate::Adapter for super::Adapter { +impl crate::Adapter for super::Adapter { + type A = super::Api; + unsafe fn open( &self, features: wgt::Features, diff --git a/third_party/rust/wgpu-hal/src/metal/command.rs b/third_party/rust/wgpu-hal/src/metal/command.rs index 6f1a0d9c2f..341712c323 100644 --- a/third_party/rust/wgpu-hal/src/metal/command.rs +++ b/third_party/rust/wgpu-hal/src/metal/command.rs @@ -168,7 +168,9 @@ impl super::CommandState { } } -impl crate::CommandEncoder for super::CommandEncoder { +impl crate::CommandEncoder for super::CommandEncoder { + type A = super::Api; + unsafe fn begin_encoding(&mut self, label: crate::Label) -> Result<(), crate::DeviceError> { let queue = &self.raw_queue.lock(); let retain_references = self.shared.settings.retain_command_buffer_references; diff --git a/third_party/rust/wgpu-hal/src/metal/device.rs b/third_party/rust/wgpu-hal/src/metal/device.rs index d7fd06c8f3..179429f5d7 100644 --- a/third_party/rust/wgpu-hal/src/metal/device.rs +++ b/third_party/rust/wgpu-hal/src/metal/device.rs @@ -273,7 +273,9 @@ impl super::Device { } } -impl crate::Device for super::Device { +impl crate::Device for super::Device { + type A = super::Api; + unsafe fn exit(self, _queue: super::Queue) {} unsafe fn create_buffer(&self, desc: &crate::BufferDescriptor) -> DeviceResult { @@ -706,7 +708,16 @@ impl crate::Device for super::Device { for (&stage, counter) in super::NAGA_STAGES.iter().zip(bg.counters.iter_mut()) { let stage_bit = map_naga_stage(stage); let mut dynamic_offsets_count = 0u32; - for (entry, layout) in desc.entries.iter().zip(desc.layout.entries.iter()) { + let layout_and_entry_iter = desc.entries.iter().map(|entry| { + let layout = desc + .layout + .entries + .iter() + .find(|layout_entry| layout_entry.binding == entry.binding) + .expect("internal error: no layout entry found with binding slot"); + (entry, layout) + }); + for (entry, layout) in layout_and_entry_iter { let size = layout.count.map_or(1, |c| c.get()); if let wgt::BindingType::Buffer { has_dynamic_offset: true, diff --git a/third_party/rust/wgpu-hal/src/metal/mod.rs b/third_party/rust/wgpu-hal/src/metal/mod.rs index 62fbf3d49d..6aeafb0f86 100644 --- a/third_party/rust/wgpu-hal/src/metal/mod.rs +++ b/third_party/rust/wgpu-hal/src/metal/mod.rs @@ -80,7 +80,9 @@ impl Instance { } } -impl crate::Instance for Instance { +impl crate::Instance for Instance { + type A = Api; + unsafe fn init(_desc: &crate::InstanceDescriptor) -> Result { profiling::scope!("Init Metal Backend"); // We do not enable metal validation based on the validation flags as it affects the entire @@ -365,7 +367,9 @@ impl std::borrow::Borrow for SurfaceTexture { unsafe impl Send for SurfaceTexture {} unsafe impl Sync for SurfaceTexture {} -impl crate::Queue for Queue { +impl crate::Queue for Queue { + type A = Api; + unsafe fn submit( &self, command_buffers: &[&CommandBuffer], diff --git a/third_party/rust/wgpu-hal/src/metal/surface.rs b/third_party/rust/wgpu-hal/src/metal/surface.rs index a97eff0aae..889e319493 100644 --- a/third_party/rust/wgpu-hal/src/metal/surface.rs +++ b/third_party/rust/wgpu-hal/src/metal/surface.rs @@ -169,7 +169,9 @@ impl super::Surface { } } -impl crate::Surface for super::Surface { +impl crate::Surface for super::Surface { + type A = super::Api; + unsafe fn configure( &self, device: &super::Device, diff --git a/third_party/rust/wgpu-hal/src/vulkan/adapter.rs b/third_party/rust/wgpu-hal/src/vulkan/adapter.rs index 83b3dfa8e5..2665463792 100644 --- a/third_party/rust/wgpu-hal/src/vulkan/adapter.rs +++ b/third_party/rust/wgpu-hal/src/vulkan/adapter.rs @@ -20,25 +20,85 @@ fn indexing_features() -> wgt::Features { | wgt::Features::PARTIALLY_BOUND_BINDING_ARRAY } -/// Aggregate of the `vk::PhysicalDevice*Features` structs used by `gfx`. +/// Features supported by a [`vk::PhysicalDevice`] and its extensions. +/// +/// This is used in two phases: +/// +/// - When enumerating adapters, this represents the features offered by the +/// adapter. [`Instance::expose_adapter`] calls `vkGetPhysicalDeviceFeatures2` +/// (or `vkGetPhysicalDeviceFeatures` if that is not available) to collect +/// this information about the `VkPhysicalDevice` represented by the +/// `wgpu_hal::ExposedAdapter`. +/// +/// - When opening a device, this represents the features we would like to +/// enable. At `wgpu_hal::Device` construction time, +/// [`PhysicalDeviceFeatures::from_extensions_and_requested_features`] +/// constructs an value of this type indicating which Vulkan features to +/// enable, based on the `wgpu_types::Features` requested. #[derive(Debug, Default)] pub struct PhysicalDeviceFeatures { + /// Basic Vulkan 1.0 features. core: vk::PhysicalDeviceFeatures, + + /// Features provided by `VK_EXT_descriptor_indexing`, promoted to Vulkan 1.2. pub(super) descriptor_indexing: Option, + + /// Features provided by `VK_KHR_imageless_framebuffer`, promoted to Vulkan 1.2. imageless_framebuffer: Option, + + /// Features provided by `VK_KHR_timeline_semaphore`, promoted to Vulkan 1.2 timeline_semaphore: Option, + + /// Features provided by `VK_EXT_image_robustness`, promoted to Vulkan 1.3 image_robustness: Option, + + /// Features provided by `VK_EXT_robustness2`. robustness2: Option, + + /// Features provided by `VK_KHR_multiview`, promoted to Vulkan 1.1. multiview: Option, + + /// Features provided by `VK_KHR_sampler_ycbcr_conversion`, promoted to Vulkan 1.1. sampler_ycbcr_conversion: Option, + + /// Features provided by `VK_EXT_texture_compression_astc_hdr`, promoted to Vulkan 1.3. astc_hdr: Option, + + /// Features provided by `VK_KHR_shader_float16_int8` (promoted to Vulkan + /// 1.2) and `VK_KHR_16bit_storage` (promoted to Vulkan 1.1). We use these + /// features together, or not at all. shader_float16: Option<( vk::PhysicalDeviceShaderFloat16Int8Features, vk::PhysicalDevice16BitStorageFeatures, )>, + + /// Features provided by `VK_KHR_acceleration_structure`. acceleration_structure: Option, + + /// Features provided by `VK_KHR_buffer_device_address`, promoted to Vulkan 1.2. + /// + /// We only use this feature for + /// [`Features::RAY_TRACING_ACCELERATION_STRUCTURE`], which requires + /// `VK_KHR_acceleration_structure`, which depends on + /// `VK_KHR_buffer_device_address`, so [`Instance::expose_adapter`] only + /// bothers to check if `VK_KHR_acceleration_structure` is available, + /// leaving this `None`. + /// + /// However, we do populate this when creating a device if + /// [`Features::RAY_TRACING_ACCELERATION_STRUCTURE`] is requested. buffer_device_address: Option, + + /// Features provided by `VK_KHR_ray_query`, + /// + /// Vulkan requires that the feature be present if the `VK_KHR_ray_query` + /// extension is present, so [`Instance::expose_adapter`] doesn't bother retrieving + /// this from `vkGetPhysicalDeviceFeatures2`. + /// + /// However, we do populate this when creating a device if ray tracing is requested. ray_query: Option, + + /// Features provided by `VK_KHR_zero_initialize_workgroup_memory`, promoted + /// to Vulkan 1.3. zero_initialize_workgroup_memory: Option, } @@ -91,9 +151,32 @@ impl PhysicalDeviceFeatures { info } - /// Create a `PhysicalDeviceFeatures` that will be used to create a logical device. + /// Create a `PhysicalDeviceFeatures` that can be used to create a logical + /// device. /// - /// `requested_features` should be the same as what was used to generate `enabled_extensions`. + /// Return a `PhysicalDeviceFeatures` value capturing all the Vulkan + /// features needed for the given [`Features`], [`DownlevelFlags`], and + /// [`PrivateCapabilities`]. You can use the returned value's + /// [`add_to_device_create_builder`] method to configure a + /// [`DeviceCreateInfoBuilder`] to build a logical device providing those + /// features. + /// + /// To ensure that the returned value is able to select all the Vulkan + /// features needed to express `requested_features`, `downlevel_flags`, and + /// `private_caps`: + /// + /// - The given `enabled_extensions` set must include all the extensions + /// selected by [`Adapter::required_device_extensions`] when passed + /// `features`. + /// + /// - The given `device_api_version` must be the Vulkan API version of the + /// physical device we will use to create the logical device. + /// + /// [`Features`]: wgt::Features + /// [`DownlevelFlags`]: wgt::DownlevelFlags + /// [`PrivateCapabilities`]: super::PrivateCapabilities + /// [`DeviceCreateInfoBuilder`]: vk::DeviceCreateInfoBuilder + /// [`Adapter::required_device_extensions`]: super::Adapter::required_device_extensions fn from_extensions_and_requested_features( device_api_version: u32, enabled_extensions: &[&'static CStr], @@ -354,11 +437,16 @@ impl PhysicalDeviceFeatures { } } + /// Compute the wgpu [`Features`] and [`DownlevelFlags`] supported by a physical device. + /// + /// Given `self`, together with the instance and physical device it was + /// built from, and a `caps` also built from those, determine which wgpu + /// features and downlevel flags the device can support. fn to_wgpu( &self, instance: &ash::Instance, phd: vk::PhysicalDevice, - caps: &PhysicalDeviceCapabilities, + caps: &PhysicalDeviceProperties, ) -> (wgt::Features, wgt::DownlevelFlags) { use crate::auxil::db; use wgt::{DownlevelFlags as Df, Features as F}; @@ -639,15 +727,52 @@ impl PhysicalDeviceFeatures { } } -/// Information gathered about a physical device capabilities. +/// Vulkan "properties" structures gathered about a physical device. +/// +/// This structure holds the properties of a [`vk::PhysicalDevice`]: +/// - the standard Vulkan device properties +/// - the `VkExtensionProperties` structs for all available extensions, and +/// - the per-extension properties structures for the available extensions that +/// `wgpu` cares about. +/// +/// Generally, if you get it from any of these functions, it's stored +/// here: +/// - `vkEnumerateDeviceExtensionProperties` +/// - `vkGetPhysicalDeviceProperties` +/// - `vkGetPhysicalDeviceProperties2` +/// +/// This also includes a copy of the device API version, since we can +/// use that as a shortcut for searching for an extension, if the +/// extension has been promoted to core in the current version. +/// +/// This does not include device features; for those, see +/// [`PhysicalDeviceFeatures`]. #[derive(Default, Debug)] -pub struct PhysicalDeviceCapabilities { +pub struct PhysicalDeviceProperties { + /// Extensions supported by the `vk::PhysicalDevice`, + /// as returned by `vkEnumerateDeviceExtensionProperties`. supported_extensions: Vec, + + /// Properties of the `vk::PhysicalDevice`, as returned by + /// `vkGetPhysicalDeviceProperties`. properties: vk::PhysicalDeviceProperties, + + /// Additional `vk::PhysicalDevice` properties from the + /// `VK_KHR_maintenance3` extension, promoted to Vulkan 1.1. maintenance_3: Option, + + /// Additional `vk::PhysicalDevice` properties from the + /// `VK_EXT_descriptor_indexing` extension, promoted to Vulkan 1.2. descriptor_indexing: Option, + + /// Additional `vk::PhysicalDevice` properties from the + /// `VK_KHR_acceleration_structure` extension. acceleration_structure: Option, + + /// Additional `vk::PhysicalDevice` properties from the + /// `VK_KHR_driver_properties` extension, promoted to Vulkan 1.2. driver: Option, + /// The device API version. /// /// Which is the version of Vulkan supported for device-level functionality. @@ -657,10 +782,10 @@ pub struct PhysicalDeviceCapabilities { } // This is safe because the structs have `p_next: *mut c_void`, which we null out/never read. -unsafe impl Send for PhysicalDeviceCapabilities {} -unsafe impl Sync for PhysicalDeviceCapabilities {} +unsafe impl Send for PhysicalDeviceProperties {} +unsafe impl Sync for PhysicalDeviceProperties {} -impl PhysicalDeviceCapabilities { +impl PhysicalDeviceProperties { pub fn properties(&self) -> vk::PhysicalDeviceProperties { self.properties } @@ -899,9 +1024,9 @@ impl super::InstanceShared { fn inspect( &self, phd: vk::PhysicalDevice, - ) -> (PhysicalDeviceCapabilities, PhysicalDeviceFeatures) { + ) -> (PhysicalDeviceProperties, PhysicalDeviceFeatures) { let capabilities = { - let mut capabilities = PhysicalDeviceCapabilities::default(); + let mut capabilities = PhysicalDeviceProperties::default(); capabilities.supported_extensions = unsafe { self.raw.enumerate_device_extension_properties(phd).unwrap() }; capabilities.properties = unsafe { self.raw.get_physical_device_properties(phd) }; @@ -923,9 +1048,10 @@ impl super::InstanceShared { let mut builder = vk::PhysicalDeviceProperties2KHR::builder(); if supports_maintenance3 { - capabilities.maintenance_3 = - Some(vk::PhysicalDeviceMaintenance3Properties::default()); - builder = builder.push_next(capabilities.maintenance_3.as_mut().unwrap()); + let next = capabilities + .maintenance_3 + .insert(vk::PhysicalDeviceMaintenance3Properties::default()); + builder = builder.push_next(next); } if supports_descriptor_indexing { @@ -1001,7 +1127,8 @@ impl super::InstanceShared { builder = builder.push_next(next); } - // `VK_KHR_imageless_framebuffer` is promoted to 1.2, but has no changes, so we can keep using the extension unconditionally. + // `VK_KHR_imageless_framebuffer` is promoted to 1.2, but has no + // changes, so we can keep using the extension unconditionally. if capabilities.supports_extension(vk::KhrImagelessFramebufferFn::name()) { let next = features .imageless_framebuffer @@ -1009,7 +1136,8 @@ impl super::InstanceShared { builder = builder.push_next(next); } - // `VK_KHR_timeline_semaphore` is promoted to 1.2, but has no changes, so we can keep using the extension unconditionally. + // `VK_KHR_timeline_semaphore` is promoted to 1.2, but has no + // changes, so we can keep using the extension unconditionally. if capabilities.supports_extension(vk::KhrTimelineSemaphoreFn::name()) { let next = features .timeline_semaphore @@ -1295,7 +1423,7 @@ impl super::Adapter { self.raw } - pub fn physical_device_capabilities(&self) -> &PhysicalDeviceCapabilities { + pub fn physical_device_capabilities(&self) -> &PhysicalDeviceProperties { &self.phd_capabilities } @@ -1320,7 +1448,20 @@ impl super::Adapter { supported_extensions } - /// `features` must be the same features used to create `enabled_extensions`. + /// Create a `PhysicalDeviceFeatures` for opening a logical device with + /// `features` from this adapter. + /// + /// The given `enabled_extensions` set must include all the extensions + /// selected by [`required_device_extensions`] when passed `features`. + /// Otherwise, the `PhysicalDeviceFeatures` value may not be able to select + /// all the Vulkan features needed to represent `features` and this + /// adapter's characteristics. + /// + /// Typically, you'd simply call `required_device_extensions`, and then pass + /// its return value and the feature set you gave it directly to this + /// function. But it's fine to add more extensions to the list. + /// + /// [`required_device_extensions`]: Self::required_device_extensions pub fn physical_device_features( &self, enabled_extensions: &[&'static CStr], @@ -1607,7 +1748,9 @@ impl super::Adapter { } } -impl crate::Adapter for super::Adapter { +impl crate::Adapter for super::Adapter { + type A = super::Api; + unsafe fn open( &self, features: wgt::Features, diff --git a/third_party/rust/wgpu-hal/src/vulkan/command.rs b/third_party/rust/wgpu-hal/src/vulkan/command.rs index 42ea907738..43a2471954 100644 --- a/third_party/rust/wgpu-hal/src/vulkan/command.rs +++ b/third_party/rust/wgpu-hal/src/vulkan/command.rs @@ -60,7 +60,9 @@ impl super::CommandEncoder { } } -impl crate::CommandEncoder for super::CommandEncoder { +impl crate::CommandEncoder for super::CommandEncoder { + type A = super::Api; + unsafe fn begin_encoding(&mut self, label: crate::Label) -> Result<(), crate::DeviceError> { if self.free.is_empty() { let vk_info = vk::CommandBufferAllocateInfo::builder() diff --git a/third_party/rust/wgpu-hal/src/vulkan/device.rs b/third_party/rust/wgpu-hal/src/vulkan/device.rs index c00c3d1d43..70028cc700 100644 --- a/third_party/rust/wgpu-hal/src/vulkan/device.rs +++ b/third_party/rust/wgpu-hal/src/vulkan/device.rs @@ -830,7 +830,9 @@ impl super::Device { } } -impl crate::Device for super::Device { +impl crate::Device for super::Device { + type A = super::Api; + unsafe fn exit(self, queue: super::Queue) { unsafe { self.mem_allocator.into_inner().cleanup(&*self.shared) }; unsafe { self.desc_allocator.into_inner().cleanup(&*self.shared) }; diff --git a/third_party/rust/wgpu-hal/src/vulkan/instance.rs b/third_party/rust/wgpu-hal/src/vulkan/instance.rs index 771938b0b0..a0d29a13a3 100644 --- a/third_party/rust/wgpu-hal/src/vulkan/instance.rs +++ b/third_party/rust/wgpu-hal/src/vulkan/instance.rs @@ -579,7 +579,9 @@ impl Drop for super::InstanceShared { } } -impl crate::Instance for super::Instance { +impl crate::Instance for super::Instance { + type A = super::Api; + unsafe fn init(desc: &crate::InstanceDescriptor) -> Result { profiling::scope!("Init Vulkan Backend"); use crate::auxil::cstr_from_bytes_until_nul; @@ -956,7 +958,9 @@ impl crate::Instance for super::Instance { } } -impl crate::Surface for super::Surface { +impl crate::Surface for super::Surface { + type A = super::Api; + unsafe fn configure( &self, device: &super::Device, diff --git a/third_party/rust/wgpu-hal/src/vulkan/mod.rs b/third_party/rust/wgpu-hal/src/vulkan/mod.rs index 1f922e83da..0cd385045c 100644 --- a/third_party/rust/wgpu-hal/src/vulkan/mod.rs +++ b/third_party/rust/wgpu-hal/src/vulkan/mod.rs @@ -189,7 +189,7 @@ pub struct Adapter { instance: Arc, //queue_families: Vec, known_memory_flags: vk::MemoryPropertyFlags, - phd_capabilities: adapter::PhysicalDeviceCapabilities, + phd_capabilities: adapter::PhysicalDeviceProperties, //phd_features: adapter::PhysicalDeviceFeatures, downlevel_flags: wgt::DownlevelFlags, private_caps: PrivateCapabilities, @@ -594,7 +594,9 @@ impl Fence { } } -impl crate::Queue for Queue { +impl crate::Queue for Queue { + type A = Api; + unsafe fn submit( &self, command_buffers: &[&CommandBuffer], diff --git a/third_party/rust/wgpu-types/.cargo-checksum.json b/third_party/rust/wgpu-types/.cargo-checksum.json index dea747bd18..928cdbab21 100644 --- a/third_party/rust/wgpu-types/.cargo-checksum.json +++ b/third_party/rust/wgpu-types/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"Cargo.toml":"c536ec4d70291834fb3e6bcd6f03900bb3a651eda9449e7adf03ef2611be96a9","LICENSE.APACHE":"a6cba85bc92e0cff7a450b1d873c0eaa2e9fc96bf472df0247a26bec77bf3ff9","LICENSE.MIT":"c7fea58d1cfe49634cd92e54fc10a9d871f4b275321a4cd8c09e449122caaeb4","src/assertions.rs":"3fe98027aa73970c8ab7874a3e13dbfd6faa87df2081beb5c83aeec4c60f372f","src/lib.rs":"bafa964caee2fdc6fc2adbb1dea540d5575e29c45946bc51e9912bfbdb13a352","src/math.rs":"4d03039736dd6926feb139bc68734cb59df34ede310427bbf059e5c925e0af3b"},"package":null} \ No newline at end of file +{"files":{"Cargo.toml":"0bcb9c2d557d01677740fea4690c79544898fe749880a72512dca33db171e590","LICENSE.APACHE":"a6cba85bc92e0cff7a450b1d873c0eaa2e9fc96bf472df0247a26bec77bf3ff9","LICENSE.MIT":"c7fea58d1cfe49634cd92e54fc10a9d871f4b275321a4cd8c09e449122caaeb4","src/assertions.rs":"3fe98027aa73970c8ab7874a3e13dbfd6faa87df2081beb5c83aeec4c60f372f","src/lib.rs":"b9fda00d1b61364cdd7e544aee731ce075ea8403349ed286711d39aa8d414c28","src/math.rs":"4d03039736dd6926feb139bc68734cb59df34ede310427bbf059e5c925e0af3b"},"package":null} \ No newline at end of file diff --git a/third_party/rust/wgpu-types/Cargo.toml b/third_party/rust/wgpu-types/Cargo.toml index 792b03dcc8..c6f8b3002d 100644 --- a/third_party/rust/wgpu-types/Cargo.toml +++ b/third_party/rust/wgpu-types/Cargo.toml @@ -11,7 +11,7 @@ [package] edition = "2021" -rust-version = "1.70" +rust-version = "1.74" name = "wgpu-types" version = "0.19.0" authors = ["gfx-rs developers"] @@ -56,10 +56,10 @@ fragile-send-sync-non-atomic-wasm = [] strict_asserts = [] [target."cfg(target_arch = \"wasm32\")".dependencies] -js-sys = "0.3.67" +js-sys = "0.3.69" [target."cfg(target_arch = \"wasm32\")".dependencies.web-sys] -version = "0.3.67" +version = "0.3.69" features = [ "ImageBitmap", "HtmlVideoElement", diff --git a/third_party/rust/wgpu-types/src/lib.rs b/third_party/rust/wgpu-types/src/lib.rs index 347aad76f9..b36801e941 100644 --- a/third_party/rust/wgpu-types/src/lib.rs +++ b/third_party/rust/wgpu-types/src/lib.rs @@ -337,7 +337,7 @@ bitflags::bitflags! { /// For arbitrary timestamp write commands on encoders refer to [`Features::TIMESTAMP_QUERY_INSIDE_ENCODERS`]. /// For arbitrary timestamp write commands on passes refer to [`Features::TIMESTAMP_QUERY_INSIDE_PASSES`]. /// - /// They must be resolved using [`CommandEncoder::resolve_query_sets`] into a buffer, + /// They must be resolved using [`CommandEncoder::resolve_query_set`] into a buffer, /// then the result must be multiplied by the timestamp period [`Queue::get_timestamp_period`] /// to get the timestamp in nanoseconds. Multiple timestamps can then be diffed to get the /// time for operations between them to finish. @@ -480,10 +480,10 @@ bitflags::bitflags! { // API: /// Enables use of Pipeline Statistics Queries. These queries tell the count of various operations - /// performed between the start and stop call. Call [`RenderPassEncoder::begin_pipeline_statistics_query`] to start - /// a query, then call [`RenderPassEncoder::end_pipeline_statistics_query`] to stop one. + /// performed between the start and stop call. Call [`RenderPass::begin_pipeline_statistics_query`] to start + /// a query, then call [`RenderPass::end_pipeline_statistics_query`] to stop one. /// - /// They must be resolved using [`CommandEncoder::resolve_query_sets`] into a buffer. + /// They must be resolved using [`CommandEncoder::resolve_query_set`] into a buffer. /// The rules on how these resolve into buffers are detailed in the documentation for [`PipelineStatisticsTypes`]. /// /// Supported Platforms: @@ -511,8 +511,8 @@ bitflags::bitflags! { /// Implies [`Features::TIMESTAMP_QUERY`] & [`Features::TIMESTAMP_QUERY_INSIDE_ENCODERS`] is supported. /// /// Additionally allows for timestamp queries to be used inside render & compute passes using: - /// - [`RenderPassEncoder::write_timestamp`] - /// - [`ComputePassEncoder::write_timestamp`] + /// - [`RenderPass::write_timestamp`] + /// - [`ComputePass::write_timestamp`] /// /// Supported platforms: /// - Vulkan @@ -682,7 +682,14 @@ bitflags::bitflags! { /// Allows the user to call [`RenderPass::set_push_constants`], provide a non-empty array /// to [`PipelineLayoutDescriptor`], and provide a non-zero limit to [`Limits::max_push_constant_size`]. /// - /// A block of push constants can be declared with `layout(push_constant) uniform Name {..}` in shaders. + /// A block of push constants can be declared in WGSL with `var`: + /// + /// ```rust,ignore + /// struct PushConstants { example: f32, } + /// var c: PushConstants; + /// ``` + /// + /// In GLSL, this corresponds to `layout(push_constant) uniform Name {..}`. /// /// Supported platforms: /// - DX12 @@ -7183,7 +7190,7 @@ mod send_sync { /// /// Corresponds to [WebGPU `GPUDeviceLostReason`](https://gpuweb.github.io/gpuweb/#enumdef-gpudevicelostreason). #[repr(u8)] -#[derive(Debug, Copy, Clone)] +#[derive(Debug, Copy, Clone, Eq, PartialEq)] pub enum DeviceLostReason { /// Triggered by driver Unknown = 0, @@ -7203,4 +7210,10 @@ pub enum DeviceLostReason { /// exactly once before it is dropped, which helps with managing the /// memory owned by the callback. ReplacedCallback = 3, + /// When setting the callback, but the device is already invalid + /// + /// As above, when the callback is provided, wgpu guarantees that it + /// will eventually be called. If the device is already invalid, wgpu + /// will call the callback immediately, with this reason. + DeviceInvalid = 4, } diff --git a/third_party/rust/zip/src/read.rs b/third_party/rust/zip/src/read.rs index dad20c260b..3f3f41010c 100644 --- a/third_party/rust/zip/src/read.rs +++ b/third_party/rust/zip/src/read.rs @@ -334,13 +334,10 @@ impl ZipArchive { // offsets all being too small. Get the amount of error by comparing // the actual file position we found the CDE at with the offset // recorded in the CDE. - let archive_offset = cde_start_pos - .checked_sub(footer.central_directory_size as u64) - .and_then(|x| x.checked_sub(footer.central_directory_offset as u64)) - .ok_or(ZipError::InvalidArchive( - "Invalid central directory size or offset", - ))?; + // Bug 1895599: omnijars nor other zips we read have data prepended to them; trust + // the offsets! + let archive_offset = 0; let directory_start = footer.central_directory_offset as u64 + archive_offset; let number_of_files = footer.number_of_files_on_this_disk as usize; Ok((archive_offset, directory_start, number_of_files)) diff --git a/third_party/rust/zip/src/types.rs b/third_party/rust/zip/src/types.rs index ad3a5700b2..896c9a6ff5 100644 --- a/third_party/rust/zip/src/types.rs +++ b/third_party/rust/zip/src/types.rs @@ -7,8 +7,6 @@ use std::convert::{TryFrom, TryInto}; target_arch = "powerpc" )))] use std::sync::atomic; -#[cfg(not(feature = "time"))] -use std::time::SystemTime; #[cfg(doc)] use {crate::read::ZipFile, crate::write::FileOptions}; diff --git a/third_party/rust/zip/src/write.rs b/third_party/rust/zip/src/write.rs index 14252b4d59..214aafecd0 100644 --- a/third_party/rust/zip/src/write.rs +++ b/third_party/rust/zip/src/write.rs @@ -7,6 +7,7 @@ use crate::spec; use crate::types::{AtomicU64, DateTime, System, ZipFileData, DEFAULT_VERSION}; use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; use crc32fast::Hasher; +#[cfg(feature = "time")] use std::convert::TryInto; use std::default::Default; use std::io; -- cgit v1.2.3