diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-06-12 05:35:29 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-06-12 05:35:29 +0000 |
commit | 59203c63bb777a3bacec32fb8830fba33540e809 (patch) | |
tree | 58298e711c0ff0575818c30485b44a2f21bf28a0 /third_party/rust/suggest | |
parent | Adding upstream version 126.0.1. (diff) | |
download | firefox-59203c63bb777a3bacec32fb8830fba33540e809.tar.xz firefox-59203c63bb777a3bacec32fb8830fba33540e809.zip |
Adding upstream version 127.0.upstream/127.0
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'third_party/rust/suggest')
-rw-r--r-- | third_party/rust/suggest/.cargo-checksum.json | 2 | ||||
-rw-r--r-- | third_party/rust/suggest/src/benchmarks/client.rs | 1 | ||||
-rw-r--r-- | third_party/rust/suggest/src/db.rs | 6 | ||||
-rw-r--r-- | third_party/rust/suggest/src/lib.rs | 2 | ||||
-rw-r--r-- | third_party/rust/suggest/src/schema.rs | 454 | ||||
-rw-r--r-- | third_party/rust/suggest/src/store.rs | 216 | ||||
-rw-r--r-- | third_party/rust/suggest/src/suggest.udl | 15 |
7 files changed, 456 insertions, 240 deletions
diff --git a/third_party/rust/suggest/.cargo-checksum.json b/third_party/rust/suggest/.cargo-checksum.json index 120a503f29..8ead78a91c 100644 --- a/third_party/rust/suggest/.cargo-checksum.json +++ b/third_party/rust/suggest/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"Cargo.toml":"05e4d7f7b3649a3e3fa441c4af53a633d18f20bb04fd761ed33fc9d461fd0dee","README.md":"fb72d0028586cab1421b853ef529d7ce78ad7316818b7733a4f3488b0fba67f7","benches/benchmark_all.rs":"c2343c9197b6d9ccb0798d7701b1b0d2569d494dd31a975d21d7ec6f26e32879","build.rs":"78780c5cccfe22c3ff4198624b9e188559c437c3e6fa1c8bb66548eee6aa66bf","src/benchmarks/README.md":"ee6d50df2c31cfd80a5bc047011b518dcf57f1ef928a811bb770f1a09f41b3de","src/benchmarks/client.rs":"5d5db3f6e132654c06532feba15f98576122f6b9572ab5fa27b0c67d5b39ecb6","src/benchmarks/ingest.rs":"1ffdc403fb945ea0b58353df9773ba45ab0e9082d61dd5330ad49fad8cbb5d9f","src/benchmarks/mod.rs":"fe1898ba4d783213525da10d92858ee84cebfd22749bad7aeb461d338fe5504a","src/bin/debug_ingestion_sizes.rs":"ce6e810be7b3fc19e826d75b622b82cfab5a1a99397a6d0833c2c4eebff2d364","src/config.rs":"206ae9dc768c755649cb0c88a7b1fc3c926c715441784f61e9dc06a8a02fc568","src/db.rs":"734f5fd9f36f03c07a508a9a353872b81107f5fe09f27294ba27d7e1249e3988","src/error.rs":"f563210a6c050d98ec85e0f6d9401e7373bfb816e865e8edabbabb23d848ba13","src/keyword.rs":"988d0ab021c0df19cfd3c519df7d37f606bf984cd14d0efca4e5a7aff88344dd","src/lib.rs":"91ebbe0e1ffb99eefde204f81bc6bb199b4941976347baf1f132fd0ede20479c","src/pocket.rs":"1316668840ec9b4ea886223921dc9d3b5a1731d1a5206c0b1089f2a6c45c1b7b","src/provider.rs":"fe76f19a223f5cac056c7d48525087ca2c26bf0629b0e11b1f8dc98d165c8bb2","src/rs.rs":"e3eabde58c859ebe1154bf8da56ca134ace135934e3f280acc8186b4204399b3","src/schema.rs":"8b21006940e872658d722b52ba171280c96789eecf614b837d8cdbc9153ab576","src/store.rs":"413779074db3ce4589c31cd4fb0a050d44d1cbad1df3c94101d03e98efdf09cb","src/suggest.udl":"de50ea5c7ece0ae0ff4798979e0e12a5227b42bf024d48b6f585ea30a5133eb3","src/suggestion.rs":"f31227779d13d1b03a622e08a417ceba4afb161885a01c2bc87a6a652b5e8be5","src/yelp.rs":"9c0dc02a994cc05df524aa4ef337d10f575d1891259193b6419fed6fe279cb54","uniffi.toml":"f26317442ddb5b3281245bef6e60ffcb78bb95d29fe4a351a56dbb88d4ec8aab"},"package":null}
\ No newline at end of file +{"files":{"Cargo.toml":"05e4d7f7b3649a3e3fa441c4af53a633d18f20bb04fd761ed33fc9d461fd0dee","README.md":"fb72d0028586cab1421b853ef529d7ce78ad7316818b7733a4f3488b0fba67f7","benches/benchmark_all.rs":"c2343c9197b6d9ccb0798d7701b1b0d2569d494dd31a975d21d7ec6f26e32879","build.rs":"78780c5cccfe22c3ff4198624b9e188559c437c3e6fa1c8bb66548eee6aa66bf","src/benchmarks/README.md":"ee6d50df2c31cfd80a5bc047011b518dcf57f1ef928a811bb770f1a09f41b3de","src/benchmarks/client.rs":"4b2125031d740ca1ab468e76bbea777ac0bc4cc221b03b7bc2da773bed61dac5","src/benchmarks/ingest.rs":"1ffdc403fb945ea0b58353df9773ba45ab0e9082d61dd5330ad49fad8cbb5d9f","src/benchmarks/mod.rs":"fe1898ba4d783213525da10d92858ee84cebfd22749bad7aeb461d338fe5504a","src/bin/debug_ingestion_sizes.rs":"ce6e810be7b3fc19e826d75b622b82cfab5a1a99397a6d0833c2c4eebff2d364","src/config.rs":"206ae9dc768c755649cb0c88a7b1fc3c926c715441784f61e9dc06a8a02fc568","src/db.rs":"a4e18b9f45e0473ea64b5ecdf6d1d67e0519f9629d495c157b0bd1b47c3e2f4f","src/error.rs":"f563210a6c050d98ec85e0f6d9401e7373bfb816e865e8edabbabb23d848ba13","src/keyword.rs":"988d0ab021c0df19cfd3c519df7d37f606bf984cd14d0efca4e5a7aff88344dd","src/lib.rs":"18f988eb49626c6e186c8bc65a51b4a40d796f36d3de8905506f76c6e5e876cd","src/pocket.rs":"1316668840ec9b4ea886223921dc9d3b5a1731d1a5206c0b1089f2a6c45c1b7b","src/provider.rs":"fe76f19a223f5cac056c7d48525087ca2c26bf0629b0e11b1f8dc98d165c8bb2","src/rs.rs":"e3eabde58c859ebe1154bf8da56ca134ace135934e3f280acc8186b4204399b3","src/schema.rs":"88ff3ae6b652fa5a5cff4dc504d11a7fc33f1b2ee9716b970f646d9f9ca90ab7","src/store.rs":"aad193774eecec739a7debd1c9e4fd46df384e7a524203e5e5f0354b93f73c1c","src/suggest.udl":"bfa653aa88c954860a9728a597daad8f4a7db8c81bc156725bf801f7cddf8459","src/suggestion.rs":"f31227779d13d1b03a622e08a417ceba4afb161885a01c2bc87a6a652b5e8be5","src/yelp.rs":"9c0dc02a994cc05df524aa4ef337d10f575d1891259193b6419fed6fe279cb54","uniffi.toml":"f26317442ddb5b3281245bef6e60ffcb78bb95d29fe4a351a56dbb88d4ec8aab"},"package":null}
\ No newline at end of file diff --git a/third_party/rust/suggest/src/benchmarks/client.rs b/third_party/rust/suggest/src/benchmarks/client.rs index f5a21fd9cc..713bd7752b 100644 --- a/third_party/rust/suggest/src/benchmarks/client.rs +++ b/third_party/rust/suggest/src/benchmarks/client.rs @@ -22,6 +22,7 @@ impl RemoteSettingsWarmUpClient { pub fn new() -> Self { Self { client: Client::new(RemoteSettingsConfig { + server: None, server_url: None, bucket_name: None, collection_name: crate::rs::REMOTE_SETTINGS_COLLECTION.into(), diff --git a/third_party/rust/suggest/src/db.rs b/third_party/rust/suggest/src/db.rs index 0412c50d8f..6b6603ab71 100644 --- a/third_party/rust/suggest/src/db.rs +++ b/third_party/rust/suggest/src/db.rs @@ -188,6 +188,12 @@ impl<'a> SuggestDao<'a> { // // These methods implement CRUD operations + pub fn suggestions_table_empty(&self) -> Result<bool> { + Ok(self + .conn + .query_one::<bool>("SELECT NOT EXISTS (SELECT 1 FROM suggestions)")?) + } + /// Fetches suggestions that match the given query from the database. pub fn fetch_suggestions(&self, query: &SuggestionQuery) -> Result<Vec<Suggestion>> { let unique_providers = query.providers.iter().collect::<HashSet<_>>(); diff --git a/third_party/rust/suggest/src/lib.rs b/third_party/rust/suggest/src/lib.rs index 15746614d0..93b456b8b4 100644 --- a/third_party/rust/suggest/src/lib.rs +++ b/third_party/rust/suggest/src/lib.rs @@ -3,7 +3,7 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -use remote_settings::RemoteSettingsConfig; +use remote_settings::{RemoteSettingsConfig, RemoteSettingsServer}; #[cfg(feature = "benchmark_api")] pub mod benchmarks; mod config; diff --git a/third_party/rust/suggest/src/schema.rs b/third_party/rust/suggest/src/schema.rs index b304363de5..76a0deed39 100644 --- a/third_party/rust/suggest/src/schema.rs +++ b/third_party/rust/suggest/src/schema.rs @@ -15,118 +15,118 @@ use sql_support::open_database::{self, ConnectionInitializer}; /// [`SuggestConnectionInitializer::upgrade_from`]. /// a. If suggestions should be re-ingested after the migration, call `clear_database()` inside /// the migration. -pub const VERSION: u32 = 18; +pub const VERSION: u32 = 19; /// The current Suggest database schema. pub const SQL: &str = " - CREATE TABLE meta( - key TEXT PRIMARY KEY, - value NOT NULL - ) WITHOUT ROWID; - - CREATE TABLE keywords( - keyword TEXT NOT NULL, - suggestion_id INTEGER NOT NULL REFERENCES suggestions(id) ON DELETE CASCADE, - full_keyword_id INTEGER NULL REFERENCES full_keywords(id) ON DELETE SET NULL, - rank INTEGER NOT NULL, - PRIMARY KEY (keyword, suggestion_id) - ) WITHOUT ROWID; - - -- full keywords are what we display to the user when a (partial) keyword matches - -- The FK to suggestion_id makes it so full keywords get deleted when the parent suggestion is deleted. - CREATE TABLE full_keywords( - id INTEGER PRIMARY KEY, - suggestion_id INTEGER NOT NULL REFERENCES suggestions(id) ON DELETE CASCADE, - full_keyword TEXT NOT NULL - ); - - CREATE TABLE prefix_keywords( - keyword_prefix TEXT NOT NULL, - keyword_suffix TEXT NOT NULL DEFAULT '', - confidence INTEGER NOT NULL DEFAULT 0, - rank INTEGER NOT NULL, - suggestion_id INTEGER NOT NULL REFERENCES suggestions(id) ON DELETE CASCADE, - PRIMARY KEY (keyword_prefix, keyword_suffix, suggestion_id) - ) WITHOUT ROWID; - - CREATE UNIQUE INDEX keywords_suggestion_id_rank ON keywords(suggestion_id, rank); - - CREATE TABLE suggestions( - id INTEGER PRIMARY KEY, - record_id TEXT NOT NULL, - provider INTEGER NOT NULL, - title TEXT NOT NULL, - url TEXT NOT NULL, - score REAL NOT NULL - ); - - CREATE TABLE amp_custom_details( - suggestion_id INTEGER PRIMARY KEY, - advertiser TEXT NOT NULL, - block_id INTEGER NOT NULL, - iab_category TEXT NOT NULL, - impression_url TEXT NOT NULL, - click_url TEXT NOT NULL, - icon_id TEXT NOT NULL, - FOREIGN KEY(suggestion_id) REFERENCES suggestions(id) ON DELETE CASCADE - ); - - CREATE TABLE wikipedia_custom_details( - suggestion_id INTEGER PRIMARY KEY REFERENCES suggestions(id) ON DELETE CASCADE, - icon_id TEXT NOT NULL - ); - - CREATE TABLE amo_custom_details( - suggestion_id INTEGER PRIMARY KEY, - description TEXT NOT NULL, - guid TEXT NOT NULL, - icon_url TEXT NOT NULL, - rating TEXT, - number_of_ratings INTEGER NOT NULL, - FOREIGN KEY(suggestion_id) REFERENCES suggestions(id) ON DELETE CASCADE - ); - - CREATE INDEX suggestions_record_id ON suggestions(record_id); - - CREATE TABLE icons( - id TEXT PRIMARY KEY, - data BLOB NOT NULL, - mimetype TEXT NOT NULL - ) WITHOUT ROWID; - - CREATE TABLE yelp_subjects( - keyword TEXT PRIMARY KEY, - record_id TEXT NOT NULL - ) WITHOUT ROWID; - - CREATE TABLE yelp_modifiers( - type INTEGER NOT NULL, - keyword TEXT NOT NULL, - record_id TEXT NOT NULL, - PRIMARY KEY (type, keyword) - ) WITHOUT ROWID; - - CREATE TABLE yelp_location_signs( - keyword TEXT PRIMARY KEY, - need_location INTEGER NOT NULL, - record_id TEXT NOT NULL - ) WITHOUT ROWID; - - CREATE TABLE yelp_custom_details( - icon_id TEXT PRIMARY KEY, - score REAL NOT NULL, - record_id TEXT NOT NULL - ) WITHOUT ROWID; - - CREATE TABLE mdn_custom_details( - suggestion_id INTEGER PRIMARY KEY, - description TEXT NOT NULL, - FOREIGN KEY(suggestion_id) REFERENCES suggestions(id) ON DELETE CASCADE - ); - - CREATE TABLE dismissed_suggestions ( - url TEXT PRIMARY KEY - ) WITHOUT ROWID; +CREATE TABLE meta( + key TEXT PRIMARY KEY, + value NOT NULL +) WITHOUT ROWID; + +CREATE TABLE keywords( + keyword TEXT NOT NULL, + suggestion_id INTEGER NOT NULL REFERENCES suggestions(id) ON DELETE CASCADE, + full_keyword_id INTEGER NULL REFERENCES full_keywords(id) ON DELETE SET NULL, + rank INTEGER NOT NULL, + PRIMARY KEY (keyword, suggestion_id) +) WITHOUT ROWID; + +-- full keywords are what we display to the user when a (partial) keyword matches +-- The FK to suggestion_id makes it so full keywords get deleted when the parent suggestion is deleted. +CREATE TABLE full_keywords( + id INTEGER PRIMARY KEY, + suggestion_id INTEGER NOT NULL REFERENCES suggestions(id) ON DELETE CASCADE, + full_keyword TEXT NOT NULL +); + +CREATE TABLE prefix_keywords( + keyword_prefix TEXT NOT NULL, + keyword_suffix TEXT NOT NULL DEFAULT '', + confidence INTEGER NOT NULL DEFAULT 0, + rank INTEGER NOT NULL, + suggestion_id INTEGER NOT NULL REFERENCES suggestions(id) ON DELETE CASCADE, + PRIMARY KEY (keyword_prefix, keyword_suffix, suggestion_id) +) WITHOUT ROWID; + +CREATE UNIQUE INDEX keywords_suggestion_id_rank ON keywords(suggestion_id, rank); + +CREATE TABLE suggestions( + id INTEGER PRIMARY KEY, + record_id TEXT NOT NULL, + provider INTEGER NOT NULL, + title TEXT NOT NULL, + url TEXT NOT NULL, + score REAL NOT NULL +); + +CREATE TABLE amp_custom_details( + suggestion_id INTEGER PRIMARY KEY, + advertiser TEXT NOT NULL, + block_id INTEGER NOT NULL, + iab_category TEXT NOT NULL, + impression_url TEXT NOT NULL, + click_url TEXT NOT NULL, + icon_id TEXT NOT NULL, + FOREIGN KEY(suggestion_id) REFERENCES suggestions(id) ON DELETE CASCADE +); + +CREATE TABLE wikipedia_custom_details( + suggestion_id INTEGER PRIMARY KEY REFERENCES suggestions(id) ON DELETE CASCADE, + icon_id TEXT NOT NULL +); + +CREATE TABLE amo_custom_details( + suggestion_id INTEGER PRIMARY KEY, + description TEXT NOT NULL, + guid TEXT NOT NULL, + icon_url TEXT NOT NULL, + rating TEXT, + number_of_ratings INTEGER NOT NULL, + FOREIGN KEY(suggestion_id) REFERENCES suggestions(id) ON DELETE CASCADE +); + +CREATE INDEX suggestions_record_id ON suggestions(record_id); + +CREATE TABLE icons( + id TEXT PRIMARY KEY, + data BLOB NOT NULL, + mimetype TEXT NOT NULL +) WITHOUT ROWID; + +CREATE TABLE yelp_subjects( + keyword TEXT PRIMARY KEY, + record_id TEXT NOT NULL +) WITHOUT ROWID; + +CREATE TABLE yelp_modifiers( + type INTEGER NOT NULL, + keyword TEXT NOT NULL, + record_id TEXT NOT NULL, + PRIMARY KEY (type, keyword) +) WITHOUT ROWID; + +CREATE TABLE yelp_location_signs( + keyword TEXT PRIMARY KEY, + need_location INTEGER NOT NULL, + record_id TEXT NOT NULL +) WITHOUT ROWID; + +CREATE TABLE yelp_custom_details( + icon_id TEXT PRIMARY KEY, + score REAL NOT NULL, + record_id TEXT NOT NULL +) WITHOUT ROWID; + +CREATE TABLE mdn_custom_details( + suggestion_id INTEGER PRIMARY KEY, + description TEXT NOT NULL, + FOREIGN KEY(suggestion_id) REFERENCES suggestions(id) ON DELETE CASCADE +); + +CREATE TABLE dismissed_suggestions ( + url TEXT PRIMARY KEY +) WITHOUT ROWID; "; /// Initializes an SQLite connection to the Suggest database, performing @@ -166,9 +166,9 @@ impl ConnectionInitializer for SuggestConnectionInitializer { 16 => { tx.execute( " - CREATE TABLE dismissed_suggestions ( - url_hash INTEGER PRIMARY KEY - ) WITHOUT ROWID;", +CREATE TABLE dismissed_suggestions ( + url_hash INTEGER PRIMARY KEY +) WITHOUT ROWID;", (), )?; Ok(()) @@ -176,14 +176,23 @@ impl ConnectionInitializer for SuggestConnectionInitializer { 17 => { tx.execute( " - DROP TABLE dismissed_suggestions; - CREATE TABLE dismissed_suggestions ( - url TEXT PRIMARY KEY - ) WITHOUT ROWID;", +DROP TABLE dismissed_suggestions; +CREATE TABLE dismissed_suggestions ( + url TEXT PRIMARY KEY +) WITHOUT ROWID;", (), )?; Ok(()) } + 18 => { + tx.execute_batch( + " +CREATE TABLE IF NOT EXISTS dismissed_suggestions ( + url TEXT PRIMARY KEY +) WITHOUT ROWID;", + )?; + Ok(()) + } _ => Err(open_database::Error::IncompatibleVersion(version)), } } @@ -212,112 +221,112 @@ mod test { // Snapshot of the v16 schema. We use this to test that we can migrate from there to the // current schema. const V16_SCHEMA: &str = r#" - CREATE TABLE meta( - key TEXT PRIMARY KEY, - value NOT NULL - ) WITHOUT ROWID; - - CREATE TABLE keywords( - keyword TEXT NOT NULL, - suggestion_id INTEGER NOT NULL REFERENCES suggestions(id) ON DELETE CASCADE, - full_keyword_id INTEGER NULL REFERENCES full_keywords(id) ON DELETE SET NULL, - rank INTEGER NOT NULL, - PRIMARY KEY (keyword, suggestion_id) - ) WITHOUT ROWID; - - -- full keywords are what we display to the user when a (partial) keyword matches - -- The FK to suggestion_id makes it so full keywords get deleted when the parent suggestion is deleted. - CREATE TABLE full_keywords( - id INTEGER PRIMARY KEY, - suggestion_id INTEGER NOT NULL REFERENCES suggestions(id) ON DELETE CASCADE, - full_keyword TEXT NOT NULL - ); - - CREATE TABLE prefix_keywords( - keyword_prefix TEXT NOT NULL, - keyword_suffix TEXT NOT NULL DEFAULT '', - confidence INTEGER NOT NULL DEFAULT 0, - rank INTEGER NOT NULL, - suggestion_id INTEGER NOT NULL REFERENCES suggestions(id) ON DELETE CASCADE, - PRIMARY KEY (keyword_prefix, keyword_suffix, suggestion_id) - ) WITHOUT ROWID; - - CREATE UNIQUE INDEX keywords_suggestion_id_rank ON keywords(suggestion_id, rank); - - CREATE TABLE suggestions( - id INTEGER PRIMARY KEY, - record_id TEXT NOT NULL, - provider INTEGER NOT NULL, - title TEXT NOT NULL, - url TEXT NOT NULL, - score REAL NOT NULL - ); - - CREATE TABLE amp_custom_details( - suggestion_id INTEGER PRIMARY KEY, - advertiser TEXT NOT NULL, - block_id INTEGER NOT NULL, - iab_category TEXT NOT NULL, - impression_url TEXT NOT NULL, - click_url TEXT NOT NULL, - icon_id TEXT NOT NULL, - FOREIGN KEY(suggestion_id) REFERENCES suggestions(id) ON DELETE CASCADE - ); - - CREATE TABLE wikipedia_custom_details( - suggestion_id INTEGER PRIMARY KEY REFERENCES suggestions(id) ON DELETE CASCADE, - icon_id TEXT NOT NULL - ); - - CREATE TABLE amo_custom_details( - suggestion_id INTEGER PRIMARY KEY, - description TEXT NOT NULL, - guid TEXT NOT NULL, - icon_url TEXT NOT NULL, - rating TEXT, - number_of_ratings INTEGER NOT NULL, - FOREIGN KEY(suggestion_id) REFERENCES suggestions(id) ON DELETE CASCADE - ); - - CREATE INDEX suggestions_record_id ON suggestions(record_id); - - CREATE TABLE icons( - id TEXT PRIMARY KEY, - data BLOB NOT NULL, - mimetype TEXT NOT NULL - ) WITHOUT ROWID; - - CREATE TABLE yelp_subjects( - keyword TEXT PRIMARY KEY, - record_id TEXT NOT NULL - ) WITHOUT ROWID; - - CREATE TABLE yelp_modifiers( - type INTEGER NOT NULL, - keyword TEXT NOT NULL, - record_id TEXT NOT NULL, - PRIMARY KEY (type, keyword) - ) WITHOUT ROWID; - - CREATE TABLE yelp_location_signs( - keyword TEXT PRIMARY KEY, - need_location INTEGER NOT NULL, - record_id TEXT NOT NULL - ) WITHOUT ROWID; - - CREATE TABLE yelp_custom_details( - icon_id TEXT PRIMARY KEY, - score REAL NOT NULL, - record_id TEXT NOT NULL - ) WITHOUT ROWID; - - CREATE TABLE mdn_custom_details( - suggestion_id INTEGER PRIMARY KEY, - description TEXT NOT NULL, - FOREIGN KEY(suggestion_id) REFERENCES suggestions(id) ON DELETE CASCADE - ); - - PRAGMA user_version=16; +CREATE TABLE meta( + key TEXT PRIMARY KEY, + value NOT NULL +) WITHOUT ROWID; + +CREATE TABLE keywords( + keyword TEXT NOT NULL, + suggestion_id INTEGER NOT NULL REFERENCES suggestions(id) ON DELETE CASCADE, + full_keyword_id INTEGER NULL REFERENCES full_keywords(id) ON DELETE SET NULL, + rank INTEGER NOT NULL, + PRIMARY KEY (keyword, suggestion_id) +) WITHOUT ROWID; + +-- full keywords are what we display to the user when a (partial) keyword matches +-- The FK to suggestion_id makes it so full keywords get deleted when the parent suggestion is deleted. +CREATE TABLE full_keywords( + id INTEGER PRIMARY KEY, + suggestion_id INTEGER NOT NULL REFERENCES suggestions(id) ON DELETE CASCADE, + full_keyword TEXT NOT NULL +); + +CREATE TABLE prefix_keywords( + keyword_prefix TEXT NOT NULL, + keyword_suffix TEXT NOT NULL DEFAULT '', + confidence INTEGER NOT NULL DEFAULT 0, + rank INTEGER NOT NULL, + suggestion_id INTEGER NOT NULL REFERENCES suggestions(id) ON DELETE CASCADE, + PRIMARY KEY (keyword_prefix, keyword_suffix, suggestion_id) +) WITHOUT ROWID; + +CREATE UNIQUE INDEX keywords_suggestion_id_rank ON keywords(suggestion_id, rank); + +CREATE TABLE suggestions( + id INTEGER PRIMARY KEY, + record_id TEXT NOT NULL, + provider INTEGER NOT NULL, + title TEXT NOT NULL, + url TEXT NOT NULL, + score REAL NOT NULL +); + +CREATE TABLE amp_custom_details( + suggestion_id INTEGER PRIMARY KEY, + advertiser TEXT NOT NULL, + block_id INTEGER NOT NULL, + iab_category TEXT NOT NULL, + impression_url TEXT NOT NULL, + click_url TEXT NOT NULL, + icon_id TEXT NOT NULL, + FOREIGN KEY(suggestion_id) REFERENCES suggestions(id) ON DELETE CASCADE +); + +CREATE TABLE wikipedia_custom_details( + suggestion_id INTEGER PRIMARY KEY REFERENCES suggestions(id) ON DELETE CASCADE, + icon_id TEXT NOT NULL +); + +CREATE TABLE amo_custom_details( + suggestion_id INTEGER PRIMARY KEY, + description TEXT NOT NULL, + guid TEXT NOT NULL, + icon_url TEXT NOT NULL, + rating TEXT, + number_of_ratings INTEGER NOT NULL, + FOREIGN KEY(suggestion_id) REFERENCES suggestions(id) ON DELETE CASCADE +); + +CREATE INDEX suggestions_record_id ON suggestions(record_id); + +CREATE TABLE icons( + id TEXT PRIMARY KEY, + data BLOB NOT NULL, + mimetype TEXT NOT NULL +) WITHOUT ROWID; + +CREATE TABLE yelp_subjects( + keyword TEXT PRIMARY KEY, + record_id TEXT NOT NULL +) WITHOUT ROWID; + +CREATE TABLE yelp_modifiers( + type INTEGER NOT NULL, + keyword TEXT NOT NULL, + record_id TEXT NOT NULL, + PRIMARY KEY (type, keyword) +) WITHOUT ROWID; + +CREATE TABLE yelp_location_signs( + keyword TEXT PRIMARY KEY, + need_location INTEGER NOT NULL, + record_id TEXT NOT NULL +) WITHOUT ROWID; + +CREATE TABLE yelp_custom_details( + icon_id TEXT PRIMARY KEY, + score REAL NOT NULL, + record_id TEXT NOT NULL +) WITHOUT ROWID; + +CREATE TABLE mdn_custom_details( + suggestion_id INTEGER PRIMARY KEY, + description TEXT NOT NULL, + FOREIGN KEY(suggestion_id) REFERENCES suggestions(id) ON DELETE CASCADE +); + +PRAGMA user_version=16; "#; /// Test running all schema upgrades from V16, which was the first schema with a "real" @@ -328,5 +337,6 @@ mod test { fn test_all_upgrades() { let db_file = MigratedDatabaseFile::new(SuggestConnectionInitializer, V16_SCHEMA); db_file.run_all_upgrades(); + db_file.assert_schema_matches_new_database(); } } diff --git a/third_party/rust/suggest/src/store.rs b/third_party/rust/suggest/src/store.rs index c55cffc7f5..19886b22b8 100644 --- a/third_party/rust/suggest/src/store.rs +++ b/third_party/rust/suggest/src/store.rs @@ -13,7 +13,8 @@ use error_support::handle_error; use once_cell::sync::OnceCell; use parking_lot::Mutex; use remote_settings::{ - self, GetItemsOptions, RemoteSettingsConfig, RemoteSettingsRecord, SortOrder, + self, GetItemsOptions, RemoteSettingsConfig, RemoteSettingsRecord, RemoteSettingsServer, + SortOrder, }; use rusqlite::{ types::{FromSql, ToSqlOutput}, @@ -50,6 +51,7 @@ pub struct SuggestStoreBuilder(Mutex<SuggestStoreBuilderInner>); #[derive(Default)] struct SuggestStoreBuilderInner { data_path: Option<String>, + remote_settings_server: Option<RemoteSettingsServer>, remote_settings_config: Option<RemoteSettingsConfig>, } @@ -79,6 +81,11 @@ impl SuggestStoreBuilder { self } + pub fn remote_settings_server(self: Arc<Self>, server: RemoteSettingsServer) -> Arc<Self> { + self.0.lock().remote_settings_server = Some(server); + self + } + #[handle_error(Error)] pub fn build(&self) -> SuggestApiResult<Arc<SuggestStore>> { let inner = self.0.lock(); @@ -86,14 +93,29 @@ impl SuggestStoreBuilder { .data_path .clone() .ok_or_else(|| Error::SuggestStoreBuilder("data_path not specified".to_owned()))?; - let settings_client = - remote_settings::Client::new(inner.remote_settings_config.clone().unwrap_or_else( - || RemoteSettingsConfig { - server_url: None, - bucket_name: None, - collection_name: REMOTE_SETTINGS_COLLECTION.into(), - }, - ))?; + let remote_settings_config = match ( + inner.remote_settings_server.as_ref(), + inner.remote_settings_config.as_ref(), + ) { + (Some(server), None) => RemoteSettingsConfig { + server: Some(server.clone()), + server_url: None, + bucket_name: None, + collection_name: REMOTE_SETTINGS_COLLECTION.into(), + }, + (None, Some(remote_settings_config)) => remote_settings_config.clone(), + (None, None) => RemoteSettingsConfig { + server: None, + server_url: None, + bucket_name: None, + collection_name: REMOTE_SETTINGS_COLLECTION.into(), + }, + (Some(_), Some(_)) => Err(Error::SuggestStoreBuilder( + "can't specify both `remote_settings_server` and `remote_settings_config`" + .to_owned(), + ))?, + }; + let settings_client = remote_settings::Client::new(remote_settings_config)?; Ok(Arc::new(SuggestStore { inner: SuggestStoreInner::new(data_path, settings_client), })) @@ -172,6 +194,7 @@ impl SuggestStore { let settings_client = || -> Result<_> { Ok(remote_settings::Client::new( settings_config.unwrap_or_else(|| RemoteSettingsConfig { + server: None, server_url: None, bucket_name: None, collection_name: REMOTE_SETTINGS_COLLECTION.into(), @@ -252,6 +275,8 @@ pub struct SuggestIngestionConstraints { /// soft limit, and the store might ingest more than requested. pub max_suggestions: Option<u64>, pub providers: Option<Vec<SuggestionProvider>>, + /// Only run ingestion if the table `suggestions` is empty + pub empty_only: bool, } /// The implementation of the store. This is generic over the Remote Settings @@ -334,6 +359,10 @@ where pub fn ingest(&self, constraints: SuggestIngestionConstraints) -> Result<()> { let writer = &self.dbs()?.writer; + if constraints.empty_only && !writer.read(|dao| dao.suggestions_table_empty())? { + return Ok(()); + } + if let Some(unparsable_records) = writer.read(|dao| dao.get_meta::<UnparsableRecords>(UNPARSABLE_RECORDS_META_KEY))? { @@ -865,6 +894,12 @@ mod tests { let store = unique_test_store(SnapshotSettingsClient::with_snapshot(snapshot)); + // suggestions_table_empty returns true before the ingestion is complete + assert!(store + .dbs()? + .reader + .read(|dao| dao.suggestions_table_empty())?); + store.ingest(SuggestIngestionConstraints::default())?; store.dbs()?.reader.read(|dao| { @@ -904,6 +939,153 @@ mod tests { Ok(()) })?; + // suggestions_table_empty returns false after the ingestion is complete + assert!(!store + .dbs()? + .reader + .read(|dao| dao.suggestions_table_empty())?); + + Ok(()) + } + + /// Tests ingesting suggestions into an empty database. + #[test] + fn ingest_empty_only() -> anyhow::Result<()> { + before_each(); + + // This ingestion should run, since the DB is empty + let snapshot = Snapshot::with_records(json!([{ + "id": "1234", + "type": "data", + "last_modified": 15, + "attachment": { + "filename": "data-1.json", + "mimetype": "application/json", + "location": "data-1.json", + "hash": "", + "size": 0, + }, + }]))? + .with_data( + "data-1.json", + json!([{ + "id": 0, + "advertiser": "Los Pollos Hermanos", + "iab_category": "8 - Food & Drink", + "keywords": ["lo", "los", "los p", "los pollos", "los pollos h", "los pollos hermanos"], + "title": "Los Pollos Hermanos - Albuquerque", + "url": "https://www.lph-nm.biz", + "icon": "5678", + "impression_url": "https://example.com/impression_url", + "click_url": "https://example.com/click_url", + "score": 0.3 + }]), + )?; + let mut store = unique_test_store(SnapshotSettingsClient::with_snapshot(snapshot)); + store.ingest(SuggestIngestionConstraints { + empty_only: true, + ..SuggestIngestionConstraints::default() + })?; + + store.dbs()?.reader.read(|dao| { + expect![[r#" + [ + Amp { + title: "Los Pollos Hermanos - Albuquerque", + url: "https://www.lph-nm.biz", + raw_url: "https://www.lph-nm.biz", + icon: None, + icon_mimetype: None, + full_keyword: "los", + block_id: 0, + advertiser: "Los Pollos Hermanos", + iab_category: "8 - Food & Drink", + impression_url: "https://example.com/impression_url", + click_url: "https://example.com/click_url", + raw_click_url: "https://example.com/click_url", + score: 0.3, + }, + ] + "#]] + .assert_debug_eq(&dao.fetch_suggestions(&SuggestionQuery { + keyword: "lo".into(), + providers: vec![SuggestionProvider::Amp], + limit: None, + })?); + + Ok(()) + })?; + + // ingestion should run with SuggestIngestionConstraints::empty_only = true, since the DB + // is empty + store.settings_client = SnapshotSettingsClient::with_snapshot(Snapshot::with_records(json!([{ + "id": "1234", + "type": "data", + "last_modified": 15, + "attachment": { + "filename": "data-1.json", + "mimetype": "application/json", + "location": "data-1.json", + "hash": "", + "size": 0, + }, + }, { + "id": "12345", + "type": "data", + "last_modified": 15, + "attachment": { + "filename": "data-2.json", + "mimetype": "application/json", + "location": "data-2.json", + "hash": "", + "size": 0, + }, + }]))? + .with_data( + "data-1.json", + json!([{ + "id": 0, + "advertiser": "Los Pollos Hermanos", + "iab_category": "8 - Food & Drink", + "keywords": ["lo", "los", "los p", "los pollos", "los pollos h", "los pollos hermanos"], + "title": "Los Pollos Hermanos - Albuquerque", + "url": "https://www.lph-nm.biz", + "icon": "5678", + "impression_url": "https://example.com/impression_url", + "click_url": "https://example.com/click_url", + "score": 0.3 + }]) + )? + .with_data("data-2.json", json!([{ + "id": 1, + "advertiser": "Good Place Eats", + "iab_category": "8 - Food & Drink", + "keywords": ["la", "las", "lasa", "lasagna", "lasagna come out tomorrow"], + "title": "Lasagna Come Out Tomorrow", + "url": "https://www.lasagna.restaurant", + "icon": "2", + "impression_url": "https://example.com/impression_url", + "click_url": "https://example.com/click_url" + }]), + )?); + store.ingest(SuggestIngestionConstraints { + empty_only: true, + ..SuggestIngestionConstraints::default() + })?; + + store.dbs()?.reader.read(|dao| { + expect![[r#" + [] + "#]] + .assert_debug_eq(&dao.fetch_suggestions(&SuggestionQuery { + keyword: "la".into(), + providers: vec![SuggestionProvider::Amp], + limit: None, + })?); + + Ok(()) + })?; + Ok(()) } @@ -2189,6 +2371,7 @@ mod tests { store.ingest(SuggestIngestionConstraints { max_suggestions: Some(max_suggestions), providers: Some(vec![SuggestionProvider::Amp]), + ..SuggestIngestionConstraints::default() })?; let actual_limit = store .settings_client @@ -5021,10 +5204,10 @@ mod tests { UnparsableRecords( { "clippy-2": UnparsableRecord { - schema_version: 18, + schema_version: 19, }, "fancy-new-suggestions-1": UnparsableRecord { - schema_version: 18, + schema_version: 19, }, }, ), @@ -5093,10 +5276,10 @@ mod tests { UnparsableRecords( { "clippy-2": UnparsableRecord { - schema_version: 18, + schema_version: 19, }, "fancy-new-suggestions-1": UnparsableRecord { - schema_version: 18, + schema_version: 19, }, }, ), @@ -5178,6 +5361,7 @@ mod tests { let constraints = SuggestIngestionConstraints { max_suggestions: Some(100), providers: Some(vec![SuggestionProvider::Amp, SuggestionProvider::Pocket]), + ..SuggestIngestionConstraints::default() }; store.ingest(constraints)?; @@ -5292,10 +5476,10 @@ mod tests { UnparsableRecords( { "clippy-2": UnparsableRecord { - schema_version: 18, + schema_version: 19, }, "fancy-new-suggestions-1": UnparsableRecord { - schema_version: 18, + schema_version: 19, }, }, ), @@ -5381,7 +5565,7 @@ mod tests { UnparsableRecords( { "invalid-attachment": UnparsableRecord { - schema_version: 18, + schema_version: 19, }, }, ), diff --git a/third_party/rust/suggest/src/suggest.udl b/third_party/rust/suggest/src/suggest.udl index 4a4e3fe9a0..0c4781b951 100644 --- a/third_party/rust/suggest/src/suggest.udl +++ b/third_party/rust/suggest/src/suggest.udl @@ -6,6 +6,9 @@ [External="remote_settings"] typedef extern RemoteSettingsConfig; +[External="remote_settings"] +typedef extern RemoteSettingsServer; + namespace suggest { boolean raw_suggestion_url_matches([ByRef] string raw_url, [ByRef] string url); @@ -103,6 +106,14 @@ dictionary SuggestionQuery { dictionary SuggestIngestionConstraints { u64? max_suggestions = null; sequence<SuggestionProvider>? providers = null; + // Only ingest if the table `suggestions` is empty. + // + // This is indented to handle periodic updates. Consumers can schedule an ingest with + // `empty_only=true` on startup and a regular ingest with `empty_only=false` to run on a long periodic schedule (maybe + // once a day). This allows ingestion to normally be run at a slow, periodic rate. However, if + // there is a schema upgrade that causes the database to be thrown away, then the + // `empty_only=true` ingestion that runs on startup will repopulate it. + boolean empty_only = false; }; dictionary SuggestGlobalConfig { @@ -155,6 +166,10 @@ interface SuggestStoreBuilder { SuggestStoreBuilder cache_path(string path); [Self=ByArc] + SuggestStoreBuilder remote_settings_server(RemoteSettingsServer server); + + // Deprecated: Use `remote_settings_server()` instead. + [Self=ByArc] SuggestStoreBuilder remote_settings_config(RemoteSettingsConfig config); [Throws=SuggestApiError] |