Adding upstream version 110.0.1.upstream/110.0.1upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

4 files changed, 850 insertions, 0 deletions

diff --git a/third_party/rust/sync15/src/bso/content.rs b/third_party/rust/sync15/src/bso/content.rs
new file mode 100644
index 0000000000..f7aa6f608b
--- /dev/null
+++ b/third_party/rust/sync15/src/bso/content.rs
@@ -0,0 +1,388 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+//! This module enhances the IncomingBso and OutgoingBso records to deal with
+//! arbitrary <T> types, which we call "content"
+//! It can:
+//! * Parse JSON into some <T> while handling tombstones and invalid json.
+//! * Turn arbitrary <T> objects with an `id` field into an OutgoingBso.
+
+use super::{IncomingBso, IncomingContent, IncomingKind, OutgoingBso, OutgoingEnvelope};
+use crate::Guid;
+use error_support::report_error;
+use serde::Serialize;
+
+// The only errors we return here are serde errors.
+type Result<T> = std::result::Result<T, serde_json::Error>;
+
+impl<T> IncomingContent<T> {
+    /// Returns Some(content) if [self.kind] is [IncomingKind::Content], None otherwise.
+    pub fn content(self) -> Option<T> {
+        match self.kind {
+            IncomingKind::Content(t) => Some(t),
+            _ => None,
+        }
+    }
+}
+
+// We don't want to force our T to be Debug, but we can be Debug if T is.
+impl<T: std::fmt::Debug> std::fmt::Debug for IncomingKind<T> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            IncomingKind::Content(r) => {
+                write!(f, "IncomingKind::Content<{:?}>", r)
+            }
+            IncomingKind::Tombstone => write!(f, "IncomingKind::Tombstone"),
+            IncomingKind::Malformed => write!(f, "IncomingKind::Malformed"),
+        }
+    }
+}
+
+impl IncomingBso {
+    /// Convert an [IncomingBso] to an [IncomingContent] possibly holding a T.
+    pub fn into_content<T: for<'de> serde::Deserialize<'de>>(self) -> IncomingContent<T> {
+        match serde_json::from_str(&self.payload) {
+            Ok(json) => {
+                // We got a good serde_json::Value, see if it's a <T>.
+                let kind = json_to_kind(json, &self.envelope.id);
+                IncomingContent {
+                    envelope: self.envelope,
+                    kind,
+                }
+            }
+            Err(e) => {
+                // payload isn't valid json.
+                log::warn!("Invalid incoming cleartext {}: {}", self.envelope.id, e);
+                IncomingContent {
+                    envelope: self.envelope,
+                    kind: IncomingKind::Malformed,
+                }
+            }
+        }
+    }
+}
+
+impl OutgoingBso {
+    /// Creates a new tombstone record.
+    /// Not all collections expect tombstones.
+    pub fn new_tombstone(envelope: OutgoingEnvelope) -> Self {
+        Self {
+            envelope,
+            payload: serde_json::json!({"deleted": true}).to_string(),
+        }
+    }
+
+    /// Creates a outgoing record from some <T>, which can be made into a JSON object
+    /// with a valid `id`. This is the most convenient way to create an outgoing
+    /// item from a <T> when the default envelope is suitable.
+    /// Will panic if there's no good `id` in the json.
+    pub fn from_content_with_id<T>(record: T) -> Result<Self>
+    where
+        T: Serialize,
+    {
+        let (json, id) = content_with_id_to_json(record)?;
+        Ok(Self {
+            envelope: id.into(),
+            payload: serde_json::to_string(&json)?,
+        })
+    }
+
+    /// Create an Outgoing record with an explicit envelope. Will panic if the
+    /// payload has an ID but it doesn't match the envelope.
+    pub fn from_content<T>(envelope: OutgoingEnvelope, record: T) -> Result<Self>
+    where
+        T: Serialize,
+    {
+        let json = content_to_json(record, &envelope.id)?;
+        Ok(Self {
+            envelope,
+            payload: serde_json::to_string(&json)?,
+        })
+    }
+}
+
+// Helpers for packing and unpacking serde objects to and from a <T>. In particular:
+// * Helping deal complications around raw json payload not having 'id' (the envelope is
+//   canonical) but needing it to exist when dealing with serde locally.
+//   For example, a record on the server after being decrypted looks like:
+//   `{"id": "a-guid", payload: {"field": "value"}}`
+//   But the `T` for this typically looks like `struct T { id: Guid, field: String}`
+//   So before we try and deserialize this record into a T, we copy the `id` field
+//   from the envelope into the payload, and when serializing from a T we do the
+//   reverse (ie, ensure the `id` in the payload is removed and placed in the envelope)
+// * Tombstones.
+
+// Deserializing json into a T
+fn json_to_kind<T>(mut json: serde_json::Value, id: &Guid) -> IncomingKind<T>
+where
+    T: for<'de> serde::Deserialize<'de>,
+{
+    // It's possible that the payload does not carry 'id', but <T> always does - so grab it from the
+    // envelope and put it into the json before deserializing the record.
+    if let serde_json::Value::Object(ref mut map) = json {
+        if map.contains_key("deleted") {
+            return IncomingKind::Tombstone;
+        }
+        match map.get("id") {
+            Some(serde_json::Value::String(content_id)) => {
+                // It exists in the payload! We treat a mismatch as malformed.
+                if content_id != id {
+                    log::trace!(
+                        "malformed incoming record: envelope id: {} payload id: {}",
+                        content_id,
+                        id
+                    );
+                    report_error!(
+                        "incoming-invalid-mismatched-ids",
+                        "Envelope and payload don't agree on the ID"
+                    );
+                    return IncomingKind::Malformed;
+                }
+                if !id.is_valid_for_sync_server() {
+                    log::trace!("malformed incoming record: id is not valid: {}", id);
+                    report_error!(
+                        "incoming-invalid-bad-payload-id",
+                        "ID in the payload is invalid"
+                    );
+                    return IncomingKind::Malformed;
+                }
+            }
+            Some(v) => {
+                // It exists in the payload but is not a string - they can't possibly be
+                // the same as the envelope uses a String, so must be malformed.
+                log::trace!("malformed incoming record: id is not a string: {}", v);
+                report_error!("incoming-invalid-wrong_type", "ID is not a string");
+                return IncomingKind::Malformed;
+            }
+            None => {
+                // Doesn't exist in the payload - add it before trying to deser a T.
+                if !id.is_valid_for_sync_server() {
+                    log::trace!("malformed incoming record: id is not valid: {}", id);
+                    report_error!(
+                        "incoming-invalid-bad-envelope-id",
+                        "ID in envelope is not valid"
+                    );
+                    return IncomingKind::Malformed;
+                }
+                map.insert("id".to_string(), id.to_string().into());
+            }
+        }
+    };
+    match serde_json::from_value(json) {
+        Ok(v) => IncomingKind::Content(v),
+        Err(e) => {
+            report_error!("invalid-incoming-content", "Invalid incoming T: {}", e);
+            IncomingKind::Malformed
+        }
+    }
+}
+
+// Serializing <T> into json with special handling of `id` (the `id` from the payload
+// is used as the envelope ID)
+fn content_with_id_to_json<T>(record: T) -> Result<(serde_json::Value, Guid)>
+where
+    T: Serialize,
+{
+    let mut json = serde_json::to_value(record)?;
+    let id = match json.as_object_mut() {
+        Some(ref mut map) => {
+            match map.get("id").as_ref().and_then(|v| v.as_str()) {
+                Some(id) => {
+                    let id: Guid = id.into();
+                    assert!(id.is_valid_for_sync_server(), "record's ID is invalid");
+                    id
+                }
+                // In practice, this is a "static" error and not influenced by runtime behavior
+                None => panic!("record does not have an ID in the payload"),
+            }
+        }
+        None => panic!("record is not a json object"),
+    };
+    Ok((json, id))
+}
+
+// Serializing <T> into json with special handling of `id` (if `id` in serialized
+// JSON already exists, we panic if it doesn't match the envelope. If the serialized
+// content does not have an `id`, it is added from the envelope)
+// is used as the envelope ID)
+fn content_to_json<T>(record: T, id: &Guid) -> Result<serde_json::Value>
+where
+    T: Serialize,
+{
+    let mut payload = serde_json::to_value(record)?;
+    if let Some(ref mut map) = payload.as_object_mut() {
+        if let Some(content_id) = map.get("id").as_ref().and_then(|v| v.as_str()) {
+            assert_eq!(content_id, id);
+            assert!(id.is_valid_for_sync_server(), "record's ID is invalid");
+        } else {
+            map.insert("id".to_string(), serde_json::Value::String(id.to_string()));
+        }
+    };
+    Ok(payload)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::bso::IncomingBso;
+    use serde::{Deserialize, Serialize};
+    use serde_json::json;
+
+    #[derive(Default, Debug, PartialEq, Serialize, Deserialize)]
+    struct TestStruct {
+        id: Guid,
+        data: u32,
+    }
+    #[test]
+    fn test_content_deser() {
+        env_logger::try_init().ok();
+        let json = json!({
+            "id": "test",
+            "payload": json!({"data": 1}).to_string(),
+        });
+        let incoming: IncomingBso = serde_json::from_value(json).unwrap();
+        assert_eq!(incoming.envelope.id, "test");
+        let record = incoming.into_content::<TestStruct>().content().unwrap();
+        let expected = TestStruct {
+            id: Guid::new("test"),
+            data: 1,
+        };
+        assert_eq!(record, expected);
+    }
+
+    #[test]
+    fn test_content_deser_empty_id() {
+        env_logger::try_init().ok();
+        let json = json!({
+            "id": "",
+            "payload": json!({"data": 1}).to_string(),
+        });
+        let incoming: IncomingBso = serde_json::from_value(json).unwrap();
+        // The envelope has an invalid ID, but it's not handled until we try and deserialize
+        // it into a T
+        assert_eq!(incoming.envelope.id, "");
+        let content = incoming.into_content::<TestStruct>();
+        assert!(matches!(content.kind, IncomingKind::Malformed));
+    }
+
+    #[test]
+    fn test_content_deser_invalid() {
+        env_logger::try_init().ok();
+        // And a non-empty but still invalid guid.
+        let json = json!({
+            "id": "X".repeat(65),
+            "payload": json!({"data": 1}).to_string(),
+        });
+        let incoming: IncomingBso = serde_json::from_value(json).unwrap();
+        let content = incoming.into_content::<TestStruct>();
+        assert!(matches!(content.kind, IncomingKind::Malformed));
+    }
+
+    #[test]
+    fn test_content_deser_not_string() {
+        env_logger::try_init().ok();
+        // A non-string id.
+        let json = json!({
+            "id": "0",
+            "payload": json!({"id": 0, "data": 1}).to_string(),
+        });
+        let incoming: IncomingBso = serde_json::from_value(json).unwrap();
+        let content = incoming.into_content::<serde_json::Value>();
+        assert!(matches!(content.kind, IncomingKind::Malformed));
+    }
+
+    #[test]
+    fn test_content_ser_with_id() {
+        env_logger::try_init().ok();
+        // When serializing, expect the ID to be in the top-level payload (ie,
+        // in the envelope) but should not appear in the cleartext `payload` part of
+        // the payload.
+        let val = TestStruct {
+            id: Guid::new("test"),
+            data: 1,
+        };
+        let outgoing = OutgoingBso::from_content_with_id(val).unwrap();
+
+        // The envelope should have our ID.
+        assert_eq!(outgoing.envelope.id, Guid::new("test"));
+
+        // and make sure `cleartext` part of the payload the data and the id.
+        let ct_value = serde_json::from_str::<serde_json::Value>(&outgoing.payload).unwrap();
+        assert_eq!(ct_value, json!({"data": 1, "id": "test"}));
+    }
+
+    #[test]
+    fn test_content_ser_with_envelope() {
+        env_logger::try_init().ok();
+        // When serializing, expect the ID to be in the top-level payload (ie,
+        // in the envelope) but should not appear in the cleartext `payload`
+        let val = TestStruct {
+            id: Guid::new("test"),
+            data: 1,
+        };
+        let envelope: OutgoingEnvelope = Guid::new("test").into();
+        let outgoing = OutgoingBso::from_content(envelope, val).unwrap();
+
+        // The envelope should have our ID.
+        assert_eq!(outgoing.envelope.id, Guid::new("test"));
+
+        // and make sure `cleartext` part of the payload has data and the id.
+        let ct_value = serde_json::from_str::<serde_json::Value>(&outgoing.payload).unwrap();
+        assert_eq!(ct_value, json!({"data": 1, "id": "test"}));
+    }
+
+    #[test]
+    #[should_panic]
+    fn test_content_ser_no_ids() {
+        env_logger::try_init().ok();
+        #[derive(Serialize)]
+        struct StructWithNoId {
+            data: u32,
+        }
+        let val = StructWithNoId { data: 1 };
+        let _ = OutgoingBso::from_content_with_id(val);
+    }
+
+    #[test]
+    #[should_panic]
+    fn test_content_ser_not_object() {
+        env_logger::try_init().ok();
+        let _ = OutgoingBso::from_content_with_id(json!("string"));
+    }
+
+    #[test]
+    #[should_panic]
+    fn test_content_ser_mismatched_ids() {
+        env_logger::try_init().ok();
+        let val = TestStruct {
+            id: Guid::new("test"),
+            data: 1,
+        };
+        let envelope: OutgoingEnvelope = Guid::new("different").into();
+        let _ = OutgoingBso::from_content(envelope, val);
+    }
+
+    #[test]
+    #[should_panic]
+    fn test_content_empty_id() {
+        env_logger::try_init().ok();
+        let val = TestStruct {
+            id: Guid::new(""),
+            data: 1,
+        };
+        let _ = OutgoingBso::from_content_with_id(val);
+    }
+
+    #[test]
+    #[should_panic]
+    fn test_content_invalid_id() {
+        env_logger::try_init().ok();
+        let val = TestStruct {
+            id: Guid::new(&"X".repeat(65)),
+            data: 1,
+        };
+        let _ = OutgoingBso::from_content_with_id(val);
+    }
+}
diff --git a/third_party/rust/sync15/src/bso/crypto.rs b/third_party/rust/sync15/src/bso/crypto.rs
new file mode 100644
index 0000000000..d572c4692b
--- /dev/null
+++ b/third_party/rust/sync15/src/bso/crypto.rs
@@ -0,0 +1,197 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+//! Support for "encrypted bso"s, as received by the storage servers.
+//! This module decrypts them into IncomingBso's suitable for use by the
+//! engines.
+use super::{IncomingBso, IncomingEnvelope, OutgoingBso, OutgoingEnvelope};
+use crate::error;
+use crate::key_bundle::KeyBundle;
+use crate::EncryptedPayload;
+use serde::{de::DeserializeOwned, Deserialize, Serialize};
+
+// The BSO implementation we use for encrypted payloads.
+// Note that this is almost identical to the IncomingBso implementations, except
+// instead of a String payload we use an EncryptedPayload. Obviously we *could*
+// just use a String payload and transform it into an EncryptedPayload - any maybe we
+// should - but this is marginally optimal in terms of deserialization.
+#[derive(Deserialize, Debug)]
+pub struct IncomingEncryptedBso {
+    #[serde(flatten)]
+    pub envelope: IncomingEnvelope,
+    #[serde(
+        with = "as_json",
+        bound(deserialize = "EncryptedPayload: DeserializeOwned")
+    )]
+    pub(crate) payload: EncryptedPayload,
+}
+
+impl IncomingEncryptedBso {
+    pub fn new(envelope: IncomingEnvelope, payload: EncryptedPayload) -> Self {
+        Self { envelope, payload }
+    }
+    /// Decrypt a BSO, consuming it into a clear-text version.
+    pub fn into_decrypted(self, key: &KeyBundle) -> error::Result<IncomingBso> {
+        Ok(IncomingBso::new(self.envelope, self.payload.decrypt(key)?))
+    }
+}
+
+#[derive(Serialize, Debug)]
+pub struct OutgoingEncryptedBso {
+    #[serde(flatten)]
+    pub envelope: OutgoingEnvelope,
+    #[serde(with = "as_json", bound(serialize = "EncryptedPayload: Serialize"))]
+    payload: EncryptedPayload,
+}
+
+impl OutgoingEncryptedBso {
+    pub fn new(envelope: OutgoingEnvelope, payload: EncryptedPayload) -> Self {
+        Self { envelope, payload }
+    }
+
+    #[inline]
+    pub fn serialized_payload_len(&self) -> usize {
+        self.payload.serialized_len()
+    }
+}
+
+impl OutgoingBso {
+    pub fn into_encrypted(self, key: &KeyBundle) -> error::Result<OutgoingEncryptedBso> {
+        Ok(OutgoingEncryptedBso {
+            envelope: self.envelope,
+            payload: EncryptedPayload::from_cleartext(key, self.payload)?,
+        })
+    }
+}
+
+// The BSOs we write to the servers expect a "payload" attribute which is a JSON serialized
+// string, rather than the JSON representation of the object.
+// ie, the serialized object is expected to look like:
+// `{"id": "some-guid", "payload": "{\"IV\": ... }"}` <-- payload is a string.
+// However, if we just serialize it directly, we end up with:
+// `{"id": "some-guid", "payload": {"IV":  ... }}` <-- payload is an object.
+// The magic here means we can serialize and deserialize directly into/from the object, correctly
+// working with the payload as a string, instead of needing to explicitly stringify/parse the
+// payload as an extra step.
+//
+// This would work for any <T>, but we only use it for EncryptedPayload - the way our cleartext
+// BSOs work mean it's not necessary there as they define the payload as a String - ie, they do
+// explicitly end up doing 2 JSON operations as an ergonomic design choice.
+mod as_json {
+    use serde::de::{self, Deserialize, DeserializeOwned, Deserializer};
+    use serde::ser::{self, Serialize, Serializer};
+
+    pub fn serialize<T, S>(t: &T, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        T: Serialize,
+        S: Serializer,
+    {
+        let j = serde_json::to_string(t).map_err(ser::Error::custom)?;
+        serializer.serialize_str(&j)
+    }
+
+    pub fn deserialize<'de, T, D>(deserializer: D) -> Result<T, D::Error>
+    where
+        T: DeserializeOwned,
+        D: Deserializer<'de>,
+    {
+        let j = String::deserialize(deserializer)?;
+        serde_json::from_str(&j).map_err(de::Error::custom)
+    }
+}
+
+// Lots of stuff for testing the sizes of encrypted records, because the servers have
+// certain limits in terms of max-POST sizes, forcing us to chunk uploads, but
+// we need to calculate based on encrypted record size rather than the raw <T> size.
+//
+// This is a little cludgey but I couldn't think of another way to have easy deserialization
+// without a bunch of wrapper types, while still only serializing a single time in the
+// postqueue.
+#[cfg(test)]
+impl OutgoingEncryptedBso {
+    /// Return the length of the serialized payload.
+    pub fn payload_serialized_len(&self) -> usize {
+        self.payload.serialized_len()
+    }
+
+    // self.payload is private, but tests want to create funky things.
+    // XXX - test only, but test in another crate :(
+    //#[cfg(test)]
+    pub fn make_test_bso(ciphertext: String) -> Self {
+        Self {
+            envelope: OutgoingEnvelope {
+                id: "".into(),
+                sortindex: None,
+                ttl: None,
+            },
+            payload: EncryptedPayload {
+                iv: "".into(),
+                hmac: "".into(),
+                ciphertext,
+            },
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::bso::OutgoingEnvelope;
+
+    #[test]
+    fn test_deserialize_enc() {
+        let serialized = r#"{
+            "id": "1234",
+            "collection": "passwords",
+            "modified": 12344321.0,
+            "payload": "{\"IV\": \"aaaaa\", \"hmac\": \"bbbbb\", \"ciphertext\": \"ccccc\"}"
+        }"#;
+        let record: IncomingEncryptedBso = serde_json::from_str(serialized).unwrap();
+        assert_eq!(&record.envelope.id, "1234");
+        assert_eq!((record.envelope.modified.0 - 12_344_321_000).abs(), 0);
+        assert_eq!(record.envelope.sortindex, None);
+        assert_eq!(&record.payload.iv, "aaaaa");
+        assert_eq!(&record.payload.hmac, "bbbbb");
+        assert_eq!(&record.payload.ciphertext, "ccccc");
+    }
+
+    #[test]
+    fn test_deserialize_autofields() {
+        let serialized = r#"{
+            "id": "1234",
+            "collection": "passwords",
+            "modified": 12344321.0,
+            "sortindex": 100,
+            "ttl": 99,
+            "payload": "{\"IV\": \"aaaaa\", \"hmac\": \"bbbbb\", \"ciphertext\": \"ccccc\"}"
+        }"#;
+        let record: IncomingEncryptedBso = serde_json::from_str(serialized).unwrap();
+        assert_eq!(record.envelope.sortindex, Some(100));
+        assert_eq!(record.envelope.ttl, Some(99));
+    }
+
+    #[test]
+    fn test_serialize_enc() {
+        let goal = r#"{"id":"1234","payload":"{\"IV\":\"aaaaa\",\"hmac\":\"bbbbb\",\"ciphertext\":\"ccccc\"}"}"#;
+        let record = OutgoingEncryptedBso {
+            envelope: OutgoingEnvelope {
+                id: "1234".into(),
+                ..Default::default()
+            },
+            payload: EncryptedPayload {
+                iv: "aaaaa".into(),
+                hmac: "bbbbb".into(),
+                ciphertext: "ccccc".into(),
+            },
+        };
+        let actual = serde_json::to_string(&record).unwrap();
+        assert_eq!(actual, goal);
+
+        let val_str_payload: serde_json::Value = serde_json::from_str(goal).unwrap();
+        assert_eq!(
+            val_str_payload["payload"].as_str().unwrap().len(),
+            record.payload.serialized_len()
+        )
+    }
+}
diff --git a/third_party/rust/sync15/src/bso/mod.rs b/third_party/rust/sync15/src/bso/mod.rs
new file mode 100644
index 0000000000..251c11fb3b
--- /dev/null
+++ b/third_party/rust/sync15/src/bso/mod.rs
@@ -0,0 +1,204 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/// This module defines our core "bso" abstractions.
+/// In the terminology of this crate:
+/// * "bso" is an acronym for "basic storage object" and used extensively in the sync server docs.
+///    the record always has a well-defined "envelope" with metadata (eg, the ID of the record,
+///    the server timestamp of the resource,  etc) and a field called `payload`.
+///    A bso is serialized to and from JSON.
+/// * There's a "cleartext" bso:
+/// ** The payload is a String, which itself is JSON encoded (ie, this string `payload` is
+///    always double JSON encoded in a server record)
+/// ** This supplies helper methods for working with the "content" (some arbitrary <T>) in the
+///    payload.
+/// * There's an "encrypted" bso
+/// ** The payload is an [crate::enc_payload::EncryptedPayload]
+/// ** Only clients use this; as soon as practical we decrypt and as late as practical we encrypt
+///    to and from encrypted bsos.
+/// ** The encrypted bsos etc are all in the [crypto] module and require the `crypto` feature.
+///
+/// Let's look at some real-world examples:
+/// # meta/global
+/// A "bso" (ie, record with an "envelope" and a "payload" with a JSON string) - but the payload
+/// is cleartext.
+/// ```json
+/// {
+///   "id":"global",
+///   "modified":1661564513.50,
+///   "payload": "{\"syncID\":\"p1z5_oDdOfLF\",\"storageVersion\":5,\"engines\":{\"passwords\":{\"version\":1,\"syncID\":\"6Y6JJkB074cF\"} /* snip */},\"declined\":[]}"
+/// }```
+///
+/// # encrypted bsos:
+/// Encrypted BSOs are still a "bso" (ie, a record with a field names `payload` which is a string)
+/// but the payload is in the form of an EncryptedPayload.
+/// For example, crypto/keys:
+/// ```json
+/// {
+///   "id":"keys",
+///   "modified":1661564513.74,
+///   "payload":"{\"IV\":\"snip-base-64==\",\"hmac\":\"snip-hex\",\"ciphertext\":\"snip-base64==\"}"
+/// }```
+/// (Note that as described above, most code working with bsos *do not* use that `payload`
+/// directly, but instead a decrypted cleartext bso.
+///
+/// Note all collection responses are the same shape as `crypto/keys` - a `payload` field with a
+/// JSON serialized EncryptedPayload, it's just that the final <T> content differs for each
+/// collection (eg, tabs and bookmarks have quite different <T>s JSON-encoded in the
+/// String payload.)
+///
+/// For completeness, some other "non-BSO" records - no "id", "modified" or "payload" fields in
+/// the response, just plain-old clear-text JSON.
+/// # Example
+/// ## `info/collections`
+/// ```json
+/// {
+///   "bookmarks":1661564648.65,
+///   "meta":1661564513.50,
+///   "addons":1661564649.09,
+///   "clients":1661564643.57,
+///   ...
+/// }```
+/// ## `info/configuration`
+/// ```json
+/// {
+///   "max_post_bytes":2097152,
+///   "max_post_records":100,
+///   "max_record_payload_bytes":2097152,
+///   ...
+/// }```
+///
+/// Given our definitions above, these are not any kind of "bso", so are
+/// not relevant to this module
+use crate::{Guid, ServerTimestamp};
+use serde::{Deserialize, Serialize};
+
+#[cfg(feature = "crypto")]
+mod crypto;
+#[cfg(feature = "crypto")]
+pub use crypto::{IncomingEncryptedBso, OutgoingEncryptedBso};
+
+mod content;
+
+// A feature for this would be ideal, but (a) the module is small and (b) it
+// doesn't really fit the "features" model for sync15 to have a dev-dependency
+// against itself but with a different feature set.
+pub mod test_utils;
+
+/// An envelope for an incoming item. Envelopes carry all the metadata for
+/// a Sync BSO record (`id`, `modified`, `sortindex`), *but not* the payload
+/// itself.
+#[derive(Debug, Clone, Deserialize)]
+pub struct IncomingEnvelope {
+    /// The ID of the record.
+    pub id: Guid,
+    // If we don't give it a default, a small handful of tests fail.
+    // XXX - we should probably fix the tests and kill this?
+    #[serde(default = "ServerTimestamp::default")]
+    pub modified: ServerTimestamp,
+    pub sortindex: Option<i32>,
+    pub ttl: Option<u32>,
+}
+
+/// An envelope for an outgoing item. This is conceptually identical to
+/// [IncomingEnvelope], but omits fields that are only set by the server,
+/// like `modified`.
+#[derive(Debug, Default, Clone, Serialize)]
+pub struct OutgoingEnvelope {
+    /// The ID of the record.
+    pub id: Guid,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub sortindex: Option<i32>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub ttl: Option<u32>,
+}
+
+/// Allow an outgoing envelope to be constructed with just a guid when default
+/// values for the other fields are OK.
+impl From<Guid> for OutgoingEnvelope {
+    fn from(id: Guid) -> Self {
+        OutgoingEnvelope {
+            id,
+            ..Default::default()
+        }
+    }
+}
+
+/// IncomingBso's can come from:
+/// * Directly from the server (ie, some records aren't encrypted, such as meta/global)
+/// * From environments where the encryption is done externally (eg, Rust syncing in Desktop
+///   Firefox has the encryption/decryption done by Firefox and the cleartext BSOs are passed in.
+/// * Read from the server as an EncryptedBso; see EncryptedBso description above.
+#[derive(Deserialize, Debug)]
+pub struct IncomingBso {
+    #[serde(flatten)]
+    pub envelope: IncomingEnvelope,
+    // payload is public for some edge-cases in some components, but in general,
+    // you should use into_content<> to get a record out of it.
+    pub payload: String,
+}
+
+impl IncomingBso {
+    pub fn new(envelope: IncomingEnvelope, payload: String) -> Self {
+        Self { envelope, payload }
+    }
+}
+
+#[derive(Serialize, Debug)]
+pub struct OutgoingBso {
+    #[serde(flatten)]
+    pub envelope: OutgoingEnvelope,
+    // payload is public for some edge-cases in some components, but in general,
+    // you should use into_content<> to get a record out of it.
+    pub payload: String,
+}
+
+impl OutgoingBso {
+    /// Most consumers will use `self.from_content` and `self.from_content_with_id`
+    /// but this exists for the few consumers for whom that doesn't make sense.
+    pub fn new<T: Serialize>(
+        envelope: OutgoingEnvelope,
+        val: &T,
+    ) -> Result<Self, serde_json::Error> {
+        Ok(Self {
+            envelope,
+            payload: serde_json::to_string(&val)?,
+        })
+    }
+}
+
+/// We also have the concept of "content", which helps work with a `T` which
+/// is represented inside the payload. Real-world examples of a `T` include
+/// Bookmarks or Tabs.
+/// See the content module for the implementations.
+///
+/// So this all flows together in the following way:
+/// * Incoming encrypted data:
+///   EncryptedIncomingBso -> IncomingBso -> [specific engine] -> IncomingContent<T>
+/// * Incoming cleartext data:
+///   IncomingBso -> IncomingContent<T>
+///   (Note that incoming cleartext only happens for a few collections managed by
+///   the sync client and never by specific engines - engine BSOs are always encryted)
+/// * Outgoing encrypted data:
+///   OutgoingBso (created in the engine) -> [this crate] -> EncryptedOutgoingBso
+///  * Outgoing cleartext data: just an OutgoingBso with no conversions needed.
+
+/// [IncomingContent] is the result of converting an [IncomingBso] into
+/// some <T> - it consumes the Bso, so you get the envelope, and the [IncomingKind]
+/// which reflects the state of parsing the json.
+#[derive(Debug)]
+pub struct IncomingContent<T> {
+    pub envelope: IncomingEnvelope,
+    pub kind: IncomingKind<T>,
+}
+
+/// The "kind" of incoming content after deserializing it.
+pub enum IncomingKind<T> {
+    /// A good, live T.
+    Content(T),
+    /// A record that used to be a T but has been replaced with a tombstone.
+    Tombstone,
+    /// Either not JSON, or can't be made into a T.
+    Malformed,
+}
diff --git a/third_party/rust/sync15/src/bso/test_utils.rs b/third_party/rust/sync15/src/bso/test_utils.rs
new file mode 100644
index 0000000000..55735afda2
--- /dev/null
+++ b/third_party/rust/sync15/src/bso/test_utils.rs
@@ -0,0 +1,61 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+//! Utilities for tests to make IncomingBsos and Content from test data.
+use super::{IncomingBso, IncomingEnvelope, OutgoingBso};
+use crate::{Guid, ServerTimestamp};
+
+/// Tests often want an IncomingBso to test, and the easiest way is often to
+/// create an OutgoingBso convert it back to an incoming.
+impl OutgoingBso {
+    // These functions would ideally consume `self` and avoid the clones, but
+    // this is more convenient for some tests and the extra overhead doesn't
+    // really matter for tests.
+    /// When a test has an [OutgoingBso] and wants it as an [IncomingBso]
+    pub fn to_test_incoming(&self) -> IncomingBso {
+        self.to_test_incoming_ts(ServerTimestamp::default())
+    }
+
+    /// When a test has an [OutgoingBso] and wants it as an [IncomingBso] with a specific timestamp.
+    pub fn to_test_incoming_ts(&self, ts: ServerTimestamp) -> IncomingBso {
+        IncomingBso {
+            envelope: IncomingEnvelope {
+                id: self.envelope.id.clone(),
+                modified: ts,
+                sortindex: self.envelope.sortindex,
+                ttl: self.envelope.ttl,
+            },
+            payload: self.payload.clone(),
+        }
+    }
+
+    /// When a test has an [OutgoingBso] and wants it as an [IncomingBso] with a specific T.
+    pub fn to_test_incoming_t<T: for<'de> serde::Deserialize<'de>>(&self) -> T {
+        self.to_test_incoming().into_content().content().unwrap()
+    }
+}
+
+/// Helpers to create an IncomingBso from some T
+impl IncomingBso {
+    /// When a test has an T and wants it as an [IncomingBso]
+    pub fn from_test_content<T: serde::Serialize>(json: T) -> Self {
+        // Go via an OutgoingBso
+        OutgoingBso::from_content_with_id(json)
+            .unwrap()
+            .to_test_incoming()
+    }
+
+    /// When a test has an T and wants it as an [IncomingBso] with a specific timestamp.
+    pub fn from_test_content_ts<T: serde::Serialize>(json: T, ts: ServerTimestamp) -> Self {
+        // Go via an OutgoingBso
+        OutgoingBso::from_content_with_id(json)
+            .unwrap()
+            .to_test_incoming_ts(ts)
+    }
+
+    /// When a test wants a new incoming tombstone.
+    pub fn new_test_tombstone(guid: Guid) -> Self {
+        OutgoingBso::new_tombstone(guid.into()).to_test_incoming()
+    }
+}