Milestone 2: persistent blob store, add/materialize/gc

iroh-blobs 0.35 fs store under <store_dir>/blobs. Add imports files or whole directory trees (as Collections, deterministic order), Materialize exports them with a real FICLONE reflink attempt and streaming-copy fallback, Gc is an explicit mark-and-sweep rooted at the pins. Pins, trusted peers and hash formats persist in meta.json (atomic writes). Store reads run on a LocalPool because iroh-blobs entry readers are not Send. Integration tests drive the real daemon binary: directory round trip byte-identical, gc keeps pinned/drops unpinned, reflink verified on the repo's own filesystem (XFS), plus a 32-case proptest round trip. Dependencies: iroh-blobs =0.35.0 (the store itself; pinned per spec), iroh-io (AsyncSliceReader traits to read store entries), reflink-copy (FICLONE with copy fallback, per spec), proptest (dev-only, round-trip property test). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-14 20:47:11 +02:00
parent 4e1a95613b
commit e256f73439
12 changed files with 5816 additions and 50 deletions
@@ -0,0 +1,251 @@
+//! Milestone 2: Add, List, Materialize, Gc against a real daemon.
+
+mod support;
+
+use std::path::Path;
+
+use varde_proto::{ErrorCode, MaterializeMode, PinPolicy, Request, ResponseData};
+
+fn add(client: &mut support::Client, path: &Path, recursive: bool) -> (String, u64) {
+    let reply = client.request(&Request::Add {
+        path: path.display().to_string(),
+        recursive,
+    });
+    assert!(reply.ok, "add failed: {:?}", reply.error);
+    match reply.data {
+        Some(ResponseData::Added { hash, bytes }) => (hash, bytes),
+        other => panic!("unexpected add reply: {other:?}"),
+    }
+}
+
+fn materialize(
+    client: &mut support::Client,
+    hash: &str,
+    dest: &Path,
+    mode: MaterializeMode,
+) -> Result<(u64, bool), ErrorCode> {
+    let reply = client.request(&Request::Materialize {
+        hash: hash.to_string(),
+        dest: dest.display().to_string(),
+        mode,
+    });
+    if !reply.ok {
+        return Err(reply.error.expect("error body").code);
+    }
+    match reply.data {
+        Some(ResponseData::Materialized { bytes, reflinked }) => Ok((bytes, reflinked)),
+        other => panic!("unexpected materialize reply: {other:?}"),
+    }
+}
+
+#[test]
+fn file_round_trips_byte_identical() {
+    let daemon = support::spawn_daemon();
+    let mut client = support::Client::connect(&daemon.socket);
+    let dir = tempfile::tempdir().unwrap();
+
+    let src = dir.path().join("input.bin");
+    let payload: Vec<u8> = (0..100_000u32).flat_map(|i| i.to_le_bytes()).collect();
+    std::fs::write(&src, &payload).unwrap();
+
+    let (hash, bytes) = add(&mut client, &src, false);
+    assert_eq!(bytes, payload.len() as u64);
+
+    let dest = dir.path().join("output.bin");
+    let (out_bytes, _) = materialize(&mut client, &hash, &dest, MaterializeMode::Copy).unwrap();
+    assert_eq!(out_bytes, payload.len() as u64);
+    assert_eq!(
+        std::fs::read(&dest).unwrap(),
+        payload,
+        "bytes differ after round trip"
+    );
+}
+
+#[test]
+fn directory_tree_round_trips() {
+    let daemon = support::spawn_daemon();
+    let mut client = support::Client::connect(&daemon.socket);
+    let dir = tempfile::tempdir().unwrap();
+
+    let src = dir.path().join("tree");
+    std::fs::create_dir_all(src.join("sub/deeper")).unwrap();
+    std::fs::write(src.join("a.txt"), b"alpha").unwrap();
+    std::fs::write(src.join("sub/b.bin"), vec![0u8; 20_000]).unwrap();
+    std::fs::write(src.join("sub/deeper/c"), b"").unwrap();
+
+    let (hash, bytes) = add(&mut client, &src, true);
+    assert_eq!(bytes, 5 + 20_000);
+
+    let dest = dir.path().join("out");
+    let (out_bytes, _) =
+        materialize(&mut client, &hash, &dest, MaterializeMode::ReflinkOrCopy).unwrap();
+    assert_eq!(out_bytes, bytes);
+
+    for rel in ["a.txt", "sub/b.bin", "sub/deeper/c"] {
+        let original = std::fs::read(src.join(rel)).unwrap();
+        let copy = std::fs::read(dest.join(rel)).unwrap();
+        assert_eq!(original, copy, "{rel} differs after round trip");
+    }
+}
+
+#[test]
+fn directory_without_recursive_is_rejected() {
+    let daemon = support::spawn_daemon();
+    let mut client = support::Client::connect(&daemon.socket);
+    let dir = tempfile::tempdir().unwrap();
+    std::fs::write(dir.path().join("f"), b"x").unwrap();
+
+    let reply = client.request(&Request::Add {
+        path: dir.path().display().to_string(),
+        recursive: false,
+    });
+    assert!(!reply.ok);
+    assert_eq!(reply.error.unwrap().code, ErrorCode::InvalidArgument);
+}
+
+#[test]
+fn materialize_unknown_hash_is_not_found() {
+    let daemon = support::spawn_daemon();
+    let mut client = support::Client::connect(&daemon.socket);
+    let dir = tempfile::tempdir().unwrap();
+
+    let missing = "0".repeat(64);
+    let err = materialize(
+        &mut client,
+        &missing,
+        &dir.path().join("out"),
+        MaterializeMode::Copy,
+    )
+    .unwrap_err();
+    assert_eq!(err, ErrorCode::NotFound);
+}
+
+#[test]
+fn gc_drops_unpinned_and_keeps_pinned() {
+    let daemon = support::spawn_daemon();
+    let mut client = support::Client::connect(&daemon.socket);
+    let dir = tempfile::tempdir().unwrap();
+
+    // A pinned directory tree (root + children + meta must all survive)
+    // and an unpinned lone file (must be collected).
+    let tree = dir.path().join("tree");
+    std::fs::create_dir_all(&tree).unwrap();
+    std::fs::write(tree.join("keep.bin"), vec![7u8; 50_000]).unwrap();
+    let (kept_root, _) = add(&mut client, &tree, true);
+
+    let loose = dir.path().join("loose.bin");
+    std::fs::write(&loose, vec![9u8; 50_000]).unwrap();
+    let (doomed, _) = add(&mut client, &loose, false);
+
+    let reply = client.request(&Request::Pin {
+        hash: kept_root.clone(),
+        policy: PinPolicy::default(),
+    });
+    assert!(reply.ok, "pin failed: {:?}", reply.error);
+
+    let reply = client.request(&Request::Gc {});
+    assert!(reply.ok, "gc failed: {:?}", reply.error);
+    match reply.data {
+        Some(ResponseData::GcDone { blobs_removed }) => assert!(blobs_removed >= 1),
+        other => panic!("unexpected gc reply: {other:?}"),
+    }
+
+    // Pinned tree still materializes; loose blob is gone.
+    materialize(
+        &mut client,
+        &kept_root,
+        &dir.path().join("out"),
+        MaterializeMode::Copy,
+    )
+    .expect("pinned content must survive gc");
+    let err = materialize(
+        &mut client,
+        &doomed,
+        &dir.path().join("out2"),
+        MaterializeMode::Copy,
+    )
+    .unwrap_err();
+    assert_eq!(err, ErrorCode::NotFound);
+
+    // Unpinning and collecting again drops the tree too.
+    let reply = client.request(&Request::Unpin {
+        hash: kept_root.clone(),
+    });
+    assert!(reply.ok);
+    let reply = client.request(&Request::Gc {});
+    assert!(reply.ok);
+    let err = materialize(
+        &mut client,
+        &kept_root,
+        &dir.path().join("out3"),
+        MaterializeMode::Copy,
+    )
+    .unwrap_err();
+    assert_eq!(err, ErrorCode::NotFound);
+}
+
+#[test]
+fn list_reports_pin_completeness() {
+    let daemon = support::spawn_daemon();
+    let mut client = support::Client::connect(&daemon.socket);
+    let dir = tempfile::tempdir().unwrap();
+
+    let src = dir.path().join("f.bin");
+    std::fs::write(&src, vec![1u8; 10_000]).unwrap();
+    let (hash, _) = add(&mut client, &src, false);
+    let reply = client.request(&Request::Pin {
+        hash: hash.clone(),
+        policy: PinPolicy { open_lan: true },
+    });
+    assert!(reply.ok);
+
+    let reply = client.request(&Request::List {});
+    assert!(reply.ok);
+    match reply.data {
+        Some(ResponseData::Pins { pins }) => {
+            assert_eq!(pins.len(), 1);
+            assert_eq!(pins[0].hash, hash);
+            assert!(pins[0].complete);
+            assert!(pins[0].policy.open_lan);
+            assert_eq!(pins[0].total_bytes, Some(10_000));
+        }
+        other => panic!("unexpected list reply: {other:?}"),
+    }
+}
+
+/// Reflink needs store and destination on one filesystem that supports
+/// FICLONE. `CARGO_TARGET_TMPDIR` lives under `target/`, so this exercises
+/// the real filesystem the repo sits on; on filesystems without reflink
+/// the test degrades into asserting the copy fallback works.
+#[test]
+fn reflink_when_supported() {
+    let base = Path::new(env!("CARGO_TARGET_TMPDIR")).join("reflink-test");
+    let _ = std::fs::remove_dir_all(&base);
+    std::fs::create_dir_all(&base).unwrap();
+    let daemon = support::spawn_daemon_at(&base);
+    let mut client = support::Client::connect(&daemon.socket);
+
+    let src = base.join("big.bin");
+    // Comfortably above the inline threshold so the blob lands as a
+    // real file in the store's data dir.
+    std::fs::write(&src, vec![42u8; 1_000_000]).unwrap();
+    let (hash, _) = add(&mut client, &src, false);
+
+    let dest = base.join("out.bin");
+    let (bytes, reflinked) =
+        materialize(&mut client, &hash, &dest, MaterializeMode::ReflinkOrCopy).unwrap();
+    assert_eq!(bytes, 1_000_000);
+    assert_eq!(std::fs::read(&dest).unwrap(), vec![42u8; 1_000_000]);
+
+    if !reflinked {
+        eprintln!("SKIP: filesystem does not support reflink; copy fallback verified instead");
+        return;
+    }
+    // Forced copy mode must not reflink.
+    let dest2 = base.join("out2.bin");
+    let (_, reflinked2) = materialize(&mut client, &hash, &dest2, MaterializeMode::Copy).unwrap();
+    assert!(!reflinked2, "copy mode must never reflink");
+
+    drop(daemon);
+    let _ = std::fs::remove_dir_all(&base);
+}