Milestone 3: iroh endpoint, tickets, pin-triggered fetch
The daemon binds an iroh endpoint (ed25519 identity at secret.key, 0600), serves its store via Blobs/Router on the standard ALPN, and fetches with the iroh-blobs Downloader rather than the rpc client to keep quic-rpc out of the tree. Relay is disabled unless wan_upload is set: LAN-only, zero WAN upload by default. TicketImport pins first (the pin is the GC root protecting in-flight data), registers the ticket's NodeAddr with the endpoint (the downloader dials by NodeId alone), then fetches in the background; completion emits pin_complete. Tests: two-daemon localhost transfers (blob + directory collection, byte-identical), and kill -9 mid-transfer followed by restart on the same store resuming to completion. Dependencies: iroh 0.35 (endpoint/router, pairs with iroh-blobs 0.35), rand 0.8 (secret key generation, same version iroh uses). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,218 @@
|
||||
//! Milestone 3: ticket export/import between two real daemons over
|
||||
//! localhost, and crash-resume behavior.
|
||||
|
||||
mod support;
|
||||
|
||||
use std::path::Path;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use varde_proto::{MaterializeMode, PinPolicy, Request, ResponseData};
|
||||
|
||||
fn add_file(client: &mut support::Client, path: &Path) -> String {
|
||||
let reply = client.request(&Request::Add {
|
||||
path: path.display().to_string(),
|
||||
recursive: false,
|
||||
});
|
||||
assert!(reply.ok, "add failed: {:?}", reply.error);
|
||||
match reply.data {
|
||||
Some(ResponseData::Added { hash, .. }) => hash,
|
||||
other => panic!("unexpected add reply: {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
fn export_ticket(client: &mut support::Client, hash: &str) -> String {
|
||||
let reply = client.request(&Request::TicketExport {
|
||||
hash: hash.to_string(),
|
||||
});
|
||||
assert!(reply.ok, "ticket export failed: {:?}", reply.error);
|
||||
match reply.data {
|
||||
Some(ResponseData::Ticket { ticket }) => ticket,
|
||||
other => panic!("unexpected ticket reply: {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
fn import_ticket(client: &mut support::Client, ticket: &str) {
|
||||
let reply = client.request(&Request::TicketImport {
|
||||
ticket: ticket.to_string(),
|
||||
pin_policy: PinPolicy::default(),
|
||||
});
|
||||
assert!(reply.ok, "ticket import failed: {:?}", reply.error);
|
||||
}
|
||||
|
||||
/// Poll per-hash status until the content is complete or the deadline
|
||||
/// passes.
|
||||
fn wait_complete(socket: &Path, hash: &str, timeout: Duration) -> bool {
|
||||
let deadline = Instant::now() + timeout;
|
||||
while Instant::now() < deadline {
|
||||
let mut client = support::Client::connect(socket);
|
||||
let reply = client.request(&Request::Status {
|
||||
hash: Some(hash.to_string()),
|
||||
});
|
||||
if reply.ok {
|
||||
if let Some(ResponseData::HashStatus(s)) = reply.data {
|
||||
if s.complete {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(100));
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ticket_transfers_between_two_daemons() {
|
||||
let provider = support::spawn_daemon();
|
||||
let fetcher = support::spawn_daemon();
|
||||
let mut provider_client = support::Client::connect(&provider.socket);
|
||||
let mut fetcher_client = support::Client::connect(&fetcher.socket);
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
|
||||
// Distinctive, incompressible-ish payload big enough to span many
|
||||
// chunks.
|
||||
let payload: Vec<u8> = (0..2_000_000u32).flat_map(|i| i.to_le_bytes()).collect();
|
||||
let src = dir.path().join("shared.bin");
|
||||
std::fs::write(&src, &payload).unwrap();
|
||||
|
||||
let hash = add_file(&mut provider_client, &src);
|
||||
// Pin on the provider so the content is a deliberate, GC-proof share.
|
||||
let reply = provider_client.request(&Request::Pin {
|
||||
hash: hash.clone(),
|
||||
policy: PinPolicy::default(),
|
||||
});
|
||||
assert!(reply.ok);
|
||||
|
||||
let ticket = export_ticket(&mut provider_client, &hash);
|
||||
import_ticket(&mut fetcher_client, &ticket);
|
||||
|
||||
assert!(
|
||||
wait_complete(&fetcher.socket, &hash, Duration::from_secs(60)),
|
||||
"fetch did not complete in time"
|
||||
);
|
||||
|
||||
// The fetched bytes materialize identically on the second daemon.
|
||||
let dest = dir.path().join("fetched.bin");
|
||||
let reply = fetcher_client.request(&Request::Materialize {
|
||||
hash: hash.clone(),
|
||||
dest: dest.display().to_string(),
|
||||
mode: MaterializeMode::Copy,
|
||||
});
|
||||
assert!(reply.ok, "materialize failed: {:?}", reply.error);
|
||||
assert_eq!(std::fs::read(&dest).unwrap(), payload);
|
||||
|
||||
// The pin created by the import shows up in list, complete.
|
||||
let reply = fetcher_client.request(&Request::List {});
|
||||
match reply.data {
|
||||
Some(ResponseData::Pins { pins }) => {
|
||||
assert_eq!(pins.len(), 1);
|
||||
assert_eq!(pins[0].hash, hash);
|
||||
assert!(pins[0].complete);
|
||||
}
|
||||
other => panic!("unexpected list reply: {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn directory_ticket_transfers_collection() {
|
||||
let provider = support::spawn_daemon();
|
||||
let fetcher = support::spawn_daemon();
|
||||
let mut provider_client = support::Client::connect(&provider.socket);
|
||||
let mut fetcher_client = support::Client::connect(&fetcher.socket);
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
|
||||
let tree = dir.path().join("tree");
|
||||
std::fs::create_dir_all(tree.join("nested")).unwrap();
|
||||
std::fs::write(tree.join("a.txt"), b"hello varde").unwrap();
|
||||
std::fs::write(tree.join("nested/b.bin"), vec![3u8; 100_000]).unwrap();
|
||||
|
||||
let reply = provider_client.request(&Request::Add {
|
||||
path: tree.display().to_string(),
|
||||
recursive: true,
|
||||
});
|
||||
let hash = match reply.data {
|
||||
Some(ResponseData::Added { hash, .. }) => hash,
|
||||
other => panic!("unexpected add reply: {other:?}"),
|
||||
};
|
||||
let reply = provider_client.request(&Request::Pin {
|
||||
hash: hash.clone(),
|
||||
policy: PinPolicy::default(),
|
||||
});
|
||||
assert!(reply.ok);
|
||||
|
||||
let ticket = export_ticket(&mut provider_client, &hash);
|
||||
import_ticket(&mut fetcher_client, &ticket);
|
||||
assert!(
|
||||
wait_complete(&fetcher.socket, &hash, Duration::from_secs(60)),
|
||||
"fetch did not complete in time"
|
||||
);
|
||||
|
||||
let dest = dir.path().join("out");
|
||||
let reply = fetcher_client.request(&Request::Materialize {
|
||||
hash,
|
||||
dest: dest.display().to_string(),
|
||||
mode: MaterializeMode::Copy,
|
||||
});
|
||||
assert!(reply.ok, "materialize failed: {:?}", reply.error);
|
||||
assert_eq!(std::fs::read(dest.join("a.txt")).unwrap(), b"hello varde");
|
||||
assert_eq!(
|
||||
std::fs::read(dest.join("nested/b.bin")).unwrap(),
|
||||
vec![3u8; 100_000]
|
||||
);
|
||||
}
|
||||
|
||||
/// The daemon must survive `kill -9` mid-transfer and resume cleanly on
|
||||
/// restart: iroh-blobs keeps partial state on disk, and re-issuing the
|
||||
/// import after restart completes the content.
|
||||
#[test]
|
||||
fn kill_dash_nine_mid_transfer_then_resume() {
|
||||
let provider = support::spawn_daemon();
|
||||
let mut provider_client = support::Client::connect(&provider.socket);
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
|
||||
// Big enough that the transfer takes long enough to interrupt.
|
||||
let payload: Vec<u8> = (0..16_000_000u32).flat_map(|i| i.to_le_bytes()).collect();
|
||||
let src = dir.path().join("big.bin");
|
||||
std::fs::write(&src, &payload).unwrap();
|
||||
let hash = add_file(&mut provider_client, &src);
|
||||
let reply = provider_client.request(&Request::Pin {
|
||||
hash: hash.clone(),
|
||||
policy: PinPolicy::default(),
|
||||
});
|
||||
assert!(reply.ok);
|
||||
let ticket = export_ticket(&mut provider_client, &hash);
|
||||
|
||||
// Fetcher gets its own persistent dir that survives the kill.
|
||||
let fetcher_dir = tempfile::tempdir().unwrap();
|
||||
let mut fetcher = support::spawn_daemon_at(fetcher_dir.path());
|
||||
let mut fetcher_client = support::Client::connect(&fetcher.socket);
|
||||
import_ticket(&mut fetcher_client, &ticket);
|
||||
|
||||
// SIGKILL shortly after the fetch starts. If the machine is fast
|
||||
// enough that the transfer already finished, the test still proves a
|
||||
// clean restart with intact state.
|
||||
std::thread::sleep(Duration::from_millis(150));
|
||||
fetcher.kill_dash_nine();
|
||||
|
||||
// Restart on the same store; the socket file is stale but the daemon
|
||||
// replaces it. Re-issue the import to resume.
|
||||
let fetcher = support::spawn_daemon_at(fetcher_dir.path());
|
||||
let mut fetcher_client = support::Client::connect(&fetcher.socket);
|
||||
import_ticket(&mut fetcher_client, &ticket);
|
||||
|
||||
assert!(
|
||||
wait_complete(&fetcher.socket, &hash, Duration::from_secs(120)),
|
||||
"fetch did not complete after restart"
|
||||
);
|
||||
let dest = dir.path().join("resumed.bin");
|
||||
let reply = fetcher_client.request(&Request::Materialize {
|
||||
hash,
|
||||
dest: dest.display().to_string(),
|
||||
mode: MaterializeMode::Copy,
|
||||
});
|
||||
assert!(reply.ok, "materialize failed: {:?}", reply.error);
|
||||
assert_eq!(
|
||||
std::fs::read(&dest).unwrap(),
|
||||
payload,
|
||||
"resumed content differs"
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user