From 32dc7f1f406e5fe43ba095a918a7a1c4b28f2335 Mon Sep 17 00:00:00 2001 From: KKRainbow <5665404+KKRainbow@users.noreply.github.com> Date: Tue, 13 Oct 2026 21:00:03 +0800 Subject: [PATCH] feat: add shared virtual nic (#2352) * Add shared virtual NIC core with per-member routing Share a named TUN across instances while keeping unnamed devices dedicated. Track member addresses, routes, and MTU, and reject overlapping destinations except the common Magic DNS host route. Dispatch packets between the physical TUN and member tunnels. Keep IPv4 and ordinary IPv6 source translation for mobile wrong-source traffic. Update platform configuration and runtime host integration, with unit and network namespace coverage for routing and lifecycle. Use smoltcp for packet parsing and checksum updates. Keep core CI matrix jobs independent. Reserve native inbound flows before source translations allocate their wire tuples. Keep active reservations under table pressure and leave forwarded subnet traffic outside the local flow table. Complete interface claim updates on the shared runtime and retain member registrations until the transaction commits. Keep address, route, and MTU cleanup ordered after cancelled callers, including device initialization. Project mobile VPN addresses, routes, and DNS from admitted native members. Prefer unchanged attached claims and report conflicting members individually. Apply each member's complete initial claims before opening the shared fd, so conflicts leave healthy members usable. Cover inbound IPv4/IPv6 collisions, cancellation during native changes and dispatcher acknowledgments, admission priority, and atomic mobile claim rejection with regression tests. Use scoped TestNetwork fixtures for shared TUN bridge setup and cleanup. Resolve public IPv6 namespace aliases through the lab's actual names so the new tests remain isolated under parallel upstream test runs. * Configure one mobile VPN from running TUN instances Use admitted native members to configure VpnService and attach its single fd to the core shared TUN. Wait for attachment completion and record the actual attached members before refreshing VPN state. Report rejected members in the GUI and preserve the VPN for unchanged healthy members. Reconcile ordinary configuration notifications without invalidating an arbitrary member. Invalidate attachments on instance lifecycle changes, including restarts that reuse an instance ID. Forward public IPv6 changes alongside proxy route changes to refresh the platform VPN configuration. Exclude confirmed IPv4-less members from readiness gating while retaining retries for unavailable status and pending addresses. Quick Settings stops all running TUN members and restores that group on the next start, skipping removed configurations and no_tun networks. Keep the FFI set_tun_fd call scoped to one instance and report attachment errors. Cover mobile configuration, member rejection and replacement, attachment completion, collection notifications, IPv4-less members, and tile stop/start restoration in GUI regression tests. Validate with 36 mobile VPN tests, vue-tsc --noEmit, Android GUI cargo check, workspace clippy, and shared TUN network namespace tests. * fix(tun): preserve shared NIC ownership and failure invariants Keep mobile address and route claims in dispatcher tables without applying desktop interface configuration to synthetic TUN names. Limit the Magic DNS overlap exemption to route claims and remove the shared route retry that bypassed claim accounting after invalidation. Invalidate and stop dispatchers on failed mobile TUN replacement or I/O. Bound TUN writes so stalled devices release pending source updates and member cleanup instead of retaining an unusable shared dispatcher. Treat missing Darwin and Windows addresses and routes as successful deletions while preserving permission and other configuration errors. Add regressions for claim conflicts, mobile configuration, failed TUN replacement, stalled writes, and platform deletion error handling. * fix(tun): forward single-member shared NIC traffic unchanged Deliver TUN packets directly to the sole attached member without destination matching or creating source translations. Keep native flow reservations so a later member cannot collide with existing connections, and continue applying mappings created before other members leave. Reuse flow preparation with the original source address. Preserve the desktop subnet forwarding bypass, including when the flow table is full. IPv4 fragments and IPv6 extension headers remain unchanged on flows that need no translation. Cover unknown sources and destinations, first and later IP fragments, native flows established before another member joins, and retained translations after returning to one member. Document the forwarding behavior and retain rejection coverage for multiple attached members. * fix(tun): preserve wildcard UDP replies on shared member ports Mark flow reservations created by incoming traffic separately from outgoing mappings. When a wildcard UDP socket replies with another member's source address, reuse the incoming reservation after source correction and preserve its service port and inbound destination. Keep exact outgoing mappings and outgoing flow collision handling ahead of this rule. Incoming UDP sessions take priority when packet tuples cannot distinguish a reply from a new client using identical endpoints. Cover IPv4 and IPv6 replies, checksum updates, repeated inbound traffic, existing outgoing mappings, TCP collisions, and single-to-shared member transitions with regression tests. * test(tun): wait for IPv6 activation in mobile configuration test Assert that the configured IPv6 address is registered before forwarding starts, then wait up to five seconds for the kernel to make it usable. Netlink acknowledges address creation before asynchronous IPv6 activation completes, so an immediate bind can fail under CI parallelism. Disable automatic link-local address generation in the test namespace to keep startup IPv6 control traffic out of member-routing assertions. Validation: the original test failed 40 of 320 runs at 32-way parallelism. The updated test passed all 320 runs; workspace fmt and clippy also pass. --- .github/workflows/core.yml | 2 +- docs/core-architecture.md | 67 + easytier-contrib/easytier-ffi/Cargo.toml | 1 + .../easytier-ffi/src/instance_api.rs | 7 +- easytier-gui/src-tauri/src/lib.rs | 142 +- easytier-gui/src/auto-imports.d.ts | 12 +- easytier-gui/src/composables/backend.ts | 18 +- easytier-gui/src/composables/config.ts | 16 +- easytier-gui/src/composables/event.ts | 11 +- .../src/composables/mobile_vpn.test.ts | 346 +- easytier-gui/src/composables/mobile_vpn.ts | 421 +- .../src/composables/mobile_vpn_tile.test.ts | 58 +- .../src/composables/mobile_vpn_tile.ts | 33 +- easytier-gui/src/pages/index.vue | 12 +- easytier/Cargo.toml | 2 +- easytier/src/common/ifcfg/darwin.rs | 95 +- easytier/src/common/ifcfg/mod.rs | 106 +- easytier/src/common/ifcfg/netlink.rs | 115 +- easytier/src/common/ifcfg/netlink_wire.rs | 24 + easytier/src/common/ifcfg/windows.rs | 43 +- easytier/src/instance/composition.rs | 10 +- easytier/src/instance/dns_server/runner.rs | 154 +- .../instance/dns_server/server_instance.rs | 83 +- easytier/src/instance/factory.rs | 18 + easytier/src/instance/factory/mobile_tun.rs | 393 ++ easytier/src/instance/mod.rs | 3 + easytier/src/instance/runtime_host.rs | 77 +- .../src/instance/runtime_host/magic_dns.rs | 7 +- .../src/instance/runtime_host/tun_common.rs | 45 +- .../src/instance/runtime_host/tun_desktop.rs | 49 +- .../src/instance/runtime_host/tun_mobile.rs | 135 +- easytier/src/instance/shared_virtual_nic.rs | 1943 ++++++++ .../instance/shared_virtual_nic/dispatcher.rs | 4160 +++++++++++++++++ easytier/src/instance/test_instance.rs | 45 +- easytier/src/instance/virtual_nic.rs | 1020 +++- easytier/src/tests/mod.rs | 3 + easytier/src/tests/shared_virtual_nic.rs | 874 ++++ easytier/src/tests/three_node.rs | 160 + .../android/src/main/java/TauriVpnService.kt | 34 +- .../android/src/main/java/VpnServicePlugin.kt | 6 +- tauri-plugin-vpnservice/guest-js/index.ts | 2 + tauri-plugin-vpnservice/src/models.rs | 2 + 42 files changed, 10052 insertions(+), 702 deletions(-) create mode 100644 easytier/src/instance/factory/mobile_tun.rs create mode 100644 easytier/src/instance/shared_virtual_nic.rs create mode 100644 easytier/src/instance/shared_virtual_nic/dispatcher.rs create mode 100644 easytier/src/tests/shared_virtual_nic.rs diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml index 3f9585da..3663f586 100644 --- a/.github/workflows/core.yml +++ b/.github/workflows/core.yml @@ -144,7 +144,7 @@ jobs: easytier-web/frontend/dist/* build: strategy: - fail-fast: true + fail-fast: false matrix: include: - TARGET: x86_64-unknown-linux-musl diff --git a/docs/core-architecture.md b/docs/core-architecture.md index f2acf670..6459f204 100644 --- a/docs/core-architecture.md +++ b/docs/core-architecture.md @@ -413,6 +413,73 @@ Process-global capability objects may contain stateless or shared platform mechanisms. They must not contain instance-specific peer, route, configuration, or connectivity state. +### Native shared TUN ownership + +The native composition owns `SharedVirtualNicRegistry`. Named desktop TUNs +are shared within a network namespace; mobile instances attach to one supplied +TUN descriptor. Members own address, route, and MTU claims, not the physical +device. Bootstrap route cleanup runs only when that device is created. Mobile +IPv6 claims are initialized before forwarding and follow public IPv6 updates. +Mobile VPN configuration is projected from admitted members' native claims. +Admission keeps unchanged attached members ahead of conflicting newcomers or +reconfigured members. Rejected members are reported individually, while the +remaining members stay attached. Each member's initial claims are applied as +one batch before it can open or replace the shared descriptor. +Mobile claims update dispatcher tables without configuring the OS interface; +the platform VPN owns its addresses, routes, and MTU. +If applying native claims fails, the shared NIC is invalidated and its +dispatcher is stopped so every member can recreate the device. Restoring only +the recorded claims would leave partially applied OS configuration in use. +TUN replacement and I/O failures also invalidate the shared NIC. TUN writes +have bounded waits so a stalled device cannot indefinitely block claim updates +or member cleanup. + +Desktop and mobile shared TUNs use the same lazily initialized native Tokio +runtime, independent of every member's executor. It creates the device's async +I/O registration, drives dispatchers, and completes member claim cleanup even +after the member executor stops or the registration is dropped outside Tokio. +The runtime has two workers on desktop to allow independent TUN dispatchers +to run concurrently, and retains one worker on mobile. It owns no registry or +member claims; those remain with each shared NIC. Cleanup remains asynchronous, +and the last member's cleanup stops the dispatcher and releases the desktop +device. A process-lifetime runtime does not extend the device's lifetime. +Unnamed desktop TUNs and instance-local work retain their existing executors. + +Only the member running the Magic DNS resolver claims its shared fake-IP +route. The claim is removed before resolver shutdown and acquired by the next +serving member after takeover. Client-only runners do not claim that route. + +The dispatcher preserves desktop kernel-forwarded subnet sources. It corrects +wrong-member local sources and, on mobile, synthetic sources. Native local +flows and translated flows reserve member-scoped transport tuples so address +translation cannot merge sockets using the same port and remote endpoint. +Collisions remap the source port or ICMP Echo identifier; replies restore the +original tuple. Unrelated route updates and member removals retain mappings. +All Echo sequence numbers for one identifier share a reservation; translation +preserves the sequence field in each request and reply. +Removing an involved address or member releases its mappings. A full member +queue drops that packet without blocking other members or control messages. + +With one attached member, packets go directly to that member without destination +matching or new source translations. Native flow reservations remain active so +later members cannot reuse their tuples. Existing translations continue in both +directions after other members leave. Unmodified IPv4 fragments and IPv6 +extension headers pass through the single-member path unchanged. + +Reservations are bounded at 4096 flows. Activity in either direction refreshes +them; table pressure reclaims reservations idle for at least 180 seconds. If +all slots remain active, new tracked flows are dropped rather than evicting a +live mapping. Desktop external-source forwarding bypasses this table. As with +other idle-expiring NAT state, very late replies after tuple reuse cannot be +distinguished from replies to the new flow. + +Translation supports unfragmented IPv4 and ordinary IPv6 TCP, UDP, and ICMP +Echo traffic. IPv4 fragment and IPv6 extension-header translation remain +unsupported. ICMP errors quoting a translated flow are not reverse-translated; +in particular, a smaller downstream MTU can prevent PMTU discovery for such +flows. This limitation does not affect unmodified flows and is intentionally +retained rather than extending the dispatcher into a full ICMP/fragment NAT. + ## Runtime configuration authority `TomlConfig` is the authoritative desired configuration used for management diff --git a/easytier-contrib/easytier-ffi/Cargo.toml b/easytier-contrib/easytier-ffi/Cargo.toml index 56293e45..5d37dee7 100644 --- a/easytier-contrib/easytier-ffi/Cargo.toml +++ b/easytier-contrib/easytier-ffi/Cargo.toml @@ -14,6 +14,7 @@ ffi-dataplane = [ "easytier/ffi-dataplane", "easytier-core/proxy-smoltcp-stack", ] +macos-ne = ["easytier/macos-ne"] [dependencies] easytier = { workspace = true, default-features = true, features = ["tracing-log"] } diff --git a/easytier-contrib/easytier-ffi/src/instance_api.rs b/easytier-contrib/easytier-ffi/src/instance_api.rs index 285d0eaa..9a488ae0 100644 --- a/easytier-contrib/easytier-ffi/src/instance_api.rs +++ b/easytier-contrib/easytier-ffi/src/instance_api.rs @@ -21,7 +21,7 @@ pub(crate) unsafe fn set_tun_fd(inst_name: *const c_char, fd: c_int) -> c_int { let inst_id = match resolve_instance_id_by_name(&inst_name) { Ok(Some(instance_id)) => instance_id, Ok(None) => { - set_error_msg("instance not found"); + set_error_msg(&format!("instance not found: {inst_name}")); return -1; } Err(error) => { @@ -32,7 +32,10 @@ pub(crate) unsafe fn set_tun_fd(inst_name: *const c_char, fd: c_int) -> c_int { match ffi_context().manager.attach_tun_fd(inst_id, fd) { Ok(_) => 0, - Err(_) => -1, + Err(e) => { + set_error_msg(&format!("failed to set tun fd: {}", e)); + -1 + } } } diff --git a/easytier-gui/src-tauri/src/lib.rs b/easytier-gui/src-tauri/src/lib.rs index fd0b106c..e344b7eb 100644 --- a/easytier-gui/src-tauri/src/lib.rs +++ b/easytier-gui/src-tauri/src/lib.rs @@ -6,6 +6,10 @@ mod elevate; use anyhow::Context; #[cfg(target_os = "android")] use easytier::instance::factory::subscribe_native_instance_event; +#[cfg(any(target_os = "android", target_os = "ios"))] +use easytier::instance::factory::{ + attach_mobile_tun_fd, get_mobile_tun_config as native_mobile_tun_config, +}; use easytier::proto::api::config::{ ConfigPatchAction, ConfigRpc, ConfigRpcClientFactory, VpnPortalClientPatch, }; @@ -25,7 +29,7 @@ use easytier::{ config::{ConfigLoader, ConfigSource, FileLoggerConfig, LoggingConfig, TomlConfigLoader}, log, }, - instance::factory::{NativeInstanceManager, native_instance_manager}, + instance::factory::{MobileTunConfig, NativeInstanceManager, native_instance_manager}, proto::rpc::standalone::{runtime_rpc_dialer, runtime_rpc_listener}, rpc_service::ApiRpcServer, utils::panic::setup_panic_handler, @@ -235,19 +239,45 @@ async fn set_logging_level(level: String) -> Result<(), String> { } #[tauri::command] -async fn set_tun_fd(fd: i32) -> Result<(), String> { +async fn set_tun_fd(fd: i32, instance_ids: Option>) -> Result { let Some(instance_manager) = INSTANCE_MANAGER.read().await.clone() else { return Err("set_tun_fd is not supported in remote mode".to_string()); }; - if let Some(uuid) = get_client_manager!()? - .get_enabled_instances_with_tun_ids() - .next() + + #[cfg(any(target_os = "android", target_os = "ios"))] + return attach_mobile_tun_fd(instance_manager.as_ref(), fd, instance_ids.as_deref()) + .await + .map_err(|error| error.to_string()); + + #[cfg(not(any(target_os = "android", target_os = "ios")))] { - instance_manager - .attach_tun_fd(uuid, fd) - .map_err(|e| e.to_string())?; + let _ = instance_ids; + if let Some(instance_id) = get_client_manager!()? + .get_enabled_instances_with_tun_ids() + .next() + { + instance_manager + .attach_tun_fd(instance_id, fd) + .map_err(|error| error.to_string())?; + } + Ok(MobileTunConfig::default()) + } +} + +#[tauri::command] +async fn get_mobile_tun_config(instance_ids: Vec) -> Result { + #[cfg(any(target_os = "android", target_os = "ios"))] + { + let Some(manager) = INSTANCE_MANAGER.read().await.clone() else { + return Err("mobile TUN configuration is not supported in remote mode".to_owned()); + }; + Ok(native_mobile_tun_config(manager.as_ref(), &instance_ids).await) + } + #[cfg(not(any(target_os = "android", target_os = "ios")))] + { + let _ = instance_ids; + Err("mobile TUN configuration is unavailable on this platform".to_owned()) } - Ok(()) } #[tauri::command] @@ -783,11 +813,6 @@ mod manager { Self::Web => ConfigSource::Web, } } - - #[cfg(any(test, target_os = "android"))] - fn is_web_like(self) -> bool { - matches!(self, Self::Web) - } } #[derive(Clone)] @@ -1008,44 +1033,21 @@ mod manager { .filter_map(|c| c.config.instance_id().parse::().ok()) } - #[cfg(target_os = "android")] - pub fn get_enabled_instances_with_web_like_tun_ids( - &self, - ) -> impl Iterator + '_ { - self.storage - .network_configs - .iter() - .filter(|v| self.storage.enabled_networks.contains(v.key())) - .filter(|v| !v.config.no_tun()) - .filter(|v| v.source.is_web_like()) - .filter_map(|c| c.config.instance_id().parse::().ok()) - } - - #[cfg(target_os = "android")] - pub(super) async fn disable_instances_with_tun( - &self, - app: &AppHandle, - web_only: bool, - ) -> Result<(), easytier_core::management::remote_client::RemoteClientError> - { - let inst_ids: Vec = if web_only { - self.get_enabled_instances_with_web_like_tun_ids().collect() - } else { - self.get_enabled_instances_with_tun_ids().collect() - }; - for inst_id in inst_ids { - self.handle_update_network_state(app.clone(), inst_id, true) - .await?; - } - Ok(()) - } - pub(super) fn notify_vpn_stop_if_no_tun(&self, app: &AppHandle) -> Result<(), String> { - let has_tun = self.get_enabled_instances_with_tun_ids().any(|_| true); - if !has_tun { - app.emit("vpn_service_stop", "") + #[cfg(target_os = "android")] + if self.get_enabled_instances_with_tun_ids().next().is_some() { + app.emit("vpn_service_config_changed", "") .map_err(|e| e.to_string())?; + return Ok(()); } + + #[cfg(not(target_os = "android"))] + if self.get_enabled_instances_with_tun_ids().next().is_some() { + return Ok(()); + } + + app.emit("vpn_service_stop", "") + .map_err(|e| e.to_string())?; Ok(()) } @@ -1059,28 +1061,6 @@ mod manager { app.emit("pre_run_network_instance", instance_id.to_string()) .map_err(|e| e.to_string())?; - #[cfg(target_os = "android")] - if !cfg.get_flags().no_tun { - match source { - PersistedConfigSource::User | PersistedConfigSource::Legacy => { - self.disable_instances_with_tun(app, false) - .await - .map_err(|e| e.to_string())?; - } - PersistedConfigSource::Web => { - self.disable_instances_with_tun(app, true) - .await - .map_err(|e| e.to_string())?; - if self.get_enabled_instances_with_tun_ids().next().is_some() { - return Err( - "Android only supports one active TUN network; user-managed VPN remains active" - .to_string(), - ); - } - } - } - } - self.storage .save_config( app, @@ -1109,10 +1089,13 @@ mod manager { let instance_id_str = instance_id_clone.to_string(); loop { match event_receiver.recv().await { - Ok(easytier::common::global_ctx::GlobalCtxEvent::DhcpIpv4Changed(_, _)) => { + Ok(easytier::common::global_ctx::GlobalCtxEvent::DhcpIpv4Changed(_, _) + | easytier::common::global_ctx::GlobalCtxEvent::DhcpIpv4Conflicted(_)) => { let _ = app_clone.emit("dhcp_ip_changed", &instance_id_str); } - Ok(easytier::common::global_ctx::GlobalCtxEvent::ProxyCidrsUpdated(_, _)) => { + Ok(easytier::common::global_ctx::GlobalCtxEvent::ProxyCidrsUpdated(_, _) + | easytier::common::global_ctx::GlobalCtxEvent::PublicIpv6Changed(_, _) + | easytier::common::global_ctx::GlobalCtxEvent::PublicIpv6RoutesUpdated(_, _)) => { let _ = app_clone.emit("proxy_cidrs_updated", &instance_id_str); } Ok(_) => {} @@ -1192,10 +1175,11 @@ mod manager { ) -> anyhow::Result<()> { self.storage.network_configs.clear(); for stored in configs { - let instance_id = stored.config.instance_id(); + let config = stored.config; + let instance_id = config.instance_id(); self.storage.network_configs.insert( instance_id.parse()?, - GUIConfig::new(instance_id.to_string(), stored.config, stored.source), + GUIConfig::new(instance_id.to_string(), config, stored.source), ); } @@ -1319,13 +1303,6 @@ mod manager { PersistedConfigSource::Web ); } - - #[test] - fn only_web_configs_are_web_like() { - assert!(!PersistedConfigSource::Legacy.is_web_like()); - assert!(!PersistedConfigSource::User.is_web_like()); - assert!(PersistedConfigSource::Web.is_web_like()); - } } } @@ -1517,6 +1494,7 @@ pub fn run_gui() -> std::process::ExitCode { patch_vpn_portal_clients, set_logging_level, set_tun_fd, + get_mobile_tun_config, easytier_version, set_dock_visibility, list_network_instance_ids, diff --git a/easytier-gui/src/auto-imports.d.ts b/easytier-gui/src/auto-imports.d.ts index 34230ae2..96bc0dff 100644 --- a/easytier-gui/src/auto-imports.d.ts +++ b/easytier-gui/src/auto-imports.d.ts @@ -29,6 +29,7 @@ declare global { const getCurrentInstance: typeof import('vue')['getCurrentInstance'] const getCurrentScope: typeof import('vue')['getCurrentScope'] const getEasytierVersion: typeof import('./composables/backend')['getEasytierVersion'] + const getMobileTunConfig: typeof import('./composables/backend')['getMobileTunConfig'] const getNetworkMetas: typeof import('./composables/backend')['getNetworkMetas'] const getServiceStatus: typeof import('./composables/backend')['getServiceStatus'] const getVpnPortalInfo: typeof import('./composables/backend')['getVpnPortalInfo'] @@ -48,6 +49,7 @@ declare global { const listenGlobalEvents: typeof import('./composables/event')['listenGlobalEvents'] const loadLastNetworkInstanceId: typeof import('./composables/config')['loadLastNetworkInstanceId'] const loadMode: typeof import('./composables/mode')['loadMode'] + const loadVpnTileStoppedInstanceIds: typeof import('./composables/config')['loadVpnTileStoppedInstanceIds'] const mapActions: typeof import('pinia')['mapActions'] const mapGetters: typeof import('pinia')['mapGetters'] const mapState: typeof import('pinia')['mapState'] @@ -67,6 +69,7 @@ declare global { const onMounted: typeof import('vue')['onMounted'] const onNetworkInstanceChange: typeof import('./composables/mobile_vpn')['onNetworkInstanceChange'] const onNetworkInstanceUpdate: typeof import('./composables/mobile_vpn')['onNetworkInstanceUpdate'] + const onNetworkInstancesChange: typeof import('./composables/mobile_vpn')['onNetworkInstancesChange'] const onRenderTracked: typeof import('vue')['onRenderTracked'] const onRenderTriggered: typeof import('vue')['onRenderTriggered'] const onScopeDispose: typeof import('vue')['onScopeDispose'] @@ -85,10 +88,12 @@ declare global { const saveLastNetworkInstanceId: typeof import('./composables/config')['saveLastNetworkInstanceId'] const saveMode: typeof import('./composables/mode')['saveMode'] const saveNetworkConfig: typeof import('./composables/backend')['saveNetworkConfig'] + const saveVpnTileStoppedInstanceIds: typeof import('./composables/config')['saveVpnTileStoppedInstanceIds'] const sendConfigs: typeof import('./composables/backend')['sendConfigs'] const setActivePinia: typeof import('pinia')['setActivePinia'] const setLoggingLevel: typeof import('./composables/backend')['setLoggingLevel'] const setMapStoreSuffix: typeof import('pinia')['setMapStoreSuffix'] + const setMobileVpnErrorHandler: typeof import('./composables/mobile_vpn')['setMobileVpnErrorHandler'] const setMobileVpnTileActionHandler: typeof import('./composables/mobile_vpn')['setMobileVpnTileActionHandler'] const setServiceStatus: typeof import('./composables/backend')['setServiceStatus'] const setTrayMenu: typeof import('./composables/tray')['setTrayMenu'] @@ -160,6 +165,7 @@ declare module 'vue' { readonly getCurrentInstance: UnwrapRef readonly getCurrentScope: UnwrapRef readonly getEasytierVersion: UnwrapRef + readonly getMobileTunConfig: UnwrapRef readonly getNetworkMetas: UnwrapRef readonly getServiceStatus: UnwrapRef readonly getVpnPortalInfo: UnwrapRef @@ -179,6 +185,7 @@ declare module 'vue' { readonly listenGlobalEvents: UnwrapRef readonly loadLastNetworkInstanceId: UnwrapRef readonly loadMode: UnwrapRef + readonly loadVpnTileStoppedInstanceIds: UnwrapRef readonly mapActions: UnwrapRef readonly mapGetters: UnwrapRef readonly mapState: UnwrapRef @@ -198,6 +205,7 @@ declare module 'vue' { readonly onMounted: UnwrapRef readonly onNetworkInstanceChange: UnwrapRef readonly onNetworkInstanceUpdate: UnwrapRef + readonly onNetworkInstancesChange: UnwrapRef readonly onRenderTracked: UnwrapRef readonly onRenderTriggered: UnwrapRef readonly onScopeDispose: UnwrapRef @@ -216,10 +224,12 @@ declare module 'vue' { readonly saveLastNetworkInstanceId: UnwrapRef readonly saveMode: UnwrapRef readonly saveNetworkConfig: UnwrapRef + readonly saveVpnTileStoppedInstanceIds: UnwrapRef readonly sendConfigs: UnwrapRef readonly setActivePinia: UnwrapRef readonly setLoggingLevel: UnwrapRef readonly setMapStoreSuffix: UnwrapRef + readonly setMobileVpnErrorHandler: UnwrapRef readonly setMobileVpnTileActionHandler: UnwrapRef readonly setServiceStatus: UnwrapRef readonly setTrayMenu: UnwrapRef @@ -256,4 +266,4 @@ declare module 'vue' { readonly watchPostEffect: UnwrapRef readonly watchSyncEffect: UnwrapRef } -} \ No newline at end of file +} diff --git a/easytier-gui/src/composables/backend.ts b/easytier-gui/src/composables/backend.ts index a66ca615..c5a47c76 100644 --- a/easytier-gui/src/composables/backend.ts +++ b/easytier-gui/src/composables/backend.ts @@ -87,8 +87,20 @@ export async function setLoggingLevel(level: string) { return await invoke('set_logging_level', { level }) } -export async function setTunFd(fd: number) { - return await invoke('set_tun_fd', { fd }) +export interface MobileTunConfig { + instance_ids: string[] + addresses: string[] + routes: string[] + dns?: string | null + rejected: { instance_id: string, error: string }[] +} + +export async function getMobileTunConfig(instanceIds: string[]) { + return await invoke('get_mobile_tun_config', { instanceIds }) +} + +export async function setTunFd(fd: number, instanceIds?: string[]) { + return await invoke('set_tun_fd', { fd, instanceIds }) } export async function getEasytierVersion() { @@ -127,7 +139,7 @@ export async function sendConfigs(enabledNetworks: string[]) { config: NetworkTypes.toBackendNetworkConfig(config), source, })), - enabledNetworks + enabledNetworks, }) } diff --git a/easytier-gui/src/composables/config.ts b/easytier-gui/src/composables/config.ts index 72e8cae9..400c094a 100644 --- a/easytier-gui/src/composables/config.ts +++ b/easytier-gui/src/composables/config.ts @@ -17,4 +17,18 @@ export function saveLastNetworkInstanceId(instanceId: string) { */ export function loadLastNetworkInstanceId(): string | null { return localStorage.getItem('last_network_instance_id') -} \ No newline at end of file +} + +export function saveVpnTileStoppedInstanceIds(instanceIds: string[]) { + localStorage.setItem('vpn_tile_stopped_instance_ids', JSON.stringify(instanceIds)) +} + +export function loadVpnTileStoppedInstanceIds(): string[] { + try { + const value: unknown = JSON.parse(localStorage.getItem('vpn_tile_stopped_instance_ids') || '[]') + return Array.isArray(value) ? value.filter((id): id is string => typeof id === 'string') : [] + } + catch { + return [] + } +} diff --git a/easytier-gui/src/composables/event.ts b/easytier-gui/src/composables/event.ts index 2b7a029e..27815dac 100644 --- a/easytier-gui/src/composables/event.ts +++ b/easytier-gui/src/composables/event.ts @@ -3,7 +3,7 @@ import { type } from "@tauri-apps/plugin-os"; import { NetworkTypes } from "easytier-frontend-lib" import { Utils } from "easytier-frontend-lib"; import { normalizeConfigSource } from './config_source' -import { onNetworkInstanceUpdate } from './mobile_vpn' +import { onNetworkInstancesChange, onNetworkInstanceUpdate } from './mobile_vpn' interface StoredGuiConfig { config: NetworkTypes.NetworkConfig @@ -15,6 +15,7 @@ const EVENTS = Object.freeze({ PRE_RUN_NETWORK_INSTANCE: 'pre_run_network_instance', POST_RUN_NETWORK_INSTANCE: 'post_run_network_instance', VPN_SERVICE_STOP: 'vpn_service_stop', + VPN_SERVICE_CONFIG_CHANGED: 'vpn_service_config_changed', DHCP_IP_CHANGED: 'dhcp_ip_changed', PROXY_CIDRS_UPDATED: 'proxy_cidrs_updated', EVENT_LAGGED: 'event_lagged', @@ -77,6 +78,13 @@ async function onVpnServiceStop(event: Event) { await syncMobileVpnService(); } +async function onVpnServiceConfigChanged(event: Event) { + console.log(`Received event '${EVENTS.VPN_SERVICE_CONFIG_CHANGED}'`, event.payload) + if (type() === 'android') { + await onNetworkInstancesChange(); + } +} + async function onDhcpIpChanged(event: Event) { const instanceId = normalizeInstanceIdPayload(event.payload) console.log(`Received event '${EVENTS.DHCP_IP_CHANGED}' for instance: ${instanceId}`); @@ -105,6 +113,7 @@ export async function listenGlobalEvents() { await listen(EVENTS.PRE_RUN_NETWORK_INSTANCE, onPreRunNetworkInstance), await listen(EVENTS.POST_RUN_NETWORK_INSTANCE, onPostRunNetworkInstance), await listen(EVENTS.VPN_SERVICE_STOP, onVpnServiceStop), + await listen(EVENTS.VPN_SERVICE_CONFIG_CHANGED, onVpnServiceConfigChanged), await listen(EVENTS.DHCP_IP_CHANGED, onDhcpIpChanged), await listen(EVENTS.PROXY_CIDRS_UPDATED, onProxyCidrsUpdated), await listen(EVENTS.EVENT_LAGGED, onEventLagged), diff --git a/easytier-gui/src/composables/mobile_vpn.test.ts b/easytier-gui/src/composables/mobile_vpn.test.ts index 19c94abe..9ee8234d 100644 --- a/easytier-gui/src/composables/mobile_vpn.test.ts +++ b/easytier-gui/src/composables/mobile_vpn.test.ts @@ -1,4 +1,6 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' +import { IPv4CidrRange } from 'ip-num/IPRange' +import type { MobileTunConfig } from './backend' const mocks = vi.hoisted(() => { const listeners = new Map Promise>() @@ -20,7 +22,10 @@ const mocks = vi.hoisted(() => { getVpnStatus: vi.fn<() => Promise>>(async () => ({ running: false })), listNetworkInstanceIds: vi.fn<() => Promise<{ running_inst_ids: unknown[] }>>(async () => ({ running_inst_ids: [] })), prepareVpn: vi.fn(async () => ({ granted: true })), - setTunFd: vi.fn(async () => undefined), + getMobileTunConfig: vi.fn(async (instanceIds: string[]) => makeMobileTunConfig(instanceIds)), + setTunFd: vi.fn(async (fd: number, instanceIds?: string[]) => fd > 0 + ? makeMobileTunConfig(instanceIds ?? [...configs.keys()]) + : { instance_ids: [], addresses: [], routes: [], rejected: [] }), startVpn: vi.fn(async () => { await listeners.get('vpn_service_start')?.({ fd: 1 }) return {} @@ -54,17 +59,46 @@ vi.mock('tauri-plugin-vpnservice-api', () => ({ vi.mock('./backend', () => ({ collectNetworkInfo: mocks.collectNetworkInfo, getConfig: mocks.getConfig, + getMobileTunConfig: mocks.getMobileTunConfig, listNetworkInstanceIds: mocks.listNetworkInstanceIds, setTunFd: mocks.setTunFd, })) -function setConfig(instanceId: string, noTun = false) { +function makeMobileTunConfig(instanceIds: string[]): MobileTunConfig { + const addresses: string[] = [] + const routes = new Set() + let dns: string | undefined + for (const id of instanceIds) { + const config = mocks.configs.get(id)! + const info = mocks.networkInfo.get(id) as any + const ipv4 = info?.my_node_info?.virtual_ipv4 + if (!ipv4?.address?.addr || info.error_msg) + continue + const address = `${ipv4.address.addr}/${ipv4.network_length || 24}` + addresses.push(address) + const range = IPv4CidrRange.fromCidr(address) + routes.add(`${range.getFirst()}/${range.getPrefix()}`) + const configuredRoutes = config.enable_manual_routes + ? config.routes as string[] + : (info.routes ?? []).flatMap((route: any) => route.proxy_cidrs ?? []) + configuredRoutes.forEach((route: string) => routes.add(route.includes('/') ? route : `${route}/32`)) + if (config.enable_magic_dns) { + routes.add('100.100.100.101/32') + dns = '100.100.100.101' + } + } + return { instance_ids: [...instanceIds].sort(), addresses: addresses.sort(), routes: [...routes].sort(), dns, rejected: [] } +} + +function setConfig(instanceId: string, noTun = false, devName?: string) { mocks.configs.set(instanceId, { no_tun: noTun, + dev_name: devName, dhcp: false, enable_magic_dns: false, routes: [], }) + mocks.listNetworkInstanceIds.mockResolvedValue({ running_inst_ids: [...mocks.configs.keys()] }) } function setReady(instanceId: string, ipv4: string) { @@ -87,6 +121,7 @@ async function loadVpnModule() { beforeEach(() => { vi.useFakeTimers() + vi.clearAllTimers() vi.resetModules() mocks.listeners.clear() mocks.configs.clear() @@ -102,34 +137,290 @@ beforeEach(() => { mocks.listNetworkInstanceIds.mockResolvedValue({ running_inst_ids: [] }) mocks.prepareVpn.mockClear() mocks.setTunFd.mockClear() + mocks.setTunFd.mockImplementation(async (fd: number, instanceIds?: string[]) => fd > 0 + ? makeMobileTunConfig(instanceIds ?? [...mocks.configs.keys()]) + : { instance_ids: [], addresses: [], routes: [], rejected: [] }) + mocks.getMobileTunConfig.mockReset() + mocks.getMobileTunConfig.mockImplementation(async instanceIds => makeMobileTunConfig(instanceIds)) mocks.startVpn.mockClear() mocks.stopVpn.mockClear() }) -describe('mobile VPN reconciliation ownership', () => { - it('stops A before retrying an unavailable B, then starts B when it becomes ready', async () => { +describe('mobile VPN reconciliation', () => { + it('preserves healthy members and reports a rejected bootstrap member', async () => { + setConfig('A') + setReady('A', '10.126.126.1') + const vpn = await loadVpnModule() + await vpn.onNetworkInstanceChange('A') + const errorHandler = vi.fn() + vpn.setMobileVpnErrorHandler(errorHandler) + mocks.startVpn.mockClear() + mocks.setTunFd.mockClear() + setConfig('B') + setReady('B', '10.126.126.2') + mocks.getMobileTunConfig.mockResolvedValue({ ...makeMobileTunConfig(['A']), rejected: [{ instance_id: 'B', error: 'overlapping subnet' }] }) + await vpn.onNetworkInstanceChange('B') + await vpn.onNetworkInstanceUpdate('B') + expect(mocks.stopVpn).not.toHaveBeenCalled() + expect(mocks.startVpn).not.toHaveBeenCalled() + expect(mocks.setTunFd).not.toHaveBeenCalled() + expect(errorHandler).toHaveBeenCalledOnce() + expect(errorHandler).toHaveBeenCalledWith('B: overlapping subnet') + }) + + it('does not invalidate healthy attachments on a collection notification', async () => { + setConfig('A') + setReady('A', '10.0.0.1') + const vpn = await loadVpnModule() + await vpn.onNetworkInstanceChange('A') + mocks.setTunFd.mockClear() + await vpn.onNetworkInstancesChange() + expect(mocks.stopVpn).not.toHaveBeenCalled() + expect(mocks.startVpn).toHaveBeenCalledOnce() + expect(mocks.setTunFd).not.toHaveBeenCalled() + }) + + it('corrects OS configuration when attachment admits fewer members', async () => { setConfig('A') setConfig('B') setReady('A', '10.0.0.1') + setReady('B', '10.0.1.1') + const actual = { ...makeMobileTunConfig(['A']), rejected: [{ instance_id: 'B', error: 'claim changed during attachment' }] } + mocks.setTunFd.mockResolvedValue(actual) + const vpn = await loadVpnModule() + await vpn.onNetworkInstanceChange('A') + expect(mocks.stopVpn).not.toHaveBeenCalled() + mocks.getMobileTunConfig.mockResolvedValue(actual) + await vi.advanceTimersByTimeAsync(2000) + expect(mocks.stopVpn).toHaveBeenCalledOnce() + expect(mocks.startVpn).toHaveBeenLastCalledWith(expect.objectContaining({ ipv4Addrs: ['10.0.0.1/24'], routes: ['10.0.0.0/24'] })) + }) + + it('waits for native attachment completion before completing startup', async () => { + setConfig('A') + setReady('A', '10.0.0.1') + let resolve!: (value: ReturnType) => void + mocks.setTunFd.mockImplementationOnce(() => new Promise(r => { resolve = r })) + const vpn = await loadVpnModule() + let completed = false + const start = vpn.onNetworkInstanceChange('A').then(() => { completed = true }) + await vi.waitFor(() => expect(mocks.setTunFd).toHaveBeenCalledWith(1, ['A'])) + expect(completed).toBe(false) + resolve(makeMobileTunConfig(['A'])) + await start + expect(completed).toBe(true) + }) + + it('stops the VPN and reports a physical TUN attachment failure', async () => { + setConfig('A') + setReady('A', '10.0.0.1') + mocks.setTunFd.mockRejectedValueOnce(new Error('physical TUN failed')) + const vpn = await loadVpnModule() + const errorHandler = vi.fn() + vpn.setMobileVpnErrorHandler(errorHandler) + await vpn.onNetworkInstanceChange('A') + expect(mocks.stopVpn).toHaveBeenCalledOnce() + expect(errorHandler).toHaveBeenCalledWith('Error: physical TUN failed') + }) + + it('keeps attached shared members during a temporary status gap', async () => { + setConfig('A', false, 'shared0') + setConfig('B', false, 'shared0') + setReady('A', '10.0.0.1') + setReady('B', '10.0.1.1') + const vpn = await loadVpnModule() + await vpn.onNetworkInstanceChange('A') + mocks.startVpn.mockClear() + + mocks.networkInfo.delete('B') + await vpn.onNetworkInstanceUpdate('B') + expect(mocks.stopVpn).not.toHaveBeenCalled() + expect(mocks.startVpn).not.toHaveBeenCalled() + + setReady('B', '10.0.1.2') + await vpn.onNetworkInstanceUpdate('B') + expect(mocks.stopVpn).toHaveBeenCalledTimes(1) + expect(mocks.startVpn).toHaveBeenLastCalledWith(expect.objectContaining({ + ipv4Addrs: ['10.0.0.1/24', '10.0.1.2/24'], + })) + }) + + it('keeps a ready member active while a new shared member awaits an IP', async () => { + setConfig('A', false, 'shared0') + setReady('A', '10.0.0.1') + const vpn = await loadVpnModule() + await vpn.onNetworkInstanceChange('A') + + setConfig('B', false, 'shared0') + await vpn.onNetworkInstanceChange('B') + expect(mocks.stopVpn).not.toHaveBeenCalled() + expect(mocks.startVpn).toHaveBeenCalledTimes(1) + + setReady('B', '10.0.1.1') + await vpn.onNetworkInstanceUpdate('B') + expect(mocks.stopVpn).toHaveBeenCalledTimes(1) + expect(mocks.startVpn).toHaveBeenLastCalledWith(expect.objectContaining({ + ipv4Addrs: ['10.0.0.1/24', '10.0.1.1/24'], + })) + }) + + it('uses one VPN for different dev_name values and keeps the remaining member', async () => { + setConfig('A', false, 'shared0') + setConfig('B', false, 'other0') + setConfig('C', true) + setReady('A', '10.0.0.1') + setReady('B', '10.0.1.1') const vpn = await loadVpnModule() await vpn.onNetworkInstanceChange('A') - expect(mocks.startVpn).toHaveBeenCalledTimes(1) + expect(mocks.startVpn).toHaveBeenCalledWith(expect.objectContaining({ + ipv4Addrs: ['10.0.0.1/24', '10.0.1.1/24'], + })) + expect(mocks.setTunFd).toHaveBeenCalledWith(1, ['A', 'B']) mocks.startVpn.mockClear() - await vpn.onNetworkInstanceChange('B') + mocks.stopVpn.mockClear() + mocks.listNetworkInstanceIds.mockResolvedValue({ running_inst_ids: ['C', 'B'] }) + await vpn.onNetworkInstanceChange('A') expect(mocks.stopVpn).toHaveBeenCalledTimes(1) - expect(mocks.startVpn).not.toHaveBeenCalled() - - setReady('B', '10.0.0.2') - await vpn.onNetworkInstanceUpdate('B') - - expect(mocks.startVpn).toHaveBeenCalledTimes(1) - expect(mocks.startVpn).toHaveBeenCalledWith(expect.objectContaining({ ipv4Addr: '10.0.0.2/24' })) + expect(mocks.startVpn).toHaveBeenCalledWith(expect.objectContaining({ + ipv4Addrs: ['10.0.1.1/24'], + })) + expect(mocks.setTunFd).toHaveBeenLastCalledWith(1, ['B']) }) - it('stops the previous owner during pre-run even if the new instance never reaches post-run', async () => { + it.each(['A', 'B'])('reattaches a replacement member %s with the same VPN configuration', async (replacementId) => { + setConfig('A') + setReady('A', '10.0.0.1') + const vpn = await loadVpnModule() + await vpn.onNetworkInstanceChange('A') + mocks.setTunFd.mockClear() + + mocks.listNetworkInstanceIds.mockResolvedValue({ running_inst_ids: [] }) + const stopA = vpn.onNetworkInstanceChange('A') + setConfig(replacementId) + setReady(replacementId, '10.0.0.1') + mocks.listNetworkInstanceIds.mockResolvedValue({ running_inst_ids: [replacementId] }) + const startReplacement = vpn.onNetworkInstanceChange(replacementId) + await Promise.all([stopA, startReplacement]) + + expect(mocks.setTunFd).toHaveBeenLastCalledWith(1, [replacementId]) + expect(mocks.startVpn).toHaveBeenCalledTimes(2) + expect(mocks.stopVpn).toHaveBeenCalledTimes(1) + + mocks.setTunFd.mockClear() + await vpn.onNetworkInstanceUpdate(replacementId) + expect(mocks.setTunFd).not.toHaveBeenCalled() + }) + + it('does not reattach unchanged members when their order changes', async () => { + setConfig('A') + setConfig('B') + setReady('A', '10.0.0.1') + setReady('B', '10.0.1.1') + const vpn = await loadVpnModule() + await vpn.onNetworkInstanceChange('A') + mocks.setTunFd.mockClear() + + mocks.listNetworkInstanceIds.mockResolvedValue({ running_inst_ids: ['B', 'A'] }) + await vpn.onNetworkInstanceUpdate('A') + + expect(mocks.setTunFd).not.toHaveBeenCalled() + expect(mocks.stopVpn).not.toHaveBeenCalled() + }) + + it('stops the VPN when only confirmed IPv4-less members remain', async () => { + setConfig('A') + setConfig('B') + setReady('A', '10.0.0.1') + mocks.networkInfo.set('B', { my_node_info: {}, routes: [] }) + const vpn = await loadVpnModule() + await vpn.onNetworkInstanceChange('A') + + mocks.listNetworkInstanceIds.mockResolvedValue({ running_inst_ids: ['B'] }) + await vpn.onNetworkInstanceChange('A') + + expect(mocks.stopVpn).toHaveBeenCalledTimes(1) + expect(mocks.setTunFd).toHaveBeenLastCalledWith(0) + expect(vi.getTimerCount()).toBe(0) + }) + + it('applies DHCP updates while another member has no configured IPv4', async () => { + setConfig('A') + setConfig('B') + mocks.configs.get('A')!.dhcp = true + setReady('A', '10.0.0.1') + mocks.networkInfo.set('B', { my_node_info: {}, routes: [] }) + const vpn = await loadVpnModule() + await vpn.onNetworkInstanceChange('A') + + setReady('A', '10.0.1.1') + await vpn.onNetworkInstanceUpdate('A') + + expect(mocks.stopVpn).toHaveBeenCalledTimes(1) + expect(mocks.startVpn).toHaveBeenLastCalledWith(expect.objectContaining({ + ipv4Addrs: ['10.0.1.1/24'], + routes: ['10.0.1.0/24'], + })) + expect(vi.getTimerCount()).toBe(0) + }) + + it.each([ + { virtualIpv4: '', info: { routes: [] } }, + { virtualIpv4: '10.0.1.1', info: { my_node_info: {}, routes: [] } }, + ])('waits for unconfirmed static IPv4 state: %j', async ({ virtualIpv4, info }) => { + setConfig('A') + setConfig('B') + mocks.configs.get('B')!.virtual_ipv4 = virtualIpv4 + setReady('A', '10.0.0.1') + mocks.networkInfo.set('B', info) + const vpn = await loadVpnModule() + await vpn.onNetworkInstanceChange('A') + + mocks.listNetworkInstanceIds.mockResolvedValue({ running_inst_ids: ['B'] }) + await vpn.onNetworkInstanceChange('A') + + expect(mocks.stopVpn).not.toHaveBeenCalled() + expect(vi.getTimerCount()).toBe(1) + }) + + it('removes a shared member when its DHCP address is withdrawn', async () => { + setConfig('A', false, 'shared0') + setConfig('B', false, 'shared0') + mocks.configs.get('B')!.dhcp = true + setReady('A', '10.0.0.1') + setReady('B', '10.0.1.1') + const vpn = await loadVpnModule() + await vpn.onNetworkInstanceChange('A') + + mocks.networkInfo.set('B', { my_node_info: {}, routes: [] }) + await vpn.onNetworkInstanceUpdate('B') + + expect(mocks.stopVpn).toHaveBeenCalledTimes(1) + expect(mocks.startVpn).toHaveBeenLastCalledWith(expect.objectContaining({ + ipv4Addrs: ['10.0.0.1/24'], + })) + expect(mocks.setTunFd).toHaveBeenLastCalledWith(1, ['A', 'B']) + }) + + it('uses manual routes instead of peer proxy routes', async () => { + setConfig('A') + mocks.configs.get('A')!.enable_manual_routes = true + mocks.configs.get('A')!.routes = ['192.0.2.0/24'] + setReady('A', '10.0.0.1') + const info = mocks.networkInfo.get('A') as { routes: unknown[] } + info.routes = [{ proxy_cidrs: ['10.9.0.0/16'] }] + const vpn = await loadVpnModule() + + await vpn.onNetworkInstanceChange('A') + + expect(mocks.startVpn).toHaveBeenCalledWith(expect.objectContaining({ + routes: ['10.0.0.0/24', '192.0.2.0/24'], + })) + }) + + it('keeps the VPN during pre-run of another instance', async () => { setConfig('A') setConfig('B') setReady('A', '10.0.0.1') @@ -140,7 +431,7 @@ describe('mobile VPN reconciliation ownership', () => { await vpn.prepareVpnService('B') - expect(mocks.stopVpn).toHaveBeenCalledTimes(1) + expect(mocks.stopVpn).not.toHaveBeenCalled() }) it('preserves the VPN while retrying the same instance', async () => { @@ -157,22 +448,25 @@ describe('mobile VPN reconciliation ownership', () => { expect(mocks.stopVpn).not.toHaveBeenCalled() }) - it('ignores an update from an instance that no longer owns the VPN', async () => { + it('reconciles all running members after any member update', async () => { setConfig('A') - setConfig('B') + setConfig('B', false, 'other0') setReady('A', '10.0.0.1') + setReady('B', '10.0.1.1') const vpn = await loadVpnModule() await vpn.onNetworkInstanceChange('A') - await vpn.onNetworkInstanceChange('B') - mocks.collectNetworkInfo.mockClear() + mocks.startVpn.mockClear() + setReady('B', '10.0.1.2') await vpn.onNetworkInstanceUpdate('A') - expect(mocks.collectNetworkInfo).not.toHaveBeenCalled() + expect(mocks.startVpn).toHaveBeenCalledWith(expect.objectContaining({ + ipv4Addrs: ['10.0.0.1/24', '10.0.1.2/24'], + })) }) - it('does not apply an in-flight result after the desired instance changes', async () => { + it('does not apply a stale in-flight network result', async () => { setConfig('A') setConfig('B') setReady('A', '10.0.0.1') @@ -195,7 +489,7 @@ describe('mobile VPN reconciliation ownership', () => { const staleUpdate = vpn.onNetworkInstanceUpdate('A') await collectStarted - const switchToB = vpn.onNetworkInstanceChange('B') + const newerUpdate = vpn.onNetworkInstanceChange('B') resolveNetworkInfo({ info: { map: { @@ -212,13 +506,13 @@ describe('mobile VPN reconciliation ownership', () => { }, }) - await Promise.all([staleUpdate, switchToB]) + await Promise.all([staleUpdate, newerUpdate]) expect(mocks.startVpn).not.toHaveBeenCalled() - expect(mocks.stopVpn).toHaveBeenCalledTimes(1) + expect(mocks.stopVpn).not.toHaveBeenCalled() }) - it('stops a native VPN with unknown ownership before retrying the selected instance', async () => { + it('preserves a native VPN while network info is unavailable', async () => { setConfig('A') mocks.getVpnStatus.mockResolvedValue({ running: true, @@ -230,7 +524,7 @@ describe('mobile VPN reconciliation ownership', () => { await vpn.syncMobileVpnService() - expect(mocks.stopVpn).toHaveBeenCalledTimes(1) + expect(mocks.stopVpn).not.toHaveBeenCalled() expect(mocks.startVpn).not.toHaveBeenCalled() }) }) diff --git a/easytier-gui/src/composables/mobile_vpn.ts b/easytier-gui/src/composables/mobile_vpn.ts index 1cd2adc3..681caab8 100644 --- a/easytier-gui/src/composables/mobile_vpn.ts +++ b/easytier-gui/src/composables/mobile_vpn.ts @@ -1,4 +1,4 @@ -import type { NetworkTypes } from 'easytier-frontend-lib' +import type { MobileTunConfig } from './backend' import { addPluginListener } from '@tauri-apps/api/core' import { Utils } from 'easytier-frontend-lib' import { @@ -9,14 +9,11 @@ import { stop_vpn, type VpnTileAction, } from 'tauri-plugin-vpnservice-api' -import { collectNetworkInfo, getConfig, listNetworkInstanceIds, setTunFd } from './backend' - -type Route = NetworkTypes.Route +import { collectNetworkInfo, getConfig, getMobileTunConfig, listNetworkInstanceIds, setTunFd } from './backend' interface vpnStatus { running: boolean - ipv4Addr: string | null | undefined - ipv4Cidr: number | null | undefined + ipv4Addrs: string[] routes: string[] dns: string | null | undefined } @@ -25,19 +22,32 @@ let vpnReconcileTimer: ReturnType | null = null const VPN_RECONCILE_INTERVAL_MS = 2000 const VPN_RECONCILE_MAX_ATTEMPTS = 60 -let desiredVpnInstanceId: string | undefined -let activeVpnInstanceId: string | undefined let vpnReconcileGeneration = 0 let vpnReconcileAttempts = 0 let vpnReconcileQueue: Promise = Promise.resolve() let vpnPermissionRequest: Promise | null = null let vpnTileActionHandler: ((action: VpnTileAction) => Promise) | undefined let vpnTileActionQueue: Promise = Promise.resolve() +let vpnAttachedInstanceIds: string[] = [] +let vpnStartingInstanceIds: string[] | undefined +let vpnAttachment: Promise | null = null +let vpnErrorHandler: ((error: string) => void) | undefined +let vpnMemberErrors = '' + +export function setMobileVpnErrorHandler(handler?: (error: string) => void) { + vpnErrorHandler = handler +} + +function reportRejectedMembers(config: MobileTunConfig) { + const errors = config.rejected.map(({ instance_id, error }) => `${instance_id}: ${error}`).sort().join('\n') + if (errors && errors !== vpnMemberErrors) + vpnErrorHandler?.(errors) + vpnMemberErrors = errors +} const curVpnStatus: vpnStatus = { running: false, - ipv4Addr: undefined, - ipv4Cidr: undefined, + ipv4Addrs: [], routes: [], dns: undefined, } @@ -108,26 +118,24 @@ function clearVpnReconcileTimer() { } } -function beginVpnReconcile(instanceId?: string) { +function beginVpnReconcile() { clearVpnReconcileTimer() - desiredVpnInstanceId = instanceId vpnReconcileAttempts = 0 vpnReconcileGeneration += 1 return vpnReconcileGeneration } -function isCurrentVpnReconcile(instanceId: string, generation: number) { - return desiredVpnInstanceId === (instanceId || undefined) && vpnReconcileGeneration === generation +function isCurrentVpnReconcile(generation: number) { + return vpnReconcileGeneration === generation } -function scheduleVpnReconcile(instanceId: string, generation: number, reason: string) { - if (!isCurrentVpnReconcile(instanceId, generation)) +function scheduleVpnReconcile(generation: number, reason: string) { + if (!isCurrentVpnReconcile(generation)) return if (vpnReconcileAttempts >= VPN_RECONCILE_MAX_ATTEMPTS) { console.error( 'vpn service reconcile stopped after maximum attempts', - instanceId, VPN_RECONCILE_MAX_ATTEMPTS, reason, ) @@ -139,7 +147,6 @@ function scheduleVpnReconcile(instanceId: string, generation: number, reason: st console.log( 'vpn service is not ready, retrying', JSON.stringify({ - instanceId, attempt: vpnReconcileAttempts, maxAttempts: VPN_RECONCILE_MAX_ATTEMPTS, delayMs: VPN_RECONCILE_INTERVAL_MS, @@ -148,13 +155,13 @@ function scheduleVpnReconcile(instanceId: string, generation: number, reason: st ) vpnReconcileTimer = setTimeout(() => { vpnReconcileTimer = null - void enqueueVpnReconcile(instanceId, generation) + void enqueueVpnReconcile(generation) }, VPN_RECONCILE_INTERVAL_MS) } function resetVpnConfigStatus() { - curVpnStatus.ipv4Addr = undefined - curVpnStatus.ipv4Cidr = undefined + vpnAttachedInstanceIds = [] + curVpnStatus.ipv4Addrs = [] curVpnStatus.routes = [] curVpnStatus.dns = undefined } @@ -162,24 +169,11 @@ function resetVpnConfigStatus() { function syncVpnStatusFromNative(status: Awaited>) { curVpnStatus.running = status?.running ?? false if (!curVpnStatus.running) { - activeVpnInstanceId = undefined resetVpnConfigStatus() return } - const ipv4WithCidr = status?.ipv4Addr - if (ipv4WithCidr?.length) { - const [ipv4Addr, cidr] = ipv4WithCidr.split('/') - curVpnStatus.ipv4Addr = ipv4Addr - - const parsedCidr = Number(cidr) - curVpnStatus.ipv4Cidr = Number.isInteger(parsedCidr) ? parsedCidr : undefined - } - else { - curVpnStatus.ipv4Addr = undefined - curVpnStatus.ipv4Cidr = undefined - } - + curVpnStatus.ipv4Addrs = [...(status?.ipv4Addrs ?? [])] curVpnStatus.routes = [...(status?.routes ?? [])] curVpnStatus.dns = status?.dns ?? undefined } @@ -194,12 +188,21 @@ async function waitVpnStatus(target_status: boolean, timeout_sec: number) { } } +async function detachTunFd() { + try { + await setTunFd(0) + } + catch (e) { + console.error('detach tun fd failed', e) + } +} + async function doStopVpn(force = false) { const wasRunning = curVpnStatus.running if (!force && !wasRunning) { - activeVpnInstanceId = undefined return } + await detachTunFd() console.log('stop vpn') const stop_ret = await stop_vpn() console.log('stop vpn', JSON.stringify((stop_ret))) @@ -207,18 +210,25 @@ async function doStopVpn(force = false) { await waitVpnStatus(false, 3) } - activeVpnInstanceId = undefined resetVpnConfigStatus() } -async function doStartVpn(instanceId: string, ipv4Addr: string, cidr: number, routes: string[], dns?: string) { +async function doStartVpn( + instanceIds: string[], + ipv4Addrs: string[], + routes: string[], + dns: string | undefined, +) { if (curVpnStatus.running) { return } - console.log('start vpn service', ipv4Addr, cidr, routes, dns) + vpnStartingInstanceIds = instanceIds + vpnAttachment = null + + console.log('start vpn service', ipv4Addrs, routes, dns) const request = { - ipv4Addr: `${ipv4Addr}/${cidr}`, + ipv4Addrs, routes, dns, disallowedApplications: ['com.kkrainbow.easytier'], @@ -239,29 +249,50 @@ async function doStartVpn(instanceId: string, ipv4Addr: string, cidr: number, ro if (start_ret?.errorMsg?.length) { throw new Error(start_ret.errorMsg) } - await waitVpnStatus(true, 3) + const actual = await waitVpnAttachment() - curVpnStatus.ipv4Addr = ipv4Addr - curVpnStatus.ipv4Cidr = cidr + curVpnStatus.ipv4Addrs = [...ipv4Addrs] curVpnStatus.routes = routes curVpnStatus.dns = dns - activeVpnInstanceId = instanceId + vpnAttachedInstanceIds = [...actual.instance_ids].sort() + if (JSON.stringify(actual.addresses) !== JSON.stringify(ipv4Addrs) + || JSON.stringify(actual.routes) !== JSON.stringify(routes) + || (actual.dns ?? undefined) !== dns) { + scheduleVpnReconcile(vpnReconcileGeneration, 'attachment_configuration_changed') + } +} + +async function waitVpnAttachment() { + if (!vpnAttachment) + await waitVpnStatus(true, 3) + const actual = await vpnAttachment + if (!actual?.instance_ids.length) + throw new Error('no mobile TUN member was attached') + return actual } async function onVpnServiceStart(payload: any) { console.log('vpn service start', JSON.stringify(payload)) curVpnStatus.running = true if (payload.fd) { - await setTunFd(payload.fd).catch((e) => { + vpnAttachment = setTunFd(payload.fd, vpnStartingInstanceIds) + await vpnAttachment.then(async (actual) => { + reportRejectedMembers(actual) + vpnAttachedInstanceIds = [...actual.instance_ids].sort() + if (!actual.instance_ids.length) + await doStopVpn(true) + }).catch(async (e) => { console.error('set tun fd failed', e) + vpnErrorHandler?.(String(e)) + await doStopVpn(true).catch(stopError => console.error('stop vpn after tun attach failure', stopError)) }) } } async function onVpnServiceStop(payload: any) { console.log('vpn service stop', JSON.stringify(payload)) + await detachTunFd() curVpnStatus.running = false - activeVpnInstanceId = undefined resetVpnConfigStatus() } @@ -290,159 +321,145 @@ async function registerVpnServiceListener() { ) } -function getRoutesForVpn(routes: Route[] | undefined, node_config: NetworkTypes.NetworkConfig): string[] { - const ret = [] - for (const r of routes ?? []) { - for (let cidr of r.proxy_cidrs ?? []) { - if (!cidr.includes('/')) { - cidr += '/32' - } - ret.push(cidr) - } - } - - for (const route of node_config.routes ?? []) { - ret.push(route) - } - - if (node_config.enable_magic_dns) { - ret.push('100.100.100.101/32') - } - - // sort and dedup - return Array.from(new Set(ret)).sort() -} - -async function stopVpnOwnedByOtherInstance(instanceId: string, generation: number) { - if (!isCurrentVpnReconcile(instanceId, generation)) - return false - - if (curVpnStatus.running && activeVpnInstanceId !== instanceId) { - console.warn('vpn service owner changed', activeVpnInstanceId, instanceId) - await doStopVpn() - } - - return isCurrentVpnReconcile(instanceId, generation) -} - -async function reconcileNetworkInstance(instanceId: string, generation: number) { - if (!isCurrentVpnReconcile(instanceId, generation)) +async function reconcileNetworkInstance(generation: number) { + if (!isCurrentVpnReconcile(generation)) return clearVpnReconcileTimer() - if (!instanceId) { - console.warn('vpn service skipped because instance id is empty') - if (curVpnStatus.running) { - await doStopVpn() - } - return - } - const config = await getConfig(instanceId) - if (!isCurrentVpnReconcile(instanceId, generation)) - return - - console.log('vpn service loaded config', instanceId, JSON.stringify({ - no_tun: config.no_tun, - dhcp: config.dhcp, - enable_magic_dns: config.enable_magic_dns, - })) - if (config.no_tun) { - console.log('vpn service skipped because no_tun is enabled', instanceId) - if (activeVpnInstanceId === instanceId) { - await doStopVpn() - } - return - } - - if (!await stopVpnOwnedByOtherInstance(instanceId, generation)) - return - - let curNetworkInfo + let instances: Awaited> try { - curNetworkInfo = (await collectNetworkInfo(instanceId))?.info?.map?.[instanceId] + instances = await findRunningTunInstances() } - catch (e) { - console.warn('vpn service network info query failed', instanceId, e) - scheduleVpnReconcile(instanceId, generation, 'network_info_query_failed') + catch (error) { + console.warn('vpn service instance query failed', error) + scheduleVpnReconcile(generation, 'instance_list_unavailable') return } - if (!isCurrentVpnReconcile(instanceId, generation)) + if (!isCurrentVpnReconcile(generation)) return - if (!curNetworkInfo) { - scheduleVpnReconcile(instanceId, generation, 'network_info_unavailable') + if (!instances.length) { + if (curVpnStatus.running) + await doStopVpn() return } - if (curNetworkInfo.error_msg?.length) { - console.warn('vpn service skipped because network instance failed', instanceId, curNetworkInfo.error_msg) + let retryReason: string | undefined + let networkInfoUnavailable = false + + for (const { instanceId: memberId, config } of instances) { + let curNetworkInfo + try { + curNetworkInfo = (await collectNetworkInfo(memberId)).info.map[memberId] + } + catch (error) { + console.warn('vpn service network info query failed', memberId, error) + retryReason ??= 'network_info_query_failed' + networkInfoUnavailable = true + continue + } + + if (!isCurrentVpnReconcile(generation)) + return + + if (!curNetworkInfo) { + console.warn('vpn service network info unavailable', memberId) + retryReason ??= 'network_info_unavailable' + networkInfoUnavailable = true + continue + } + + if (curNetworkInfo.error_msg?.length) { + console.warn('vpn service network failed', memberId, curNetworkInfo.error_msg) + retryReason ??= 'network_failed' + continue + } + + const virtualIpv4 = curNetworkInfo.my_node_info?.virtual_ipv4 + const virtualIp = virtualIpv4?.address?.addr ? Utils.ipv4ToString(virtualIpv4.address) : undefined + if (!virtualIp) { + if (!config.dhcp && !config.virtual_ipv4 && curNetworkInfo.my_node_info) + continue + retryReason ??= config.dhcp ? 'dhcp_ipv4_unavailable' : 'static_ipv4_unavailable' + if (!config.dhcp) + networkInfoUnavailable = true + continue + } + + } + + if (!isCurrentVpnReconcile(generation)) + return + + if (networkInfoUnavailable && curVpnStatus.running) { + scheduleVpnReconcile(generation, retryReason || 'network_info_unavailable') + return + } + + const preferredIds = [...instances].sort((a, b) => + Number(vpnAttachedInstanceIds.includes(b.instanceId)) - Number(vpnAttachedInstanceIds.includes(a.instanceId)) + || a.instanceId.localeCompare(b.instanceId)).map(({ instanceId }) => instanceId) + let config: MobileTunConfig + try { + config = await getMobileTunConfig(preferredIds) + } + catch (error) { + console.error('mobile TUN configuration query failed', error) + scheduleVpnReconcile(generation, 'tun_configuration_unavailable') + return + } + if (!isCurrentVpnReconcile(generation)) + return + reportRejectedMembers(config) + if (!config.addresses.length) { + if (retryReason) + scheduleVpnReconcile(generation, retryReason) + if (curVpnStatus.running) + await doStopVpn() + return + } + + if (retryReason) + scheduleVpnReconcile(generation, retryReason) + else vpnReconcileAttempts = 0 - await doStopVpn() - return - } + const sortedIpv4Addrs = [...config.addresses].sort() + const sortedRoutes = [...config.routes].sort() + const instanceIds = [...config.instance_ids].sort() + const dns = config.dns ?? undefined + // A replacement runtime needs its own attachment even with identical OS routes. + const membersChanged = JSON.stringify(instanceIds) !== JSON.stringify(vpnAttachedInstanceIds) + const configChanged + = JSON.stringify(sortedIpv4Addrs) !== JSON.stringify(curVpnStatus.ipv4Addrs) + || JSON.stringify(sortedRoutes) !== JSON.stringify(curVpnStatus.routes) + || dns !== curVpnStatus.dns - const virtualIpv4 = curNetworkInfo.my_node_info?.virtual_ipv4 - const virtual_ip = virtualIpv4?.address?.addr ? Utils.ipv4ToString(virtualIpv4.address) : undefined - - if (!virtual_ip || !virtual_ip.length) { - scheduleVpnReconcile( - instanceId, - generation, - config.dhcp ? 'dhcp_ipv4_unavailable' : 'static_ipv4_unavailable', - ) - return - } - - vpnReconcileAttempts = 0 - - let network_length = virtualIpv4?.network_length - if (!network_length) { - network_length = 24 - } - - const routes = getRoutesForVpn(curNetworkInfo?.routes, config) - - const dns = config.enable_magic_dns ? '100.100.100.101' : undefined - - const ipChanged = virtual_ip !== curVpnStatus.ipv4Addr - const cidrChanged = network_length !== curVpnStatus.ipv4Cidr - const routesChanged = JSON.stringify(routes) !== JSON.stringify(curVpnStatus.routes) - const dnsChanged = dns != curVpnStatus.dns - const configChanged = ipChanged || cidrChanged || routesChanged || dnsChanged - const shouldStartVpn = !curVpnStatus.running - - if (shouldStartVpn || configChanged) { - console.info('vpn service virtual ip changed', JSON.stringify(curVpnStatus), virtual_ip) + if (!curVpnStatus.running || configChanged || membersChanged) { if (curVpnStatus.running) { try { await doStopVpn() } - catch (e) { - console.error(e) + catch (error) { + console.error('stop vpn service failed', error) } } + if (!isCurrentVpnReconcile(generation)) + return + try { - if (!isCurrentVpnReconcile(instanceId, generation)) - return - - await doStartVpn(instanceId, virtual_ip, network_length, routes, dns) - if (!isCurrentVpnReconcile(instanceId, generation) && activeVpnInstanceId === instanceId) { + await doStartVpn(instanceIds, sortedIpv4Addrs, sortedRoutes, dns) + if (!isCurrentVpnReconcile(generation)) await doStopVpn() - } } - catch (e) { - if (e instanceof Error && e.message === 'need_prepare') { - console.info('vpn permission is required before starting the Android VPN service') - return - } - if (e instanceof Error && e.message === 'vpn_permission_denied') { + catch (error) { + if (error instanceof Error && error.message === 'vpn_permission_denied') { console.info('vpn permission request was denied or dismissed') return } - console.error('start vpn service failed', e) + console.error('start vpn service failed', error) } } } @@ -459,42 +476,24 @@ function enqueueVpnTask(task: () => Promise) { return run } -function enqueueVpnReconcile(instanceId: string, generation: number) { - return enqueueVpnTask(() => reconcileNetworkInstance(instanceId, generation)) +function enqueueVpnReconcile(generation: number) { + return enqueueVpnTask(() => reconcileNetworkInstance(generation)) } export async function onNetworkInstanceChange(instanceId: string) { - const generation = beginVpnReconcile(instanceId || undefined) + // A restarted instance keeps its ID but loses the previous runtime's TUN attachment. + vpnAttachedInstanceIds = vpnAttachedInstanceIds.filter(id => id !== instanceId) + await onNetworkInstancesChange() +} - if (instanceId && await isNoTunEnabled(instanceId)) { - if (vpnReconcileGeneration !== generation) - return - - if (activeVpnInstanceId === instanceId) { - desiredVpnInstanceId = undefined - await enqueueVpnReconcile('', generation) - return - } - - desiredVpnInstanceId = activeVpnInstanceId - if (activeVpnInstanceId) { - await enqueueVpnReconcile(activeVpnInstanceId, generation) - } - return - } - - if (vpnReconcileGeneration !== generation) - return - - await enqueueVpnReconcile(instanceId, generation) +export async function onNetworkInstancesChange() { + await enqueueVpnReconcile(beginVpnReconcile()) } export async function onNetworkInstanceUpdate(instanceId: string) { - if (!instanceId || instanceId !== desiredVpnInstanceId) + if (!instanceId) return - - const generation = beginVpnReconcile(instanceId) - await enqueueVpnReconcile(instanceId, generation) + await onNetworkInstancesChange() } async function isNoTunEnabled(instanceId: string | undefined) { @@ -504,20 +503,19 @@ async function isNoTunEnabled(instanceId: string | undefined) { return (await getConfig(instanceId)).no_tun ?? false } -async function findRunningTunInstanceId() { +async function findRunningTunInstances() { const instanceIds = await listNetworkInstanceIds() const runningIds = (instanceIds.running_inst_ids ?? []).map(Utils.UuidToStr) - console.log('vpn service sync running instances', JSON.stringify(runningIds)) + const runningTunInstances = [] for (const instanceId of runningIds) { - if (await isNoTunEnabled(instanceId)) { + const config = await getConfig(instanceId) + if (config.no_tun) continue - } - - return instanceId + runningTunInstances.push({ instanceId, config }) } - return undefined + return runningTunInstances } export async function initMobileVpnService() { @@ -529,21 +527,10 @@ export async function prepareVpnService(instanceId: string) { return } - const generation = beginVpnReconcile(instanceId) - const stopPreviousOwner = enqueueVpnTask(async () => { - await stopVpnOwnedByOtherInstance(instanceId, generation) - }) - await Promise.all([requestVpnPermission(), stopPreviousOwner]) + await requestVpnPermission() } export async function syncMobileVpnService() { syncVpnStatusFromNative(await get_vpn_status()) - const instanceId = await findRunningTunInstanceId() - if (instanceId) { - console.log('vpn service sync selected instance', instanceId) - await onNetworkInstanceChange(instanceId) - return - } - await onNetworkInstanceChange('') } diff --git a/easytier-gui/src/composables/mobile_vpn_tile.test.ts b/easytier-gui/src/composables/mobile_vpn_tile.test.ts index 71d8d5ed..8b5376a0 100644 --- a/easytier-gui/src/composables/mobile_vpn_tile.test.ts +++ b/easytier-gui/src/composables/mobile_vpn_tile.test.ts @@ -36,6 +36,45 @@ describe('vpn quick settings tile actions', () => { syncVpnService.mockClear() }) + it('restores the whole stopped group across tile stop and start', async () => { + const running = ['first', 'last', 'headless'] + const disabled: string[] = [] + const { api, configs } = createApi(running, disabled, ['headless']) + let stoppedInstanceIds: string[] = [] + const saveStoppedInstanceIds = (ids: string[]) => { stoppedInstanceIds = ids } + const result = await executeVpnTileAction('stop', api, { lastInstanceId: 'last', syncVpnService, saveStoppedInstanceIds }) + expect(stoppedInstanceIds).toEqual(['first', 'last']) + disabled.push('first', 'last') + running.splice(0, 2) + await executeVpnTileAction('start', api, { lastInstanceId: result.instanceId, stoppedInstanceIds, syncVpnService, saveStoppedInstanceIds }) + expect(api.run_network).toHaveBeenCalledWith(configs.get('first'), true) + expect(api.run_network).toHaveBeenCalledWith(configs.get('last'), true) + expect(api.run_network).toHaveBeenCalledTimes(2) + expect(stoppedInstanceIds).toEqual([]) + }) + + it('skips removed and no_tun members when restoring the stopped group', async () => { + const { api, configs } = createApi(['running'], ['first', 'headless'], ['headless']) + await executeVpnTileAction('start', api, { stoppedInstanceIds: ['removed', 'headless', 'first', 'running', 'first'], syncVpnService }) + expect(api.run_network).toHaveBeenCalledOnce() + expect(api.run_network).toHaveBeenCalledWith(configs.get('first'), true) + }) + + it.each(['removed', 'headless'])('does not start another network when the saved group only contains %s', async (stoppedId) => { + const { api } = createApi([], ['unrelated', 'headless'], ['headless']) + const saveStoppedInstanceIds = vi.fn() + const result = await executeVpnTileAction('start', api, { + lastInstanceId: 'unrelated', + stoppedInstanceIds: [stoppedId], + syncVpnService, + saveStoppedInstanceIds, + }) + expect(api.run_network).not.toHaveBeenCalled() + expect(saveStoppedInstanceIds).toHaveBeenCalledWith([]) + expect(syncVpnService).not.toHaveBeenCalled() + expect(result).toEqual({ action: 'start', changed: false }) + }) + it('starts the last configured network and reconciles the Android VPN', async () => { const { api, configs } = createApi([], ['first', 'last']) @@ -88,7 +127,7 @@ describe('vpn quick settings tile actions', () => { expect(result.changed).toBe(false) }) - it('stops the last running network, falling back to the first running network', async () => { + it('stops all running TUN networks when the last network is disabled', async () => { const { api } = createApi(['first', 'second'], ['disabled']) const result = await executeVpnTileAction('stop', api, { @@ -97,10 +136,27 @@ describe('vpn quick settings tile actions', () => { }) expect(api.update_network_instance_state).toHaveBeenCalledWith('first', true) + expect(api.update_network_instance_state).toHaveBeenCalledWith('second', true) + expect(api.update_network_instance_state).toHaveBeenCalledTimes(2) expect(syncVpnService).toHaveBeenCalledOnce() expect(result).toEqual({ action: 'stop', instanceId: 'first', changed: true }) }) + it('stops all running TUN networks while preserving no_tun networks', async () => { + const { api } = createApi(['headless', 'first', 'last'], ['disabled'], ['headless']) + + const result = await executeVpnTileAction('stop', api, { + lastInstanceId: 'last', + syncVpnService, + }) + + expect(api.update_network_instance_state).toHaveBeenCalledWith('first', true) + expect(api.update_network_instance_state).toHaveBeenCalledWith('last', true) + expect(api.update_network_instance_state).toHaveBeenCalledTimes(2) + expect(syncVpnService).toHaveBeenCalledOnce() + expect(result).toEqual({ action: 'stop', instanceId: 'last', changed: true }) + }) + it('does not select a running no_tun network for stop', async () => { const { api } = createApi(['headless', 'vpn'], [], ['headless']) diff --git a/easytier-gui/src/composables/mobile_vpn_tile.ts b/easytier-gui/src/composables/mobile_vpn_tile.ts index 47ee1b3a..4b68ff5f 100644 --- a/easytier-gui/src/composables/mobile_vpn_tile.ts +++ b/easytier-gui/src/composables/mobile_vpn_tile.ts @@ -10,6 +10,8 @@ export interface VpnTileActionResult { export interface VpnTileActionOptions { lastInstanceId?: string | null + stoppedInstanceIds?: string[] + saveStoppedInstanceIds?: (instanceIds: string[]) => void syncVpnService: () => Promise } @@ -31,22 +33,39 @@ export async function executeVpnTileAction( })), ) const candidates = candidateIds.filter(instanceId => !candidateConfigs.get(instanceId)?.no_tun) - const instanceId = options.lastInstanceId && candidates.includes(options.lastInstanceId) + const stoppedCandidates = [...new Set(options.stoppedInstanceIds ?? [])] + .filter(instanceId => candidates.includes(instanceId)) + const restoringStoppedGroup = action === 'start' && !!options.stoppedInstanceIds?.length + const selectedCandidates = restoringStoppedGroup + ? stoppedCandidates + : candidates + const instanceId = options.lastInstanceId && selectedCandidates.includes(options.lastInstanceId) ? options.lastInstanceId - : candidates[0] + : selectedCandidates[0] if (!instanceId) { + if (restoringStoppedGroup) + options.saveStoppedInstanceIds?.([]) return { action, changed: false } } let changed = false - if (action === 'start' && !runningIds.includes(instanceId)) { - const config = candidateConfigs.get(instanceId)! - await api.run_network(config, true) - changed = true + if (action === 'start') { + const startIds = restoringStoppedGroup ? stoppedCandidates : [instanceId] + for (const candidateId of startIds) { + if (!runningIds.includes(candidateId)) { + await api.run_network(candidateConfigs.get(candidateId)!, true) + changed = true + } + } + options.saveStoppedInstanceIds?.([]) } else if (action === 'stop') { - await api.update_network_instance_state(instanceId, true) + // Save before stopping so a partially completed action can also be resumed. + options.saveStoppedInstanceIds?.([...candidates]) + for (const candidateId of candidates) { + await api.update_network_instance_state(candidateId, true) + } changed = true } diff --git a/easytier-gui/src/pages/index.vue b/easytier-gui/src/pages/index.vue index 2ce5d455..53b98a78 100644 --- a/easytier-gui/src/pages/index.vue +++ b/easytier-gui/src/pages/index.vue @@ -13,6 +13,7 @@ import { consumePendingMobileVpnTileAction, initMobileVpnService, setMobileVpnTileActionHandler, + setMobileVpnErrorHandler, syncMobileVpnService, } from '~/composables/mobile_vpn' import { executeVpnTileAction } from '~/composables/mobile_vpn_tile' @@ -20,7 +21,12 @@ import { GUIRemoteClient } from '~/modules/api' import { useToast, useConfirm } from 'primevue' import { loadMode, saveMode, WebClientConfig, type Mode } from '~/composables/mode' -import { saveLastNetworkInstanceId, loadLastNetworkInstanceId } from '~/composables/config' +import { + saveLastNetworkInstanceId, + loadLastNetworkInstanceId, + loadVpnTileStoppedInstanceIds, + saveVpnTileStoppedInstanceIds, +} from '~/composables/config' import ModeSwitcher from '~/components/ModeSwitcher.vue' import { getEasytierVersion, getServiceStatus } from '~/composables/backend' @@ -227,6 +233,8 @@ onMounted(async () => { const cleanupFns: Array<() => void> = [] if (type() === 'android') { + setMobileVpnErrorHandler(error => toast.add({ severity: 'error', summary: t('error'), detail: error, life: 10000 })) + cleanupFns.push(() => setMobileVpnErrorHandler()) try { await initMobileVpnService() } catch (e: any) { @@ -265,6 +273,8 @@ async function handleMobileVpnTileAction(action: 'start' | 'stop') { try { const result = await executeVpnTileAction(action, remoteClient.value, { lastInstanceId: loadLastNetworkInstanceId(), + stoppedInstanceIds: loadVpnTileStoppedInstanceIds(), + saveStoppedInstanceIds: saveVpnTileStoppedInstanceIds, syncVpnService: syncMobileVpnService, }) diff --git a/easytier/Cargo.toml b/easytier/Cargo.toml index 6939019e..595c7fff 100644 --- a/easytier/Cargo.toml +++ b/easytier/Cargo.toml @@ -365,7 +365,7 @@ mimalloc = ["dep:mimalloc"] aes-gcm = ["easytier-core/aes-gcm"] openssl-crypto = ["easytier-core/openssl-crypto"] ring-crypto = ["easytier-core/ring-crypto"] -tun = ["dep:tun", "linux-netlink"] +tun = ["dep:tun", "dep:smoltcp", "linux-netlink", "tokio/rt-multi-thread"] linux-netlink = ["dep:netlink-sys"] proxy-cidr-monitor = ["easytier-core/proxy-cidr-monitor"] websocket = [ diff --git a/easytier/src/common/ifcfg/darwin.rs b/easytier/src/common/ifcfg/darwin.rs index 0412d9e2..d4e5e51a 100644 --- a/easytier/src/common/ifcfg/darwin.rs +++ b/easytier/src/common/ifcfg/darwin.rs @@ -1,10 +1,44 @@ -use std::net::Ipv4Addr; +use std::{collections::BTreeSet, net::Ipv4Addr}; -use super::{Error, IfConfiguerTrait, cidr_to_subnet_mask, run_shell_cmd}; +use super::{ + Error, IfConfiguerTrait, cidr_to_subnet_mask, ignore_removed_shell_cmd_not_found, run_shell_cmd, +}; use async_trait::async_trait; use cidr::{Ipv4Inet, Ipv6Inet}; +use tokio::sync::Mutex; + +#[derive(Default)] +pub struct MacIfConfiger { + configured_ipv4: Mutex>, +} + +impl MacIfConfiger { + fn build_add_ipv4_cmd(name: &str, addr: Ipv4Inet, has_configured_ipv4: bool) -> String { + let address = addr.address(); + if has_configured_ipv4 { + format!( + "ifconfig {} alias {:?} {:?} netmask {}", + name, + address, + address, + cidr_to_subnet_mask(addr.network_length()) + ) + } else { + format!( + "ifconfig {} {:?}/{:?} {:?} up", + name, + address, + addr.network_length(), + address, + ) + } + } +} + +async fn run_delete_shell_cmd(cmd: &str) -> Result<(), Error> { + ignore_removed_shell_cmd_not_found(run_shell_cmd(cmd).await) +} -pub struct MacIfConfiger {} #[async_trait] impl IfConfiguerTrait for MacIfConfiger { async fn add_ipv4_route( @@ -14,12 +48,28 @@ impl IfConfiguerTrait for MacIfConfiger { cidr_prefix: u8, cost: Option, ) -> Result<(), Error> { + self.add_ipv4_route_with_source_hint(name, address, cidr_prefix, cost, None) + .await + } + + async fn add_ipv4_route_with_source_hint( + &self, + name: &str, + address: Ipv4Addr, + cidr_prefix: u8, + cost: Option, + source_hint: Option, + ) -> Result<(), Error> { + let source_hint = source_hint + .map(|source| format!(" -ifa {}", source)) + .unwrap_or_default(); run_shell_cmd( format!( - "route -n add {} -netmask {} -interface {} -hopcount {}", + "route -n add {} -netmask {} -interface {}{} -hopcount {}", address, cidr_to_subnet_mask(cidr_prefix), name, + source_hint, cost.unwrap_or(7) ) .as_str(), @@ -33,7 +83,7 @@ impl IfConfiguerTrait for MacIfConfiger { address: Ipv4Addr, cidr_prefix: u8, ) -> Result<(), Error> { - run_shell_cmd( + run_delete_shell_cmd( format!( "route -n delete {} -netmask {} -interface {}", address, @@ -51,14 +101,15 @@ impl IfConfiguerTrait for MacIfConfiger { address: Ipv4Addr, cidr_prefix: u8, ) -> Result<(), Error> { - run_shell_cmd( - format!( - "ifconfig {} {:?}/{:?} {:?} up", - name, address, cidr_prefix, address, - ) - .as_str(), - ) - .await + let addr = Ipv4Inet::new(address, cidr_prefix).map_err(|err| { + anyhow::anyhow!("invalid IPv4 address {address}/{cidr_prefix}: {err:?}") + })?; + let mut configured_ipv4 = self.configured_ipv4.lock().await; + let cmd = Self::build_add_ipv4_cmd(name, addr, !configured_ipv4.is_empty()); + + run_shell_cmd(cmd.as_str()).await?; + configured_ipv4.insert(addr); + Ok(()) } async fn set_link_status(&self, name: &str, up: bool) -> Result<(), Error> { @@ -67,11 +118,18 @@ impl IfConfiguerTrait for MacIfConfiger { } async fn remove_ip(&self, name: &str, ip: Option) -> Result<(), Error> { + let mut configured_ipv4 = self.configured_ipv4.lock().await; if let Some(ip) = ip { - run_shell_cmd(format!("ifconfig {} inet {} delete", name, ip.address()).as_str()).await + run_delete_shell_cmd( + format!("ifconfig {} inet {} delete", name, ip.address()).as_str(), + ) + .await?; + configured_ipv4.remove(&ip); } else { - run_shell_cmd(format!("ifconfig {} inet delete", name).as_str()).await + run_delete_shell_cmd(format!("ifconfig {} inet delete", name).as_str()).await?; + configured_ipv4.clear(); } + Ok(()) } async fn set_mtu(&self, name: &str, mtu: u32) -> Result<(), Error> { @@ -90,7 +148,10 @@ impl IfConfiguerTrait for MacIfConfiger { async fn remove_ipv6(&self, name: &str, ip: Option) -> Result<(), Error> { if let Some(ip) = ip { - run_shell_cmd(format!("ifconfig {} inet6 {} delete", name, ip.address()).as_str()).await + run_delete_shell_cmd( + format!("ifconfig {} inet6 {} delete", name, ip.address()).as_str(), + ) + .await } else { // Remove all IPv6 addresses is more complex on macOS, just succeed Ok(()) @@ -124,7 +185,7 @@ impl IfConfiguerTrait for MacIfConfiger { address: std::net::Ipv6Addr, cidr_prefix: u8, ) -> Result<(), Error> { - run_shell_cmd( + run_delete_shell_cmd( format!( "route -n delete -inet6 {}/{} -interface {}", address, cidr_prefix, name diff --git a/easytier/src/common/ifcfg/mod.rs b/easytier/src/common/ifcfg/mod.rs index 0a64a33c..0069f176 100644 --- a/easytier/src/common/ifcfg/mod.rs +++ b/easytier/src/common/ifcfg/mod.rs @@ -40,6 +40,16 @@ pub trait IfConfiguerTrait: Send + Sync { ) -> Result<(), Error> { Ok(()) } + async fn add_ipv4_route_with_source_hint( + &self, + name: &str, + address: Ipv4Addr, + cidr_prefix: u8, + cost: Option, + _source_hint: Option, + ) -> Result<(), Error> { + self.add_ipv4_route(name, address, cidr_prefix, cost).await + } async fn remove_ipv4_route( &self, _name: &str, @@ -48,6 +58,16 @@ pub trait IfConfiguerTrait: Send + Sync { ) -> Result<(), Error> { Ok(()) } + async fn remove_ipv4_route_with_cost_and_source_hint( + &self, + name: &str, + address: Ipv4Addr, + cidr_prefix: u8, + _cost: Option, + _source_hint: Option, + ) -> Result<(), Error> { + self.remove_ipv4_route(name, address, cidr_prefix).await + } async fn add_ipv4_ip( &self, _name: &str, @@ -142,7 +162,12 @@ async fn run_shell_cmd(cmd: &str) -> Result<(), Error> { #[cfg(not(target_os = "windows"))] { - cmd_out = Command::new("sh").arg("-c").arg(cmd).output().await?; + cmd_out = Command::new("sh") + .env("LC_ALL", "C") + .arg("-c") + .arg(cmd) + .output() + .await?; stdout = String::from_utf8_lossy(cmd_out.stdout.as_slice()).to_string(); stderr = String::from_utf8_lossy(cmd_out.stderr.as_slice()).to_string(); }; @@ -157,6 +182,34 @@ async fn run_shell_cmd(cmd: &str) -> Result<(), Error> { Ok(()) } +#[cfg(any( + all(target_os = "macos", not(feature = "macos-ne")), + target_os = "freebsd", + test +))] +fn ignore_removed_shell_cmd_not_found(result: Result<(), Error>) -> Result<(), Error> { + match result { + Err(Error::ShellCommandError(output)) + if !output.trim().is_empty() + && output + .lines() + .filter(|line| !line.trim().is_empty()) + .all(|line| { + (line.starts_with("ifconfig: ") + && line.ends_with(": Can't assign requested address")) + || (line.starts_with("delete ") && line.ends_with(": not in table")) + || line == "route: route has not been found" + || line == "route: writing to routing socket: not in table" + || line == "route: writing to routing socket: No such process" + }) => + { + Ok(()) + } + other => other, + } +} + +#[derive(Default)] pub struct DummyIfConfiger {} #[async_trait] impl IfConfiguerTrait for DummyIfConfiger {} @@ -221,3 +274,54 @@ pub(crate) fn list_ipv6_ndp_proxy( ) -> Result, Error> { netlink::NetlinkIfConfiger::list_ipv6_ndp_proxy(name) } + +#[cfg(test)] +mod tests { + use super::{Error, ignore_removed_shell_cmd_not_found}; + + #[test] + fn removed_shell_cmd_ignores_missing_addresses_and_routes() { + for output in [ + "ifconfig: ioctl (SIOCDIFADDR): Can't assign requested address\n", + "ifconfig: ioctl (SIOCDIFADDR_IN6): Can't assign requested address\n", + "delete net 10.0.0.0: gateway tun0: not in table\n", + "delete net 10.0.0.0: gateway tun0 fib 0: not in table\nroute: route has not been found\n", + "delete net 10.0.0.0: gateway tun0 fib 0: not in table\nroute: writing to routing socket: No such process\n", + "delete net 10.0.0.0: gateway tun0: not in table\nroute: writing to routing socket: not in table\n", + ] { + assert!( + ignore_removed_shell_cmd_not_found(Err(Error::ShellCommandError( + output.to_owned() + ))) + .is_ok() + ); + } + } + + #[test] + fn removed_shell_cmd_preserves_other_errors() { + for output in [ + "ifconfig: ioctl (SIOCDIFADDR): Operation not permitted\n", + "route: writing to routing socket: Permission denied\n", + "route: route has not been found: Permission denied\n", + "route: route has not been found: No such process\n", + "delete net 10.0.0.0: gateway tun0: not in table\nroute: Permission denied\n", + "delete net 10.0.0.0: gateway tun0: not in table\nroute: writing to routing socket: No such process\nroute: Permission denied\n", + "unexpected command: not in table\n", + "", + ] { + assert!(matches!( + ignore_removed_shell_cmd_not_found(Err(Error::ShellCommandError(output.to_owned()))), + Err(Error::ShellCommandError(actual)) if actual == output + )); + } + + assert!(matches!( + ignore_removed_shell_cmd_not_found(Err(std::io::Error::from( + std::io::ErrorKind::PermissionDenied + ) + .into())), + Err(Error::IOError(_)) + )); + } +} diff --git a/easytier/src/common/ifcfg/netlink.rs b/easytier/src/common/ifcfg/netlink.rs index a401e744..d5068325 100644 --- a/easytier/src/common/ifcfg/netlink.rs +++ b/easytier/src/common/ifcfg/netlink.rs @@ -133,6 +133,7 @@ fn dump_netlink_messages( receive_netlink_dump(builder) } +#[derive(Default)] pub struct NetlinkIfConfiger {} impl NetlinkIfConfiger { @@ -335,6 +336,27 @@ impl NetlinkIfConfiger { }) .collect()) } + + fn ipv4_route_message( + ifindex: u32, + address: Ipv4Addr, + cidr_prefix: u8, + cost: Option, + source_hint: Option, + ) -> RouteMessage { + let mut builder = RouteMessageBuilder::new(libc::AF_INET as u8) + .destination(IpAddr::V4(address), cidr_prefix) + .oif(ifindex) + .priority(cost.unwrap_or(65535) as u32) + .table(libc::RT_TABLE_MAIN.into()) + .static_protocol() + .universe_scope() + .route_type(RouteType::Unicast); + if let Some(source_hint) = source_hint { + builder = builder.preferred_source(IpAddr::V4(source_hint)); + } + builder.build() + } } #[async_trait] @@ -346,15 +368,25 @@ impl IfConfiguerTrait for NetlinkIfConfiger { cidr_prefix: u8, cost: Option, ) -> Result<(), Error> { - let message = RouteMessageBuilder::new(libc::AF_INET as u8) - .destination(IpAddr::V4(address), cidr_prefix) - .oif(Self::get_interface_index(name)?) - .priority(cost.unwrap_or(65535) as u32) - .table(libc::RT_TABLE_MAIN.into()) - .static_protocol() - .universe_scope() - .route_type(RouteType::Unicast) - .build(); + self.add_ipv4_route_with_source_hint(name, address, cidr_prefix, cost, None) + .await + } + + async fn add_ipv4_route_with_source_hint( + &self, + name: &str, + address: Ipv4Addr, + cidr_prefix: u8, + cost: Option, + source_hint: Option, + ) -> Result<(), Error> { + let message = NetlinkIfConfiger::ipv4_route_message( + NetlinkIfConfiger::get_interface_index(name)?, + address, + cidr_prefix, + cost, + source_hint, + ); let request = message_request( RTM_NEWROUTE, NLM_F_ACK | NLM_F_CREATE | NLM_F_EXCL | NLM_F_REQUEST, @@ -390,6 +422,28 @@ impl IfConfiguerTrait for NetlinkIfConfiger { Ok(()) } + async fn remove_ipv4_route_with_cost_and_source_hint( + &self, + name: &str, + address: Ipv4Addr, + cidr_prefix: u8, + cost: Option, + source_hint: Option, + ) -> Result<(), Error> { + let message = Self::ipv4_route_message( + Self::get_interface_index(name)?, + address, + cidr_prefix, + cost, + source_hint, + ); + let request = message_request(RTM_DELROUTE, NLM_F_ACK | NLM_F_REQUEST, &message)?; + match send_netlink_req_and_wait_ack(request) { + Err(Error::IOError(err)) if err.raw_os_error() == Some(libc::ESRCH) => Ok(()), + result => result, + } + } + async fn add_ipv4_ip( &self, name: &str, @@ -696,6 +750,49 @@ mod tests { assert!(!routes.contains(&IpAddr::V4("10.5.5.0".parse().unwrap()))); } + #[serial_test::serial] + #[tokio::test] + async fn remove_ipv4_route_with_source_hint_keeps_other_metric() { + let iface = test_iface_name("rm"); + let _link = ScopedDummyLink::new(&iface); + let ifcfg = NetlinkIfConfiger {}; + let address = "10.231.1.1".parse().unwrap(); + let destination = "10.99.0.0".parse().unwrap(); + + ifcfg.add_ipv4_ip(&iface, address, 24).await.unwrap(); + for cost in [123, 124] { + ifcfg + .add_ipv4_route_with_source_hint(&iface, destination, 24, Some(cost), Some(address)) + .await + .unwrap(); + } + + ifcfg + .remove_ipv4_route_with_cost_and_source_hint( + &iface, + destination, + 24, + Some(123), + Some(address), + ) + .await + .unwrap(); + let routes = run_cmd(&format!("ip -4 route show 10.99.0.0/24 dev {iface}")); + assert!(!routes.contains("metric 123")); + assert!(routes.contains("metric 124")); + + ifcfg + .remove_ipv4_route_with_cost_and_source_hint( + &iface, + destination, + 24, + Some(123), + Some(address), + ) + .await + .unwrap(); + } + #[serial_test::serial] #[tokio::test] async fn ipv6_addr_readback_test() { diff --git a/easytier/src/common/ifcfg/netlink_wire.rs b/easytier/src/common/ifcfg/netlink_wire.rs index 6bd92925..7dbb7284 100644 --- a/easytier/src/common/ifcfg/netlink_wire.rs +++ b/easytier/src/common/ifcfg/netlink_wire.rs @@ -37,6 +37,7 @@ const RTA_DST: u16 = 1; const RTA_SRC: u16 = 2; const RTA_OIF: u16 = 4; const RTA_PRIORITY: u16 = 6; +const RTA_PREFSRC: u16 = 7; const RTA_TABLE: u16 = 15; const NDA_DST: u16 = 1; @@ -488,6 +489,13 @@ impl RouteMessageBuilder { self } + pub(crate) fn preferred_source(mut self, address: IpAddr) -> Self { + self.message + .attributes + .push(Attribute::new(RTA_PREFSRC, ip_bytes(address))); + self + } + pub(crate) fn table(mut self, table: u32) -> Self { if let Ok(table) = u8::try_from(table) { self.message.table = table; @@ -632,6 +640,22 @@ mod tests { assert_eq!(encode(&decoded), bytes); } + #[test] + fn route_builder_encodes_preferred_source() { + let source = "10.231.1.1".parse().unwrap(); + let message = RouteMessageBuilder::new(libc::AF_INET as u8) + .destination("10.99.0.0".parse().unwrap(), 24) + .preferred_source(source) + .oif(7) + .table(libc::RT_TABLE_MAIN.into()) + .build(); + let bytes = encode(&message); + let attributes = parse_attributes(&bytes[12..]).unwrap(); + assert!(attributes.iter().any(|attribute| { + attribute.kind == RTA_PREFSRC && attribute.value == ip_bytes(source) + })); + } + #[test] fn route_parser_reads_ipv6_source_prefix() { let mut bytes = vec![ diff --git a/easytier/src/common/ifcfg/windows.rs b/easytier/src/common/ifcfg/windows.rs index 15e4b23d..fc14e966 100644 --- a/easytier/src/common/ifcfg/windows.rs +++ b/easytier/src/common/ifcfg/windows.rs @@ -9,7 +9,7 @@ use std::{ }; use windows::Win32::NetworkManagement::IpHelper::INTERNAL_IF_OPER_STATUS; use windows::Win32::{ - Foundation::NO_ERROR, + Foundation::{ERROR_NOT_FOUND, NO_ERROR}, NetworkManagement::IpHelper::{GetIfEntry, MIB_IFROW, SetIfEntry}, System::Diagnostics::Debug::{ FORMAT_MESSAGE_FROM_SYSTEM, FORMAT_MESSAGE_IGNORE_INSERTS, FormatMessageW, @@ -22,8 +22,16 @@ use winreg::{ }; use super::{Error, IfConfiguerTrait}; +#[derive(Default)] pub struct WindowsIfConfiger {} +fn ignore_removed_not_found(result: Result<(), u32>) -> Result<(), u32> { + match result { + Err(error) if error == ERROR_NOT_FOUND.0 => Ok(()), + other => other, + } +} + fn format_win_error(error: u32) -> String { // use FormatMessageW to get the error message let mut buffer = vec![0; 1024]; @@ -74,7 +82,7 @@ impl WindowsIfConfiger { anyhow::anyhow!("Failed to get interface luid: {}", format_win_error(e)) })?; if let Some(addr) = addr { - luid.delete_ipv4_address(&addr).map_err(|e| { + ignore_removed_not_found(luid.delete_ipv4_address(&addr)).map_err(|e| { anyhow::anyhow!("Failed to delete IPv4 address: {}", format_win_error(e)) })?; } else { @@ -167,7 +175,7 @@ impl WindowsIfConfiger { anyhow::anyhow!("Failed to get interface luid: {}", format_win_error(e)) })?; if let Some(addr) = addr { - luid.delete_ipv6_address(&addr).map_err(|e| { + ignore_removed_not_found(luid.delete_ipv6_address(&addr)).map_err(|e| { anyhow::anyhow!("Failed to delete IPv6 address: {}", format_win_error(e)) })?; } else { @@ -216,10 +224,10 @@ impl IfConfiguerTrait for WindowsIfConfiger { let luid = InterfaceLuid::luid_from_index(if_index).map_err(|e| { anyhow::anyhow!("Failed to get interface luid: {}", format_win_error(e)) })?; - luid.delete_route_ipv4( + ignore_removed_not_found(luid.delete_route_ipv4( &Ipv4Inet::new(address, cidr_prefix).unwrap(), &Ipv4Addr::UNSPECIFIED, - ) + )) .map_err(|e| anyhow::anyhow!("Failed to delete route: {}", format_win_error(e)))?; Ok(()) } @@ -306,10 +314,10 @@ impl IfConfiguerTrait for WindowsIfConfiger { anyhow::anyhow!("Failed to get interface luid: {}", format_win_error(e)) })?; - luid.delete_route_ipv6( + ignore_removed_not_found(luid.delete_route_ipv6( &Ipv6Inet::new(address, cidr_prefix).unwrap(), &Ipv6Addr::UNSPECIFIED, - ) + )) .map_err(|e| anyhow::anyhow!("Failed to delete route: {}", format_win_error(e)))?; Ok(()) } @@ -530,3 +538,24 @@ impl RegistryManager { ) } } + +#[cfg(test)] +mod tests { + use super::{ERROR_NOT_FOUND, ignore_removed_not_found}; + use windows::Win32::Foundation::{ + ERROR_ACCESS_DENIED, ERROR_FILE_NOT_FOUND, ERROR_INVALID_PARAMETER, + }; + + #[test] + fn removed_windows_ifcfg_ignores_only_not_found() { + assert_eq!(ignore_removed_not_found(Ok(())), Ok(())); + assert_eq!(ignore_removed_not_found(Err(ERROR_NOT_FOUND.0)), Ok(())); + for error in [ + ERROR_ACCESS_DENIED.0, + ERROR_INVALID_PARAMETER.0, + ERROR_FILE_NOT_FOUND.0, + ] { + assert_eq!(ignore_removed_not_found(Err(error)), Err(error)); + } + } +} diff --git a/easytier/src/instance/composition.rs b/easytier/src/instance/composition.rs index f6cc4a73..c1bc0078 100644 --- a/easytier/src/instance/composition.rs +++ b/easytier/src/instance/composition.rs @@ -31,6 +31,8 @@ use crate::{ instance::runtime_host::NativeInstanceRuntimeHost, }; +#[cfg(feature = "tun")] +use super::shared_virtual_nic::ArcSharedVirtualNicRegistry; use super::{ host::{NativeInstanceHost, native_instance_host}, runtime_executor::NativeInstanceExecutor, @@ -48,6 +50,7 @@ pub(crate) type NativeCoreInstance = CoreInstance; pub(super) fn compose_native_core_instance( toml_config: TomlConfig, process_runtime: Arc, + #[cfg(feature = "tun")] shared_virtual_nic_registry: ArcSharedVirtualNicRegistry, compact_runtime: bool, executor: Option, ) -> anyhow::Result> { @@ -62,7 +65,12 @@ pub(super) fn compose_native_core_instance( &normalized, &host_config, )); - let runtime_host = NativeInstanceRuntimeHost::new_with_executor(global_ctx.clone(), executor); + let runtime_host = NativeInstanceRuntimeHost::new_with_executor( + global_ctx.clone(), + #[cfg(feature = "tun")] + shared_virtual_nic_registry, + executor, + ); let mut adapters = runtime_core_host_adapters_with_packet_egress_and_config( global_ctx.clone(), process_runtime, diff --git a/easytier/src/instance/dns_server/runner.rs b/easytier/src/instance/dns_server/runner.rs index 76791c68..8e79806f 100644 --- a/easytier/src/instance/dns_server/runner.rs +++ b/easytier/src/instance/dns_server/runner.rs @@ -5,7 +5,7 @@ use std::{net::Ipv4Addr, sync::Arc, time::Duration}; use easytier_core::instance::CorePacketPlane; -use crate::common::global_ctx::ArcGlobalCtx; +use crate::{common::global_ctx::ArcGlobalCtx, instance::virtual_nic::NicBackend}; use super::{client_instance::MagicDnsClientInstance, server_instance::MagicDnsServerInstance}; @@ -17,6 +17,34 @@ pub struct DnsRunner { tun_dev: Option, tun_inet: Ipv4Inet, fake_ip: Ipv4Addr, + shared_route_backend: Option, + active_fake_ip_route: Option, +} + +struct MagicDnsFakeIpRouteClaim { + fake_ip: Ipv4Addr, + route_backend: NicBackend, +} + +impl MagicDnsFakeIpRouteClaim { + async fn add(&self) -> anyhow::Result<()> { + let cost = if cfg!(target_os = "windows") { + Some(4) + } else { + None + }; + + self.route_backend + .add_route_with_cost(self.fake_ip, 32, cost) + .await?; + Ok(()) + } + + async fn remove(&self) { + if let Err(err) = self.route_backend.remove_route(self.fake_ip, 32).await { + tracing::warn!(?err, fake_ip = ?self.fake_ip, "remove magic dns route failed"); + } + } } impl DnsRunner { @@ -35,29 +63,74 @@ impl DnsRunner { tun_dev, tun_inet, fake_ip, + shared_route_backend: None, + active_fake_ip_route: None, } } + pub(crate) fn with_shared_route_backend(mut self, route_backend: Option) -> Self { + self.shared_route_backend = route_backend; + self + } + async fn clean_env(&mut self) { + if let Some(claim) = self.active_fake_ip_route.take() { + claim.remove().await; + } if let Some(server) = self.server.take() { server.clean_env().await; } self.client.take(); } - async fn run_once(&mut self) -> anyhow::Result<()> { + fn should_manage_fake_ip_route_externally(&self) -> bool { + self.shared_route_backend.is_some() && !self.tun_inet.contains(&self.fake_ip) + } + + fn fake_ip_route_claim(&self) -> Option { + if !self.should_manage_fake_ip_route_externally() { + return None; + } + + Some(MagicDnsFakeIpRouteClaim { + fake_ip: self.fake_ip, + route_backend: self.shared_route_backend.clone()?, + }) + } + + async fn run_once(&mut self, cancel_token: &CancellationToken) -> anyhow::Result<()> { // try server first - match MagicDnsServerInstance::new( - self.packet_plane.clone(), - self.global_ctx.clone(), - self.tun_dev.clone(), - self.tun_inet, - self.fake_ip, - ) - .await - { + let server_result = if self.should_manage_fake_ip_route_externally() { + MagicDnsServerInstance::new_with_external_fake_ip_route( + self.packet_plane.clone(), + self.global_ctx.clone(), + self.tun_dev.clone(), + self.tun_inet, + self.fake_ip, + ) + .await + } else { + MagicDnsServerInstance::new( + self.packet_plane.clone(), + self.global_ctx.clone(), + self.tun_dev.clone(), + self.tun_inet, + self.fake_ip, + ) + .await + }; + + match server_result { Ok(server) => { self.server = Some(server); + // Keep the initialized resolver owned by the runner before adding + // its route, so route failure still reaches async server cleanup. + self.active_fake_ip_route = self.fake_ip_route_claim(); + if let Some(claim) = &self.active_fake_ip_route { + claim.add().await.map_err(|err| { + anyhow::anyhow!("failed to add magic dns fake-ip route: {err}") + })?; + } tracing::info!("DnsRunner::run_once: server started"); } Err(e) => { @@ -65,34 +138,37 @@ impl DnsRunner { } } - // every runner must run a client - let client = MagicDnsClientInstance::new(self.packet_plane.clone()).await?; - self.client = Some(client); - self.client.as_mut().unwrap().run_and_wait().await; - - Err(anyhow::anyhow!("Client instance exit")) - } - - pub async fn run(&mut self, canel_token: CancellationToken) { - loop { - tracing::info!("DnsRunner::run: start"); - tokio::select! { - _ = canel_token.cancelled() => { - self.clean_env().await; - tracing::info!("DnsRunner::run: cancelled"); - return; - } - - ret = self.run_once() => { - self.clean_env().await; - if let Err(e) = ret { - tracing::error!("DnsRunner::run: {:?}", e); - } else { - tracing::info!("DnsRunner::run: unexpected exit, server may be down"); - } - tokio::time::sleep(Duration::from_millis(500)).await; - } - } + // Every runner must run a client. Only this phase is cancellable: server + // initialization and route updates must finish before async cleanup. + tokio::select! { + _ = cancel_token.cancelled() => Ok(()), + ret = async { + let client = MagicDnsClientInstance::new(self.packet_plane.clone()).await?; + self.client = Some(client); + self.client.as_mut().unwrap().run_and_wait().await; + Err(anyhow::anyhow!("Client instance exit")) + } => ret, } } + + pub async fn run(&mut self, cancel_token: CancellationToken) { + while !cancel_token.is_cancelled() { + tracing::info!("DnsRunner::run: start"); + let ret = self.run_once(&cancel_token).await; + self.clean_env().await; + if cancel_token.is_cancelled() { + break; + } + if let Err(e) = ret { + tracing::error!("DnsRunner::run: {:?}", e); + } else { + tracing::info!("DnsRunner::run: unexpected exit, server may be down"); + } + tokio::select! { + _ = cancel_token.cancelled() => break, + _ = tokio::time::sleep(Duration::from_millis(500)) => {} + } + } + tracing::info!("DnsRunner::run: cancelled"); + } } diff --git a/easytier/src/instance/dns_server/server_instance.rs b/easytier/src/instance/dns_server/server_instance.rs index 4c1c3c00..2908f222 100644 --- a/easytier/src/instance/dns_server/server_instance.rs +++ b/easytier/src/instance/dns_server/server_instance.rs @@ -14,8 +14,10 @@ use super::{ }; use crate::{ common::{ + error::Error as EtError, global_ctx::ArcGlobalCtx, ifcfg::{IfConfiger, IfConfiguerTrait}, + netns::NetNS, }, instance::dns_server::{ config::{Record, RecordBuilder, RecordType}, @@ -51,7 +53,9 @@ use std::{collections::BTreeMap, io, net::Ipv4Addr, str::FromStr, sync::Arc, tim pub(super) struct MagicDnsServerInstanceData { dns_server: Server, tun_dev: Option, + net_ns: NetNS, fake_ip: Ipv4Addr, + manage_fake_ip_route: bool, route_store: MagicDnsRecordStore, record_apply: tokio::sync::Mutex<()>, @@ -356,12 +360,66 @@ fn get_system_config( } impl MagicDnsServerInstance { + async fn add_fake_ip_route( + tun_dev_name: &str, + fake_ip: Ipv4Addr, + net_ns: &NetNS, + cost: Option, + ) -> Result<(), anyhow::Error> { + let ifcfg = IfConfiger::default(); + let _guard = net_ns.guard(); + match ifcfg.add_ipv4_route(tun_dev_name, fake_ip, 32, cost).await { + Err(EtError::IOError(err)) if err.kind() == io::ErrorKind::AlreadyExists => { + ifcfg.remove_ipv4_route(tun_dev_name, fake_ip, 32).await?; + ifcfg + .add_ipv4_route(tun_dev_name, fake_ip, 32, cost) + .await?; + Ok(()) + } + ret => ret.map_err(Into::into), + } + } + + async fn remove_fake_ip_route(tun_dev_name: &str, fake_ip: Ipv4Addr, net_ns: &NetNS) { + let ifcfg = IfConfiger::default(); + let _guard = net_ns.guard(); + if let Err(err) = ifcfg.remove_ipv4_route(tun_dev_name, fake_ip, 32).await { + tracing::warn!( + ?err, + ?tun_dev_name, + ?fake_ip, + "remove magic dns route failed" + ); + } + } + pub(crate) async fn new( packet_plane: Arc, global_ctx: ArcGlobalCtx, tun_dev: Option, tun_inet: Ipv4Inet, fake_ip: Ipv4Addr, + ) -> Result { + Self::new_inner(packet_plane, global_ctx, tun_dev, tun_inet, fake_ip, true).await + } + + pub(crate) async fn new_with_external_fake_ip_route( + packet_plane: Arc, + global_ctx: ArcGlobalCtx, + tun_dev: Option, + tun_inet: Ipv4Inet, + fake_ip: Ipv4Addr, + ) -> Result { + Self::new_inner(packet_plane, global_ctx, tun_dev, tun_inet, fake_ip, false).await + } + + async fn new_inner( + packet_plane: Arc, + global_ctx: ArcGlobalCtx, + tun_dev: Option, + tun_inet: Ipv4Inet, + fake_ip: Ipv4Addr, + manage_fake_ip_route: bool, ) -> Result { let tcp_listener = runtime_rpc_listener(MAGIC_DNS_INSTANCE_SOCKET_ADDR.parse()?); let mut rpc_server = StandAloneServer::new(tcp_listener); @@ -374,7 +432,8 @@ impl MagicDnsServerInstance { let mut dns_server = Server::new(dns_config); dns_server.run().await?; - if !tun_inet.contains(&fake_ip) + if manage_fake_ip_route + && !tun_inet.contains(&fake_ip) && let Some(tun_dev_name) = &tun_dev { let cost = if cfg!(target_os = "windows") { @@ -382,16 +441,15 @@ impl MagicDnsServerInstance { } else { None }; - let ifcfg = IfConfiger {}; - ifcfg - .add_ipv4_route(tun_dev_name, fake_ip, 32, cost) - .await?; + Self::add_fake_ip_route(tun_dev_name, fake_ip, &global_ctx.net_ns, cost).await?; } let data = Arc::new(MagicDnsServerInstanceData { dns_server, tun_dev: tun_dev.clone(), + net_ns: global_ctx.net_ns.clone(), fake_ip, + manage_fake_ip_route, route_store: MagicDnsRecordStore::default(), record_apply: tokio::sync::Mutex::new(()), system_config: get_system_config(tun_dev.as_deref())?, @@ -436,14 +494,13 @@ impl MagicDnsServerInstance { if let Err(e) = ret { tracing::error!("Failed to close system config: {:?}", e); } - if !self.tun_inet.contains(&self.data.fake_ip) - && let Some(tun_dev_name) = &self.data.tun_dev - { - let ifcfg = IfConfiger {}; - let _ = ifcfg - .remove_ipv4_route(tun_dev_name, self.data.fake_ip, 32) - .await; - } + } + + if self.data.manage_fake_ip_route + && !self.tun_inet.contains(&self.data.fake_ip) + && let Some(tun_dev_name) = &self.data.tun_dev + { + Self::remove_fake_ip_route(tun_dev_name, self.data.fake_ip, &self.data.net_ns).await; } self.packet_filter.close().await; diff --git a/easytier/src/instance/factory.rs b/easytier/src/instance/factory.rs index 71c341ce..059f1ea9 100644 --- a/easytier/src/instance/factory.rs +++ b/easytier/src/instance/factory.rs @@ -1,5 +1,8 @@ use std::sync::Arc; +#[cfg(feature = "tun")] +use tokio::sync::Mutex; + #[cfg(any(feature = "management-rpc", test))] use easytier_core::instance::manager::InstanceManager; #[cfg(feature = "management-rpc")] @@ -12,12 +15,20 @@ use easytier_core::{ use crate::common::global_ctx::EventBusSubscriber; +#[cfg(feature = "tun")] +use super::shared_virtual_nic::{ArcSharedVirtualNicRegistry, SharedVirtualNicRegistry}; use super::{ composition::compose_native_core_instance, host::NativeInstanceHost, runtime_executor::NativeInstanceExecutor, runtime_host::NativeInstanceRuntimeHost, }; pub type NativeCoreInstance = CoreInstance; +#[cfg(all(feature = "management-rpc", feature = "tun"))] +mod mobile_tun; +#[cfg(all(feature = "management-rpc", feature = "tun"))] +pub use mobile_tun::MobileTunConfig; +#[cfg(all(feature = "management-rpc", feature = "tun", mobile))] +pub use mobile_tun::{attach_mobile_tun_fd, get_mobile_tun_config}; #[cfg(feature = "management-rpc")] pub type NativeInstanceManager = InstanceManager; #[cfg(feature = "management")] @@ -97,6 +108,8 @@ fn native_instance_manager_with_optional_runtime( /// Native construction Adapter for the canonical core InstanceManager. pub struct NativeInstanceFactory { process_runtime: Arc, + #[cfg(feature = "tun")] + shared_virtual_nic_registry: ArcSharedVirtualNicRegistry, runtime_handle: Option, compact_runtime: bool, #[cfg(feature = "logging")] @@ -107,6 +120,8 @@ impl NativeInstanceFactory { pub fn new(process_runtime: Arc) -> Self { Self { process_runtime, + #[cfg(feature = "tun")] + shared_virtual_nic_registry: Arc::new(Mutex::new(SharedVirtualNicRegistry::new())), runtime_handle: None, compact_runtime: false, #[cfg(feature = "logging")] @@ -126,6 +141,7 @@ impl NativeInstanceFactory { self } + #[cfg(feature = "management-rpc")] fn with_compact_runtime(mut self) -> Self { self.compact_runtime = true; self @@ -159,6 +175,8 @@ impl InstanceFactory for NativeInstanceFactory { let instance = compose_native_core_instance( config, self.process_runtime.clone(), + #[cfg(feature = "tun")] + self.shared_virtual_nic_registry.clone(), self.compact_runtime, executor, )?; diff --git a/easytier/src/instance/factory/mobile_tun.rs b/easytier/src/instance/factory/mobile_tun.rs new file mode 100644 index 00000000..da1f9ad9 --- /dev/null +++ b/easytier/src/instance/factory/mobile_tun.rs @@ -0,0 +1,393 @@ +#[cfg(any(mobile, test))] +use std::collections::BTreeSet; + +#[cfg(mobile)] +use super::{NativeInstanceManager, NativeInstanceRuntimeHost}; +#[cfg(any(mobile, test))] +use crate::instance::shared_virtual_nic::{ + SharedIfConfig, SharedIfConfigClaims, SharedIfConfigConflict, SharedIpv4Route, +}; +#[cfg(mobile)] +use easytier_core::instance::CoreInstanceState; + +#[derive(Clone, Debug, Default, serde::Serialize)] +pub struct MobileTunConfig { + pub instance_ids: Vec, + pub addresses: Vec, + pub routes: Vec, + pub dns: Option, + pub rejected: Vec, +} + +#[derive(Clone, Debug, serde::Serialize)] +pub struct MobileTunMemberError { + pub instance_id: uuid::Uuid, + pub error: String, +} + +#[cfg(any(mobile, test))] +struct MobileTunMember { + instance_id: uuid::Uuid, + claims: SharedIfConfigClaims, + current_claims: Option, + dns: bool, +} + +#[cfg(any(mobile, test))] +fn magic_dns_route() -> SharedIpv4Route { + SharedIpv4Route::new( + crate::instance::dns_server::MAGIC_DNS_FAKE_IP + .parse() + .unwrap(), + 32, + Some(1), + ) +} + +#[cfg(any(mobile, test))] +fn comparable_claims(mut claims: SharedIfConfigClaims) -> SharedIfConfigClaims { + claims.mtu = None; + let magic_dns = magic_dns_route(); + claims + .ipv4_routes + .retain(|route| route.address != magic_dns.address || route.prefix != magic_dns.prefix); + claims +} + +#[cfg(any(mobile, test))] +fn select_members( + mut members: Vec, + mut rejected: Vec, +) -> (Vec, MobileTunConfig) { + // Keep unchanged attached members ahead of new or reconfigured members. + members.sort_by_key(|member| { + !member.current_claims.as_ref().is_some_and(|current| { + comparable_claims(current.clone()) == comparable_claims(member.claims.clone()) + }) + }); + let mut ifcfg = SharedIfConfig::default(); + let mut accepted = Vec::new(); + for member in members { + let mut claims = member.claims.clone(); + if member.dns { + claims.ipv4_routes.insert(magic_dns_route()); + } + if let Err(error) = ifcfg.ensure_disjoint(member.instance_id, &claims) { + rejected.push(MobileTunMemberError { + instance_id: member.instance_id, + error: error.to_string(), + }); + continue; + } + ifcfg.apply_member_claims(member.instance_id, claims); + accepted.push(member); + } + let config = config_for_members(&accepted, rejected); + (accepted, config) +} + +#[cfg(any(mobile, test))] +fn config_for_members( + members: &[MobileTunMember], + rejected: Vec, +) -> MobileTunConfig { + let mut config = MobileTunConfig { + rejected, + ..Default::default() + }; + let mut addresses = BTreeSet::new(); + let mut routes = BTreeSet::new(); + for member in members { + config.instance_ids.push(member.instance_id); + for address in &member.claims.ipv4_addresses { + addresses.insert(format!( + "{}/{}", + address.address(), + address.network_length() + )); + routes.insert(format!( + "{}/{}", + address.network().first_address(), + address.network_length() + )); + } + for address in &member.claims.ipv6_addresses { + addresses.insert(format!( + "{}/{}", + address.address(), + address.network_length() + )); + routes.insert(format!( + "{}/{}", + address.network().first_address(), + address.network_length() + )); + } + for route in &member.claims.ipv4_routes { + if let Ok(route) = cidr::Ipv4Inet::new(route.address, route.prefix) { + routes.insert(format!( + "{}/{}", + route.network().first_address(), + route.network_length() + )); + } + } + for route in &member.claims.ipv6_routes { + if let Ok(route) = cidr::Ipv6Inet::new(route.address, route.prefix) { + routes.insert(format!( + "{}/{}", + route.network().first_address(), + route.network_length() + )); + } + } + if member.dns { + config.dns = Some(crate::instance::dns_server::MAGIC_DNS_FAKE_IP.to_owned()); + routes.insert(format!( + "{}/32", + crate::instance::dns_server::MAGIC_DNS_FAKE_IP + )); + } + } + config.instance_ids.sort(); + config.addresses = addresses.into_iter().collect(); + config.routes = routes.into_iter().collect(); + config +} + +#[cfg(mobile)] +async fn collect_members( + manager: &NativeInstanceManager, + instance_ids: &[uuid::Uuid], +) -> (Vec, Vec) { + let mut members = Vec::new(); + let mut rejected = Vec::new(); + let mut seen = BTreeSet::new(); + for &instance_id in instance_ids { + if !seen.insert(instance_id) { + continue; + } + let instance = manager.instance(instance_id); + let runtime = instance + .as_ref() + .filter(|instance| instance.state() == CoreInstanceState::Running) + .and_then(|instance| instance.runtime_host::()) + .filter(|runtime| runtime.tun_enabled()); + let Some(runtime) = runtime else { + rejected.push(MobileTunMemberError { + instance_id, + error: "TUN-enabled instance is not running".to_owned(), + }); + continue; + }; + let (claims, current_claims, dns) = runtime + .mobile_tun_claims(&instance.as_ref().unwrap().packet_plane()) + .await; + members.push(MobileTunMember { + instance_id, + claims, + current_claims, + dns, + }); + } + (members, rejected) +} + +#[cfg(mobile)] +pub async fn get_mobile_tun_config( + manager: &NativeInstanceManager, + instance_ids: &[uuid::Uuid], +) -> MobileTunConfig { + let (members, rejected) = collect_members(manager, instance_ids).await; + select_members(members, rejected).1 +} + +#[cfg(any(mobile, test))] +fn is_claim_conflict(error: &anyhow::Error) -> bool { + error + .chain() + .any(|error| error.is::()) +} + +#[cfg(mobile)] +pub async fn attach_mobile_tun_fd( + manager: &NativeInstanceManager, + fd: i32, + instance_ids: Option<&[uuid::Uuid]>, +) -> anyhow::Result { + let all_ids = manager + .instances() + .into_iter() + .filter(|instance| { + instance.state() == CoreInstanceState::Running + && instance + .runtime_host::() + .is_some_and(NativeInstanceRuntimeHost::tun_enabled) + }) + .map(|instance| instance.instance_id()) + .collect::>() + .into_iter() + .collect::>(); + if fd <= 0 { + let mut errors = Vec::new(); + for id in all_ids { + if let Err(error) = attach_member(manager, id, 0, false, None).await { + errors.push(error.to_string()); + } + } + anyhow::ensure!( + errors.is_empty(), + "failed to detach mobile TUN fd: {}", + errors.join("; ") + ); + return Ok(MobileTunConfig::default()); + } + let (members, rejected) = collect_members(manager, instance_ids.unwrap_or(&all_ids)).await; + let (members, mut config) = select_members(members, rejected); + let mut attached = Vec::new(); + // Open once; a rejected member never replaces or tears down this shared fd. + for member in members { + match attach_member( + manager, + member.instance_id, + fd, + attached.is_empty(), + Some(member.claims.clone()), + ) + .await + { + Ok(()) => attached.push(member), + Err(error) if is_claim_conflict(&error) => { + attach_member(manager, member.instance_id, 0, false, None).await?; + config.rejected.push(MobileTunMemberError { + instance_id: member.instance_id, + error: error.to_string(), + }); + } + Err(error) => { + for id in &all_ids { + let _ = attach_member(manager, *id, 0, false, None).await; + } + return Err(error.context("failed to attach mobile TUN fd")); + } + } + } + Ok(config_for_members(&attached, config.rejected)) +} + +#[cfg(mobile)] +async fn attach_member( + manager: &NativeInstanceManager, + instance_id: uuid::Uuid, + fd: i32, + replace_tun_fd: bool, + claims: Option, +) -> anyhow::Result<()> { + let instance = manager + .instance(instance_id) + .ok_or_else(|| anyhow::anyhow!("instance {instance_id} is unavailable"))?; + let runtime = instance + .runtime_host::() + .ok_or_else(|| anyhow::anyhow!("native runtime host is unavailable"))?; + runtime + .attach_mobile_tun_fd(fd, replace_tun_fd, claims) + .await +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::instance::shared_virtual_nic::SharedIpv6Route; + + fn member(id: u128, address: &str) -> MobileTunMember { + MobileTunMember { + instance_id: uuid::Uuid::from_u128(id), + claims: SharedIfConfigClaims { + ipv4_addresses: BTreeSet::from([address.parse().unwrap()]), + ..Default::default() + }, + current_claims: None, + dns: false, + } + } + + #[test] + fn keeps_attached_member_when_bootstrap_subnets_overlap() { + let mut existing = member(2, "10.126.126.2/24"); + existing.current_claims = Some(existing.claims.clone()); + let (_, config) = select_members(vec![member(1, "10.126.126.1/24"), existing], Vec::new()); + assert_eq!(config.instance_ids, vec![uuid::Uuid::from_u128(2)]); + assert_eq!(config.addresses, vec!["10.126.126.2/24"]); + assert_eq!(config.routes, vec!["10.126.126.0/24"]); + assert_eq!(config.rejected[0].instance_id, uuid::Uuid::from_u128(1)); + } + + #[test] + fn reconfigured_member_does_not_displace_unchanged_member() { + let mut changed = member(1, "10.0.2.1/24"); + changed.current_claims = Some(member(1, "10.0.1.1/24").claims); + let mut unchanged = member(2, "10.0.2.2/24"); + unchanged.current_claims = Some(unchanged.claims.clone()); + unchanged.current_claims.as_mut().unwrap().mtu = Some(1300); + let (_, config) = select_members(vec![changed, unchanged], Vec::new()); + assert_eq!(config.instance_ids, vec![uuid::Uuid::from_u128(2)]); + assert_eq!(config.rejected[0].instance_id, uuid::Uuid::from_u128(1)); + } + + #[test] + fn projects_ipv6_and_shared_dns_and_rejects_overlapping_proxy_routes() { + let mut first = member(1, "10.0.1.1/24"); + first.dns = true; + first + .claims + .ipv6_addresses + .insert("2001:db8::1/128".parse().unwrap()); + first + .claims + .ipv6_routes + .insert(SharedIpv6Route::new("::".parse().unwrap(), 0, Some(5))); + let mut second = member(2, "10.0.2.1/24"); + second.dns = true; + let (_, config) = select_members( + vec![first, second, member(3, "100.100.100.101/32")], + Vec::new(), + ); + assert_eq!(config.instance_ids.len(), 2); + assert!(config.addresses.contains(&"2001:db8::1/128".to_owned())); + assert!(config.routes.contains(&"::/0".to_owned())); + assert_eq!( + config + .routes + .iter() + .filter(|route| *route == "100.100.100.101/32") + .count(), + 1 + ); + assert_eq!(config.dns.as_deref(), Some("100.100.100.101")); + assert_eq!(config.rejected[0].instance_id, uuid::Uuid::from_u128(3)); + let mut route_owner = member(2, "10.0.2.1/24"); + route_owner.claims.ipv4_routes.insert(SharedIpv4Route::new( + "10.0.1.0".parse().unwrap(), + 24, + None, + )); + let (_, config) = select_members(vec![member(1, "10.0.1.1/24"), route_owner], Vec::new()); + assert_eq!(config.instance_ids.len(), 1); + assert_eq!(config.rejected[0].instance_id, uuid::Uuid::from_u128(2)); + } + + #[test] + fn distinguishes_claim_conflicts_from_device_failures_through_context() { + let mut claims = SharedIfConfig::default(); + claims.apply_member_claims(uuid::Uuid::from_u128(1), member(1, "10.0.1.1/24").claims); + let conflict = claims + .ensure_disjoint(uuid::Uuid::from_u128(2), &member(2, "10.0.1.2/24").claims) + .unwrap_err(); + let error = anyhow::Error::new(crate::common::error::Error::from(conflict)) + .context("attach mobile TUN failed"); + assert!(is_claim_conflict(&error)); + assert!(!is_claim_conflict(&anyhow::Error::new( + std::io::Error::from_raw_os_error(19) + ))); + } +} diff --git a/easytier/src/instance/mod.rs b/easytier/src/instance/mod.rs index 4657ba43..1d3ceebf 100644 --- a/easytier/src/instance/mod.rs +++ b/easytier/src/instance/mod.rs @@ -19,6 +19,9 @@ pub(crate) mod listeners; #[cfg(feature = "public-ipv6-provider")] pub(crate) mod public_ipv6_provider; +#[cfg(feature = "tun")] +pub mod shared_virtual_nic; + #[cfg(feature = "tun")] pub mod virtual_nic; diff --git a/easytier/src/instance/runtime_host.rs b/easytier/src/instance/runtime_host.rs index aa22a702..6cd93e1a 100644 --- a/easytier/src/instance/runtime_host.rs +++ b/easytier/src/instance/runtime_host.rs @@ -1,13 +1,16 @@ use std::sync::Arc; +#[cfg(feature = "web-client")] +use easytier_core::config::runtime::CoreInstanceRuntimeConfig; use easytier_core::{ - config::runtime::CoreInstanceRuntimeConfig, gateway::dhcp::DhcpIpv4Host, - host::packet::HostPacketReceiver, instance::CorePacketPlane, + gateway::dhcp::DhcpIpv4Host, host::packet::HostPacketReceiver, instance::CorePacketPlane, }; use tokio::sync::Mutex; use tokio_util::sync::CancellationToken; use crate::common::global_ctx::ArcGlobalCtx; +#[cfg(feature = "tun")] +use crate::instance::shared_virtual_nic::ArcSharedVirtualNicRegistry; mod event_journal; mod implementation; @@ -42,16 +45,30 @@ pub(crate) struct NativeInstanceRuntimeHost { impl NativeInstanceRuntimeHost { #[cfg(test)] - pub(crate) fn new(global_ctx: ArcGlobalCtx) -> Arc { - Self::new_with_executor(global_ctx, None) + pub(crate) fn new( + global_ctx: ArcGlobalCtx, + #[cfg(feature = "tun")] shared_virtual_nic_registry: ArcSharedVirtualNicRegistry, + ) -> Arc { + Self::new_with_executor( + global_ctx, + #[cfg(feature = "tun")] + shared_virtual_nic_registry, + None, + ) } pub(super) fn new_with_executor( global_ctx: ArcGlobalCtx, + #[cfg(feature = "tun")] shared_virtual_nic_registry: ArcSharedVirtualNicRegistry, executor: Option, ) -> Arc { let cancel = CancellationToken::new(); - let tun = NativeTunRuntime::new(global_ctx.clone(), cancel.clone()); + let tun = NativeTunRuntime::new( + global_ctx.clone(), + cancel.clone(), + #[cfg(feature = "tun")] + shared_virtual_nic_registry, + ); let event_journal = EventJournal::new(&global_ctx); Arc::new(Self { global_ctx, @@ -128,10 +145,44 @@ impl NativeInstanceRuntimeHost { self.global_ctx.subscribe() } + #[cfg(all(feature = "tun", mobile))] + pub(crate) fn tun_enabled(&self) -> bool { + !self.global_ctx.get_flags().no_tun + } + fn attach_runtime_tun_fd(&self, fd: i32) -> anyhow::Result<()> { self.tun.attach_fd(fd) } + #[cfg(all(feature = "tun", mobile))] + pub(crate) async fn attach_mobile_tun_fd( + &self, + fd: i32, + replace_tun_fd: bool, + claims: Option, + ) -> anyhow::Result<()> { + self.tun.attach_mobile_fd(fd, replace_tun_fd, claims).await + } + + #[cfg(all(feature = "tun", mobile))] + pub(crate) async fn mobile_tun_claims( + &self, + packet_plane: &CorePacketPlane, + ) -> ( + crate::instance::shared_virtual_nic::SharedIfConfigClaims, + Option, + bool, + ) { + ( + crate::instance::virtual_nic::NicCtx::mobile_tun_claims(&self.global_ctx, packet_plane) + .await, + self.tun.current_mobile_tun_claims().await, + cfg!(feature = "magic-dns") + && self.global_ctx.get_flags().accept_dns + && self.global_ctx.get_ipv4().is_some(), + ) + } + fn install_packet_receiver(&self, receiver: HostPacketReceiver) -> anyhow::Result<()> { self.tun.install_packet_receiver(receiver) } @@ -145,6 +196,16 @@ mod tests { global_ctx::{GlobalCtx, GlobalCtxEvent}, }; + fn runtime_host(global_ctx: ArcGlobalCtx) -> Arc { + NativeInstanceRuntimeHost::new( + global_ctx, + #[cfg(feature = "tun")] + Arc::new(tokio::sync::Mutex::new( + crate::instance::shared_virtual_nic::SharedVirtualNicRegistry::new(), + )), + ) + } + #[cfg(feature = "web-client")] fn runtime_config(config: &TomlConfig) -> CoreInstanceRuntimeConfig { let normalized = easytier_core::instance::CoreInstanceConfig::from_toml(config).unwrap(); @@ -157,7 +218,7 @@ mod tests { #[test] fn runtime_host_owns_event_subscription_context() { let global_ctx = Arc::new(GlobalCtx::new(TomlConfig::default())); - let runtime_host = NativeInstanceRuntimeHost::new(global_ctx.clone()); + let runtime_host = runtime_host(global_ctx.clone()); let mut events = runtime_host.subscribe_event(); global_ctx.issue_event(GlobalCtxEvent::CredentialChanged); @@ -178,7 +239,7 @@ mod tests { config.set_ipv4(Some("10.20.0.1/24".parse().unwrap())); config.set_ipv6(Some("fd00::1/64".parse().unwrap())); let global_ctx = Arc::new(GlobalCtx::new(config.clone())); - let runtime_host = NativeInstanceRuntimeHost::new(global_ctx.clone()); + let runtime_host = runtime_host(global_ctx.clone()); assert_eq!(global_ctx.get_hostname(), "before"); assert_eq!(global_ctx.get_ipv4(), Some("10.20.0.1/24".parse().unwrap())); @@ -220,7 +281,7 @@ mod tests { let config = TomlConfig::default(); config.set_dhcp(true); let global_ctx = Arc::new(GlobalCtx::new(config.clone())); - let runtime_host = NativeInstanceRuntimeHost::new(global_ctx.clone()); + let runtime_host = runtime_host(global_ctx.clone()); let lease = "10.20.0.7/24".parse().unwrap(); global_ctx.set_ipv4(Some(lease)); diff --git a/easytier/src/instance/runtime_host/magic_dns.rs b/easytier/src/instance/runtime_host/magic_dns.rs index 8ee17048..0035ee2d 100644 --- a/easytier/src/instance/runtime_host/magic_dns.rs +++ b/easytier/src/instance/runtime_host/magic_dns.rs @@ -3,9 +3,9 @@ use easytier_core::instance::CorePacketPlane; #[cfg(feature = "magic-dns")] use tokio_util::{sync::CancellationToken, task::AbortOnDropHandle}; -use crate::common::global_ctx::ArcGlobalCtx; #[cfg(feature = "magic-dns")] use crate::instance::dns_server::{MAGIC_DNS_FAKE_IP, runner::DnsRunner}; +use crate::{common::global_ctx::ArcGlobalCtx, instance::virtual_nic::NicBackend}; #[derive(Default)] pub(super) struct MagicDnsRuntime { @@ -26,6 +26,7 @@ impl MagicDnsRuntime { packet_plane: std::sync::Arc, tun_dev: Option, tun_ip: Ipv4Inet, + shared_route_backend: Option, ) -> Self { let active = global_ctx.get_flags().accept_dns.then(|| { let mut runner = DnsRunner::new( @@ -34,7 +35,8 @@ impl MagicDnsRuntime { tun_dev, tun_ip, MAGIC_DNS_FAKE_IP.parse().unwrap(), - ); + ) + .with_shared_route_backend(shared_route_backend); let cancel = CancellationToken::new(); let task_cancel = cancel.clone(); let task = tokio::spawn(async move { @@ -54,6 +56,7 @@ impl MagicDnsRuntime { _packet_plane: std::sync::Arc, _tun_dev: Option, _tun_ip: Ipv4Inet, + _shared_route_backend: Option, ) -> Self { Self::default() } diff --git a/easytier/src/instance/runtime_host/tun_common.rs b/easytier/src/instance/runtime_host/tun_common.rs index a86aead9..053a14db 100644 --- a/easytier/src/instance/runtime_host/tun_common.rs +++ b/easytier/src/instance/runtime_host/tun_common.rs @@ -3,11 +3,43 @@ use std::{ sync::{Arc, OnceLock}, }; -use easytier_core::host::packet::HostPacketReceiver; +use easytier_core::{host::packet::HostPacketReceiver, instance::CorePacketPlane}; use tokio::{sync::Mutex, task::JoinSet}; use super::MagicDnsRuntime; -use crate::instance::virtual_nic::NicCtx; +use crate::{ + common::{error::Error, global_ctx::ArcGlobalCtx}, + instance::{shared_virtual_nic::ArcSharedVirtualNicRegistry, virtual_nic::NicCtx}, +}; + +pub(super) async fn create_nic_ctx( + global_ctx: ArcGlobalCtx, + packet_plane: Arc, + receiver: Arc>, + close_notifier: Arc, + registry: ArcSharedVirtualNicRegistry, +) -> Result { + #[cfg(not(mobile))] + if global_ctx.get_flags().dev_name.is_empty() { + return Ok(NicCtx::new( + global_ctx, + packet_plane, + receiver, + close_notifier, + )); + } + + let member_id = global_ctx.get_id(); + NicCtx::new_shared( + global_ctx, + packet_plane, + receiver, + close_notifier, + registry, + member_id, + ) + .await +} struct NicCtxContainer { _nic_ctx: Option>, @@ -65,6 +97,15 @@ impl TunNicState { drop(old); } + #[cfg(mobile)] + pub(super) async fn mobile_tun_claims( + &self, + ) -> Option { + let nic = self.nic_ctx.lock().await; + let nic = nic.as_ref()?._nic_ctx.as_ref()?.downcast_ref::()?; + nic.current_mobile_tun_claims().await + } + pub(super) async fn drain(&self) { self.stop().await; let receiver = self.receiver(); diff --git a/easytier/src/instance/runtime_host/tun_desktop.rs b/easytier/src/instance/runtime_host/tun_desktop.rs index 94629a31..f179750c 100644 --- a/easytier/src/instance/runtime_host/tun_desktop.rs +++ b/easytier/src/instance/runtime_host/tun_desktop.rs @@ -14,28 +14,37 @@ use tokio::{ }; use tokio_util::sync::CancellationToken; -use super::{MagicDnsRuntime, tun_common::TunNicState}; +use super::{ + MagicDnsRuntime, + tun_common::{TunNicState, create_nic_ctx}, +}; use crate::{ common::{ error::Error, global_ctx::{ArcGlobalCtx, GlobalCtxEvent}, }, - instance::virtual_nic::NicCtx, + instance::shared_virtual_nic::ArcSharedVirtualNicRegistry, }; pub(super) struct NativeTunRuntime { global_ctx: ArcGlobalCtx, cancel: CancellationToken, nic: TunNicState, + shared_virtual_nic_registry: ArcSharedVirtualNicRegistry, static_ip_task: Mutex>>, } impl NativeTunRuntime { - pub(super) fn new(global_ctx: ArcGlobalCtx, cancel: CancellationToken) -> Self { + pub(super) fn new( + global_ctx: ArcGlobalCtx, + cancel: CancellationToken, + shared_virtual_nic_registry: ArcSharedVirtualNicRegistry, + ) -> Self { Self { global_ctx, cancel, nic: TunNicState::empty(), + shared_virtual_nic_registry, static_ip_task: Mutex::new(None), } } @@ -67,6 +76,7 @@ impl NativeTunRuntime { let cancel = self.cancel.clone(); let global_ctx = self.global_ctx.clone(); let receiver = self.nic.receiver(); + let shared_virtual_nic_registry = self.shared_virtual_nic_registry.clone(); let (output, first_round) = oneshot::channel(); let task = tokio::spawn(async move { let mut output = Some(output); @@ -76,12 +86,29 @@ impl NativeTunRuntime { return; } let closed = Arc::new(Notify::new()); - let mut nic = NicCtx::new( + let mut nic = match create_nic_ctx( global_ctx.clone(), packet_plane.clone(), receiver.clone(), closed.clone(), - ); + shared_virtual_nic_registry.clone(), + ) + .await + { + Ok(nic) => nic, + Err(error) => { + if let Some(output) = output.take() { + let _ = output.send(Err(error)); + return; + } + tracing::error!(?error, "failed to create native interface context"); + tokio::select! { + _ = cancel.cancelled() => return, + _ = tokio::time::sleep(Duration::from_secs(1)) => {} + } + continue; + } + }; let result = tokio::select! { biased; _ = cancel.cancelled() => { @@ -104,11 +131,13 @@ impl NativeTunRuntime { } let magic_dns = if let Some(ip) = ipv4 { + let shared_route_backend = nic.shared_route_backend_for_dns(); MagicDnsRuntime::start( global_ctx.clone(), packet_plane.clone(), nic.ifname().await, ip, + shared_route_backend, ) } else { MagicDnsRuntime::default() @@ -161,6 +190,7 @@ impl NativeTunRuntime { nic: self.nic.clone(), closed: Arc::new(Notify::new()), packet_plane, + shared_virtual_nic_registry: self.shared_virtual_nic_registry.clone(), }) } } @@ -172,6 +202,7 @@ struct NativeDhcpIpv4Host { nic: TunNicState, closed: Arc, packet_plane: Arc, + shared_virtual_nic_registry: ArcSharedVirtualNicRegistry, } impl NativeDhcpIpv4Host { @@ -199,21 +230,25 @@ impl NativeDhcpIpv4Host { return Ok(Some(ip)); } - let mut nic = NicCtx::new( + let mut nic = create_nic_ctx( self.global_ctx.clone(), self.packet_plane.clone(), self.nic.receiver(), self.closed.clone(), - ); + self.shared_virtual_nic_registry.clone(), + ) + .await?; tokio::select! { _ = self.cancel.cancelled() => anyhow::bail!("instance is closing; DHCP update cancelled"), result = nic.run(Some(ip), self.global_ctx.get_ipv6()) => result?, } + let shared_route_backend = nic.shared_route_backend_for_dns(); let magic_dns = MagicDnsRuntime::start( self.global_ctx.clone(), self.packet_plane.clone(), nic.ifname().await, ip, + shared_route_backend, ); self.nic.install(nic, magic_dns).await; self.global_ctx.set_ipv4(Some(ip)); diff --git a/easytier/src/instance/runtime_host/tun_mobile.rs b/easytier/src/instance/runtime_host/tun_mobile.rs index 4f459326..1200f2ae 100644 --- a/easytier/src/instance/runtime_host/tun_mobile.rs +++ b/easytier/src/instance/runtime_host/tun_mobile.rs @@ -8,26 +8,41 @@ use easytier_core::{ instance::CorePacketPlane, }; use futures::FutureExt as _; -use tokio::sync::{Mutex, Notify, mpsc}; +use tokio::sync::{Mutex, Notify, mpsc, oneshot}; use tokio_util::sync::CancellationToken; -use super::{MagicDnsRuntime, tun_common::TunNicState}; +use super::{ + MagicDnsRuntime, + tun_common::{TunNicState, create_nic_ctx}, +}; use crate::{ common::global_ctx::{ArcGlobalCtx, GlobalCtxEvent}, - instance::virtual_nic::NicCtx, + instance::shared_virtual_nic::ArcSharedVirtualNicRegistry, }; +struct MobileTunAttachment { + fd: i32, + replace_tun_fd: bool, + completion: Option>>, + claims: Option, +} + pub(super) struct NativeTunRuntime { global_ctx: ArcGlobalCtx, cancel: CancellationToken, nic: TunNicState, - tun_fd: mpsc::Sender, - tun_fd_receiver: Mutex>>, + tun_fd: mpsc::Sender, + tun_fd_receiver: Mutex>>, task: Mutex>>, + shared_virtual_nic_registry: ArcSharedVirtualNicRegistry, } impl NativeTunRuntime { - pub(super) fn new(global_ctx: ArcGlobalCtx, cancel: CancellationToken) -> Self { + pub(super) fn new( + global_ctx: ArcGlobalCtx, + cancel: CancellationToken, + shared_virtual_nic_registry: ArcSharedVirtualNicRegistry, + ) -> Self { let (tun_fd, tun_fd_receiver) = mpsc::channel(16); Self { global_ctx, @@ -36,6 +51,7 @@ impl NativeTunRuntime { tun_fd, tun_fd_receiver: Mutex::new(Some(tun_fd_receiver)), task: Mutex::new(None), + shared_virtual_nic_registry, } } @@ -51,23 +67,39 @@ impl NativeTunRuntime { global_ctx: ArcGlobalCtx, packet_plane: Arc, fd: i32, + replace_tun_fd: bool, + shared_virtual_nic_registry: ArcSharedVirtualNicRegistry, + claims: Option, ) -> anyhow::Result<()> { nic_state.drain().await; if fd <= 0 { return Ok(()); } let closed = Arc::new(Notify::new()); - let mut nic = NicCtx::new( + let mut nic = create_nic_ctx( global_ctx.clone(), packet_plane.clone(), nic_state.receiver(), closed, - ); - nic.run_for_mobile(fd).await.context("add ip failed")?; - let magic_dns = global_ctx - .get_ipv4() - .map(|ip| MagicDnsRuntime::start(global_ctx, packet_plane, None, ip)) - .unwrap_or_default(); + shared_virtual_nic_registry, + ) + .await?; + let claims = match claims { + Some(claims) => claims, + None => { + crate::instance::virtual_nic::NicCtx::mobile_tun_claims(&global_ctx, &packet_plane) + .await + } + }; + nic.run_for_mobile(fd, replace_tun_fd, claims) + .await + .context("attach mobile TUN failed")?; + let magic_dns = if let Some(ip) = global_ctx.get_ipv4() { + let shared_route_backend = nic.shared_route_backend_for_dns(); + MagicDnsRuntime::start(global_ctx, packet_plane, None, ip, shared_route_backend) + } else { + MagicDnsRuntime::default() + }; nic_state.install(nic, magic_dns).await; Ok(()) } @@ -80,28 +112,44 @@ impl NativeTunRuntime { let nic_state = self.nic.clone(); let global_ctx = self.global_ctx.clone(); let cancel = self.cancel.clone(); + let shared_virtual_nic_registry = self.shared_virtual_nic_registry.clone(); self.task.lock().await.replace(tokio::spawn(async move { loop { - let fd = tokio::select! { + let attachment = tokio::select! { _ = cancel.cancelled() => return, - fd = tun_fds.recv() => match fd { Some(fd) => fd, None => return }, + attachment = tun_fds.recv() => match attachment { + Some(attachment) => attachment, + None => return, + }, }; - if let Err(error) = Self::install_mobile_tun( - nic_state.clone(), - global_ctx.clone(), - packet_plane.clone(), - fd, - ) - .await - { + let result = if attachment.fd <= 0 { + nic_state.drain().await; + Ok(()) + } else { + Self::install_mobile_tun( + nic_state.clone(), + global_ctx.clone(), + packet_plane.clone(), + attachment.fd, + attachment.replace_tun_fd, + shared_virtual_nic_registry.clone(), + attachment.claims, + ) + .await + }; + if let Err(error) = &result { tracing::error!(?error, "failed to attach mobile TUN fd"); } + if let Some(completion) = attachment.completion { + let _ = completion.send(result); + } } })); Ok(()) } pub(super) async fn shutdown(&self) { + self.cancel.cancel(); if let Some(task) = self.task.lock().await.take() { let _ = task.await; } @@ -110,10 +158,43 @@ impl NativeTunRuntime { pub(super) fn attach_fd(&self, fd: i32) -> anyhow::Result<()> { self.tun_fd - .try_send(fd) + .try_send(MobileTunAttachment { + fd, + replace_tun_fd: true, + completion: None, + claims: None, + }) .map_err(|error| anyhow::anyhow!("failed to send TUN fd: {error}")) } + pub(super) async fn attach_mobile_fd( + &self, + fd: i32, + replace_tun_fd: bool, + claims: Option, + ) -> anyhow::Result<()> { + if self.task.lock().await.is_none() { + anyhow::bail!("mobile TUN runtime is not running"); + } + + let (completion, result) = oneshot::channel(); + tokio::select! { + _ = self.cancel.cancelled() => anyhow::bail!("instance is closing; TUN attachment cancelled"), + send_result = self.tun_fd.send(MobileTunAttachment { + fd, + replace_tun_fd, + completion: Some(completion), + claims, + }) => send_result.map_err(|error| anyhow::anyhow!("failed to send TUN fd: {error}"))?, + } + + tokio::select! { + _ = self.cancel.cancelled() => anyhow::bail!("instance is closing; TUN attachment cancelled"), + result = result => result + .map_err(|_| anyhow::anyhow!("mobile TUN runtime stopped before attachment completed"))?, + } + } + pub(super) fn dhcp_host( &self, operation: Arc>, @@ -127,6 +208,12 @@ impl NativeTunRuntime { closed: Arc::new(Notify::new()), }) } + + pub(super) async fn current_mobile_tun_claims( + &self, + ) -> Option { + self.nic.mobile_tun_claims().await + } } struct NativeDhcpIpv4Host { diff --git a/easytier/src/instance/shared_virtual_nic.rs b/easytier/src/instance/shared_virtual_nic.rs new file mode 100644 index 00000000..7e6b750b --- /dev/null +++ b/easytier/src/instance/shared_virtual_nic.rs @@ -0,0 +1,1943 @@ +use std::{ + collections::{BTreeMap, BTreeSet}, + net::{Ipv4Addr, Ipv6Addr}, + sync::{ + Arc, LazyLock, + atomic::{AtomicBool, Ordering}, + }, +}; + +use cidr::{Ipv4Cidr, Ipv4Inet, Ipv6Cidr, Ipv6Inet}; +use tokio::{ + runtime::{Builder, Runtime}, + sync::{Mutex, Notify}, +}; +use tokio_util::task::AbortOnDropHandle; + +use crate::common::error::Error; +use easytier_core::tunnel::{Tunnel, ring::create_ring_tunnel_pair}; + +use super::virtual_nic::{VirtualNic, VirtualNicConfig}; + +#[cfg(not(target_os = "linux"))] +use crate::common::ifcfg::IfConfiger; + +mod dispatcher; + +use dispatcher::{SharedVirtualNicDispatcher, SharedVirtualNicMemberTunnelTable}; + +pub type SharedVirtualNicMemberId = uuid::Uuid; +pub(super) type SharedVirtualNicMemberRegistrationId = uuid::Uuid; +pub(crate) type ArcSharedVirtualNicRegistry = Arc>; + +// The shared device and its cleanup must outlive every member's executor. +static SHARED_VIRTUAL_NIC_RUNTIME: LazyLock = LazyLock::new(|| { + Builder::new_multi_thread() + .worker_threads(if cfg!(mobile) { 1 } else { 2 }) + .thread_name("easytier-shared-tun") + .enable_all() + .build() + .expect("failed to build shared virtual nic runtime") +}); + +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub struct SharedIpv4Route { + pub address: Ipv4Addr, + pub prefix: u8, + pub cost: Option, +} + +impl SharedIpv4Route { + pub fn new(address: Ipv4Addr, prefix: u8, cost: Option) -> Self { + Self { + address, + prefix, + cost, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub struct SharedIpv6Route { + pub address: Ipv6Addr, + pub prefix: u8, + pub cost: Option, +} + +impl SharedIpv6Route { + pub fn new(address: Ipv6Addr, prefix: u8, cost: Option) -> Self { + Self { + address, + prefix, + cost, + } + } +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct SharedIfConfigClaims { + pub ipv4_addresses: BTreeSet, + pub ipv6_addresses: BTreeSet, + pub ipv4_routes: BTreeSet, + pub ipv6_routes: BTreeSet, + pub mtu: Option, +} + +impl SharedIfConfigClaims { + fn ipv4_destinations(&self) -> impl Iterator + '_ { + self.ipv4_addresses + .iter() + .map(|ip| (ip.network(), false)) + .chain( + self.ipv4_routes + .iter() + .filter_map(|route| Ipv4Inet::new(route.address, route.prefix).ok()) + .map(|route| (route.network(), true)), + ) + } + + fn ipv6_destinations(&self) -> impl Iterator + '_ { + self.ipv6_addresses.iter().map(Ipv6Inet::network).chain( + self.ipv6_routes + .iter() + .filter_map(|route| Ipv6Inet::new(route.address, route.prefix).ok()) + .map(|route| route.network()), + ) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct OwnedItemDelta { + pub added: BTreeSet, + pub removed: BTreeSet, +} + +impl Default for OwnedItemDelta { + fn default() -> Self { + Self { + added: BTreeSet::new(), + removed: BTreeSet::new(), + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SharedMtuChange { + pub old: Option, + pub new: Option, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct SharedIfConfigDelta { + pub ipv4_addresses: OwnedItemDelta, + pub ipv6_addresses: OwnedItemDelta, + pub ipv4_routes: OwnedItemDelta, + pub ipv6_routes: OwnedItemDelta, + pub mtu: Option, +} + +impl SharedIfConfigDelta { + fn between(old: &EffectiveIfConfig, new: &EffectiveIfConfig) -> Self { + Self { + ipv4_addresses: item_delta(&old.ipv4_addresses, &new.ipv4_addresses), + ipv6_addresses: item_delta(&old.ipv6_addresses, &new.ipv6_addresses), + ipv4_routes: item_delta(&old.ipv4_routes, &new.ipv4_routes), + ipv6_routes: item_delta(&old.ipv6_routes, &new.ipv6_routes), + mtu: mtu_delta(old.effective_mtu, new.effective_mtu), + } + } +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +struct EffectiveIfConfig { + ipv4_addresses: BTreeSet, + ipv6_addresses: BTreeSet, + ipv4_routes: BTreeSet, + ipv6_routes: BTreeSet, + effective_mtu: Option, +} + +#[derive(Clone, Debug, Default)] +pub struct SharedIfConfig { + member_claims: BTreeMap, +} + +#[derive(Debug, thiserror::Error)] +#[error( + "shared virtual NIC destination {requested} overlaps member {member_id} destination {existing}" +)] +pub(crate) struct SharedIfConfigConflict { + requested: String, + member_id: SharedVirtualNicMemberId, + existing: String, +} + +impl SharedIfConfig { + pub(crate) fn ensure_disjoint( + &self, + member_id: SharedVirtualNicMemberId, + claims: &SharedIfConfigClaims, + ) -> anyhow::Result<()> { + let magic_dns = Ipv4Cidr::new(crate::instance::dns_server::MAGIC_DNS_FAKE_IP.parse()?, 32)?; + for (other_id, other) in &self.member_claims { + if *other_id == member_id { + continue; + } + for (requested, requested_is_route) in claims.ipv4_destinations() { + if let Some((existing, _)) = + other + .ipv4_destinations() + .find(|(existing, existing_is_route)| { + !(requested_is_route + && *existing_is_route + && requested == magic_dns + && *existing == magic_dns) + && ipv4_cidrs_overlap(requested, *existing) + }) + { + return Err(SharedIfConfigConflict { + requested: requested.to_string(), + member_id: *other_id, + existing: existing.to_string(), + } + .into()); + } + } + for requested in claims.ipv6_destinations() { + if let Some(existing) = other + .ipv6_destinations() + .find(|existing| ipv6_cidrs_overlap(requested, *existing)) + { + return Err(SharedIfConfigConflict { + requested: requested.to_string(), + member_id: *other_id, + existing: existing.to_string(), + } + .into()); + } + } + } + Ok(()) + } + + pub(crate) fn apply_member_claims( + &mut self, + member_id: SharedVirtualNicMemberId, + claims: SharedIfConfigClaims, + ) -> SharedIfConfigDelta { + let old = self.effective(); + self.member_claims.insert(member_id, claims); + SharedIfConfigDelta::between(&old, &self.effective()) + } + + pub fn remove_member( + &mut self, + member_id: SharedVirtualNicMemberId, + ) -> Option { + let old = self.effective(); + self.member_claims.remove(&member_id)?; + Some(SharedIfConfigDelta::between(&old, &self.effective())) + } + + pub fn effective_mtu(&self) -> Option { + self.member_claims + .values() + .filter_map(|claims| claims.mtu) + .min() + } + + pub fn owners_of_ipv4_route( + &self, + route: &SharedIpv4Route, + ) -> BTreeSet { + owners_for(&self.member_claims, |claims| &claims.ipv4_routes, route) + } + + pub fn owners_of_ipv4_address(&self, address: &Ipv4Inet) -> BTreeSet { + owners_for( + &self.member_claims, + |claims| &claims.ipv4_addresses, + address, + ) + } + + fn effective(&self) -> EffectiveIfConfig { + EffectiveIfConfig { + ipv4_addresses: self + .member_claims + .values() + .flat_map(|claims| claims.ipv4_addresses.iter().copied()) + .collect(), + ipv6_addresses: self + .member_claims + .values() + .flat_map(|claims| claims.ipv6_addresses.iter().copied()) + .collect(), + ipv4_routes: self + .member_claims + .values() + .flat_map(|claims| claims.ipv4_routes.iter().cloned()) + .collect(), + ipv6_routes: self + .member_claims + .values() + .flat_map(|claims| claims.ipv6_routes.iter().cloned()) + .collect(), + effective_mtu: self.effective_mtu(), + } + } + + fn claims_of(&self, member_id: SharedVirtualNicMemberId) -> SharedIfConfigClaims { + self.member_claims + .get(&member_id) + .cloned() + .unwrap_or_default() + } + + fn ipv4_route_source_hint(&self, route: &SharedIpv4Route) -> Option { + let route_inet = Ipv4Inet::new(route.address, route.prefix).ok(); + let mut fallback = None; + + for claims in self.member_claims.values() { + if !claims.ipv4_routes.contains(route) { + continue; + } + + for address in &claims.ipv4_addresses { + fallback.get_or_insert(address.address()); + if route_inet + .as_ref() + .is_some_and(|route_inet| route_inet.contains(&address.address())) + { + return Some(address.address()); + } + } + } + + fallback + } + + fn changed_ipv4_route_sources(&self, next: &Self) -> BTreeSet { + let old_routes = self.effective().ipv4_routes; + let next_routes = next.effective().ipv4_routes; + old_routes + .iter() + .filter(|route| { + next_routes.contains(*route) + && self.ipv4_route_source_hint(route) != next.ipv4_route_source_hint(route) + }) + .cloned() + .collect() + } +} + +pub struct SharedVirtualNic { + nic: Arc>, + ifcfg: SharedIfConfig, + valid: Arc, + member_tunnel_table: SharedVirtualNicMemberTunnelTable, + member_registrations: BTreeMap, + dispatcher: Option, +} + +impl SharedVirtualNic { + pub fn new(config: VirtualNicConfig) -> Self { + Self { + nic: Arc::new(Mutex::new(VirtualNic::new(config))), + ifcfg: SharedIfConfig::default(), + valid: Arc::new(AtomicBool::new(true)), + member_tunnel_table: SharedVirtualNicMemberTunnelTable::default(), + member_registrations: BTreeMap::new(), + dispatcher: None, + } + } + + pub fn mark_invalid(&self) { + self.valid.store(false, Ordering::Release); + } + + pub fn is_valid(&self) -> bool { + self.valid.load(Ordering::Acquire) + } + + pub fn ifcfg(&self) -> &SharedIfConfig { + &self.ifcfg + } + + pub fn ifcfg_mut(&mut self) -> &mut SharedIfConfig { + &mut self.ifcfg + } + + pub fn nic(&self) -> Arc> { + self.nic.clone() + } + + #[cfg(not(target_os = "linux"))] + async fn ifcfg_and_ifname(&self) -> Result<(IfConfiger, String), Error> { + self.ensure_valid()?; + let nic = self.nic.lock().await; + Ok((nic.get_ifcfg(), nic.ifname().to_owned())) + } + + async fn link_up(&self) -> Result<(), Error> { + self.ensure_valid()?; + #[cfg(not(mobile))] + self.nic.lock().await.link_up().await?; + Ok(()) + } + + async fn attach_member_registration( + &mut self, + member_id: SharedVirtualNicMemberId, + registration_id: SharedVirtualNicMemberRegistrationId, + ) -> Result<(), Error> { + self.ensure_valid()?; + + match self.member_registrations.get(&member_id).copied() { + Some(old_registration_id) if old_registration_id == registration_id => { + return Ok(()); + } + Some(_) => { + if let Err(err) = self.remove_member_claims(member_id).await { + self.invalidate_and_shutdown_dispatcher().await; + return Err(err); + } + } + None => {} + } + + self.member_registrations.insert(member_id, registration_id); + Ok(()) + } + + fn is_current_member_registration( + &self, + member_id: SharedVirtualNicMemberId, + registration_id: SharedVirtualNicMemberRegistrationId, + ) -> bool { + self.member_registrations + .get(&member_id) + .is_some_and(|current| *current == registration_id) + } + + async fn apply_member_claims_for_registration( + &mut self, + member_id: SharedVirtualNicMemberId, + registration_id: SharedVirtualNicMemberRegistrationId, + claims: SharedIfConfigClaims, + ) -> Result<(), Error> { + if !self.is_current_member_registration(member_id, registration_id) { + return Ok(()); + } + #[cfg(mobile)] + return self.apply_member_claims_for_mobile(member_id, claims).await; + #[cfg(not(mobile))] + self.apply_member_claims(member_id, claims).await + } + + async fn remove_member_registration_claims( + &mut self, + member_id: SharedVirtualNicMemberId, + registration_id: SharedVirtualNicMemberRegistrationId, + ) -> Result<(), Error> { + if !self.is_current_member_registration(member_id, registration_id) { + return Ok(()); + } + + if let Err(err) = self.remove_member_claims(member_id).await { + self.member_registrations.remove(&member_id); + self.invalidate_and_shutdown_dispatcher().await; + return Err(err); + } + + self.member_registrations.remove(&member_id); + self.shutdown_dispatcher_if_idle().await; + Ok(()) + } + + async fn apply_member_claims( + &mut self, + member_id: SharedVirtualNicMemberId, + claims: SharedIfConfigClaims, + ) -> Result<(), Error> { + self.ensure_valid()?; + self.ifcfg.ensure_disjoint(member_id, &claims)?; + + let mut next_ifcfg = self.ifcfg.clone(); + let old_claims = self.ifcfg.claims_of(member_id); + let next_claims = claims.clone(); + let delta = next_ifcfg.apply_member_claims(member_id, claims); + self.sync_dispatcher_sources_for_ifcfg_update(member_id, &old_claims, &next_claims) + .await?; + + if let Err(err) = self.apply_ifcfg_delta(&delta, &next_ifcfg).await { + // Native operations may have already changed addresses or routes. + // Recreate the device rather than retaining inconsistent claims. + self.invalidate_and_shutdown_dispatcher().await; + return Err(err); + } + + self.sync_dispatcher_sources_for_claims(member_id, &next_claims) + .await?; + self.ifcfg = next_ifcfg; + + Ok(()) + } + + #[cfg(any(mobile, test))] + async fn apply_member_claims_for_mobile( + &mut self, + member_id: SharedVirtualNicMemberId, + claims: SharedIfConfigClaims, + ) -> Result<(), Error> { + self.ensure_valid()?; + self.ifcfg.ensure_disjoint(member_id, &claims)?; + + let mut next_ifcfg = self.ifcfg.clone(); + let next_claims = claims.clone(); + next_ifcfg.apply_member_claims(member_id, claims); + + self.sync_dispatcher_sources_for_claims(member_id, &next_claims) + .await?; + self.ifcfg = next_ifcfg; + + Ok(()) + } + + async fn apply_member_mtu_for_registration( + &mut self, + member_id: SharedVirtualNicMemberId, + registration_id: SharedVirtualNicMemberRegistrationId, + mtu: u32, + ) -> Result<(), Error> { + if !self.is_current_member_registration(member_id, registration_id) { + return Ok(()); + } + + let mut claims = self.ifcfg.claims_of(member_id); + claims.mtu = Some(mtu); + self.apply_member_claims(member_id, claims).await + } + + #[cfg(mobile)] + async fn apply_member_mtu_for_mobile_registration( + &mut self, + member_id: SharedVirtualNicMemberId, + registration_id: SharedVirtualNicMemberRegistrationId, + mtu: u32, + ) -> Result<(), Error> { + if !self.is_current_member_registration(member_id, registration_id) { + return Ok(()); + } + + let mut claims = self.ifcfg.claims_of(member_id); + claims.mtu = Some(mtu); + self.apply_member_claims_for_mobile(member_id, claims).await + } + + async fn remove_member_claims( + &mut self, + member_id: SharedVirtualNicMemberId, + ) -> Result<(), Error> { + self.ensure_valid()?; + + let mut next_ifcfg = self.ifcfg.clone(); + let Some(delta) = next_ifcfg.remove_member(member_id) else { + return Ok(()); + }; + #[cfg(not(mobile))] + self.apply_ifcfg_delta(&delta, &next_ifcfg).await?; + #[cfg(mobile)] + drop(delta); + if let Some(dispatcher) = &self.dispatcher { + dispatcher.remove_sources(member_id).await?; + } + self.ifcfg = next_ifcfg; + + Ok(()) + } + + async fn shutdown_dispatcher_if_idle(&mut self) { + if !self.member_registrations.is_empty() { + return; + } + + self.shutdown_dispatcher().await; + } + + async fn invalidate_and_shutdown_dispatcher(&mut self) { + self.mark_invalid(); + self.shutdown_dispatcher().await; + } + + async fn shutdown_dispatcher(&mut self) { + if let Some(dispatcher) = self.dispatcher.take() { + dispatcher.shutdown_without_invalidation().await; + } + } + + async fn apply_ifcfg_delta( + &self, + delta: &SharedIfConfigDelta, + next_ifcfg: &SharedIfConfig, + ) -> Result<(), Error> { + let changed_routes = self.ifcfg.changed_ipv4_route_sources(next_ifcfg); + #[cfg(target_os = "linux")] + let next_effective = next_ifcfg.effective(); + let nic = self.nic.lock().await; + + for route in &delta.ipv4_routes.removed { + let source_hint = self.ifcfg.ipv4_route_source_hint(route); + ignore_removed_ifcfg_not_found( + remove_shared_ipv4_route(&nic, route, source_hint).await, + )?; + } + for route in &changed_routes { + let source_hint = self.ifcfg.ipv4_route_source_hint(route); + ignore_removed_ifcfg_not_found( + remove_shared_ipv4_route(&nic, route, source_hint).await, + )?; + } + for route in &delta.ipv6_routes.removed { + ignore_removed_ifcfg_not_found( + nic.remove_ipv6_route(route.address, route.prefix).await, + )?; + } + for ip in &delta.ipv4_addresses.removed { + ignore_removed_ifcfg_not_found(nic.remove_ip(Some(*ip)).await)?; + } + for ip in &delta.ipv6_addresses.removed { + ignore_removed_ifcfg_not_found(nic.remove_ipv6(Some(*ip)).await)?; + } + + for ip in &delta.ipv4_addresses.added { + nic.add_ip(ip.address(), ip.network_length() as i32).await?; + } + for ip in &delta.ipv6_addresses.added { + nic.add_ipv6(ip.address(), ip.network_length() as i32) + .await?; + } + for route in &delta.ipv4_routes.added { + add_shared_ipv4_route(&nic, route, next_ifcfg).await?; + } + for route in &changed_routes { + add_shared_ipv4_route(&nic, route, next_ifcfg).await?; + } + for route in &delta.ipv6_routes.added { + nic.add_ipv6_route_with_cost(route.address, route.prefix, route.cost) + .await?; + } + + if let Some(mtu) = &delta.mtu { + nic.set_mtu(mtu.new.unwrap_or_else(|| nic.configured_mtu())) + .await?; + } + + #[cfg(target_os = "linux")] + if !delta.ipv4_addresses.removed.is_empty() { + for route in &next_effective.ipv4_routes { + ignore_added_ifcfg_already_exists( + add_shared_ipv4_route(&nic, route, next_ifcfg).await, + )?; + } + } + + #[cfg(target_os = "linux")] + if !delta.ipv6_addresses.removed.is_empty() { + for route in &next_effective.ipv6_routes { + ignore_added_ifcfg_already_exists( + nic.add_ipv6_route_with_cost(route.address, route.prefix, route.cost) + .await, + )?; + } + } + + Ok(()) + } + + fn ensure_valid(&self) -> Result<(), Error> { + if self.is_valid() { + return Ok(()); + } + + Err(anyhow::anyhow!("shared virtual nic is invalid").into()) + } + + fn member_tunnel_table(&self) -> SharedVirtualNicMemberTunnelTable { + self.member_tunnel_table.clone() + } + + fn valid_flag(&self) -> Arc { + self.valid.clone() + } + + async fn ensure_dispatcher(&mut self) -> Result<(), Error> { + self.ensure_valid()?; + + if self.dispatcher.is_some() { + return Ok(()); + } + + // AsyncDevice registers with the current I/O driver when it is created, + // so moving only the dispatcher would still tie it to the first member. + let nic = self.nic.clone(); + let open = AbortOnDropHandle::new( + SHARED_VIRTUAL_NIC_RUNTIME.spawn(async move { nic.lock().await.create_dev().await }), + ); + let tunnel = open.await.map_err(anyhow::Error::from)??; + let dispatcher = SharedVirtualNicDispatcher::start( + tunnel, + self.member_tunnel_table.clone(), + self.valid.clone(), + ); + self.sync_dispatcher_sources(&dispatcher).await?; + self.dispatcher = Some(dispatcher); + Ok(()) + } + + #[cfg(mobile)] + async fn ensure_dispatcher_for_mobile( + &mut self, + tun_fd: std::os::fd::RawFd, + replace_tun_fd: bool, + ) -> Result<(), Error> { + self.ensure_valid()?; + + if let Some(dispatcher) = &self.dispatcher { + if replace_tun_fd { + if let Err(err) = dispatcher.update_mobile_tun_fd(tun_fd).await { + self.invalidate_and_shutdown_dispatcher().await; + return Err(err); + } + } + return Ok(()); + } + + let dispatcher = SharedVirtualNicDispatcher::start_for_mobile( + self.nic.clone(), + tun_fd, + self.member_tunnel_table.clone(), + self.valid.clone(), + ) + .await?; + self.sync_dispatcher_sources(&dispatcher).await?; + self.dispatcher = Some(dispatcher); + Ok(()) + } + + async fn sync_dispatcher_sources( + &self, + dispatcher: &SharedVirtualNicDispatcher, + ) -> Result<(), Error> { + for (member_id, claims) in &self.ifcfg.member_claims { + dispatcher.update_sources(*member_id, claims).await?; + } + Ok(()) + } + + async fn sync_dispatcher_sources_for_ifcfg_update( + &self, + member_id: SharedVirtualNicMemberId, + old_claims: &SharedIfConfigClaims, + next_claims: &SharedIfConfigClaims, + ) -> Result<(), Error> { + let active_claims = dispatcher_claims_for_ifcfg_transition(old_claims, next_claims); + self.sync_dispatcher_sources_for_claims(member_id, &active_claims) + .await + } + + async fn sync_dispatcher_sources_for_claims( + &self, + member_id: SharedVirtualNicMemberId, + claims: &SharedIfConfigClaims, + ) -> Result<(), Error> { + if let Some(dispatcher) = &self.dispatcher { + dispatcher.update_sources(member_id, claims).await?; + } + Ok(()) + } +} + +fn dispatcher_claims_for_ifcfg_transition( + old_claims: &SharedIfConfigClaims, + next_claims: &SharedIfConfigClaims, +) -> SharedIfConfigClaims { + SharedIfConfigClaims { + ipv4_addresses: merged_items(&old_claims.ipv4_addresses, &next_claims.ipv4_addresses), + ipv6_addresses: merged_items(&old_claims.ipv6_addresses, &next_claims.ipv6_addresses), + ipv4_routes: merged_items(&old_claims.ipv4_routes, &next_claims.ipv4_routes), + ipv6_routes: merged_items(&old_claims.ipv6_routes, &next_claims.ipv6_routes), + mtu: None, + } +} + +async fn add_shared_ipv4_route( + nic: &VirtualNic, + route: &SharedIpv4Route, + ifcfg: &SharedIfConfig, +) -> Result<(), Error> { + nic.add_route_with_cost_and_source_hint( + route.address, + route.prefix, + route.cost, + ifcfg.ipv4_route_source_hint(route), + ) + .await +} + +async fn remove_shared_ipv4_route( + nic: &VirtualNic, + route: &SharedIpv4Route, + source_hint: Option, +) -> Result<(), Error> { + nic.remove_route_with_cost_and_source_hint(route.address, route.prefix, route.cost, source_hint) + .await +} + +fn merged_items(old_items: &BTreeSet, new_items: &BTreeSet) -> BTreeSet +where + T: Ord + Clone, +{ + old_items.union(new_items).cloned().collect() +} + +fn remove_claimed_item(items: &mut BTreeSet, item: Option) +where + T: Ord, +{ + match item { + Some(item) => { + items.remove(&item); + } + None => { + items.clear(); + } + } +} + +fn ignore_removed_ifcfg_not_found(result: Result<(), Error>) -> Result<(), Error> { + match result { + Err(Error::NotFound) => Ok(()), + other => other, + } +} + +#[cfg(target_os = "linux")] +fn ignore_added_ifcfg_already_exists(result: Result<(), Error>) -> Result<(), Error> { + match result { + Err(Error::IOError(err)) if err.kind() == std::io::ErrorKind::AlreadyExists => Ok(()), + other => other, + } +} + +struct SharedVirtualNicMemberRegistration { + member_id: SharedVirtualNicMemberId, + registration_id: SharedVirtualNicMemberRegistrationId, + shared_nic: Arc>, + member_tunnel_table: SharedVirtualNicMemberTunnelTable, +} + +impl SharedVirtualNicMemberRegistration { + fn register_tunnel( + &self, + tunnel: Box, + close_notifier: Arc, + ) -> Result<(), Error> { + self.member_tunnel_table.register( + self.member_id, + self.registration_id, + tunnel, + close_notifier, + ) + } +} + +impl Drop for SharedVirtualNicMemberRegistration { + fn drop(&mut self) { + self.member_tunnel_table + .unregister(self.member_id, self.registration_id); + let shared_nic = self.shared_nic.clone(); + let member_id = self.member_id; + let registration_id = self.registration_id; + + SHARED_VIRTUAL_NIC_RUNTIME.spawn(async move { + let mut shared_nic = shared_nic.lock().await; + if let Err(err) = shared_nic + .remove_member_registration_claims(member_id, registration_id) + .await + { + tracing::warn!( + ?member_id, + ?err, + "failed to clean shared virtual nic member claims" + ); + } + }); + } +} + +#[derive(Clone)] +pub struct SharedVirtualNicMember { + member_id: SharedVirtualNicMemberId, + configured_mtu: u32, + shared_nic: Arc>, + close_notifier: Arc, + registration: Arc, +} + +impl SharedVirtualNicMember { + fn new( + member_id: SharedVirtualNicMemberId, + configured_mtu: u32, + shared_nic: Arc>, + close_notifier: Arc, + member_tunnel_table: SharedVirtualNicMemberTunnelTable, + ) -> Self { + let registration_id = uuid::Uuid::new_v4(); + Self { + member_id, + configured_mtu, + shared_nic: shared_nic.clone(), + close_notifier, + registration: Arc::new(SharedVirtualNicMemberRegistration { + member_id, + registration_id, + shared_nic: shared_nic.clone(), + member_tunnel_table, + }), + } + } + + pub fn member_id(&self) -> SharedVirtualNicMemberId { + self.member_id + } + + #[cfg(mobile)] + pub(crate) async fn claims(&self) -> SharedIfConfigClaims { + self.shared_nic.lock().await.ifcfg.claims_of(self.member_id) + } + + #[cfg(any(mobile, test))] + pub(crate) async fn replace_claims( + &self, + mut claims: SharedIfConfigClaims, + ) -> Result<(), Error> { + claims.mtu = Some(self.configured_mtu); + let member = self.clone(); + SHARED_VIRTUAL_NIC_RUNTIME + .spawn(async move { + let mut shared_nic = member.shared_nic.lock().await; + shared_nic + .attach_member_registration( + member.member_id, + member.registration.registration_id, + ) + .await?; + shared_nic + .apply_member_claims_for_registration( + member.member_id, + member.registration.registration_id, + claims, + ) + .await + }) + .await + .map_err(anyhow::Error::from)? + } + + pub fn shared_nic(&self) -> Arc> { + self.shared_nic.clone() + } + + pub fn close_notifier(&self) -> Arc { + self.close_notifier.clone() + } + + #[cfg(test)] + fn configured_mtu_for_test(&self) -> u32 { + self.configured_mtu + } + + pub async fn create_dev(&self) -> Result, Error> { + let (member_tunnel, shared_tunnel) = create_ring_tunnel_pair(); + let member = self.clone(); + SHARED_VIRTUAL_NIC_RUNTIME + .spawn(async move { + let mut shared_nic = member.shared_nic.lock().await; + shared_nic + .attach_member_registration( + member.member_id, + member.registration.registration_id, + ) + .await?; + shared_nic.ensure_dispatcher().await?; + shared_nic + .apply_member_mtu_for_registration( + member.member_id, + member.registration.registration_id, + member.configured_mtu, + ) + .await + }) + .await + .map_err(anyhow::Error::from)??; + self.registration + .register_tunnel(shared_tunnel, self.close_notifier.clone())?; + Ok(member_tunnel) + } + + #[cfg(mobile)] + pub async fn create_dev_for_mobile( + &self, + tun_fd: std::os::fd::RawFd, + replace_tun_fd: bool, + ) -> Result, Error> { + let (member_tunnel, shared_tunnel) = create_ring_tunnel_pair(); + let member = self.clone(); + SHARED_VIRTUAL_NIC_RUNTIME + .spawn(async move { + let mut shared_nic = member.shared_nic.lock().await; + shared_nic + .attach_member_registration( + member.member_id, + member.registration.registration_id, + ) + .await?; + shared_nic + .ensure_dispatcher_for_mobile(tun_fd, replace_tun_fd) + .await?; + shared_nic + .apply_member_mtu_for_mobile_registration( + member.member_id, + member.registration.registration_id, + member.configured_mtu, + ) + .await + }) + .await + .map_err(anyhow::Error::from)??; + self.registration + .register_tunnel(shared_tunnel, self.close_notifier.clone())?; + Ok(member_tunnel) + } + + #[cfg(not(target_os = "linux"))] + pub async fn ifcfg_and_ifname(&self) -> Result<(IfConfiger, String), Error> { + self.shared_nic.lock().await.ifcfg_and_ifname().await + } + + pub async fn link_up(&self) -> Result<(), Error> { + self.shared_nic.lock().await.link_up().await + } + + pub async fn add_ip(&self, ip: Ipv4Addr, cidr: i32) -> Result<(), Error> { + let ip = ipv4_inet(ip, cidr)?; + self.update_claims(move |claims| { + claims.ipv4_addresses.insert(ip); + }) + .await + } + + pub async fn remove_ip(&self, ip: Option) -> Result<(), Error> { + self.update_claims(move |claims| { + remove_claimed_item(&mut claims.ipv4_addresses, ip); + }) + .await + } + + pub async fn add_ipv6(&self, ip: Ipv6Addr, cidr: i32) -> Result<(), Error> { + let ip = ipv6_inet(ip, cidr)?; + self.update_claims(move |claims| { + claims.ipv6_addresses.insert(ip); + }) + .await + } + + pub async fn remove_ipv6(&self, ip: Option) -> Result<(), Error> { + self.update_claims(move |claims| { + remove_claimed_item(&mut claims.ipv6_addresses, ip); + }) + .await + } + + pub async fn add_route(&self, address: Ipv4Addr, cidr: u8) -> Result<(), Error> { + self.add_route_with_cost(address, cidr, None).await + } + + pub async fn add_route_with_cost( + &self, + address: Ipv4Addr, + cidr: u8, + cost: Option, + ) -> Result<(), Error> { + self.update_claims(move |claims| { + claims + .ipv4_routes + .insert(SharedIpv4Route::new(address, cidr, cost)); + }) + .await + } + + pub async fn remove_route(&self, address: Ipv4Addr, cidr: u8) -> Result<(), Error> { + self.update_claims(move |claims| { + claims + .ipv4_routes + .retain(|route| route.address != address || route.prefix != cidr); + }) + .await + } + + pub async fn add_ipv6_route(&self, address: Ipv6Addr, cidr: u8) -> Result<(), Error> { + self.add_ipv6_route_with_cost(address, cidr, None).await + } + + pub async fn add_ipv6_route_with_cost( + &self, + address: Ipv6Addr, + cidr: u8, + cost: Option, + ) -> Result<(), Error> { + self.update_claims(move |claims| { + claims + .ipv6_routes + .insert(SharedIpv6Route::new(address, cidr, cost)); + }) + .await + } + + pub async fn remove_ipv6_route(&self, address: Ipv6Addr, cidr: u8) -> Result<(), Error> { + self.update_claims(move |claims| { + claims + .ipv6_routes + .retain(|route| route.address != address || route.prefix != cidr); + }) + .await + } + + async fn update_claims(&self, update: F) -> Result<(), Error> + where + F: FnOnce(&mut SharedIfConfigClaims) + Send + 'static, + { + // Complete OS changes and commit their claims even if the caller is cancelled. + // Holding the registration also defers member cleanup until the commit. + let member = self.clone(); + SHARED_VIRTUAL_NIC_RUNTIME + .spawn(async move { + let mut shared_nic = member.shared_nic.lock().await; + let mut claims = shared_nic.ifcfg.claims_of(member.member_id); + update(&mut claims); + shared_nic + .apply_member_claims_for_registration( + member.member_id, + member.registration.registration_id, + claims, + ) + .await + }) + .await + .map_err(anyhow::Error::from)? + } +} + +#[derive(Default)] +pub struct SharedVirtualNicRegistry { + nics: BTreeMap, +} + +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +struct SharedVirtualNicRegistryKey { + net_ns: Option, + dev_name: String, +} + +impl SharedVirtualNicRegistryKey { + fn new(dev_name: String, config: &VirtualNicConfig) -> Self { + Self { + net_ns: config.net_ns_name(), + dev_name, + } + } +} + +struct SharedVirtualNicRegistryEntry { + nic: Arc>, + valid: Arc, + member_tunnel_table: SharedVirtualNicMemberTunnelTable, +} + +impl SharedVirtualNicRegistryEntry { + fn new(nic: SharedVirtualNic) -> Self { + Self { + valid: nic.valid_flag(), + member_tunnel_table: nic.member_tunnel_table(), + nic: Arc::new(Mutex::new(nic)), + } + } + + fn is_valid(&self) -> bool { + self.valid.load(Ordering::Acquire) + } + + fn nic(&self) -> Arc> { + self.nic.clone() + } + + fn member_tunnel_table(&self) -> SharedVirtualNicMemberTunnelTable { + self.member_tunnel_table.clone() + } +} + +impl SharedVirtualNicRegistry { + pub fn new() -> Self { + Self::default() + } + + pub fn get( + &self, + dev_name: &str, + config: &VirtualNicConfig, + ) -> Option>> { + let key = SharedVirtualNicRegistryKey::new(dev_name.to_owned(), config); + self.nics + .get(&key) + .filter(|entry| entry.is_valid()) + .map(|entry| entry.nic()) + } + + #[cfg(test)] + pub fn get_by_dev_name_for_test(&self, dev_name: &str) -> Option>> { + let mut matches = self + .nics + .iter() + .filter(|(key, entry)| key.dev_name == dev_name && entry.is_valid()) + .map(|(_, entry)| entry.nic()); + let first = matches.next()?; + if matches.next().is_some() { + return None; + } + Some(first) + } + + pub fn get_or_create( + &mut self, + dev_name: String, + config: VirtualNicConfig, + ) -> Arc> { + self.get_or_create_entry(dev_name, config).nic() + } + + fn get_or_create_entry( + &mut self, + dev_name: String, + config: VirtualNicConfig, + ) -> &SharedVirtualNicRegistryEntry { + let key = SharedVirtualNicRegistryKey::new(dev_name, &config); + let needs_new_entry = self.nics.get(&key).is_none_or(|entry| !entry.is_valid()); + if needs_new_entry { + let entry = SharedVirtualNicRegistryEntry::new(SharedVirtualNic::new(config)); + self.nics.insert(key.clone(), entry); + } + + self.nics + .get(&key) + .expect("shared virtual nic registry entry should exist") + } + + pub fn create_member( + &mut self, + dev_name: String, + config: VirtualNicConfig, + member_id: SharedVirtualNicMemberId, + close_notifier: Arc, + ) -> SharedVirtualNicMember { + let configured_mtu = config.mtu(); + let entry = self.get_or_create_entry(dev_name, config); + SharedVirtualNicMember::new( + member_id, + configured_mtu, + entry.nic(), + close_notifier, + entry.member_tunnel_table(), + ) + } +} + +fn item_delta(old: &BTreeSet, new: &BTreeSet) -> OwnedItemDelta +where + T: Ord + Clone, +{ + OwnedItemDelta { + added: new.difference(old).cloned().collect(), + removed: old.difference(new).cloned().collect(), + } +} + +fn owners_for( + claims: &BTreeMap, + select: fn(&SharedIfConfigClaims) -> &BTreeSet, + item: &T, +) -> BTreeSet +where + T: Ord, +{ + claims + .iter() + .filter_map(|(member_id, claims)| select(claims).contains(item).then_some(*member_id)) + .collect() +} + +fn mtu_delta(old: Option, new: Option) -> Option { + (old != new).then_some(SharedMtuChange { old, new }) +} + +fn ipv4_cidrs_overlap(left: Ipv4Cidr, right: Ipv4Cidr) -> bool { + left.contains(&right.first_address()) || right.contains(&left.first_address()) +} + +fn ipv6_cidrs_overlap(left: Ipv6Cidr, right: Ipv6Cidr) -> bool { + left.contains(&right.first_address()) || right.contains(&left.first_address()) +} + +fn ipv4_inet(address: Ipv4Addr, prefix: i32) -> Result { + let prefix = u8::try_from(prefix) + .map_err(|_| anyhow::anyhow!("invalid IPv4 prefix length {}", prefix))?; + Ipv4Inet::new(address, prefix).map_err(|err| { + anyhow::anyhow!("invalid IPv4 address {}/{}: {:?}", address, prefix, err).into() + }) +} + +fn ipv6_inet(address: Ipv6Addr, prefix: i32) -> Result { + let prefix = u8::try_from(prefix) + .map_err(|_| anyhow::anyhow!("invalid IPv6 prefix length {}", prefix))?; + Ipv6Inet::new(address, prefix).map_err(|err| { + anyhow::anyhow!("invalid IPv6 address {}/{}: {:?}", address, prefix, err).into() + }) +} + +#[cfg(test)] +mod tests { + use std::str::FromStr as _; + + use crate::common::{ifcfg::IfConfiguerTrait, netns::NetNS}; + use tokio::sync::Notify; + + use super::*; + + struct FailingRemoveIpIfConfiger; + + #[async_trait::async_trait] + impl IfConfiguerTrait for FailingRemoveIpIfConfiger { + async fn remove_ip(&self, _name: &str, _ip: Option) -> Result<(), Error> { + Err(anyhow::anyhow!("forced remove_ip failure").into()) + } + } + + fn member_id(n: u128) -> SharedVirtualNicMemberId { + uuid::Uuid::from_u128(n) + } + + fn claims_with_ipv4_route(route: SharedIpv4Route, mtu: Option) -> SharedIfConfigClaims { + SharedIfConfigClaims { + ipv4_routes: BTreeSet::from([route]), + mtu, + ..Default::default() + } + } + + fn claims_with_ipv4_address_and_route( + address: Ipv4Inet, + route: SharedIpv4Route, + ) -> SharedIfConfigClaims { + SharedIfConfigClaims { + ipv4_addresses: BTreeSet::from([address]), + ipv4_routes: BTreeSet::from([route]), + ..Default::default() + } + } + + fn virtual_nic_config() -> VirtualNicConfig { + VirtualNicConfig::new(String::new(), 1500, NetNS::new(None)) + } + + fn virtual_nic_config_with_mtu(mtu: u32) -> VirtualNicConfig { + VirtualNicConfig::new(String::new(), mtu, NetNS::new(None)) + } + + fn virtual_nic_config_in_netns(net_ns: &str) -> VirtualNicConfig { + VirtualNicConfig::new(String::new(), 1500, NetNS::new(Some(net_ns.to_owned()))) + } + + #[test] + fn duplicate_routes_keep_owner_sets_and_single_os_delta() { + let route = SharedIpv4Route::new(Ipv4Addr::new(100, 100, 100, 101), 32, None); + let first = member_id(1); + let second = member_id(2); + let mut ifcfg = SharedIfConfig::default(); + + let first_delta = + ifcfg.apply_member_claims(first, claims_with_ipv4_route(route.clone(), Some(1400))); + let second_delta = + ifcfg.apply_member_claims(second, claims_with_ipv4_route(route.clone(), Some(1300))); + + assert_eq!( + first_delta.ipv4_routes.added, + BTreeSet::from([route.clone()]) + ); + assert!(second_delta.ipv4_routes.added.is_empty()); + assert_eq!( + ifcfg.owners_of_ipv4_route(&route), + BTreeSet::from([first, second]) + ); + assert_eq!(ifcfg.effective_mtu(), Some(1300)); + } + + #[test] + fn removing_one_owner_keeps_shared_route_until_last_owner_leaves() { + let route = SharedIpv4Route::new(Ipv4Addr::new(100, 100, 100, 101), 32, None); + let first = member_id(1); + let second = member_id(2); + let mut ifcfg = SharedIfConfig::default(); + ifcfg.apply_member_claims(first, claims_with_ipv4_route(route.clone(), None)); + ifcfg.apply_member_claims(second, claims_with_ipv4_route(route.clone(), None)); + + let first_delta = ifcfg.remove_member(first).unwrap(); + let second_delta = ifcfg.remove_member(second).unwrap(); + + assert!(first_delta.ipv4_routes.removed.is_empty()); + assert_eq!( + second_delta.ipv4_routes.removed, + BTreeSet::from([route.clone()]) + ); + assert!(ifcfg.owners_of_ipv4_route(&route).is_empty()); + } + + #[tokio::test] + async fn overlapping_member_claims_leave_existing_configuration_intact() { + let first = member_id(1); + let second = member_id(2); + let mut shared_nic = SharedVirtualNic::new(virtual_nic_config()); + shared_nic.ifcfg_mut().apply_member_claims( + first, + SharedIfConfigClaims { + ipv4_routes: BTreeSet::from([SharedIpv4Route::new( + Ipv4Addr::new(10, 144, 0, 0), + 16, + None, + )]), + ..Default::default() + }, + ); + let original = shared_nic.ifcfg().effective(); + + let err = shared_nic + .apply_member_claims( + second, + SharedIfConfigClaims { + ipv4_addresses: BTreeSet::from(["10.144.1.2/24".parse().unwrap()]), + ..Default::default() + }, + ) + .await + .unwrap_err(); + + assert!(err.to_string().contains("overlaps member")); + assert_eq!(shared_nic.ifcfg().effective(), original); + } + + #[test] + fn shared_magic_dns_route_is_not_a_member_conflict() { + let first = member_id(1); + let second = member_id(2); + let route = SharedIpv4Route::new(Ipv4Addr::new(100, 100, 100, 101), 32, None); + let claims = claims_with_ipv4_route(route, None); + let mut ifcfg = SharedIfConfig::default(); + ifcfg.apply_member_claims(first, claims.clone()); + + assert!(ifcfg.ensure_disjoint(second, &claims).is_ok()); + } + + #[test] + fn magic_dns_address_conflicts_with_other_members() { + let address_claims = SharedIfConfigClaims { + ipv4_addresses: BTreeSet::from(["100.100.100.101/32".parse().unwrap()]), + ..Default::default() + }; + let route_claims = claims_with_ipv4_route( + SharedIpv4Route::new(Ipv4Addr::new(100, 100, 100, 101), 32, None), + None, + ); + + for (first, second) in [ + (&address_claims, &address_claims), + (&address_claims, &route_claims), + (&route_claims, &address_claims), + ] { + let mut ifcfg = SharedIfConfig::default(); + ifcfg.apply_member_claims(member_id(1), first.clone()); + + assert!(ifcfg.ensure_disjoint(member_id(2), second).is_err()); + } + } + + #[tokio::test] + async fn mobile_claims_do_not_configure_the_os_interface() { + // There is no OS interface or ifname; native ifcfg calls would fail. + let mut shared_nic = SharedVirtualNic::new(virtual_nic_config()); + let member = member_id(1); + let claims = SharedIfConfigClaims { + ipv4_addresses: BTreeSet::from(["10.144.1.1/24".parse().unwrap()]), + ipv6_addresses: BTreeSet::from(["fd00:1::1/64".parse().unwrap()]), + ipv4_routes: BTreeSet::from([SharedIpv4Route::new( + Ipv4Addr::new(10, 145, 0, 0), + 16, + None, + )]), + ipv6_routes: BTreeSet::from([SharedIpv6Route::new( + "fd00:2::".parse().unwrap(), + 64, + None, + )]), + mtu: Some(1300), + }; + + #[cfg(mobile)] + { + let registration = uuid::Uuid::new_v4(); + shared_nic.member_registrations.insert(member, registration); + shared_nic.link_up().await.unwrap(); + shared_nic + .apply_member_claims_for_registration(member, registration, claims.clone()) + .await + .unwrap(); + } + #[cfg(not(mobile))] + shared_nic + .apply_member_claims_for_mobile(member, claims.clone()) + .await + .unwrap(); + assert_eq!(shared_nic.ifcfg().claims_of(member), claims); + assert!(shared_nic.is_valid()); + + shared_nic + .apply_member_claims_for_mobile(member, SharedIfConfigClaims::default()) + .await + .unwrap(); + assert_eq!( + shared_nic.ifcfg().claims_of(member), + SharedIfConfigClaims::default() + ); + } + + #[tokio::test] + async fn conflicting_mobile_member_leaves_no_partial_claims() { + let mut shared_nic = SharedVirtualNic::new(virtual_nic_config()); + let healthy = SharedIfConfigClaims { + ipv4_addresses: BTreeSet::from(["10.144.1.1/24".parse().unwrap()]), + ipv6_addresses: BTreeSet::from(["fd00:1::1/64".parse().unwrap()]), + ..Default::default() + }; + shared_nic + .apply_member_claims_for_mobile(member_id(1), healthy.clone()) + .await + .unwrap(); + let conflicting = SharedIfConfigClaims { + // IPv4 is disjoint, but the later IPv6 claim overlaps the healthy member. + ipv4_addresses: BTreeSet::from(["10.144.2.1/24".parse().unwrap()]), + ipv6_addresses: BTreeSet::from(["fd00:1::2/64".parse().unwrap()]), + ..Default::default() + }; + assert!( + shared_nic + .apply_member_claims_for_mobile(member_id(2), conflicting) + .await + .is_err() + ); + assert_eq!(shared_nic.ifcfg().claims_of(member_id(1)), healthy); + assert_eq!( + shared_nic.ifcfg().claims_of(member_id(2)), + SharedIfConfigClaims::default() + ); + assert!(shared_nic.is_valid()); + } + + #[test] + fn member_claim_update_tracks_ip_ownership() { + let first_ip = Ipv4Inet::from_str("10.30.0.2/24").unwrap(); + let second_ip = Ipv4Inet::from_str("10.30.0.3/24").unwrap(); + let member = member_id(1); + let mut ifcfg = SharedIfConfig::default(); + + let first_delta = ifcfg.apply_member_claims( + member, + SharedIfConfigClaims { + ipv4_addresses: BTreeSet::from([first_ip]), + ..Default::default() + }, + ); + let second_delta = ifcfg.apply_member_claims( + member, + SharedIfConfigClaims { + ipv4_addresses: BTreeSet::from([second_ip]), + ..Default::default() + }, + ); + + assert_eq!(first_delta.ipv4_addresses.added, BTreeSet::from([first_ip])); + assert_eq!( + second_delta.ipv4_addresses.removed, + BTreeSet::from([first_ip]) + ); + assert_eq!( + second_delta.ipv4_addresses.added, + BTreeSet::from([second_ip]) + ); + assert_eq!( + ifcfg.owners_of_ipv4_address(&second_ip), + BTreeSet::from([member]) + ); + } + + #[test] + fn ipv4_route_source_hint_prefers_address_inside_route() { + let route = SharedIpv4Route::new(Ipv4Addr::new(10, 90, 1, 0), 24, None); + let member = member_id(1); + let mut ifcfg = SharedIfConfig::default(); + + ifcfg.apply_member_claims( + member, + SharedIfConfigClaims { + ipv4_addresses: BTreeSet::from([ + Ipv4Inet::from_str("10.1.1.1/24").unwrap(), + Ipv4Inet::from_str("10.90.1.1/24").unwrap(), + ]), + ipv4_routes: BTreeSet::from([route.clone()]), + ..Default::default() + }, + ); + + assert_eq!( + ifcfg.ipv4_route_source_hint(&route), + Some(Ipv4Addr::new(10, 90, 1, 1)) + ); + } + + #[test] + fn adding_better_ipv4_route_owner_marks_source_change() { + let route = SharedIpv4Route::new(Ipv4Addr::new(10, 90, 2, 0), 24, None); + let first = member_id(1); + let second = member_id(2); + let mut ifcfg = SharedIfConfig::default(); + ifcfg.apply_member_claims( + first, + claims_with_ipv4_address_and_route( + Ipv4Inet::from_str("10.1.2.1/24").unwrap(), + route.clone(), + ), + ); + let old = ifcfg.clone(); + + let delta = ifcfg.apply_member_claims( + second, + claims_with_ipv4_address_and_route( + Ipv4Inet::from_str("10.90.2.1/24").unwrap(), + route.clone(), + ), + ); + + assert!(delta.ipv4_routes.added.is_empty()); + assert_eq!( + old.changed_ipv4_route_sources(&ifcfg), + BTreeSet::from([route.clone()]) + ); + assert_eq!( + old.ipv4_route_source_hint(&route), + Some(Ipv4Addr::new(10, 1, 2, 1)) + ); + assert_eq!( + ifcfg.ipv4_route_source_hint(&route), + Some(Ipv4Addr::new(10, 90, 2, 1)) + ); + } + + #[test] + fn removing_ipv4_route_owner_marks_source_change_when_route_remains() { + let route = SharedIpv4Route::new(Ipv4Addr::new(10, 90, 3, 0), 24, None); + let first = member_id(1); + let second = member_id(2); + let mut ifcfg = SharedIfConfig::default(); + ifcfg.apply_member_claims( + first, + claims_with_ipv4_address_and_route( + Ipv4Inet::from_str("10.1.3.1/24").unwrap(), + route.clone(), + ), + ); + ifcfg.apply_member_claims( + second, + claims_with_ipv4_address_and_route( + Ipv4Inet::from_str("10.90.3.1/24").unwrap(), + route.clone(), + ), + ); + let old = ifcfg.clone(); + + let delta = ifcfg.remove_member(second).unwrap(); + + assert!(delta.ipv4_routes.removed.is_empty()); + assert_eq!( + old.changed_ipv4_route_sources(&ifcfg), + BTreeSet::from([route.clone()]) + ); + assert_eq!( + old.ipv4_route_source_hint(&route), + Some(Ipv4Addr::new(10, 90, 3, 1)) + ); + assert_eq!( + ifcfg.ipv4_route_source_hint(&route), + Some(Ipv4Addr::new(10, 1, 3, 1)) + ); + } + + #[test] + fn removing_ipv4_route_keeps_old_source_hint_available() { + let kept_route = SharedIpv4Route::new(Ipv4Addr::new(10, 90, 4, 0), 24, Some(10)); + let removed_route = SharedIpv4Route::new(Ipv4Addr::new(10, 90, 4, 0), 24, Some(20)); + let member = member_id(1); + let address = Ipv4Inet::from_str("10.90.4.1/24").unwrap(); + let mut ifcfg = SharedIfConfig::default(); + ifcfg.apply_member_claims( + member, + SharedIfConfigClaims { + ipv4_addresses: BTreeSet::from([address]), + ipv4_routes: BTreeSet::from([kept_route.clone(), removed_route.clone()]), + ..Default::default() + }, + ); + let old = ifcfg.clone(); + + let delta = ifcfg.apply_member_claims( + member, + SharedIfConfigClaims { + ipv4_addresses: BTreeSet::from([address]), + ipv4_routes: BTreeSet::from([kept_route.clone()]), + ..Default::default() + }, + ); + + assert_eq!( + delta.ipv4_routes.removed, + BTreeSet::from([removed_route.clone()]) + ); + assert_eq!( + old.ipv4_route_source_hint(&removed_route), + Some(Ipv4Addr::new(10, 90, 4, 1)) + ); + } + + #[test] + fn shared_virtual_nic_wraps_virtual_nic_and_tracks_ifcfg() { + let mut shared_nic = SharedVirtualNic::new(virtual_nic_config()); + let member = member_id(1); + let route = SharedIpv4Route::new(Ipv4Addr::new(10, 40, 0, 0), 24, None); + + shared_nic + .ifcfg_mut() + .apply_member_claims(member, claims_with_ipv4_route(route.clone(), None)); + + assert_eq!( + shared_nic.ifcfg().owners_of_ipv4_route(&route), + BTreeSet::from([member]) + ); + drop(shared_nic.nic()); + } + + #[tokio::test] + async fn stale_member_registration_cleanup_keeps_current_claims() { + let mut shared_nic = SharedVirtualNic::new(virtual_nic_config()); + let member = member_id(1); + let old_registration = uuid::Uuid::from_u128(10); + let current_registration = uuid::Uuid::from_u128(11); + let ip = Ipv4Inet::from_str("10.50.0.2/24").unwrap(); + + shared_nic + .member_registrations + .insert(member, current_registration); + shared_nic.ifcfg_mut().apply_member_claims( + member, + SharedIfConfigClaims { + ipv4_addresses: BTreeSet::from([ip]), + ..Default::default() + }, + ); + + shared_nic + .remove_member_registration_claims(member, old_registration) + .await + .unwrap(); + + assert_eq!( + shared_nic.ifcfg().owners_of_ipv4_address(&ip), + BTreeSet::from([member]) + ); + assert_eq!( + shared_nic.member_registrations.get(&member), + Some(¤t_registration) + ); + } + + #[tokio::test] + async fn failed_member_registration_cleanup_invalidates_shared_nic() { + let mut shared_nic = SharedVirtualNic::new(virtual_nic_config()); + let member = member_id(1); + let registration = uuid::Uuid::from_u128(10); + let ip = Ipv4Inet::from_str("10.60.0.2/24").unwrap(); + + shared_nic.member_registrations.insert(member, registration); + shared_nic.ifcfg_mut().apply_member_claims( + member, + SharedIfConfigClaims { + ipv4_addresses: BTreeSet::from([ip]), + ..Default::default() + }, + ); + let nic = shared_nic.nic(); + let mut nic = nic.lock().await; + nic.set_ifname_for_test("et0".to_string()); + nic.set_ifcfg_for_test(Box::new(FailingRemoveIpIfConfiger)); + drop(nic); + + let result = shared_nic + .remove_member_registration_claims(member, registration) + .await; + + assert!(result.is_err()); + assert!(!shared_nic.is_valid()); + assert!(!shared_nic.member_registrations.contains_key(&member)); + assert_eq!( + shared_nic.ifcfg().owners_of_ipv4_address(&ip), + BTreeSet::from([member]) + ); + } + + #[tokio::test] + async fn failed_registration_replacement_keeps_old_registration_and_invalidates() { + let mut shared_nic = SharedVirtualNic::new(virtual_nic_config()); + let member = member_id(1); + let old_registration = uuid::Uuid::from_u128(10); + let next_registration = uuid::Uuid::from_u128(11); + let ip = Ipv4Inet::from_str("10.70.0.2/24").unwrap(); + + shared_nic + .member_registrations + .insert(member, old_registration); + shared_nic.ifcfg_mut().apply_member_claims( + member, + SharedIfConfigClaims { + ipv4_addresses: BTreeSet::from([ip]), + ..Default::default() + }, + ); + let nic = shared_nic.nic(); + let mut nic = nic.lock().await; + nic.set_ifname_for_test("et0".to_string()); + nic.set_ifcfg_for_test(Box::new(FailingRemoveIpIfConfiger)); + drop(nic); + + let result = shared_nic + .attach_member_registration(member, next_registration) + .await; + + assert!(result.is_err()); + assert!(!shared_nic.is_valid()); + assert_eq!( + shared_nic.member_registrations.get(&member), + Some(&old_registration) + ); + } + + #[test] + fn registry_reuses_shared_virtual_nic_for_same_dev_name_and_netns() { + let mut registry = SharedVirtualNicRegistry::new(); + + let first = registry.get_or_create("et0".to_string(), virtual_nic_config()); + let second = registry.get_or_create("et0".to_string(), virtual_nic_config()); + + assert!(Arc::ptr_eq(&first, &second)); + } + + #[test] + fn registry_keeps_same_dev_name_in_different_netns_separate() { + let mut registry = SharedVirtualNicRegistry::new(); + + let first = registry.get_or_create("et0".to_string(), virtual_nic_config_in_netns("net-a")); + let second = + registry.get_or_create("et0".to_string(), virtual_nic_config_in_netns("net-b")); + + assert!(!Arc::ptr_eq(&first, &second)); + } + + #[test] + fn registry_keeps_different_dev_names_separate() { + let mut registry = SharedVirtualNicRegistry::new(); + + let first = registry.get_or_create("et0".to_string(), virtual_nic_config()); + let second = registry.get_or_create("et1".to_string(), virtual_nic_config()); + + assert!(!Arc::ptr_eq(&first, &second)); + } + + #[test] + fn registry_replaces_invalid_shared_virtual_nic() { + let mut registry = SharedVirtualNicRegistry::new(); + + let first = registry.get_or_create("et0".to_string(), virtual_nic_config()); + first.try_lock().unwrap().mark_invalid(); + let second = registry.get_or_create("et0".to_string(), virtual_nic_config()); + + assert!(!Arc::ptr_eq(&first, &second)); + assert!( + registry + .get("et0", &virtual_nic_config()) + .is_some_and(|nic| Arc::ptr_eq(&nic, &second)) + ); + } + + #[test] + fn registry_create_member_uses_registered_shared_virtual_nic() { + let mut registry = SharedVirtualNicRegistry::new(); + let member_id = member_id(1); + + let member = registry.create_member( + "et0".to_string(), + virtual_nic_config(), + member_id, + Arc::new(Notify::new()), + ); + let shared_nic = registry.get("et0", &virtual_nic_config()).unwrap(); + + assert_eq!(member.member_id(), member_id); + assert!(Arc::ptr_eq(&member.shared_nic(), &shared_nic)); + } + + #[test] + fn registry_create_member_keeps_member_configured_mtu() { + let mut registry = SharedVirtualNicRegistry::new(); + + let first = registry.create_member( + "et0".to_string(), + virtual_nic_config_with_mtu(1400), + member_id(1), + Arc::new(Notify::new()), + ); + let second = registry.create_member( + "et0".to_string(), + virtual_nic_config_with_mtu(1300), + member_id(2), + Arc::new(Notify::new()), + ); + + assert_eq!(first.configured_mtu_for_test(), 1400); + assert_eq!(second.configured_mtu_for_test(), 1300); + assert!(Arc::ptr_eq(&first.shared_nic(), &second.shared_nic())); + } +} diff --git a/easytier/src/instance/shared_virtual_nic/dispatcher.rs b/easytier/src/instance/shared_virtual_nic/dispatcher.rs new file mode 100644 index 00000000..9e1f3af6 --- /dev/null +++ b/easytier/src/instance/shared_virtual_nic/dispatcher.rs @@ -0,0 +1,4160 @@ +use std::{ + collections::{BTreeMap, BTreeSet, HashMap}, + pin::Pin, + sync::{ + Arc, Mutex as StdMutex, + atomic::{AtomicBool, Ordering}, + }, + time::{Duration, Instant}, +}; + +use cidr::{Ipv4Inet, Ipv6Inet}; +use futures::{SinkExt, StreamExt}; +use smoltcp::wire::{ + Icmpv4Message, Icmpv4Packet, Icmpv6Message, Icmpv6Packet, IpAddress, IpProtocol, Ipv4Packet, + Ipv6Packet, TcpPacket, UdpPacket, +}; +#[cfg(any(mobile, all(test, unix)))] +use tokio::sync::Mutex; +use tokio::sync::{Notify, mpsc, oneshot}; +use tokio_util::task::AbortOnDropHandle; + +use crate::common::error::Error; +#[cfg(any(mobile, all(test, unix)))] +use crate::instance::virtual_nic::VirtualNic; +use easytier_core::{ + packet::ZCPacket, + tunnel::{Tunnel, ZCPacketSink, ZCPacketStream}, +}; + +use super::{ + SHARED_VIRTUAL_NIC_RUNTIME, SharedIfConfigClaims, SharedIpv4Route, SharedIpv6Route, + SharedVirtualNicMemberId, SharedVirtualNicMemberRegistrationId, +}; + +const MEMBER_TUNNEL_BUFFER_SIZE: usize = 1024; +const FLOW_OWNER_LIMIT: usize = 4096; +// Match the core UDP proxy's idle lifetime; reclaim lazily under table pressure. +const FLOW_IDLE_TIMEOUT: Duration = Duration::from_secs(180); +const DISPATCHER_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(1); +const DISPATCHER_TUN_WRITE_TIMEOUT: Duration = Duration::from_secs(1); + +struct SharedVirtualNicMemberPacket { + member_id: SharedVirtualNicMemberId, + packet: ZCPacket, +} + +#[cfg(any(mobile, all(test, unix)))] +struct SharedVirtualNicMobileTunUpdate { + tun_fd: std::os::fd::RawFd, + ack: oneshot::Sender>, +} + +enum SharedVirtualNicControl { + Register { + member_id: SharedVirtualNicMemberId, + entry: SharedVirtualNicMemberTunnelEntry, + }, + Unregister { + member_id: SharedVirtualNicMemberId, + registration_id: SharedVirtualNicMemberRegistrationId, + }, + UpdateSources { + member_id: SharedVirtualNicMemberId, + sources: SharedVirtualNicMemberSources, + ack: oneshot::Sender<()>, + }, + Shutdown { + invalidate: bool, + ack: oneshot::Sender<()>, + }, +} + +#[derive(Clone, Default)] +pub(super) struct SharedVirtualNicMemberTunnelTable { + state: Arc>, +} + +#[derive(Default)] +struct SharedVirtualNicMemberTunnelTableState { + to_tun_sender: Option>, + control_sender: Option>, +} + +struct SharedVirtualNicMemberTunnelEntry { + registration_id: SharedVirtualNicMemberRegistrationId, + sender: mpsc::Sender, + close_notifier: Arc, + _tasks: Vec>, +} + +impl SharedVirtualNicMemberTunnelTable { + fn attach_dispatcher( + &self, + to_tun_sender: mpsc::Sender, + control_sender: mpsc::UnboundedSender, + ) { + let mut state = self.state.lock().unwrap(); + state.to_tun_sender = Some(to_tun_sender); + state.control_sender = Some(control_sender); + } + + pub(super) fn detach_dispatcher(&self) { + let mut state = self.state.lock().unwrap(); + state.to_tun_sender.take(); + state.control_sender.take(); + } + + pub(super) fn register( + &self, + member_id: SharedVirtualNicMemberId, + registration_id: SharedVirtualNicMemberRegistrationId, + tunnel: Box, + close_notifier: Arc, + ) -> Result<(), Error> { + let channels = self + .dispatcher_channels() + .ok_or_else(|| anyhow::anyhow!("shared virtual nic dispatcher is not running"))?; + let (to_tun_sender, control_sender) = channels; + let (mut member_stream, mut member_sink) = tunnel.split(); + let (to_member_sender, mut to_member_receiver) = mpsc::channel(MEMBER_TUNNEL_BUFFER_SIZE); + let (reader_start_sender, reader_start_receiver) = oneshot::channel(); + + let reader_control_sender = control_sender.clone(); + let reader_close_notifier = close_notifier.clone(); + let reader_task = AbortOnDropHandle::new(tokio::spawn(async move { + if reader_start_receiver.await.is_err() { + return; + } + + while let Some(packet) = member_stream.next().await { + let packet = match packet { + Ok(packet) => packet, + Err(err) => { + tracing::error!(?member_id, ?err, "shared member tunnel read failed"); + break; + } + }; + + if to_tun_sender + .send(SharedVirtualNicMemberPacket { member_id, packet }) + .await + .is_err() + { + break; + } + } + + notify_member_tunnel_closed( + &reader_control_sender, + &reader_close_notifier, + member_id, + registration_id, + ); + })); + + let writer_control_sender = control_sender.clone(); + let writer_close_notifier = close_notifier.clone(); + let writer_task = AbortOnDropHandle::new(tokio::spawn(async move { + while let Some(packet) = to_member_receiver.recv().await { + if let Err(err) = member_sink.send(packet).await { + tracing::error!(?member_id, ?err, "shared member tunnel write failed"); + notify_member_tunnel_closed( + &writer_control_sender, + &writer_close_notifier, + member_id, + registration_id, + ); + break; + } + } + })); + + let entry = SharedVirtualNicMemberTunnelEntry { + registration_id, + sender: to_member_sender, + close_notifier, + _tasks: vec![reader_task, writer_task], + }; + control_sender + .send(SharedVirtualNicControl::Register { member_id, entry }) + .map_err(|_| anyhow::anyhow!("shared virtual nic dispatcher is not running"))?; + let _ = reader_start_sender.send(()); + + Ok(()) + } + + pub(super) fn unregister( + &self, + member_id: SharedVirtualNicMemberId, + registration_id: SharedVirtualNicMemberRegistrationId, + ) { + let Some(control_sender) = self.control_sender() else { + return; + }; + let _ = control_sender.send(SharedVirtualNicControl::Unregister { + member_id, + registration_id, + }); + } + + fn dispatcher_channels( + &self, + ) -> Option<( + mpsc::Sender, + mpsc::UnboundedSender, + )> { + let state = self.state.lock().unwrap(); + Some((state.to_tun_sender.clone()?, state.control_sender.clone()?)) + } + + fn control_sender(&self) -> Option> { + self.state.lock().unwrap().control_sender.clone() + } +} + +fn notify_member_tunnel_closed( + control_sender: &mpsc::UnboundedSender, + close_notifier: &Notify, + member_id: SharedVirtualNicMemberId, + registration_id: SharedVirtualNicMemberRegistrationId, +) { + let _ = control_sender.send(SharedVirtualNicControl::Unregister { + member_id, + registration_id, + }); + close_notifier.notify_one(); +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +enum SharedVirtualNicFlowAddr { + V4(u32), + V6([u8; 16]), +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +struct SharedVirtualNicTransportPorts { + src: u16, + dst: u16, +} + +impl SharedVirtualNicTransportPorts { + fn reversed(self) -> Self { + Self { + src: self.dst, + dst: self.src, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +struct SharedVirtualNicFlowKey { + src: SharedVirtualNicFlowAddr, + dst: SharedVirtualNicFlowAddr, + protocol: u8, + ports: Option, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +struct SharedVirtualNicMemberSources { + ipv4_addresses: BTreeSet, + ipv6_addresses: BTreeSet, + ipv4_routes: BTreeSet, + ipv6_routes: BTreeSet, +} + +impl SharedVirtualNicMemberSources { + fn from_claims(claims: &SharedIfConfigClaims) -> Self { + let ipv4_addresses = claims + .ipv4_addresses + .iter() + .filter(|addr| !addr.address().is_unspecified()) + .copied() + .collect::>(); + let ipv6_addresses = claims + .ipv6_addresses + .iter() + .filter(|addr| !addr.address().is_unspecified()) + .copied() + .collect::>(); + let ipv4_routes = claims + .ipv4_routes + .iter() + .filter_map(ipv4_route_to_inet) + .collect::>(); + let ipv6_routes = claims + .ipv6_routes + .iter() + .filter_map(ipv6_route_to_inet) + .collect::>(); + + Self { + ipv4_addresses, + ipv6_addresses, + ipv4_routes, + ipv6_routes, + } + } + + fn is_empty(&self) -> bool { + self.ipv4_addresses.is_empty() + && self.ipv6_addresses.is_empty() + && self.ipv4_routes.is_empty() + && self.ipv6_routes.is_empty() + } + + fn owns_source(&self, source: SharedVirtualNicFlowAddr) -> bool { + match source { + SharedVirtualNicFlowAddr::V4(source) => self + .ipv4_addresses + .iter() + .any(|address| u32::from(address.address()) == source), + SharedVirtualNicFlowAddr::V6(source) => self + .ipv6_addresses + .iter() + .any(|address| address.address().octets() == source), + } + } +} + +fn ipv4_route_to_inet(route: &SharedIpv4Route) -> Option { + Ipv4Inet::new(route.address, route.prefix).ok() +} + +fn ipv6_route_to_inet(route: &SharedIpv6Route) -> Option { + Ipv6Inet::new(route.address, route.prefix).ok() +} + +impl From for SharedVirtualNicFlowAddr { + fn from(addr: std::net::Ipv4Addr) -> Self { + Self::V4(u32::from_be_bytes(addr.octets())) + } +} + +impl From for SharedVirtualNicFlowAddr { + fn from(addr: std::net::Ipv6Addr) -> Self { + Self::V6(addr.octets()) + } +} + +impl SharedVirtualNicFlowAddr { + fn as_ipv4(self) -> Option { + match self { + Self::V4(addr) => Some(std::net::Ipv4Addr::from(addr)), + Self::V6(_) => None, + } + } + + fn as_ipv6(self) -> Option { + match self { + Self::V4(_) => None, + Self::V6(addr) => Some(std::net::Ipv6Addr::from(addr)), + } + } + fn packet_addrs(packet: &ZCPacket) -> Option<(Self, Self)> { + let payload = packet.payload(); + match payload.first()? >> 4 { + 4 => { + let ipv4 = Ipv4Packet::new_checked(payload).ok()?; + Some((ipv4.src_addr().into(), ipv4.dst_addr().into())) + } + 6 => { + let ipv6 = Ipv6Packet::new_checked(payload).ok()?; + Some((ipv6.src_addr().into(), ipv6.dst_addr().into())) + } + _ => None, + } + } +} + +impl SharedVirtualNicFlowKey { + fn from_packet(packet: &ZCPacket) -> Option { + let payload = packet.payload(); + let version = payload.first()? >> 4; + match version { + 4 => Self::from_ipv4_payload(payload), + 6 => Self::from_ipv6_payload(payload), + _ => None, + } + } + + fn from_ipv4_payload(payload: &[u8]) -> Option { + let ipv4 = Ipv4Packet::new_checked(payload).ok()?; + if ipv4.version() != 4 { + return None; + } + + let protocol = ipv4.next_header().into(); + Some(Self { + src: ipv4.src_addr().into(), + dst: ipv4.dst_addr().into(), + protocol, + ports: if ipv4.frag_offset() == 0 { + transport_ports(protocol, ipv4.payload()) + } else { + None + }, + }) + } + + fn from_ipv6_payload(payload: &[u8]) -> Option { + let ipv6 = Ipv6Packet::new_checked(payload).ok()?; + if ipv6.version() != 6 { + return None; + } + + let protocol = ipv6.next_header().into(); + Some(Self { + src: ipv6.src_addr().into(), + dst: ipv6.dst_addr().into(), + protocol, + ports: transport_ports(protocol, ipv6.payload()), + }) + } + + fn reversed(&self) -> Self { + Self { + src: self.dst, + dst: self.src, + protocol: self.protocol, + ports: self.ports.map(|ports| { + if matches!( + IpProtocol::from(self.protocol), + IpProtocol::Icmp | IpProtocol::Icmpv6 + ) { + ports + } else { + ports.reversed() + } + }), + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct SharedVirtualNicNatEntry { + original: SharedVirtualNicFlowKey, + wire: SharedVirtualNicFlowKey, + source_owner: Option, + incoming: bool, + last_active: Instant, +} + +#[derive(Default)] +struct SharedVirtualNicNatTable { + entries: HashMap<(SharedVirtualNicMemberId, SharedVirtualNicFlowKey), SharedVirtualNicNatEntry>, + replies: HashMap<(SharedVirtualNicMemberId, SharedVirtualNicFlowKey), SharedVirtualNicFlowKey>, +} + +impl SharedVirtualNicNatTable { + fn prepare_packet( + &mut self, + member_id: SharedVirtualNicMemberId, + source_owner: Option, + original: SharedVirtualNicFlowKey, + packet: &mut ZCPacket, + translated_src: SharedVirtualNicFlowAddr, + ) -> bool { + if original.ports.is_none() { + return original.src == translated_src; + } + let now = Instant::now(); + if let Some(entry) = self.entries.get_mut(&(member_id, original)) { + entry.last_active = now; + return rewrite_packet_flow(packet, original, entry.wire, RewriteIpAddr::Source); + } + let mut wire = original; + wire.src = translated_src; + // Wildcard UDP sockets may reply with another member's source address. + // Prefer an incoming reservation after source correction, preserving its + // service port and incoming destination. Exact outgoing mappings win above; + // packet tuples cannot distinguish a new client with the same endpoints. + if IpProtocol::from(original.protocol) == IpProtocol::Udp + && let Some(entry) = self.entries.get_mut(&(member_id, wire)) + && entry.incoming + { + entry.last_active = now; + return rewrite_packet_flow(packet, original, entry.wire, RewriteIpAddr::Source); + } + if !self.ensure_capacity(now) { + return false; + } + + while self.replies.contains_key(&(member_id, wire.reversed())) { + let Some(ports) = wire.ports.as_mut() else { + return false; + }; + ports.src = ports.src.wrapping_add(1).max(1); + if Some(*ports) == original.ports { + return false; + } + } + if !rewrite_packet_flow(packet, original, wire, RewriteIpAddr::Source) { + return false; + } + self.entries.insert( + (member_id, original), + SharedVirtualNicNatEntry { + original, + wire, + source_owner, + incoming: false, + last_active: now, + }, + ); + self.replies.insert((member_id, wire.reversed()), original); + true + } + + fn reserve_reply( + &mut self, + member_id: SharedVirtualNicMemberId, + reply: SharedVirtualNicFlowKey, + ) -> bool { + if reply.ports.is_none() || self.replies.contains_key(&(member_id, reply)) { + return true; + } + let original = reply.reversed(); + // An outgoing native flow may already have been assigned a different port. + if self.entries.contains_key(&(member_id, original)) { + return false; + } + let now = Instant::now(); + if !self.ensure_capacity(now) { + return false; + } + self.entries.insert( + (member_id, original), + SharedVirtualNicNatEntry { + original, + wire: original, + source_owner: Some(member_id), + incoming: true, + last_active: now, + }, + ); + self.replies.insert((member_id, reply), original); + true + } + + fn ensure_capacity(&mut self, now: Instant) -> bool { + // Reuse only idle reservations, never an active flow's wire tuple. + if self.entries.len() >= FLOW_OWNER_LIMIT { + self.retain(|_, entry| now.duration_since(entry.last_active) < FLOW_IDLE_TIMEOUT); + } + self.entries.len() < FLOW_OWNER_LIMIT + } + + fn translate_reply( + &mut self, + member_id: SharedVirtualNicMemberId, + packet: &mut ZCPacket, + ) -> bool { + let Some(key) = SharedVirtualNicFlowKey::from_packet(packet) else { + return false; + }; + let Some(original) = self.replies.get(&(member_id, key)) else { + return false; + }; + self.entries + .get_mut(&(member_id, *original)) + .unwrap() + .last_active = Instant::now(); + rewrite_packet_flow(packet, key, original.reversed(), RewriteIpAddr::Destination) + } + + fn remove_member(&mut self, member_id: SharedVirtualNicMemberId) { + self.retain(|owner, entry| owner != member_id && entry.source_owner != Some(member_id)); + } + + fn remove_changed_addresses( + &mut self, + previous: &SharedVirtualNicMemberSources, + current: &SharedVirtualNicMemberSources, + ) { + self.retain(|_, entry| { + [entry.original.src, entry.wire.src] + .into_iter() + .all(|addr| !previous.owns_source(addr) || current.owns_source(addr)) + }); + } + + fn retain( + &mut self, + keep: impl Fn(SharedVirtualNicMemberId, &SharedVirtualNicNatEntry) -> bool, + ) { + self.entries.retain(|(member_id, _), entry| { + if keep(*member_id, entry) { + true + } else { + self.replies.remove(&(*member_id, entry.wire.reversed())); + false + } + }); + } + + fn clear(&mut self) { + self.entries.clear(); + self.replies.clear(); + } +} + +pub(super) struct SharedVirtualNicDispatcher { + _task: AbortOnDropHandle<()>, + control_sender: mpsc::UnboundedSender, + #[cfg(mobile)] + mobile_tun_update_sender: Option>, +} + +impl SharedVirtualNicDispatcher { + pub(super) fn start( + tunnel: Box, + member_tunnel_table: SharedVirtualNicMemberTunnelTable, + valid: Arc, + ) -> Self { + let (tun_stream, tun_sink) = tunnel.split(); + let (to_tun_sender, to_tun_receiver) = mpsc::channel(MEMBER_TUNNEL_BUFFER_SIZE); + let (control_sender, control_receiver) = mpsc::unbounded_channel(); + member_tunnel_table.attach_dispatcher(to_tun_sender, control_sender.clone()); + + let task = SharedVirtualNicDispatcherTask { + tun_stream, + tun_sink, + to_tun_receiver, + control_receiver, + member_tunnel_table, + valid, + state: SharedVirtualNicDispatcherState::default(), + }; + + Self { + _task: AbortOnDropHandle::new(SHARED_VIRTUAL_NIC_RUNTIME.spawn(task.run())), + control_sender, + #[cfg(mobile)] + mobile_tun_update_sender: None, + } + } + + #[cfg(mobile)] + pub(super) async fn start_for_mobile( + nic: Arc>, + tun_fd: std::os::fd::RawFd, + member_tunnel_table: SharedVirtualNicMemberTunnelTable, + valid: Arc, + ) -> Result { + let (to_tun_sender, to_tun_receiver) = mpsc::channel(MEMBER_TUNNEL_BUFFER_SIZE); + let (control_sender, control_receiver) = mpsc::unbounded_channel(); + let (mobile_tun_update_sender, mobile_tun_update_receiver) = mpsc::unbounded_channel(); + let (first_open_sender, first_open_receiver) = oneshot::channel(); + let task_member_tunnel_table = member_tunnel_table.clone(); + member_tunnel_table.attach_dispatcher(to_tun_sender, control_sender.clone()); + + let task = SHARED_VIRTUAL_NIC_RUNTIME.spawn(async move { + let task = SharedVirtualNicMobileDispatcherTask { + nic, + tun_stream: None, + tun_sink: None, + tun_fd, + tun_update_receiver: mobile_tun_update_receiver, + to_tun_receiver, + control_receiver, + member_tunnel_table: task_member_tunnel_table, + valid, + state: SharedVirtualNicDispatcherState { + translate_unknown_sources: true, + ..Default::default() + }, + }; + + task.run(first_open_sender).await; + }); + + let dispatcher = Self { + _task: AbortOnDropHandle::new(task), + control_sender, + mobile_tun_update_sender: Some(mobile_tun_update_sender), + }; + await_mobile_open(first_open_receiver) + .await + .map(|()| dispatcher) + } + + #[cfg(mobile)] + pub(super) async fn update_mobile_tun_fd( + &self, + tun_fd: std::os::fd::RawFd, + ) -> Result<(), Error> { + let sender = self.mobile_tun_update_sender.as_ref().ok_or_else(|| { + Error::from(anyhow::anyhow!( + "shared virtual nic mobile dispatcher is not running" + )) + })?; + let (ack, receiver) = oneshot::channel(); + sender + .send(SharedVirtualNicMobileTunUpdate { tun_fd, ack }) + .map_err(|_| { + Error::from(anyhow::anyhow!( + "shared virtual nic mobile dispatcher is not running" + )) + })?; + await_mobile_open(receiver).await + } + + pub(super) async fn update_sources( + &self, + member_id: SharedVirtualNicMemberId, + claims: &SharedIfConfigClaims, + ) -> Result<(), Error> { + self.send_source_update( + member_id, + SharedVirtualNicMemberSources::from_claims(claims), + ) + .await + } + + pub(super) async fn remove_sources( + &self, + member_id: SharedVirtualNicMemberId, + ) -> Result<(), Error> { + self.send_source_update(member_id, SharedVirtualNicMemberSources::default()) + .await + } + + async fn send_source_update( + &self, + member_id: SharedVirtualNicMemberId, + sources: SharedVirtualNicMemberSources, + ) -> Result<(), Error> { + let (ack, rx) = oneshot::channel(); + self.control_sender + .send(SharedVirtualNicControl::UpdateSources { + member_id, + sources, + ack, + }) + .map_err(|_| anyhow::anyhow!("shared virtual nic dispatcher is not running"))?; + rx.await + .map_err(|_| anyhow::anyhow!("shared virtual nic dispatcher is not running").into()) + } + + pub(super) async fn shutdown_without_invalidation(self) { + let (ack, rx) = oneshot::channel(); + if self + .control_sender + .send(SharedVirtualNicControl::Shutdown { + invalidate: false, + ack, + }) + .is_err() + { + return; + } + + if tokio::time::timeout(DISPATCHER_SHUTDOWN_TIMEOUT, rx) + .await + .is_err() + { + tracing::warn!("timed out shutting down shared virtual nic dispatcher"); + } + } +} + +enum DispatcherControlResult { + Continue, + Stop { + invalidate: bool, + ack: Option>, + }, +} + +fn handle_dispatcher_control( + state: &mut SharedVirtualNicDispatcherState, + control: Option, +) -> DispatcherControlResult { + let Some(control) = control else { + return DispatcherControlResult::Stop { + invalidate: true, + ack: None, + }; + }; + + match control { + SharedVirtualNicControl::Shutdown { invalidate, ack } => DispatcherControlResult::Stop { + invalidate, + ack: Some(ack), + }, + other => { + state.handle_control(other); + DispatcherControlResult::Continue + } + } +} + +fn acknowledge_dispatcher_shutdown(ack: Option>) { + if let Some(ack) = ack { + let _ = ack.send(()); + } +} + +async fn send_packet_to_tun( + tun_sink: &mut Pin>, + packet: ZCPacket, +) -> Result<(), Error> { + tokio::time::timeout(DISPATCHER_TUN_WRITE_TIMEOUT, tun_sink.send(packet)).await??; + Ok(()) +} + +struct SharedVirtualNicDispatcherTask { + tun_stream: Pin>, + tun_sink: Pin>, + to_tun_receiver: mpsc::Receiver, + control_receiver: mpsc::UnboundedReceiver, + member_tunnel_table: SharedVirtualNicMemberTunnelTable, + valid: Arc, + state: SharedVirtualNicDispatcherState, +} + +impl SharedVirtualNicDispatcherTask { + async fn run(mut self) { + loop { + tokio::select! { + control = self.control_receiver.recv() => { + if let DispatcherControlResult::Stop { invalidate, ack } = + handle_dispatcher_control(&mut self.state, control) + { + self.cleanup(invalidate); + acknowledge_dispatcher_shutdown(ack); + return; + } + } + member_packet = self.to_tun_receiver.recv() => { + let Some(member_packet) = member_packet else { + break; + }; + if !self.forward_member_packet_to_tun(member_packet).await { + break; + } + } + packet = self.tun_stream.next() => { + let Some(packet) = packet else { + break; + }; + let packet = match packet { + Ok(packet) => packet, + Err(err) => { + tracing::error!(?err, "shared virtual nic read from tun failed"); + break; + } + }; + self.state.forward_tun_packet_to_member(packet).await; + } + } + } + + self.cleanup(true); + } + + fn cleanup(&mut self, invalidate: bool) { + if invalidate { + self.valid.store(false, Ordering::Release); + } + self.member_tunnel_table.detach_dispatcher(); + self.state.close_all(); + } + + async fn forward_member_packet_to_tun( + &mut self, + member_packet: SharedVirtualNicMemberPacket, + ) -> bool { + let Some(packet) = self + .state + .prepare_member_packet_to_tun(member_packet.member_id, member_packet.packet) + else { + return true; + }; + if let Err(err) = send_packet_to_tun(&mut self.tun_sink, packet).await { + tracing::error!(?err, "shared virtual nic write to tun failed"); + return false; + } + true + } +} + +#[cfg(any(mobile, all(test, unix)))] +struct SharedVirtualNicMobileDispatcherTask { + nic: Arc>, + tun_stream: Option>>, + tun_sink: Option>>, + tun_fd: std::os::fd::RawFd, + tun_update_receiver: mpsc::UnboundedReceiver, + to_tun_receiver: mpsc::Receiver, + control_receiver: mpsc::UnboundedReceiver, + member_tunnel_table: SharedVirtualNicMemberTunnelTable, + valid: Arc, + state: SharedVirtualNicDispatcherState, +} + +#[cfg(any(mobile, all(test, unix)))] +impl SharedVirtualNicMobileDispatcherTask { + #[cfg(mobile)] + async fn run(mut self, first_open_sender: oneshot::Sender>) { + match self.open_tun().await { + Ok(()) => { + if first_open_sender.send(Ok(())).is_err() { + self.cleanup(false); + return; + } + } + Err(err) => { + self.cleanup(false); + let _ = first_open_sender.send(Err(err)); + return; + } + } + + self.run_opened_tun().await; + } + + async fn run_opened_tun(mut self) { + loop { + tokio::select! { + control = self.control_receiver.recv() => { + if !self.handle_control(control) { + return; + } + } + member_packet = self.to_tun_receiver.recv() => { + let Some(member_packet) = member_packet else { + self.cleanup(true); + return; + }; + if !self.forward_member_packet_to_tun(member_packet).await { + self.cleanup(true); + return; + } + } + packet = async { + match self.tun_stream.as_mut() { + Some(stream) => stream.next().await, + None => std::future::pending().await, + } + } => { + let Some(packet) = packet else { + tracing::error!("shared virtual nic mobile tun stream closed"); + self.cleanup(true); + return; + }; + match packet { + Ok(packet) => self.state.forward_tun_packet_to_member(packet).await, + Err(err) => { + tracing::error!(?err, "shared virtual nic read from mobile tun failed"); + self.cleanup(true); + return; + } + } + } + update = self.tun_update_receiver.recv() => { + let Some(update) = update else { + self.cleanup(true); + return; + }; + if !self.apply_tun_update(update).await { + return; + } + } + } + } + } + + async fn open_tun(&mut self) -> Result<(), Error> { + let tun_fd = self.tun_fd; + let tunnel = self.nic.lock().await.create_dev_for_mobile(tun_fd).await?; + let (tun_stream, tun_sink) = tunnel.split(); + self.tun_stream = Some(tun_stream); + self.tun_sink = Some(tun_sink); + tracing::info!(fd = tun_fd, "opened shared virtual nic mobile tun"); + Ok(()) + } + + async fn apply_tun_update(&mut self, update: SharedVirtualNicMobileTunUpdate) -> bool { + self.drop_tun(); + self.tun_fd = update.tun_fd; + match self.open_tun().await { + Ok(()) => { + let _ = update.ack.send(Ok(())); + true + } + Err(err) => { + tracing::error!(fd = self.tun_fd, ?err, "failed to replace mobile tun"); + self.cleanup(true); + let _ = update.ack.send(Err(err)); + false + } + } + } + + async fn forward_member_packet_to_tun( + &mut self, + member_packet: SharedVirtualNicMemberPacket, + ) -> bool { + let Some(packet) = self + .state + .prepare_member_packet_to_tun(member_packet.member_id, member_packet.packet) + else { + return true; + }; + let Some(tun_sink) = self.tun_sink.as_mut() else { + return false; + }; + + if let Err(err) = send_packet_to_tun(tun_sink, packet).await { + tracing::error!(?err, "shared virtual nic write to mobile tun failed"); + return false; + } + true + } + + fn handle_control(&mut self, control: Option) -> bool { + match handle_dispatcher_control(&mut self.state, control) { + DispatcherControlResult::Continue => true, + DispatcherControlResult::Stop { invalidate, ack } => { + self.cleanup(invalidate); + acknowledge_dispatcher_shutdown(ack); + false + } + } + } + + fn drop_tun(&mut self) { + self.tun_stream.take(); + self.tun_sink.take(); + } + + fn cleanup(&mut self, invalidate: bool) { + self.drop_tun(); + if invalidate { + self.valid.store(false, Ordering::Release); + } + self.member_tunnel_table.detach_dispatcher(); + self.state.close_all(); + } +} + +#[cfg(any(mobile, test))] +async fn await_mobile_open(receiver: oneshot::Receiver>) -> Result<(), Error> { + receiver.await.map_err(|_| { + Error::from(anyhow::anyhow!( + "shared virtual nic mobile dispatcher stopped before opening TUN" + )) + })? +} + +#[derive(Default)] +struct SharedVirtualNicDispatcherState { + members: BTreeMap, + nat_table: SharedVirtualNicNatTable, + source_table: SharedVirtualNicSourceTable, + // Mobile VPNs can supply synthetic sources, unlike desktop subnet forwarding. + translate_unknown_sources: bool, +} + +impl SharedVirtualNicDispatcherState { + fn handle_control(&mut self, control: SharedVirtualNicControl) { + match control { + SharedVirtualNicControl::Register { member_id, entry } => { + self.register(member_id, entry); + } + SharedVirtualNicControl::Unregister { + member_id, + registration_id, + } => { + self.unregister(member_id, registration_id); + } + SharedVirtualNicControl::UpdateSources { + member_id, + sources, + ack, + } => { + if let Some(previous) = self.source_table.member_sources.get(&member_id) { + self.nat_table.remove_changed_addresses(previous, &sources); + } + self.source_table.update_member_sources(member_id, sources); + let _ = ack.send(()); + } + SharedVirtualNicControl::Shutdown { .. } => { + unreachable!("dispatcher shutdown is handled by the dispatcher task") + } + } + } + + fn register( + &mut self, + member_id: SharedVirtualNicMemberId, + entry: SharedVirtualNicMemberTunnelEntry, + ) { + let old_entry = self.members.insert(member_id, entry); + if old_entry.is_some() { + self.nat_table.remove_member(member_id); + } + drop(old_entry); + } + + fn unregister( + &mut self, + member_id: SharedVirtualNicMemberId, + registration_id: SharedVirtualNicMemberRegistrationId, + ) { + if self + .members + .get(&member_id) + .map(|entry| entry.registration_id) + != Some(registration_id) + { + return; + } + + let entry = self.members.remove(&member_id); + drop(entry); + self.nat_table.remove_member(member_id); + self.source_table.remove_owner(member_id); + } + + fn close_all(&mut self) { + let members = std::mem::take(&mut self.members); + self.nat_table.clear(); + self.source_table.clear(); + + for entry in members.into_values() { + entry.close_notifier.notify_one(); + } + } + + fn prepare_member_packet_to_tun( + &mut self, + member_id: SharedVirtualNicMemberId, + mut packet: ZCPacket, + ) -> Option { + // A local socket can receive before it ever sends. Reserve its reverse + // tuple before delivering the packet, so later translations cannot take it. + if let Some(key) = SharedVirtualNicFlowKey::from_packet(&packet) + && self + .source_table + .member_sources + .get(&member_id) + .is_some_and(|sources| sources.owns_source(key.dst)) + && !self.nat_table.reserve_reply(member_id, key) + { + return None; + } + self.nat_table.translate_reply(member_id, &mut packet); + Some(packet) + } + + async fn forward_tun_packet_to_member(&mut self, packet: ZCPacket) { + if !self.send_packet(packet).await { + tracing::trace!("shared virtual nic dropped packet without active member"); + } + } + + async fn send_packet(&mut self, packet: ZCPacket) -> bool { + let source_owner = self.source_table.owner_of_source(&packet, &self.members); + if self.members.len() == 1 { + let member_id = *self.members.keys().next().unwrap(); + // Keep native flow reservations for later members, without creating + // translations while this member has exclusive use of the TUN. + let source_owner = source_owner + .active_member() + .or_else(|| self.translate_unknown_sources.then_some(member_id)); + return self + .send_packet_to_member_with_translation(member_id, source_owner, packet, false) + .await + .is_ok(); + } + let preferred_destination_owner = source_owner.active_member(); + let destination_owner = self.source_table.owner_of_destination( + &packet, + &self.members, + preferred_destination_owner, + ); + if let Some(member_id) = destination_owner { + let translate_source = should_translate_source_for_member( + source_owner, + member_id, + self.translate_unknown_sources, + ); + return self + .send_packet_to_member_with_translation( + member_id, + source_owner.active_member(), + packet, + translate_source, + ) + .await + .is_ok(); + } + + match source_owner { + SourceOwner::Active(member_id) => { + return self + .send_packet_to_member_with_translation( + member_id, + Some(member_id), + packet, + false, + ) + .await + .is_ok(); + } + SourceOwner::Inactive => return false, + SourceOwner::None => {} + } + + false + } + + async fn send_packet_to_member_with_translation( + &mut self, + member_id: SharedVirtualNicMemberId, + source_owner: Option, + mut packet: ZCPacket, + translate_source: bool, + ) -> Result<(), ZCPacket> { + let Some((registration_id, sender)) = self + .members + .get(&member_id) + .map(|entry| (entry.registration_id, entry.sender.clone())) + else { + return Err(packet); + }; + // Reserve queue capacity before allocating a flow, and never await a slow member. + let permit = match sender.try_reserve() { + Ok(permit) => permit, + Err(mpsc::error::TrySendError::Full(())) => return Err(packet), + Err(mpsc::error::TrySendError::Closed(())) => { + self.unregister(member_id, registration_id); + return Err(packet); + } + }; + let key = SharedVirtualNicFlowKey::from_packet(&packet); + // Forwarded subnet sources cannot collide with a member's translated address. + // Keep applying an existing translation after other members leave. + if !translate_source + && source_owner.is_none() + && !key.is_some_and(|key| self.nat_table.entries.contains_key(&(member_id, key))) + { + permit.send(packet); + return Ok(()); + } + let Some(key) = key else { + return Err(packet); + }; + let translated_src = if translate_source { + self.source_table + .source_for_member_destination(member_id, key.dst) + .unwrap_or(key.src) + } else { + key.src + }; + // Native flows need reservations too, regardless of which flow arrives first. + if !self + .nat_table + .prepare_packet(member_id, source_owner, key, &mut packet, translated_src) + { + return Err(packet); + } + permit.send(packet); + Ok(()) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum SourceOwner { + Active(SharedVirtualNicMemberId), + Inactive, + None, +} + +impl SourceOwner { + fn active_member(self) -> Option { + match self { + Self::Active(member_id) => Some(member_id), + Self::Inactive | Self::None => None, + } + } +} + +fn should_translate_source_for_member( + source_owner: SourceOwner, + member_id: SharedVirtualNicMemberId, + translate_unknown_sources: bool, +) -> bool { + match source_owner { + SourceOwner::Active(source) => source != member_id, + SourceOwner::Inactive => true, + SourceOwner::None => translate_unknown_sources, + } +} + +#[cfg(test)] +fn rewrite_packet_source( + packet: &mut ZCPacket, + expected_source: SharedVirtualNicFlowAddr, + new_source: SharedVirtualNicFlowAddr, +) -> bool { + rewrite_ip_addr( + packet, + expected_source, + new_source, + RewriteIpAddr::Source, + None, + ) +} + +fn rewrite_packet_flow( + packet: &mut ZCPacket, + original: SharedVirtualNicFlowKey, + translated: SharedVirtualNicFlowKey, + rewrite: RewriteIpAddr, +) -> bool { + if original == translated { + return true; + } + let new_port = (original.ports != translated.ports).then(|| { + let ports = translated.ports.unwrap(); + match (rewrite, IpProtocol::from(translated.protocol)) { + (_, IpProtocol::Icmp | IpProtocol::Icmpv6) | (RewriteIpAddr::Source, _) => ports.src, + (RewriteIpAddr::Destination, _) => ports.dst, + } + }); + let (old_addr, new_addr) = match rewrite { + RewriteIpAddr::Source => (original.src, translated.src), + RewriteIpAddr::Destination => (original.dst, translated.dst), + }; + rewrite_ip_addr(packet, old_addr, new_addr, rewrite, new_port) +} + +#[derive(Clone, Copy)] +enum RewriteIpAddr { + Source, + Destination, +} + +fn rewrite_ip_addr( + packet: &mut ZCPacket, + expected_addr: SharedVirtualNicFlowAddr, + new_addr: SharedVirtualNicFlowAddr, + rewrite: RewriteIpAddr, + new_port: Option, +) -> bool { + match (expected_addr, new_addr) { + (SharedVirtualNicFlowAddr::V4(_), SharedVirtualNicFlowAddr::V4(_)) => { + rewrite_ipv4_addr(packet, expected_addr, new_addr, rewrite, new_port) + } + (SharedVirtualNicFlowAddr::V6(_), SharedVirtualNicFlowAddr::V6(_)) => { + rewrite_ipv6_addr(packet, expected_addr, new_addr, rewrite, new_port) + } + _ => false, + } +} + +fn rewrite_ipv4_addr( + packet: &mut ZCPacket, + expected_addr: SharedVirtualNicFlowAddr, + new_addr: SharedVirtualNicFlowAddr, + rewrite: RewriteIpAddr, + new_port: Option, +) -> bool { + let Some(expected_addr) = expected_addr.as_ipv4() else { + return false; + }; + let Some(new_addr) = new_addr.as_ipv4() else { + return false; + }; + + let payload = packet.mut_payload(); + let Ok(mut ipv4_packet) = Ipv4Packet::new_checked(payload) else { + return false; + }; + + if ipv4_packet.version() != 4 || ipv4_packet.frag_offset() != 0 || ipv4_packet.more_frags() { + return false; + } + + let valid_transport = match ipv4_packet.next_header() { + IpProtocol::Tcp => TcpPacket::new_checked(ipv4_packet.payload_mut()).is_ok(), + IpProtocol::Udp => UdpPacket::new_checked(ipv4_packet.payload_mut()).is_ok(), + IpProtocol::Icmp => Icmpv4Packet::new_checked(ipv4_packet.payload_mut()).is_ok(), + _ => true, + }; + if !valid_transport { + return false; + } + + match rewrite { + RewriteIpAddr::Source if ipv4_packet.src_addr() == expected_addr => { + ipv4_packet.set_src_addr(new_addr); + } + RewriteIpAddr::Destination if ipv4_packet.dst_addr() == expected_addr => { + ipv4_packet.set_dst_addr(new_addr); + } + _ => return false, + } + + if let Some(port) = new_port { + rewrite_transport_port( + ipv4_packet.next_header(), + ipv4_packet.payload_mut(), + port, + rewrite, + ); + } + update_ipv4_transport_checksum(&mut ipv4_packet); + ipv4_packet.fill_checksum(); + true +} + +fn rewrite_ipv6_addr( + packet: &mut ZCPacket, + expected_addr: SharedVirtualNicFlowAddr, + new_addr: SharedVirtualNicFlowAddr, + rewrite: RewriteIpAddr, + new_port: Option, +) -> bool { + let Some(expected_addr) = expected_addr.as_ipv6() else { + return false; + }; + let Some(new_addr) = new_addr.as_ipv6() else { + return false; + }; + + let payload = packet.mut_payload(); + let Ok(mut ipv6_packet) = Ipv6Packet::new_checked(payload) else { + return false; + }; + if ipv6_packet.version() != 6 { + return false; + } + let protocol = ipv6_packet.next_header(); + let valid_transport = match protocol { + IpProtocol::Tcp => TcpPacket::new_checked(ipv6_packet.payload_mut()).is_ok(), + IpProtocol::Udp => UdpPacket::new_checked(ipv6_packet.payload_mut()).is_ok(), + IpProtocol::Icmpv6 => Icmpv6Packet::new_checked(ipv6_packet.payload_mut()).is_ok(), + _ => false, + }; + if !valid_transport { + return false; + } + + match rewrite { + RewriteIpAddr::Source if ipv6_packet.src_addr() == expected_addr => { + ipv6_packet.set_src_addr(new_addr); + } + RewriteIpAddr::Destination if ipv6_packet.dst_addr() == expected_addr => { + ipv6_packet.set_dst_addr(new_addr); + } + _ => return false, + } + + if let Some(port) = new_port { + rewrite_transport_port(protocol, ipv6_packet.payload_mut(), port, rewrite); + } + let source = ipv6_packet.src_addr(); + let destination = ipv6_packet.dst_addr(); + match protocol { + IpProtocol::Tcp => TcpPacket::new_checked(ipv6_packet.payload_mut()) + .unwrap() + .fill_checksum(&IpAddress::Ipv6(source), &IpAddress::Ipv6(destination)), + IpProtocol::Udp => UdpPacket::new_checked(ipv6_packet.payload_mut()) + .unwrap() + .fill_checksum(&IpAddress::Ipv6(source), &IpAddress::Ipv6(destination)), + IpProtocol::Icmpv6 => Icmpv6Packet::new_checked(ipv6_packet.payload_mut()) + .unwrap() + .fill_checksum(&source, &destination), + _ => unreachable!(), + } + true +} + +// Callers have already checked the transport header before changing the address. +fn rewrite_transport_port( + protocol: IpProtocol, + payload: &mut [u8], + port: u16, + rewrite: RewriteIpAddr, +) { + match protocol { + IpProtocol::Tcp => { + let mut packet = TcpPacket::new_unchecked(payload); + match rewrite { + RewriteIpAddr::Source => packet.set_src_port(port), + RewriteIpAddr::Destination => packet.set_dst_port(port), + } + } + IpProtocol::Udp => { + let mut packet = UdpPacket::new_unchecked(payload); + match rewrite { + RewriteIpAddr::Source => packet.set_src_port(port), + RewriteIpAddr::Destination => packet.set_dst_port(port), + } + } + IpProtocol::Icmp => Icmpv4Packet::new_unchecked(payload).set_echo_ident(port), + IpProtocol::Icmpv6 => Icmpv6Packet::new_unchecked(payload).set_echo_ident(port), + _ => unreachable!(), + } +} + +fn update_ipv4_transport_checksum(ipv4_packet: &mut Ipv4Packet<&mut [u8]>) { + let source = IpAddress::Ipv4(ipv4_packet.src_addr()); + let destination = IpAddress::Ipv4(ipv4_packet.dst_addr()); + let protocol = ipv4_packet.next_header(); + let transport_payload = ipv4_packet.payload_mut(); + + match protocol { + IpProtocol::Tcp => { + TcpPacket::new_checked(transport_payload) + .unwrap() + .fill_checksum(&source, &destination); + } + IpProtocol::Udp => { + let mut udp_packet = UdpPacket::new_checked(transport_payload).unwrap(); + if udp_packet.checksum() == 0 { + return; + } + udp_packet.fill_checksum(&source, &destination); + } + IpProtocol::Icmp => { + Icmpv4Packet::new_checked(transport_payload) + .unwrap() + .fill_checksum(); + } + _ => {} + } +} + +#[derive(Default)] +struct SharedVirtualNicSourceTable { + member_sources: BTreeMap, +} + +impl SharedVirtualNicSourceTable { + fn update_member_sources( + &mut self, + member_id: SharedVirtualNicMemberId, + sources: SharedVirtualNicMemberSources, + ) { + if sources.is_empty() { + self.member_sources.remove(&member_id); + } else { + self.member_sources.insert(member_id, sources); + } + } + + fn remove_owner(&mut self, member_id: SharedVirtualNicMemberId) { + self.member_sources.remove(&member_id); + } + + fn clear(&mut self) { + self.member_sources.clear(); + } + + fn owner_of_source( + &self, + packet: &ZCPacket, + active_members: &BTreeMap, + ) -> SourceOwner { + let Some((source, _)) = SharedVirtualNicFlowAddr::packet_addrs(packet) else { + return SourceOwner::None; + }; + let mut found = false; + for (member_id, sources) in &self.member_sources { + if sources.owns_source(source) { + found = true; + if active_members.contains_key(member_id) { + return SourceOwner::Active(*member_id); + } + } + } + if found { + SourceOwner::Inactive + } else { + SourceOwner::None + } + } + + fn owner_of_destination( + &self, + packet: &ZCPacket, + active_members: &BTreeMap, + preferred_member: Option, + ) -> Option { + let (_, dst) = SharedVirtualNicFlowAddr::packet_addrs(packet)?; + + let mut first = None; + for (member_id, sources) in &self.member_sources { + if !active_members.contains_key(member_id) || !sources.has_destination(dst) { + continue; + } + if Some(*member_id) == preferred_member { + return Some(*member_id); + } + first.get_or_insert(*member_id); + } + first + } + + fn source_for_member_destination( + &self, + member_id: SharedVirtualNicMemberId, + dst: SharedVirtualNicFlowAddr, + ) -> Option { + let sources = self.member_sources.get(&member_id)?; + match dst { + SharedVirtualNicFlowAddr::V4(dst) => sources + .ipv4_source_for_destination(std::net::Ipv4Addr::from(dst)) + .map(SharedVirtualNicFlowAddr::from), + SharedVirtualNicFlowAddr::V6(dst) => sources + .ipv6_source_for_destination(std::net::Ipv6Addr::from(dst)) + .map(SharedVirtualNicFlowAddr::from), + } + } +} + +impl SharedVirtualNicMemberSources { + fn has_destination(&self, dst: SharedVirtualNicFlowAddr) -> bool { + match dst { + SharedVirtualNicFlowAddr::V4(dst) => { + let dst = std::net::Ipv4Addr::from(dst); + self.ipv4_addresses.iter().any(|addr| addr.contains(&dst)) + || self.ipv4_routes.iter().any(|route| route.contains(&dst)) + } + SharedVirtualNicFlowAddr::V6(dst) => { + let dst = std::net::Ipv6Addr::from(dst); + self.ipv6_addresses.iter().any(|addr| addr.contains(&dst)) + || self.ipv6_routes.iter().any(|route| route.contains(&dst)) + } + } + } + + fn ipv4_source_for_destination(&self, dst: std::net::Ipv4Addr) -> Option { + let mut best = None; + for addr in &self.ipv4_addresses { + if addr.contains(&dst) { + update_best_source(&mut best, addr.network_length(), addr.address()); + } + } + for route in &self.ipv4_routes { + if route.contains(&dst) { + let Some(source) = self.ipv4_source_for_route(route) else { + continue; + }; + update_best_source(&mut best, route.network_length(), source); + } + } + best.map(|(_, source)| source) + } + + fn ipv4_source_for_route(&self, route: &Ipv4Inet) -> Option { + let mut default_source = None; + for addr in &self.ipv4_addresses { + default_source.get_or_insert(addr.address()); + if route.contains(&addr.address()) { + return Some(addr.address()); + } + } + default_source + } + fn ipv6_source_for_destination(&self, dst: std::net::Ipv6Addr) -> Option { + let mut best = None; + for addr in &self.ipv6_addresses { + if addr.contains(&dst) { + update_best_source(&mut best, addr.network_length(), addr.address()); + } + } + for route in &self.ipv6_routes { + if route.contains(&dst) { + let Some(source) = self.ipv6_source_for_route(route) else { + continue; + }; + update_best_source(&mut best, route.network_length(), source); + } + } + best.map(|(_, source)| source) + } + + fn ipv6_source_for_route(&self, route: &Ipv6Inet) -> Option { + let mut default_source = None; + for addr in &self.ipv6_addresses { + default_source.get_or_insert(addr.address()); + if route.contains(&addr.address()) { + return Some(addr.address()); + } + } + default_source + } +} + +fn update_best_source(best: &mut Option<(u8, T)>, prefix: u8, source: T) { + if best + .map(|(best_prefix, _)| prefix > best_prefix) + .unwrap_or(true) + { + *best = Some((prefix, source)); + } +} + +// ICMP Echo sessions use only the identifier; sequence numbers stay in the packet. +fn transport_ports(protocol: u8, payload: &[u8]) -> Option { + let (src, dst) = match IpProtocol::from(protocol) { + IpProtocol::Tcp => { + let tcp = TcpPacket::new_checked(payload).ok()?; + (tcp.src_port(), tcp.dst_port()) + } + IpProtocol::Udp => { + let udp = UdpPacket::new_checked(payload).ok()?; + (udp.src_port(), udp.dst_port()) + } + IpProtocol::Icmp => { + let icmp = Icmpv4Packet::new_checked(payload).ok()?; + if !matches!( + icmp.msg_type(), + Icmpv4Message::EchoRequest | Icmpv4Message::EchoReply + ) { + return None; + } + (icmp.echo_ident(), 0) + } + IpProtocol::Icmpv6 => { + let icmp = Icmpv6Packet::new_checked(payload).ok()?; + if !matches!( + icmp.msg_type(), + Icmpv6Message::EchoRequest | Icmpv6Message::EchoReply + ) { + return None; + } + (icmp.echo_ident(), 0) + } + _ => return None, + }; + Some(SharedVirtualNicTransportPorts { src, dst }) +} + +#[cfg(test)] +mod tests { + use std::{ + net::{Ipv4Addr, Ipv6Addr}, + time::Duration, + }; + + use super::*; + use easytier_core::tunnel::{ + TunnelError, ring::create_ring_tunnel_pair, wrapper::TunnelWrapper, + }; + const IPV4_HEADER_MIN_LEN: usize = 20; + const IPV6_HEADER_LEN: usize = 40; + const TCP_HEADER_MIN_LEN: usize = 20; + const UDP_HEADER_LEN: usize = 8; + const ICMP_ECHO_HEADER_LEN: usize = 8; + + #[tokio::test] + async fn mobile_open_handshake_waits_for_dispatcher_result() { + let (sender, receiver) = oneshot::channel(); + let waiter = tokio::spawn(await_mobile_open(receiver)); + + tokio::task::yield_now().await; + assert!(!waiter.is_finished()); + + sender.send(Ok(())).unwrap(); + waiter.await.unwrap().unwrap(); + } + + #[tokio::test] + async fn mobile_open_handshake_preserves_original_error() { + let (sender, receiver) = oneshot::channel(); + let waiter = tokio::spawn(await_mobile_open(receiver)); + let errno = 9; + + sender + .send(Err(std::io::Error::from_raw_os_error(errno).into())) + .unwrap(); + let err = waiter.await.unwrap().unwrap_err(); + assert!(matches!(err, Error::IOError(ref io_err) if io_err.raw_os_error() == Some(errno))); + } + + fn ipv6_packet(src: Ipv6Addr, dst: Ipv6Addr) -> ZCPacket { + let mut payload = vec![0; IPV6_HEADER_LEN]; + let mut ipv6 = Ipv6Packet::new_unchecked(&mut payload); + ipv6.set_version(6); + ipv6.set_next_header(IpProtocol::Icmpv6); + ipv6.set_src_addr(src); + ipv6.set_dst_addr(dst); + ZCPacket::new_with_payload(&payload) + } + + fn ipv6_udp_packet_with_ports( + src: Ipv6Addr, + dst: Ipv6Addr, + src_port: u16, + dst_port: u16, + ) -> ZCPacket { + let mut payload = vec![0; IPV6_HEADER_LEN + UDP_HEADER_LEN]; + { + let mut ipv6_packet = Ipv6Packet::new_unchecked(&mut payload); + ipv6_packet.set_version(6); + ipv6_packet.set_payload_len(UDP_HEADER_LEN as u16); + ipv6_packet.set_next_header(IpProtocol::Udp); + ipv6_packet.set_hop_limit(64); + ipv6_packet.set_src_addr(src); + ipv6_packet.set_dst_addr(dst); + } + { + let mut udp_packet = UdpPacket::new_unchecked(&mut payload[IPV6_HEADER_LEN..]); + udp_packet.set_src_port(src_port); + udp_packet.set_dst_port(dst_port); + udp_packet.set_len(UDP_HEADER_LEN as u16); + udp_packet.fill_checksum(&IpAddress::Ipv6(src), &IpAddress::Ipv6(dst)); + } + ZCPacket::new_with_payload(&payload) + } + + fn ipv6_tcp_packet_with_ports( + src: Ipv6Addr, + dst: Ipv6Addr, + src_port: u16, + dst_port: u16, + ) -> ZCPacket { + let mut payload = vec![0; IPV6_HEADER_LEN + TCP_HEADER_MIN_LEN]; + { + let mut ipv6_packet = Ipv6Packet::new_unchecked(&mut payload); + ipv6_packet.set_version(6); + ipv6_packet.set_payload_len(TCP_HEADER_MIN_LEN as u16); + ipv6_packet.set_next_header(IpProtocol::Tcp); + ipv6_packet.set_hop_limit(64); + ipv6_packet.set_src_addr(src); + ipv6_packet.set_dst_addr(dst); + } + { + let mut tcp_packet = TcpPacket::new_unchecked(&mut payload[IPV6_HEADER_LEN..]); + tcp_packet.set_src_port(src_port); + tcp_packet.set_dst_port(dst_port); + tcp_packet.set_header_len(TCP_HEADER_MIN_LEN as u8); + tcp_packet.fill_checksum(&IpAddress::Ipv6(src), &IpAddress::Ipv6(dst)); + } + ZCPacket::new_with_payload(&payload) + } + + fn ipv6_icmp_packet_with_id( + src: Ipv6Addr, + dst: Ipv6Addr, + reply: bool, + identifier: u16, + sequence: u16, + ) -> ZCPacket { + let mut payload = vec![0; IPV6_HEADER_LEN + ICMP_ECHO_HEADER_LEN]; + { + let mut ipv6_packet = Ipv6Packet::new_unchecked(&mut payload); + ipv6_packet.set_version(6); + ipv6_packet.set_payload_len(ICMP_ECHO_HEADER_LEN as u16); + ipv6_packet.set_next_header(IpProtocol::Icmpv6); + ipv6_packet.set_hop_limit(64); + ipv6_packet.set_src_addr(src); + ipv6_packet.set_dst_addr(dst); + } + { + let mut icmp_packet = Icmpv6Packet::new_unchecked(&mut payload[IPV6_HEADER_LEN..]); + icmp_packet.set_msg_type(if reply { + Icmpv6Message::EchoReply + } else { + Icmpv6Message::EchoRequest + }); + icmp_packet.set_echo_ident(identifier); + icmp_packet.set_echo_seq_no(sequence); + icmp_packet.fill_checksum(&src, &dst); + } + ZCPacket::new_with_payload(&payload) + } + + fn ipv4_udp_packet(src: Ipv4Addr, dst: Ipv4Addr) -> ZCPacket { + ipv4_udp_packet_with_ports(src, dst, 1234, 5678) + } + + fn ipv4_icmp_echo_packet(src: Ipv4Addr, dst: Ipv4Addr) -> ZCPacket { + ipv4_icmp_packet_with_id(src, dst, Icmpv4Message::EchoRequest, 0, 0) + } + + fn ipv4_icmp_packet_with_id( + src: Ipv4Addr, + dst: Ipv4Addr, + icmp_type: Icmpv4Message, + identifier: u16, + sequence: u16, + ) -> ZCPacket { + let mut payload = vec![0; IPV4_HEADER_MIN_LEN + 8]; + let payload_len = payload.len(); + { + let mut ipv4_packet = Ipv4Packet::new_unchecked(&mut payload); + ipv4_packet.set_version(4); + ipv4_packet.set_header_len(IPV4_HEADER_MIN_LEN as u8); + ipv4_packet.set_total_len(payload_len as u16); + ipv4_packet.set_hop_limit(64); + ipv4_packet.set_next_header(IpProtocol::Icmp); + ipv4_packet.set_src_addr(src); + ipv4_packet.set_dst_addr(dst); + } + { + let mut icmp_packet = Icmpv4Packet::new_unchecked(&mut payload[IPV4_HEADER_MIN_LEN..]); + icmp_packet.set_msg_type(icmp_type); + icmp_packet.set_msg_code(0); + icmp_packet.set_echo_ident(identifier); + icmp_packet.set_echo_seq_no(sequence); + icmp_packet.fill_checksum(); + } + { + Ipv4Packet::new_unchecked(&mut payload).fill_checksum(); + } + ZCPacket::new_with_payload(&payload) + } + + fn ipv4_udp_packet_with_ports( + src: Ipv4Addr, + dst: Ipv4Addr, + src_port: u16, + dst_port: u16, + ) -> ZCPacket { + let mut payload = vec![0; IPV4_HEADER_MIN_LEN + UDP_HEADER_LEN]; + let payload_len = payload.len(); + { + let mut ipv4_packet = Ipv4Packet::new_unchecked(&mut payload); + ipv4_packet.set_version(4); + ipv4_packet.set_header_len(IPV4_HEADER_MIN_LEN as u8); + ipv4_packet.set_total_len(payload_len as u16); + ipv4_packet.set_hop_limit(64); + ipv4_packet.set_next_header(IpProtocol::Udp); + ipv4_packet.set_src_addr(src); + ipv4_packet.set_dst_addr(dst); + } + { + let mut udp_packet = UdpPacket::new_unchecked(&mut payload[IPV4_HEADER_MIN_LEN..]); + udp_packet.set_src_port(src_port); + udp_packet.set_dst_port(dst_port); + udp_packet.set_len(UDP_HEADER_LEN as u16); + udp_packet.fill_checksum(&IpAddress::Ipv4(src), &IpAddress::Ipv4(dst)); + } + { + Ipv4Packet::new_unchecked(&mut payload).fill_checksum(); + } + ZCPacket::new_with_payload(&payload) + } + + fn ipv4_tcp_packet_with_ports( + src: Ipv4Addr, + dst: Ipv4Addr, + src_port: u16, + dst_port: u16, + ) -> ZCPacket { + let mut payload = vec![0; IPV4_HEADER_MIN_LEN + TCP_HEADER_MIN_LEN]; + let total_len = payload.len() as u16; + let mut ipv4 = Ipv4Packet::new_unchecked(&mut payload); + ipv4.set_version(4); + ipv4.set_header_len(IPV4_HEADER_MIN_LEN as u8); + ipv4.set_total_len(total_len); + ipv4.set_hop_limit(64); + ipv4.set_next_header(IpProtocol::Tcp); + ipv4.set_src_addr(src); + ipv4.set_dst_addr(dst); + let mut tcp = TcpPacket::new_unchecked(ipv4.payload_mut()); + tcp.set_header_len(TCP_HEADER_MIN_LEN as u8); + tcp.set_src_port(src_port); + tcp.set_dst_port(dst_port); + tcp.fill_checksum(&IpAddress::Ipv4(src), &IpAddress::Ipv4(dst)); + ipv4.fill_checksum(); + ZCPacket::new_with_payload(&payload) + } + + fn member_sources(ipv4: &[&str]) -> SharedVirtualNicMemberSources { + SharedVirtualNicMemberSources::from_claims(&member_claims(ipv4, &[], &[], &[])) + } + + fn member_sources_with_ipv4_routes( + ipv4: &[&str], + ipv4_routes: &[&str], + ) -> SharedVirtualNicMemberSources { + SharedVirtualNicMemberSources::from_claims(&member_claims(ipv4, &[], ipv4_routes, &[])) + } + + fn member_sources_with_ipv6(ipv6: &[&str]) -> SharedVirtualNicMemberSources { + SharedVirtualNicMemberSources::from_claims(&member_claims(&[], ipv6, &[], &[])) + } + + fn member_sources_with_ipv6_routes( + ipv6: &[&str], + ipv6_routes: &[&str], + ) -> SharedVirtualNicMemberSources { + SharedVirtualNicMemberSources::from_claims(&member_claims(&[], ipv6, &[], ipv6_routes)) + } + + fn member_claims( + ipv4: &[&str], + ipv6: &[&str], + ipv4_routes: &[&str], + ipv6_routes: &[&str], + ) -> SharedIfConfigClaims { + SharedIfConfigClaims { + ipv4_addresses: ipv4.iter().map(|addr| addr.parse().unwrap()).collect(), + ipv6_addresses: ipv6.iter().map(|addr| addr.parse().unwrap()).collect(), + ipv4_routes: ipv4_routes + .iter() + .map(|route| { + let inet = route.parse::().unwrap(); + SharedIpv4Route::new(inet.address(), inet.network_length(), None) + }) + .collect(), + ipv6_routes: ipv6_routes + .iter() + .map(|route| { + let inet = route.parse::().unwrap(); + SharedIpv6Route::new(inet.address(), inet.network_length(), None) + }) + .collect(), + mtu: None, + } + } + + fn member_entry(sender: mpsc::Sender) -> SharedVirtualNicMemberTunnelEntry { + member_entry_with_registration(sender, uuid::Uuid::from_u128(1)) + } + + fn member_entry_with_registration( + sender: mpsc::Sender, + registration_id: SharedVirtualNicMemberRegistrationId, + ) -> SharedVirtualNicMemberTunnelEntry { + SharedVirtualNicMemberTunnelEntry { + registration_id, + sender, + close_notifier: Arc::new(Notify::new()), + _tasks: Vec::new(), + } + } + + #[cfg(unix)] + fn start_mobile_dispatcher_for_test( + tunnel: Box, + member_tunnel_table: SharedVirtualNicMemberTunnelTable, + valid: Arc, + ) -> ( + SharedVirtualNicDispatcher, + mpsc::UnboundedSender, + ) { + use crate::{common::netns::NetNS, instance::virtual_nic::VirtualNicConfig}; + + let (tun_stream, tun_sink) = tunnel.split(); + let (to_tun_sender, to_tun_receiver) = mpsc::channel(MEMBER_TUNNEL_BUFFER_SIZE); + let (control_sender, control_receiver) = mpsc::unbounded_channel(); + let (tun_update_sender, tun_update_receiver) = mpsc::unbounded_channel(); + member_tunnel_table.attach_dispatcher(to_tun_sender, control_sender.clone()); + let task = SharedVirtualNicMobileDispatcherTask { + nic: Arc::new(Mutex::new(VirtualNic::new(VirtualNicConfig::new( + String::new(), + 1300, + NetNS::new(None), + )))), + tun_stream: Some(tun_stream), + tun_sink: Some(tun_sink), + tun_fd: -1, + tun_update_receiver, + to_tun_receiver, + control_receiver, + member_tunnel_table, + valid, + state: SharedVirtualNicDispatcherState { + translate_unknown_sources: true, + ..Default::default() + }, + }; + let dispatcher = SharedVirtualNicDispatcher { + _task: AbortOnDropHandle::new(tokio::spawn(task.run_opened_tun())), + control_sender, + #[cfg(mobile)] + mobile_tun_update_sender: Some(tun_update_sender.clone()), + }; + (dispatcher, tun_update_sender) + } + + async fn register_dispatcher_test_members( + dispatcher: &SharedVirtualNicDispatcher, + member_tunnel_table: &SharedVirtualNicMemberTunnelTable, + ) -> (Vec>, Vec>) { + let mut tunnels = Vec::new(); + let mut close_notifiers = Vec::new(); + for id in 1..=2 { + let member_id = uuid::Uuid::from_u128(id); + let close_notifier = Arc::new(Notify::new()); + let (member_tunnel, shared_tunnel) = create_ring_tunnel_pair(); + member_tunnel_table + .register( + member_id, + uuid::Uuid::new_v4(), + shared_tunnel, + close_notifier.clone(), + ) + .unwrap(); + dispatcher + .update_sources(member_id, &SharedIfConfigClaims::default()) + .await + .unwrap(); + tunnels.push(member_tunnel); + close_notifiers.push(close_notifier); + } + (tunnels, close_notifiers) + } + + fn stalled_tun_for_test() -> (Box, Arc) { + let writing = Arc::new(Notify::new()); + let sink_writing = writing.clone(); + let tun_sink = futures::sink::unfold((), move |(), _packet: ZCPacket| { + let writing = sink_writing.clone(); + async move { + writing.notify_one(); + std::future::pending::>().await + } + }); + let tun_stream = futures::stream::pending::>(); + ( + Box::new(TunnelWrapper::new(tun_stream, tun_sink, None)), + writing, + ) + } + + async fn check_dispatcher_tun_write_timeout( + dispatcher: SharedVirtualNicDispatcher, + member_tunnel_table: SharedVirtualNicMemberTunnelTable, + valid: Arc, + writing: Arc, + ) { + let (members, close_notifiers) = + register_dispatcher_test_members(&dispatcher, &member_tunnel_table).await; + let (_, mut member_sink) = members[0].split(); + member_sink + .send(ipv4_udp_packet( + Ipv4Addr::new(10, 231, 1, 1), + Ipv4Addr::new(203, 0, 113, 1), + )) + .await + .unwrap(); + tokio::time::timeout(Duration::from_secs(1), writing.notified()) + .await + .unwrap(); + + let result = tokio::time::timeout( + DISPATCHER_TUN_WRITE_TIMEOUT + Duration::from_secs(1), + dispatcher.remove_sources(uuid::Uuid::from_u128(1)), + ) + .await + .unwrap(); + assert!(result.is_err()); + for close_notifier in close_notifiers { + tokio::time::timeout(Duration::from_secs(1), close_notifier.notified()) + .await + .unwrap(); + } + assert!(!valid.load(Ordering::Acquire)); + assert!(member_tunnel_table.dispatcher_channels().is_none()); + tokio::time::timeout(Duration::from_secs(1), dispatcher._task) + .await + .unwrap() + .unwrap(); + } + + #[tokio::test] + async fn dispatcher_tun_write_timeout_closes_members_and_source_updates() { + let (tunnel, writing) = stalled_tun_for_test(); + let member_tunnel_table = SharedVirtualNicMemberTunnelTable::default(); + let valid = Arc::new(AtomicBool::new(true)); + let dispatcher = + SharedVirtualNicDispatcher::start(tunnel, member_tunnel_table.clone(), valid.clone()); + + check_dispatcher_tun_write_timeout(dispatcher, member_tunnel_table, valid, writing).await; + } + + #[cfg(unix)] + #[tokio::test] + async fn mobile_dispatcher_tun_write_timeout_closes_members_and_source_updates() { + let (tunnel, writing) = stalled_tun_for_test(); + let member_tunnel_table = SharedVirtualNicMemberTunnelTable::default(); + let valid = Arc::new(AtomicBool::new(true)); + let (dispatcher, _updates) = + start_mobile_dispatcher_for_test(tunnel, member_tunnel_table.clone(), valid.clone()); + + check_dispatcher_tun_write_timeout(dispatcher, member_tunnel_table, valid, writing).await; + } + + #[cfg(unix)] + #[tokio::test] + async fn mobile_dispatcher_invalidates_shared_nic_when_tun_replacement_fails() { + let (_tun, tunnel) = create_ring_tunnel_pair(); + let member_tunnel_table = SharedVirtualNicMemberTunnelTable::default(); + let valid = Arc::new(AtomicBool::new(true)); + let (dispatcher, updates) = + start_mobile_dispatcher_for_test(tunnel, member_tunnel_table.clone(), valid.clone()); + let (_members, close_notifiers) = + register_dispatcher_test_members(&dispatcher, &member_tunnel_table).await; + let (ack, receiver) = oneshot::channel(); + + updates + .send(SharedVirtualNicMobileTunUpdate { tun_fd: -1, ack }) + .unwrap(); + + assert!( + tokio::time::timeout(Duration::from_secs(1), await_mobile_open(receiver)) + .await + .unwrap() + .is_err() + ); + for close_notifier in close_notifiers { + tokio::time::timeout(Duration::from_secs(1), close_notifier.notified()) + .await + .unwrap(); + } + assert!(!valid.load(Ordering::Acquire)); + assert!(member_tunnel_table.dispatcher_channels().is_none()); + assert!( + dispatcher + .remove_sources(uuid::Uuid::from_u128(1)) + .await + .is_err() + ); + tokio::time::timeout(Duration::from_secs(1), dispatcher._task) + .await + .unwrap() + .unwrap(); + } + + #[cfg(unix)] + #[tokio::test] + async fn mobile_dispatcher_invalidates_shared_nic_when_tun_read_closes() { + for read_error in [false, true] { + let (tun_tx, tun_rx) = mpsc::unbounded_channel(); + let tun_stream = futures::stream::unfold(tun_rx, |mut receiver| async move { + receiver.recv().await.map(|packet| (packet, receiver)) + }); + let tun_sink = futures::sink::unfold((), |(), _packet: ZCPacket| async { + Ok::<(), TunnelError>(()) + }); + let tunnel = Box::new(TunnelWrapper::new(tun_stream, tun_sink, None)); + let member_tunnel_table = SharedVirtualNicMemberTunnelTable::default(); + let valid = Arc::new(AtomicBool::new(true)); + let (dispatcher, _updates) = start_mobile_dispatcher_for_test( + tunnel, + member_tunnel_table.clone(), + valid.clone(), + ); + let (_members, close_notifiers) = + register_dispatcher_test_members(&dispatcher, &member_tunnel_table).await; + + if read_error { + tun_tx.send(Err(TunnelError::Shutdown)).unwrap(); + } + drop(tun_tx); + + for close_notifier in close_notifiers { + tokio::time::timeout(Duration::from_secs(1), close_notifier.notified()) + .await + .unwrap(); + } + assert!(!valid.load(Ordering::Acquire)); + assert!(member_tunnel_table.dispatcher_channels().is_none()); + tokio::time::timeout(Duration::from_secs(1), dispatcher._task) + .await + .unwrap() + .unwrap(); + } + } + + #[test] + fn source_table_selects_ipv6_source_owner() { + let first = uuid::Uuid::from_u128(1); + let second = uuid::Uuid::from_u128(2); + let second_addr = "2001:db8::2".parse::().unwrap(); + let dst = "2001:db8:ffff::1".parse::().unwrap(); + let mut table = SharedVirtualNicSourceTable::default(); + let (first_sender, _first_receiver) = mpsc::channel(1); + let (second_sender, _second_receiver) = mpsc::channel(1); + let mut members = BTreeMap::new(); + members.insert(first, member_entry(first_sender)); + members.insert(second, member_entry(second_sender)); + + table.update_member_sources(first, member_sources_with_ipv6(&["2001:db8::1/64"])); + table.update_member_sources(second, member_sources_with_ipv6(&["2001:db8::2/64"])); + + assert_eq!( + table.owner_of_source(&ipv6_packet(second_addr, dst), &members), + SourceOwner::Active(second) + ); + + table.remove_owner(second); + assert_eq!( + table.owner_of_source(&ipv6_packet(second_addr, dst), &members), + SourceOwner::None + ); + } + + #[test] + fn source_table_selects_ipv4_destination_owner_from_route_claim() { + let first = uuid::Uuid::from_u128(1); + let second = uuid::Uuid::from_u128(2); + let src = Ipv4Addr::new(100, 64, 0, 1); + let dst = Ipv4Addr::new(10, 99, 0, 2); + let mut table = SharedVirtualNicSourceTable::default(); + let (first_sender, _first_receiver) = mpsc::channel(1); + let (second_sender, _second_receiver) = mpsc::channel(1); + let mut members = BTreeMap::new(); + members.insert(first, member_entry(first_sender)); + members.insert(second, member_entry(second_sender)); + + table.update_member_sources(first, member_sources(&["10.231.1.1/24"])); + table.update_member_sources( + second, + member_sources_with_ipv4_routes(&["10.231.2.1/24"], &["10.99.0.0/24"]), + ); + + assert_eq!( + table.owner_of_destination(&ipv4_udp_packet(src, dst), &members, None), + Some(second) + ); + } + + #[tokio::test] + async fn dispatcher_prefers_source_owner_for_shared_magic_dns() { + let first = uuid::Uuid::from_u128(1); + let source_owner = uuid::Uuid::from_u128(2); + let first_ip = Ipv4Addr::new(10, 231, 1, 1); + let source_owner_ip = Ipv4Addr::new(10, 231, 2, 1); + let remote_ip = Ipv4Addr::new(100, 100, 100, 101); + let (first_sender, mut first_receiver) = mpsc::channel(1); + let (source_sender, mut source_receiver) = mpsc::channel(1); + let mut state = SharedVirtualNicDispatcherState::default(); + + state.register(first, member_entry(first_sender)); + state.register(source_owner, member_entry(source_sender)); + state.source_table.update_member_sources( + first, + member_sources_with_ipv4_routes(&["10.231.1.1/24"], &["100.100.100.101/32"]), + ); + state.source_table.update_member_sources( + source_owner, + member_sources_with_ipv4_routes(&["10.231.2.1/24"], &["100.100.100.101/32"]), + ); + + state + .forward_tun_packet_to_member(ipv4_udp_packet(source_owner_ip, remote_ip)) + .await; + + assert!(first_receiver.try_recv().is_err()); + let packet = source_receiver.try_recv().unwrap(); + let ipv4 = Ipv4Packet::new_checked(packet.payload()).unwrap(); + assert_eq!(ipv4.src_addr(), source_owner_ip); + assert_ne!(ipv4.src_addr(), first_ip); + assert_eq!(ipv4.dst_addr(), remote_ip); + } + + #[test] + fn rewrite_ipv4_source_rejects_fragments_without_changing_packet() { + let src = Ipv4Addr::new(10, 231, 1, 1); + let dst = Ipv4Addr::new(10, 231, 2, 2); + for (more_frags, offset) in [(true, 0), (false, 8)] { + let mut packet = ipv4_udp_packet(src, dst); + let mut ipv4 = Ipv4Packet::new_checked(packet.mut_payload()).unwrap(); + ipv4.set_more_frags(more_frags); + ipv4.set_frag_offset(offset); + let original = packet.payload().to_vec(); + + assert!(!rewrite_packet_source( + &mut packet, + SharedVirtualNicFlowAddr::from(src), + SharedVirtualNicFlowAddr::from(Ipv4Addr::new(10, 231, 2, 1)) + )); + assert_eq!(packet.payload(), original); + } + } + + #[tokio::test] + async fn dispatcher_prefers_source_owner_over_first_member() { + let first = uuid::Uuid::from_u128(1); + let owner = uuid::Uuid::from_u128(2); + let source = "2001:db8::2".parse::().unwrap(); + let dst = "2001:db8:ffff::1".parse::().unwrap(); + let (first_sender, mut first_receiver) = mpsc::channel(1); + let (owner_sender, mut owner_receiver) = mpsc::channel(1); + let mut state = SharedVirtualNicDispatcherState::default(); + + state.register(first, member_entry(first_sender)); + state.register(owner, member_entry(owner_sender)); + state + .source_table + .update_member_sources(owner, member_sources_with_ipv6(&["2001:db8::2/64"])); + state + .forward_tun_packet_to_member(ipv6_packet(source, dst)) + .await; + + assert!(first_receiver.try_recv().is_err()); + assert!(owner_receiver.try_recv().is_ok()); + } + + #[tokio::test] + async fn dispatcher_forwards_external_ipv6_to_route_only_owner() { + let route_owner = uuid::Uuid::from_u128(1); + let source = "2001:db8:ffff::2".parse::().unwrap(); + let destination = "2001:db8:100::2".parse::().unwrap(); + let (sender, mut receiver) = mpsc::channel(1); + let mut state = SharedVirtualNicDispatcherState::default(); + + state.register(route_owner, member_entry(sender)); + state.source_table.update_member_sources( + route_owner, + member_sources_with_ipv6_routes(&[], &["2001:db8:100::2/128"]), + ); + // An exhausted local-flow table must not throttle kernel subnet forwarding. + let local = "2001:db8:100::1".parse::().unwrap(); + for port in 1..=FLOW_OWNER_LIMIT as u16 { + let mut packet = ipv6_tcp_packet_with_ports(local, destination, port, 80); + assert!(state.nat_table.prepare_packet( + route_owner, + Some(route_owner), + SharedVirtualNicFlowKey::from_packet(&packet).unwrap(), + &mut packet, + local.into(), + )); + } + for packet in [ + ipv6_packet(source, destination), + ipv6_tcp_packet_with_ports(source, destination, 443, 22001), + ] { + let unchanged = packet.payload().to_vec(); + assert!(state.send_packet(packet).await); + assert_eq!(receiver.try_recv().unwrap().payload(), unchanged); + } + } + + #[tokio::test] + async fn dispatcher_drops_inactive_source_owner_without_fallback() { + let fallback = uuid::Uuid::from_u128(1); + let owner = uuid::Uuid::from_u128(2); + let other = uuid::Uuid::from_u128(3); + let source = "2001:db8::2".parse::().unwrap(); + let dst = "2001:db8:ffff::1".parse::().unwrap(); + let (fallback_sender, mut fallback_receiver) = mpsc::channel(1); + let (owner_sender, _owner_receiver) = mpsc::channel(1); + let (other_sender, _other_receiver) = mpsc::channel(1); + let mut state = SharedVirtualNicDispatcherState::default(); + + state.register(fallback, member_entry(fallback_sender)); + state.register(owner, member_entry(owner_sender)); + state.register(other, member_entry(other_sender)); + state + .source_table + .update_member_sources(owner, member_sources_with_ipv6(&["2001:db8::2/64"])); + state.unregister(owner, uuid::Uuid::from_u128(1)); + state + .forward_tun_packet_to_member(ipv6_packet(source, dst)) + .await; + + assert!(fallback_receiver.try_recv().is_err()); + } + + #[tokio::test] + async fn dispatcher_drops_unknown_source_and_destination_without_fallback() { + let member_id = uuid::Uuid::from_u128(1); + let other = uuid::Uuid::from_u128(2); + let unknown_source = Ipv4Addr::new(100, 64, 0, 1); + let unknown_destination = Ipv4Addr::new(203, 0, 113, 1); + let (sender, mut receiver) = mpsc::channel(1); + let (other_sender, _other_receiver) = mpsc::channel(1); + let mut state = SharedVirtualNicDispatcherState::default(); + + state.register(member_id, member_entry(sender)); + state.register(other, member_entry(other_sender)); + state + .source_table + .update_member_sources(member_id, member_sources(&["10.231.1.1/24"])); + state + .forward_tun_packet_to_member(ipv4_udp_packet(unknown_source, unknown_destination)) + .await; + + assert!(receiver.try_recv().is_err()); + } + + #[tokio::test] + async fn dispatcher_single_member_forwards_unknown_addresses_unchanged() { + let member_id = uuid::Uuid::from_u128(1); + for translate_unknown_sources in [false, true] { + let (sender, mut receiver) = mpsc::channel(1); + let mut state = SharedVirtualNicDispatcherState { + translate_unknown_sources, + ..Default::default() + }; + state.register(member_id, member_entry(sender)); + for packet in [ + ipv4_udp_packet(Ipv4Addr::new(100, 64, 0, 1), Ipv4Addr::new(203, 0, 113, 1)), + ipv6_udp_packet_with_ports( + "2001:db8::1".parse().unwrap(), + "2001:db8:ffff::1".parse().unwrap(), + 1234, + 5678, + ), + ] { + let original = packet.payload().to_vec(); + assert!(state.send_packet(packet).await); + assert_eq!(receiver.try_recv().unwrap().payload(), original); + } + } + } + + #[tokio::test] + async fn dispatcher_single_member_preserves_fragmented_packets() { + let member_id = uuid::Uuid::from_u128(1); + for translate_unknown_sources in [false, true] { + let (sender, mut receiver) = mpsc::channel(1); + let mut state = SharedVirtualNicDispatcherState { + translate_unknown_sources, + ..Default::default() + }; + state.register(member_id, member_entry(sender)); + state.source_table.update_member_sources( + member_id, + SharedVirtualNicMemberSources::from_claims(&member_claims( + &["10.231.1.1/24"], + &["2001:db8:1::1/64"], + &[], + &[], + )), + ); + for source in [Ipv4Addr::new(10, 231, 1, 1), Ipv4Addr::new(100, 64, 0, 1)] { + for (more_frags, offset) in [(true, 0), (false, 8)] { + let mut packet = ipv4_udp_packet(source, Ipv4Addr::new(10, 231, 1, 2)); + let mut ipv4 = Ipv4Packet::new_checked(packet.mut_payload()).unwrap(); + ipv4.set_more_frags(more_frags); + ipv4.set_frag_offset(offset); + ipv4.fill_checksum(); + let original = packet.payload().to_vec(); + assert!(state.send_packet(packet).await); + assert_eq!(receiver.try_recv().unwrap().payload(), original); + } + } + for source in ["2001:db8:1::1", "2001:db8:ffff::1"] { + for fragment_offset in [1u16, 8u16] { + let packet = ipv6_udp_packet_with_ports( + source.parse().unwrap(), + "2001:db8:1::2".parse().unwrap(), + 1234, + 5678, + ); + let mut payload = packet.payload()[..IPV6_HEADER_LEN].to_vec(); + payload.extend_from_slice(&[17, 0]); // Fragment's next header is UDP. + payload.extend_from_slice(&fragment_offset.to_be_bytes()); + payload.extend_from_slice(&1u32.to_be_bytes()); + payload.extend_from_slice(&packet.payload()[IPV6_HEADER_LEN..]); + let mut ipv6 = Ipv6Packet::new_unchecked(&mut payload); + ipv6.set_next_header(IpProtocol::from(44)); // Fragment header. + ipv6.set_payload_len((8 + UDP_HEADER_LEN) as u16); + assert!( + state + .send_packet(ZCPacket::new_with_payload(&payload)) + .await + ); + assert_eq!(receiver.try_recv().unwrap().payload(), payload); + } + } + } + } + + #[tokio::test] + async fn dispatcher_single_member_reserves_flows_before_another_member_joins() { + let first = uuid::Uuid::from_u128(1); + let second = uuid::Uuid::from_u128(2); + let native = Ipv4Addr::new(10, 231, 1, 1); + let source = Ipv4Addr::new(10, 231, 2, 1); + let remote = Ipv4Addr::new(10, 231, 1, 2); + for incoming_first in [false, true] { + let (sender, mut receiver) = mpsc::channel(1); + let mut state = SharedVirtualNicDispatcherState::default(); + state.register(first, member_entry(sender)); + state + .source_table + .update_member_sources(first, member_sources(&["10.231.1.1/24"])); + let original = ipv4_udp_packet(native, remote); + let original = original.payload().to_vec(); + if incoming_first { + let packet = ipv4_udp_packet_with_ports(remote, native, 5678, 1234); + let unchanged = packet.payload().to_vec(); + assert_eq!( + state + .prepare_member_packet_to_tun(first, packet) + .unwrap() + .payload(), + unchanged + ); + } else { + assert!( + state + .send_packet(ZCPacket::new_with_payload(&original)) + .await + ); + assert_eq!(receiver.try_recv().unwrap().payload(), original); + } + + let (sender, _second_receiver) = mpsc::channel(1); + state.register(second, member_entry(sender)); + state + .source_table + .update_member_sources(second, member_sources(&["10.231.2.1/24"])); + assert!(state.send_packet(ipv4_udp_packet(source, remote)).await); + let translated = receiver.try_recv().unwrap(); + let wire = SharedVirtualNicFlowKey::from_packet(&translated).unwrap(); + assert_eq!(wire.src, native.into()); + // Keep incoming UDP replies on their service port after a member joins. + // A native outgoing flow still reserves its port against new clients. + assert_eq!( + wire.ports.unwrap().src, + if incoming_first { 1234 } else { 1235 } + ); + + assert!( + state + .send_packet(ZCPacket::new_with_payload(&original)) + .await + ); + assert_eq!(receiver.try_recv().unwrap().payload(), original); + for (port, destination) in [ + (1234, native), + ( + wire.ports.unwrap().src, + if incoming_first { native } else { source }, + ), + ] { + let reply = state + .prepare_member_packet_to_tun( + first, + ipv4_udp_packet_with_ports(remote, native, 5678, port), + ) + .unwrap(); + let key = SharedVirtualNicFlowKey::from_packet(&reply).unwrap(); + assert_eq!(key.dst, destination.into()); + assert_eq!(key.ports.unwrap().dst, 1234); + } + } + } + + #[tokio::test] + async fn dispatcher_single_member_keeps_existing_translations_after_other_member_leaves() { + let member_id = uuid::Uuid::from_u128(1); + let other = uuid::Uuid::from_u128(2); + let native = Ipv4Addr::new(10, 231, 1, 1); + let synthetic = Ipv4Addr::new(100, 64, 0, 1); + let remote = Ipv4Addr::new(10, 231, 1, 2); + let (sender, mut receiver) = mpsc::channel(1); + let (other_sender, _other_receiver) = mpsc::channel(1); + let mut state = SharedVirtualNicDispatcherState { + translate_unknown_sources: true, + ..Default::default() + }; + state.register(member_id, member_entry(sender)); + state.register(other, member_entry(other_sender)); + state + .source_table + .update_member_sources(member_id, member_sources(&["10.231.1.1/24"])); + assert!(state.send_packet(ipv4_udp_packet(synthetic, remote)).await); + let translated = receiver.try_recv().unwrap().payload().to_vec(); + assert_eq!( + Ipv4Packet::new_checked(&translated).unwrap().src_addr(), + native + ); + + state.unregister(other, uuid::Uuid::from_u128(1)); + assert!(state.send_packet(ipv4_udp_packet(synthetic, remote)).await); + assert_eq!(receiver.try_recv().unwrap().payload(), translated); + let reply = state + .prepare_member_packet_to_tun( + member_id, + ipv4_udp_packet_with_ports(remote, native, 5678, 1234), + ) + .unwrap(); + assert_eq!( + Ipv4Packet::new_checked(reply.payload()).unwrap().dst_addr(), + synthetic + ); + + let packet = ipv4_udp_packet_with_ports(synthetic, remote, 4321, 5678); + let original = packet.payload().to_vec(); + assert!(state.send_packet(packet).await); + assert_eq!(receiver.try_recv().unwrap().payload(), original); + } + + #[test] + fn rewrite_ipv4_flow_preserves_zero_udp_checksum() { + let source = Ipv4Addr::new(10, 231, 1, 1); + let translated_source = Ipv4Addr::new(10, 231, 2, 1); + let destination = Ipv4Addr::new(10, 231, 2, 2); + let request = ipv4_udp_packet(source, destination); + let reply = ipv4_udp_packet_with_ports(destination, translated_source, 5678, 4321); + let original = SharedVirtualNicFlowKey::from_packet(&request).unwrap(); + let wire = SharedVirtualNicFlowKey::from_packet(&reply) + .unwrap() + .reversed(); + for (mut packet, from, to, direction) in [ + (request, original, wire, RewriteIpAddr::Source), + ( + reply, + wire.reversed(), + original.reversed(), + RewriteIpAddr::Destination, + ), + ] { + UdpPacket::new_checked(&mut packet.mut_payload()[IPV4_HEADER_MIN_LEN..]) + .unwrap() + .set_checksum(0); + assert!(rewrite_packet_flow(&mut packet, from, to, direction)); + assert_eq!(SharedVirtualNicFlowKey::from_packet(&packet), Some(to)); + let ipv4 = Ipv4Packet::new_checked(packet.payload()).unwrap(); + assert!(ipv4.verify_checksum()); + assert_eq!( + UdpPacket::new_checked(ipv4.payload()).unwrap().checksum(), + 0 + ); + } + } + + #[test] + fn rewrite_ipv4_tcp_flow_updates_checksums_and_restores_ports() { + let source = Ipv4Addr::new(10, 231, 1, 1); + let translated_source = Ipv4Addr::new(10, 231, 2, 1); + let destination = Ipv4Addr::new(10, 231, 2, 2); + let request = ipv4_tcp_packet_with_ports(source, destination, 1234, 5678); + let reply = ipv4_tcp_packet_with_ports(destination, translated_source, 5678, 4321); + let original = SharedVirtualNicFlowKey::from_packet(&request).unwrap(); + let wire = SharedVirtualNicFlowKey::from_packet(&reply) + .unwrap() + .reversed(); + for (mut packet, from, to, direction) in [ + (request, original, wire, RewriteIpAddr::Source), + ( + reply, + wire.reversed(), + original.reversed(), + RewriteIpAddr::Destination, + ), + ] { + assert!(rewrite_packet_flow(&mut packet, from, to, direction)); + assert_eq!(SharedVirtualNicFlowKey::from_packet(&packet), Some(to)); + let ipv4 = Ipv4Packet::new_checked(packet.payload()).unwrap(); + assert!(ipv4.verify_checksum()); + let tcp = TcpPacket::new_checked(ipv4.payload()).unwrap(); + assert!(tcp.verify_checksum( + &IpAddress::Ipv4(ipv4.src_addr()), + &IpAddress::Ipv4(ipv4.dst_addr()) + )); + } + } + + #[tokio::test] + async fn dispatcher_translates_wrong_local_ipv6_source_to_destination_member() { + let source_owner = uuid::Uuid::from_u128(1); + let destination_owner = uuid::Uuid::from_u128(2); + let source_owner_ip = "2001:db8:1::1".parse::().unwrap(); + let destination_owner_ip = "2001:db8:2::1".parse::().unwrap(); + let remote_ip = "2001:db8:99::2".parse::().unwrap(); + let (source_sender, mut source_receiver) = mpsc::channel(1); + let (destination_sender, mut destination_receiver) = mpsc::channel(1); + let mut state = SharedVirtualNicDispatcherState::default(); + + state.register(source_owner, member_entry(source_sender)); + state.register(destination_owner, member_entry(destination_sender)); + state.source_table.update_member_sources( + source_owner, + member_sources_with_ipv6(&["2001:db8:1::1/64"]), + ); + state.source_table.update_member_sources( + destination_owner, + member_sources_with_ipv6_routes(&["2001:db8:2::1/64"], &["2001:db8:99::/64"]), + ); + + // Reserve the native tuple so the translated request must also change its port. + assert!( + state + .send_packet(ipv6_udp_packet_with_ports( + destination_owner_ip, + remote_ip, + 1234, + 5678, + )) + .await + ); + destination_receiver.try_recv().unwrap(); + + state + .forward_tun_packet_to_member(ipv6_udp_packet_with_ports( + source_owner_ip, + remote_ip, + 1234, + 5678, + )) + .await; + + assert!(source_receiver.try_recv().is_err()); + let translated = destination_receiver.try_recv().unwrap(); + let translated_ipv6 = Ipv6Packet::new_checked(translated.payload()).unwrap(); + assert_eq!(translated_ipv6.src_addr(), destination_owner_ip); + assert_eq!(translated_ipv6.dst_addr(), remote_ip); + let translated_udp = UdpPacket::new_checked(translated_ipv6.payload()).unwrap(); + assert_eq!(translated_udp.src_port(), 1235); + assert_ne!(translated_udp.checksum(), 0); + assert!(translated_udp.verify_checksum( + &IpAddress::Ipv6(destination_owner_ip), + &IpAddress::Ipv6(remote_ip) + )); + + let reply = state + .prepare_member_packet_to_tun( + destination_owner, + ipv6_udp_packet_with_ports(remote_ip, destination_owner_ip, 5678, 1235), + ) + .unwrap(); + let reply_ipv6 = Ipv6Packet::new_checked(reply.payload()).unwrap(); + assert_eq!(reply_ipv6.src_addr(), remote_ip); + assert_eq!(reply_ipv6.dst_addr(), source_owner_ip); + let reply_udp = UdpPacket::new_checked(reply_ipv6.payload()).unwrap(); + assert_eq!(reply_udp.dst_port(), 1234); + assert_ne!(reply_udp.checksum(), 0); + assert!(reply_udp.verify_checksum( + &IpAddress::Ipv6(remote_ip), + &IpAddress::Ipv6(source_owner_ip) + )); + } + + #[test] + fn rewrite_ipv6_tcp_flow_updates_checksums_and_restores_ports() { + let source = "2001:db8:1::1".parse::().unwrap(); + let translated_source = "2001:db8:2::1".parse::().unwrap(); + let destination = "2001:db8:99::2".parse::().unwrap(); + let request = ipv6_tcp_packet_with_ports(source, destination, 1234, 5678); + let reply = ipv6_tcp_packet_with_ports(destination, translated_source, 5678, 4321); + let original = SharedVirtualNicFlowKey::from_packet(&request).unwrap(); + let wire = SharedVirtualNicFlowKey::from_packet(&reply) + .unwrap() + .reversed(); + for (mut packet, from, to, direction) in [ + (request, original, wire, RewriteIpAddr::Source), + ( + reply, + wire.reversed(), + original.reversed(), + RewriteIpAddr::Destination, + ), + ] { + assert!(rewrite_packet_flow(&mut packet, from, to, direction)); + assert_eq!(SharedVirtualNicFlowKey::from_packet(&packet), Some(to)); + let ipv6 = Ipv6Packet::new_checked(packet.payload()).unwrap(); + let tcp = TcpPacket::new_checked(ipv6.payload()).unwrap(); + assert!(tcp.verify_checksum( + &IpAddress::Ipv6(ipv6.src_addr()), + &IpAddress::Ipv6(ipv6.dst_addr()) + )); + } + } + + #[test] + fn icmpv6_nat_replies_keep_distinct_source_networks() { + let member_id = uuid::Uuid::from_u128(1); + let first_source = "2001:db8:1::1".parse::().unwrap(); + let second_source = "2001:db8:2::1".parse::().unwrap(); + let translated_source = "2001:db8:3::1".parse::().unwrap(); + let destination = "2001:db8:3::2".parse::().unwrap(); + let mut nat = SharedVirtualNicNatTable::default(); + + for (identifier, source, wire_identifier, sequence) in [ + (100, first_source, 100, 1), + (200, second_source, 200, 1), + (100, second_source, 101, 1), + (100, second_source, 101, 2), + (100, second_source, 101, u16::MAX), + (100, second_source, 101, 0), + ] { + let mut packet = + ipv6_icmp_packet_with_id(source, destination, false, identifier, sequence); + let original = SharedVirtualNicFlowKey::from_packet(&packet).unwrap(); + assert!(nat.prepare_packet( + member_id, + None, + original, + &mut packet, + translated_source.into(), + )); + let ipv6 = Ipv6Packet::new_checked(packet.payload()).unwrap(); + assert_eq!(ipv6.src_addr(), translated_source); + let icmp = Icmpv6Packet::new_checked(ipv6.payload()).unwrap(); + assert_eq!(icmp.echo_ident(), wire_identifier); + assert_eq!(icmp.echo_seq_no(), sequence); + assert!(icmp.verify_checksum(&translated_source, &destination)); + + let mut reply = ipv6_icmp_packet_with_id( + destination, + translated_source, + true, + wire_identifier, + sequence, + ); + assert!(nat.translate_reply(member_id, &mut reply)); + assert_eq!( + SharedVirtualNicFlowKey::from_packet(&reply), + Some(original.reversed()) + ); + let ipv6 = Ipv6Packet::new_checked(reply.payload()).unwrap(); + let icmp = Icmpv6Packet::new_checked(ipv6.payload()).unwrap(); + assert_eq!(icmp.echo_seq_no(), sequence); + assert!(icmp.verify_checksum(&destination, &source)); + } + } + + #[test] + fn rewrite_ipv6_source_rejects_extension_headers_without_changing_packet() { + let source = "2001:db8:1::1".parse::().unwrap(); + let destination = "2001:db8:99::2".parse::().unwrap(); + let mut packet = ipv6_packet(source, destination); + packet.mut_payload()[6] = 0; // Hop-by-hop options header. + let original = packet.payload().to_vec(); + + assert!(!rewrite_packet_source( + &mut packet, + SharedVirtualNicFlowAddr::from(source), + SharedVirtualNicFlowAddr::from("2001:db8:2::1".parse::().unwrap()), + )); + assert_eq!(packet.payload(), original); + } + + #[tokio::test] + async fn dispatcher_changed_members_clear_nat_translation() { + let source_owner = uuid::Uuid::from_u128(1); + let destination_owner = uuid::Uuid::from_u128(2); + let source_owner_ip = Ipv4Addr::new(10, 231, 1, 1); + let destination_owner_ip = Ipv4Addr::new(10, 231, 2, 1); + let remote_ip = Ipv4Addr::new(10, 231, 2, 2); + for (member_id, unregister) in [ + (source_owner, true), + (destination_owner, true), + (source_owner, false), + (destination_owner, false), + ] { + let (source_sender, _source_receiver) = mpsc::channel(1); + let (destination_sender, mut destination_receiver) = mpsc::channel(1); + let mut state = SharedVirtualNicDispatcherState::default(); + state.register(source_owner, member_entry(source_sender)); + state.register(destination_owner, member_entry(destination_sender)); + state + .source_table + .update_member_sources(source_owner, member_sources(&["10.231.1.1/24"])); + state + .source_table + .update_member_sources(destination_owner, member_sources(&["10.231.2.1/24"])); + assert!( + state + .send_packet(ipv4_udp_packet(source_owner_ip, remote_ip)) + .await + ); + destination_receiver.try_recv().unwrap(); + + if unregister { + state.unregister(member_id, uuid::Uuid::from_u128(1)); + } else { + let (ack, _rx) = oneshot::channel(); + state.handle_control(SharedVirtualNicControl::UpdateSources { + member_id, + sources: member_sources(&["10.231.3.1/24"]), + ack, + }); + } + let reply = state + .prepare_member_packet_to_tun( + destination_owner, + ipv4_udp_packet_with_ports(remote_ip, destination_owner_ip, 5678, 1234), + ) + .unwrap(); + let reply_ipv4 = Ipv4Packet::new_checked(reply.payload()).unwrap(); + assert_eq!(reply_ipv4.src_addr(), remote_ip); + assert_eq!(reply_ipv4.dst_addr(), destination_owner_ip); + } + } + + #[tokio::test] + async fn dispatcher_preserves_external_ipv4_source_to_route_owner() { + let first = uuid::Uuid::from_u128(1); + let route_owner = uuid::Uuid::from_u128(2); + let synthetic_source = Ipv4Addr::new(100, 64, 0, 1); + let remote_ip = Ipv4Addr::new(10, 99, 0, 2); + let (first_sender, mut first_receiver) = mpsc::channel(1); + let (route_sender, mut route_receiver) = mpsc::channel(1); + let mut state = SharedVirtualNicDispatcherState::default(); + + state.register(first, member_entry(first_sender)); + state.register(route_owner, member_entry(route_sender)); + state + .source_table + .update_member_sources(first, member_sources(&["10.231.1.1/24"])); + state.source_table.update_member_sources( + route_owner, + member_sources_with_ipv4_routes(&["10.231.2.1/24"], &["10.99.0.0/24"]), + ); + + // An exhausted local-flow table must not throttle kernel subnet forwarding. + let local = Ipv4Addr::new(10, 231, 2, 1); + for port in 1..=FLOW_OWNER_LIMIT as u16 { + assert!( + state + .send_packet(ipv4_tcp_packet_with_ports(local, remote_ip, port, 80)) + .await + ); + route_receiver.try_recv().unwrap(); + } + for packet in [ + ipv4_udp_packet(synthetic_source, remote_ip), + ipv4_tcp_packet_with_ports(synthetic_source, remote_ip, 443, 22001), + ] { + let unchanged = packet.payload().to_vec(); + assert!(state.send_packet(packet).await); + assert_eq!(route_receiver.try_recv().unwrap().payload(), unchanged); + } + let incoming = ipv4_udp_packet_with_ports(remote_ip, synthetic_source, 5678, 1234); + let unchanged = incoming.payload().to_vec(); + assert_eq!( + state + .prepare_member_packet_to_tun(route_owner, incoming) + .unwrap() + .payload(), + unchanged + ); + assert!(first_receiver.try_recv().is_err()); + } + + #[tokio::test] + async fn dispatcher_translates_android_synthetic_icmp_source_to_destination_member() { + let first = uuid::Uuid::from_u128(1); + let destination_owner = uuid::Uuid::from_u128(2); + let synthetic_source = Ipv4Addr::new(100, 64, 0, 1); + let destination_owner_ip = Ipv4Addr::new(10, 231, 1, 1); + let remote_ip = Ipv4Addr::new(10, 231, 1, 2); + let (first_sender, mut first_receiver) = mpsc::channel(1); + let (destination_sender, mut destination_receiver) = mpsc::channel(1); + let mut state = SharedVirtualNicDispatcherState { + translate_unknown_sources: true, + ..Default::default() + }; + + state.register(first, member_entry(first_sender)); + state.register(destination_owner, member_entry(destination_sender)); + state + .source_table + .update_member_sources(first, member_sources(&["10.231.2.1/24"])); + state + .source_table + .update_member_sources(destination_owner, member_sources(&["10.231.1.1/24"])); + + state + .forward_tun_packet_to_member(ipv4_icmp_echo_packet(synthetic_source, remote_ip)) + .await; + + assert!(first_receiver.try_recv().is_err()); + let translated = destination_receiver.try_recv().unwrap(); + let translated_ipv4 = Ipv4Packet::new_checked(translated.payload()).unwrap(); + assert_eq!(translated_ipv4.src_addr(), destination_owner_ip); + assert_eq!(translated_ipv4.dst_addr(), remote_ip); + + let reply = state + .prepare_member_packet_to_tun( + destination_owner, + ipv4_icmp_echo_packet(remote_ip, destination_owner_ip), + ) + .unwrap(); + let reply_ipv4 = Ipv4Packet::new_checked(reply.payload()).unwrap(); + assert_eq!(reply_ipv4.src_addr(), remote_ip); + assert_eq!(reply_ipv4.dst_addr(), synthetic_source); + } + + #[tokio::test] + async fn dispatcher_icmp_sequences_preserve_other_members_flow_capacity() { + let first = uuid::Uuid::from_u128(1); + let second = uuid::Uuid::from_u128(2); + for ipv6 in [false, true] { + let (first_sender, mut first_receiver) = mpsc::channel(1); + let (second_sender, mut second_receiver) = mpsc::channel(1); + let mut state = SharedVirtualNicDispatcherState::default(); + state.register(first, member_entry(first_sender)); + state.register(second, member_entry(second_sender)); + state.source_table.update_member_sources( + first, + SharedVirtualNicMemberSources::from_claims(&member_claims( + &["10.231.1.1/24"], + &["2001:db8:1::1/64"], + &[], + &[], + )), + ); + state + .source_table + .update_member_sources(second, member_sources(&["10.231.2.1/24"])); + + for sequence in 0..=FLOW_OWNER_LIMIT as u16 { + let packet = if ipv6 { + ipv6_icmp_packet_with_id( + "2001:db8:1::1".parse().unwrap(), + "2001:db8:1::2".parse().unwrap(), + false, + 100, + sequence, + ) + } else { + ipv4_icmp_packet_with_id( + Ipv4Addr::new(10, 231, 1, 1), + Ipv4Addr::new(10, 231, 1, 2), + Icmpv4Message::EchoRequest, + 100, + sequence, + ) + }; + let original = packet.payload().to_vec(); + assert!(state.send_packet(packet).await); + assert_eq!(first_receiver.try_recv().unwrap().payload(), original); + } + + let packet = + ipv4_udp_packet(Ipv4Addr::new(10, 231, 2, 1), Ipv4Addr::new(10, 231, 2, 2)); + let original = packet.payload().to_vec(); + assert!(state.send_packet(packet).await); + assert_eq!(second_receiver.try_recv().unwrap().payload(), original); + } + } + + #[test] + fn icmpv4_nat_replies_keep_distinct_source_networks() { + let member_id = uuid::Uuid::from_u128(1); + let first_source = Ipv4Addr::new(10, 231, 1, 1); + let second_source = Ipv4Addr::new(10, 231, 2, 1); + let translated_source = Ipv4Addr::new(10, 231, 3, 1); + let destination = Ipv4Addr::new(10, 231, 3, 2); + let mut nat = SharedVirtualNicNatTable::default(); + for (identifier, source, wire_identifier, sequence) in [ + (100, first_source, 100, 1), + (200, second_source, 200, 1), + (100, second_source, 101, 1), + (100, second_source, 101, 2), + (100, second_source, 101, u16::MAX), + (100, second_source, 101, 0), + ] { + let mut packet = ipv4_icmp_packet_with_id( + source, + destination, + Icmpv4Message::EchoRequest, + identifier, + sequence, + ); + let original = SharedVirtualNicFlowKey::from_packet(&packet).unwrap(); + assert!(nat.prepare_packet( + member_id, + None, + original, + &mut packet, + translated_source.into(), + )); + let ipv4 = Ipv4Packet::new_checked(packet.payload()).unwrap(); + assert_eq!(ipv4.src_addr(), translated_source); + assert!(ipv4.verify_checksum()); + let icmp = Icmpv4Packet::new_checked(ipv4.payload()).unwrap(); + assert_eq!(icmp.echo_ident(), wire_identifier); + assert_eq!(icmp.echo_seq_no(), sequence); + assert!(icmp.verify_checksum()); + + let mut reply = ipv4_icmp_packet_with_id( + destination, + translated_source, + Icmpv4Message::EchoReply, + wire_identifier, + sequence, + ); + assert!(nat.translate_reply(member_id, &mut reply)); + assert_eq!( + SharedVirtualNicFlowKey::from_packet(&reply), + Some(original.reversed()) + ); + let ipv4 = Ipv4Packet::new_checked(reply.payload()).unwrap(); + assert!(ipv4.verify_checksum()); + let icmp = Icmpv4Packet::new_checked(ipv4.payload()).unwrap(); + assert_eq!(icmp.echo_seq_no(), sequence); + assert!(icmp.verify_checksum()); + } + } + + #[tokio::test] + async fn dispatcher_preserves_owned_ipv4_source_with_unspecified_placeholder() { + let member_id = uuid::Uuid::from_u128(1); + let local_ip = Ipv4Addr::new(10, 231, 1, 1); + let remote_ip = Ipv4Addr::new(10, 231, 1, 2); + let (sender, mut receiver) = mpsc::channel(1); + let mut state = SharedVirtualNicDispatcherState::default(); + + state.register(member_id, member_entry(sender)); + state + .source_table + .update_member_sources(member_id, member_sources(&["0.0.0.0/0", "10.231.1.1/24"])); + state + .forward_tun_packet_to_member(ipv4_udp_packet(local_ip, remote_ip)) + .await; + + let packet = receiver.try_recv().unwrap(); + let ipv4 = Ipv4Packet::new_checked(packet.payload()).unwrap(); + assert_eq!(ipv4.src_addr(), local_ip); + assert_eq!(ipv4.dst_addr(), remote_ip); + } + + #[tokio::test] + async fn dispatcher_ignores_stale_member_unregister() { + let member_id = uuid::Uuid::from_u128(1); + let stale_registration = uuid::Uuid::from_u128(10); + let current_registration = uuid::Uuid::from_u128(11); + let src = "2001:db8::1".parse::().unwrap(); + let dst = "2001:db8:ffff::1".parse::().unwrap(); + let (sender, mut receiver) = mpsc::channel(1); + let mut state = SharedVirtualNicDispatcherState::default(); + + state.register( + member_id, + member_entry_with_registration(sender, current_registration), + ); + state + .source_table + .update_member_sources(member_id, member_sources_with_ipv6(&["2001:db8::1/64"])); + state.unregister(member_id, stale_registration); + state + .forward_tun_packet_to_member(ipv6_packet(src, dst)) + .await; + + assert!(receiver.try_recv().is_ok()); + } + + #[tokio::test] + async fn dispatcher_invalidates_shared_nic_when_tun_read_fails() { + let member_id = uuid::Uuid::from_u128(1); + let (tun_tx, tun_rx) = mpsc::unbounded_channel(); + let tun_stream = futures::stream::unfold(tun_rx, |mut receiver| async move { + receiver.recv().await.map(|packet| (packet, receiver)) + }); + let tun_sink = futures::sink::unfold((), |(), _packet: ZCPacket| async { + Ok::<(), TunnelError>(()) + }); + let tunnel = TunnelWrapper::new(tun_stream, tun_sink, None); + let member_tunnel_table = SharedVirtualNicMemberTunnelTable::default(); + let valid = Arc::new(AtomicBool::new(true)); + let dispatcher = SharedVirtualNicDispatcher::start( + Box::new(tunnel), + member_tunnel_table.clone(), + valid.clone(), + ); + let close_notifier = Arc::new(Notify::new()); + let (_member_tunnel, shared_tunnel) = create_ring_tunnel_pair(); + + member_tunnel_table + .register( + member_id, + uuid::Uuid::from_u128(1), + shared_tunnel, + close_notifier.clone(), + ) + .unwrap(); + let empty_claims = SharedIfConfigClaims::default(); + dispatcher + .update_sources(member_id, &empty_claims) + .await + .unwrap(); + + tun_tx.send(Err(TunnelError::Shutdown)).unwrap(); + + tokio::time::timeout(Duration::from_secs(1), close_notifier.notified()) + .await + .unwrap(); + assert!(!valid.load(Ordering::Acquire)); + assert!(member_tunnel_table.dispatcher_channels().is_none()); + } + + #[tokio::test] + async fn dispatcher_shutdown_for_replacement_keeps_shared_nic_valid() { + let member_id = uuid::Uuid::from_u128(1); + let (_tun_tx, tun_rx) = mpsc::unbounded_channel(); + let tun_stream = futures::stream::unfold(tun_rx, |mut receiver| async move { + receiver.recv().await.map(|packet| (packet, receiver)) + }); + let tun_sink = futures::sink::unfold((), |(), _packet: ZCPacket| async { + Ok::<(), TunnelError>(()) + }); + let tunnel = TunnelWrapper::new(tun_stream, tun_sink, None); + let member_tunnel_table = SharedVirtualNicMemberTunnelTable::default(); + let valid = Arc::new(AtomicBool::new(true)); + let dispatcher = SharedVirtualNicDispatcher::start( + Box::new(tunnel), + member_tunnel_table.clone(), + valid.clone(), + ); + let close_notifier = Arc::new(Notify::new()); + let (_member_tunnel, shared_tunnel) = create_ring_tunnel_pair(); + + member_tunnel_table + .register( + member_id, + uuid::Uuid::from_u128(1), + shared_tunnel, + close_notifier.clone(), + ) + .unwrap(); + let empty_claims = SharedIfConfigClaims::default(); + dispatcher + .update_sources(member_id, &empty_claims) + .await + .unwrap(); + + dispatcher.shutdown_without_invalidation().await; + + tokio::time::timeout(Duration::from_secs(1), close_notifier.notified()) + .await + .unwrap(); + assert!(valid.load(Ordering::Acquire)); + assert!(member_tunnel_table.dispatcher_channels().is_none()); + } + + #[tokio::test] + async fn cancelled_member_updates_finish_before_registration_cleanup() { + use super::super::{SharedVirtualNic, SharedVirtualNicMember}; + use crate::{ + common::{ifcfg::IfConfiguerTrait, netns::NetNS}, + instance::virtual_nic::VirtualNicConfig, + }; + + #[derive(Clone)] + struct PausingIfConfiger { + claims: Arc>, + pause: Arc, + applied: Arc, + resume: Arc, + } + + impl PausingIfConfiger { + async fn after_apply(&self) { + if self.pause.swap(false, Ordering::AcqRel) { + self.applied.notify_one(); + self.resume.notified().await; + } + } + } + + #[async_trait::async_trait] + impl IfConfiguerTrait for PausingIfConfiger { + async fn add_ipv4_ip( + &self, + _name: &str, + address: Ipv4Addr, + prefix: u8, + ) -> Result<(), Error> { + self.claims + .lock() + .unwrap() + .ipv4_addresses + .insert(Ipv4Inet::new(address, prefix).unwrap()); + self.after_apply().await; + Ok(()) + } + + async fn remove_ip(&self, _name: &str, ip: Option) -> Result<(), Error> { + self.claims + .lock() + .unwrap() + .ipv4_addresses + .remove(&ip.unwrap()); + Ok(()) + } + + async fn add_ipv4_route( + &self, + _name: &str, + address: Ipv4Addr, + prefix: u8, + cost: Option, + ) -> Result<(), Error> { + self.claims + .lock() + .unwrap() + .ipv4_routes + .insert(SharedIpv4Route::new(address, prefix, cost)); + self.after_apply().await; + Ok(()) + } + + async fn remove_ipv4_route( + &self, + _name: &str, + address: Ipv4Addr, + prefix: u8, + ) -> Result<(), Error> { + self.claims + .lock() + .unwrap() + .ipv4_routes + .retain(|route| route.address != address || route.prefix != prefix); + Ok(()) + } + + async fn set_mtu(&self, _name: &str, mtu: u32) -> Result<(), Error> { + self.claims.lock().unwrap().mtu = Some(mtu); + self.after_apply().await; + Ok(()) + } + } + + for operation in ["address", "route", "mtu"] { + for pause_in_ifcfg in [false, true] { + let member_id = uuid::Uuid::from_u128(1); + let other_member_id = uuid::Uuid::from_u128(2); + let applied = Arc::new(Notify::new()); + let resume = Arc::new(Notify::new()); + let initial_claims = SharedIfConfigClaims { + mtu: Some(1500), + ..Default::default() + }; + let ifcfg = PausingIfConfiger { + claims: Arc::new(StdMutex::new(initial_claims.clone())), + pause: Arc::new(AtomicBool::new(pause_in_ifcfg)), + applied: applied.clone(), + resume: resume.clone(), + }; + let mut shared_nic = SharedVirtualNic::new(VirtualNicConfig::new( + String::new(), + 1500, + NetNS::new(None), + )); + { + let mut nic = shared_nic.nic.lock().await; + nic.set_ifname_for_test("et0".to_owned()); + nic.set_ifcfg_for_test(Box::new(ifcfg.clone())); + } + // A surviving member keeps the device alive after the caller exits. + shared_nic + .member_registrations + .insert(other_member_id, uuid::Uuid::new_v4()); + shared_nic + .ifcfg + .apply_member_claims(other_member_id, initial_claims.clone()); + let (control_sender, mut controls) = mpsc::unbounded_channel(); + let ack_applied = applied.clone(); + let ack_resume = resume.clone(); + let task = SHARED_VIRTUAL_NIC_RUNTIME.spawn(async move { + let mut updates = 0; + while let Some(control) = controls.recv().await { + match control { + SharedVirtualNicControl::UpdateSources { ack, .. } => { + updates += 1; + if updates == 2 && !pause_in_ifcfg { + ack_applied.notify_one(); + ack_resume.notified().await; + } + let _ = ack.send(()); + } + SharedVirtualNicControl::Shutdown { ack, .. } => { + let _ = ack.send(()); + break; + } + _ => {} + } + } + }); + shared_nic.dispatcher = Some(SharedVirtualNicDispatcher { + _task: AbortOnDropHandle::new(task), + control_sender, + #[cfg(mobile)] + mobile_tun_update_sender: None, + }); + let table = shared_nic.member_tunnel_table.clone(); + let shared_nic = Arc::new(tokio::sync::Mutex::new(shared_nic)); + let member = SharedVirtualNicMember::new( + member_id, + 1300, + shared_nic.clone(), + Arc::new(Notify::new()), + table, + ); + shared_nic + .lock() + .await + .member_registrations + .insert(member_id, member.registration.registration_id); + + let caller = tokio::spawn(async move { + match operation { + "address" => member.add_ip(Ipv4Addr::new(10, 231, 1, 1), 24).await, + "route" => member.add_route(Ipv4Addr::new(10, 99, 0, 0), 24).await, + "mtu" => member.create_dev().await.map(|_| ()), + _ => unreachable!(), + } + }); + tokio::time::timeout(Duration::from_secs(5), applied.notified()) + .await + .unwrap(); + caller.abort(); + assert!(caller.await.unwrap_err().is_cancelled()); + resume.notify_one(); + + tokio::time::timeout(Duration::from_secs(5), async { + while shared_nic + .lock() + .await + .member_registrations + .contains_key(&member_id) + { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert_eq!( + *ifcfg.claims.lock().unwrap(), + initial_claims, + "operation={operation}, pause_in_ifcfg={pause_in_ifcfg}" + ); + let mut shared_nic = shared_nic.lock().await; + assert!(shared_nic.is_valid()); + assert!(!shared_nic.ifcfg.member_claims.contains_key(&member_id)); + shared_nic.shutdown_dispatcher().await; + } + } + } + + #[tokio::test] + async fn dispatcher_preserves_incoming_udp_reply_ports_with_wrong_source() { + let source_owner = uuid::Uuid::from_u128(1); + let destination_owner = uuid::Uuid::from_u128(2); + for (source_sources, native_sources, incoming, outgoing) in [ + ( + member_sources(&["10.231.1.1/24"]), + member_sources(&["10.231.2.1/24"]), + ipv4_udp_packet_with_ports( + Ipv4Addr::new(10, 231, 2, 2), + Ipv4Addr::new(10, 231, 2, 1), + 5678, + 1234, + ), + ipv4_udp_packet(Ipv4Addr::new(10, 231, 1, 1), Ipv4Addr::new(10, 231, 2, 2)), + ), + ( + member_sources_with_ipv6(&["2001:db8:1::1/64"]), + member_sources_with_ipv6(&["2001:db8:2::1/64"]), + ipv6_udp_packet_with_ports( + "2001:db8:2::2".parse().unwrap(), + "2001:db8:2::1".parse().unwrap(), + 5678, + 1234, + ), + ipv6_udp_packet_with_ports( + "2001:db8:1::1".parse().unwrap(), + "2001:db8:2::2".parse().unwrap(), + 1234, + 5678, + ), + ), + ] { + let (source_sender, mut source_receiver) = mpsc::channel(1); + let (sender, mut receiver) = mpsc::channel(1); + let mut state = SharedVirtualNicDispatcherState::default(); + state.register(source_owner, member_entry(source_sender)); + state.register(destination_owner, member_entry(sender)); + state + .source_table + .update_member_sources(source_owner, source_sources); + state + .source_table + .update_member_sources(destination_owner, native_sources); + + let incoming_payload = incoming.payload().to_vec(); + let expected_reply = SharedVirtualNicFlowKey::from_packet(&incoming) + .unwrap() + .reversed(); + for _ in 0..3 { + // A wildcard UDP socket receives on B but replies with A's source IP. + let received = state + .prepare_member_packet_to_tun( + destination_owner, + ZCPacket::new_with_payload(&incoming_payload), + ) + .unwrap(); + assert_eq!(received.payload(), incoming_payload); + assert!( + state + .send_packet(ZCPacket::new_with_payload(outgoing.payload())) + .await + ); + let reply = receiver.try_recv().unwrap(); + assert_eq!( + SharedVirtualNicFlowKey::from_packet(&reply), + Some(expected_reply) + ); + match expected_reply.src { + SharedVirtualNicFlowAddr::V4(_) => { + let ipv4 = Ipv4Packet::new_checked(reply.payload()).unwrap(); + assert!(ipv4.verify_checksum()); + assert!( + UdpPacket::new_checked(ipv4.payload()) + .unwrap() + .verify_checksum( + &IpAddress::Ipv4(ipv4.src_addr()), + &IpAddress::Ipv4(ipv4.dst_addr()) + ) + ); + } + SharedVirtualNicFlowAddr::V6(_) => { + let ipv6 = Ipv6Packet::new_checked(reply.payload()).unwrap(); + assert!( + UdpPacket::new_checked(ipv6.payload()) + .unwrap() + .verify_checksum( + &IpAddress::Ipv6(ipv6.src_addr()), + &IpAddress::Ipv6(ipv6.dst_addr()) + ) + ); + } + } + assert!(source_receiver.try_recv().is_err()); + } + } + } + + #[tokio::test] + async fn dispatcher_reserves_native_incoming_tcp_flows_before_translation() { + let source_owner = uuid::Uuid::from_u128(1); + let destination_owner = uuid::Uuid::from_u128(2); + for (source_sources, native_sources, incoming, outgoing) in [ + ( + member_sources(&["10.231.1.1/24"]), + member_sources(&["10.231.2.1/24"]), + ipv4_tcp_packet_with_ports( + Ipv4Addr::new(10, 231, 2, 2), + Ipv4Addr::new(10, 231, 2, 1), + 5678, + 1234, + ), + ipv4_tcp_packet_with_ports( + Ipv4Addr::new(10, 231, 1, 1), + Ipv4Addr::new(10, 231, 2, 2), + 1234, + 5678, + ), + ), + ( + member_sources_with_ipv6(&["2001:db8:1::1/64"]), + member_sources_with_ipv6(&["2001:db8:2::1/64"]), + ipv6_tcp_packet_with_ports( + "2001:db8:2::2".parse().unwrap(), + "2001:db8:2::1".parse().unwrap(), + 5678, + 1234, + ), + ipv6_tcp_packet_with_ports( + "2001:db8:1::1".parse().unwrap(), + "2001:db8:2::2".parse().unwrap(), + 1234, + 5678, + ), + ), + ] { + let (source_sender, _source_receiver) = mpsc::channel(1); + let (sender, mut receiver) = mpsc::channel(1); + let mut state = SharedVirtualNicDispatcherState::default(); + state.register(source_owner, member_entry(source_sender)); + state.register(destination_owner, member_entry(sender)); + state + .source_table + .update_member_sources(source_owner, source_sources); + state + .source_table + .update_member_sources(destination_owner, native_sources); + + let incoming_payload = incoming.payload().to_vec(); + let native_reply = state + .prepare_member_packet_to_tun(destination_owner, incoming) + .unwrap(); + assert_eq!(native_reply.payload(), incoming_payload); + + // The native socket has not replied when another member starts a flow. + let original = SharedVirtualNicFlowKey::from_packet(&outgoing).unwrap(); + assert!(state.send_packet(outgoing).await); + let translated = receiver.try_recv().unwrap(); + let wire = SharedVirtualNicFlowKey::from_packet(&translated).unwrap(); + assert_ne!(wire.ports.unwrap().src, original.ports.unwrap().src); + + let native_reply = state + .prepare_member_packet_to_tun( + destination_owner, + ZCPacket::new_with_payload(&incoming_payload), + ) + .unwrap(); + assert_eq!(native_reply.payload(), incoming_payload); + + let translated_reply = match (wire.dst, wire.src) { + (SharedVirtualNicFlowAddr::V4(src), SharedVirtualNicFlowAddr::V4(dst)) => { + ipv4_tcp_packet_with_ports( + src.into(), + dst.into(), + 5678, + wire.ports.unwrap().src, + ) + } + (SharedVirtualNicFlowAddr::V6(src), SharedVirtualNicFlowAddr::V6(dst)) => { + ipv6_tcp_packet_with_ports( + src.into(), + dst.into(), + 5678, + wire.ports.unwrap().src, + ) + } + _ => unreachable!(), + }; + let reply = state + .prepare_member_packet_to_tun(destination_owner, translated_reply) + .unwrap(); + assert_eq!( + SharedVirtualNicFlowKey::from_packet(&reply), + Some(original.reversed()) + ); + } + } + + #[test] + fn nat_existing_outgoing_udp_mapping_precedes_incoming_reservation() { + let member_id = uuid::Uuid::from_u128(1); + let source = Ipv4Addr::new(10, 231, 1, 1); + let native = Ipv4Addr::new(10, 231, 2, 1); + let other_native = Ipv4Addr::new(10, 231, 2, 3); + let remote = Ipv4Addr::new(10, 231, 2, 2); + let mut nat = SharedVirtualNicNatTable::default(); + let mut outgoing = ipv4_udp_packet(source, remote); + let original = SharedVirtualNicFlowKey::from_packet(&outgoing).unwrap(); + assert!(nat.prepare_packet(member_id, None, original, &mut outgoing, native.into())); + let wire = SharedVirtualNicFlowKey::from_packet(&outgoing).unwrap(); + + let incoming = ipv4_udp_packet_with_ports(remote, other_native, 5678, 1234); + assert!(nat.reserve_reply( + member_id, + SharedVirtualNicFlowKey::from_packet(&incoming).unwrap() + )); + let mut outgoing = ipv4_udp_packet(source, remote); + assert!(nat.prepare_packet( + member_id, + None, + original, + &mut outgoing, + other_native.into() + )); + assert_eq!(SharedVirtualNicFlowKey::from_packet(&outgoing), Some(wire)); + let mut reply = ipv4_udp_packet_with_ports(remote, native, 5678, 1234); + assert!(nat.translate_reply(member_id, &mut reply)); + assert_eq!( + SharedVirtualNicFlowKey::from_packet(&reply), + Some(original.reversed()) + ); + } + + #[tokio::test] + async fn dispatcher_incoming_flow_pressure_keeps_active_reservations() { + let member_id = uuid::Uuid::from_u128(1); + let native = Ipv4Addr::new(10, 231, 1, 1); + let remote = Ipv4Addr::new(10, 231, 1, 2); + let mut state = SharedVirtualNicDispatcherState::default(); + state + .source_table + .update_member_sources(member_id, member_sources(&["10.231.1.1/24"])); + + for port in 1..=FLOW_OWNER_LIMIT as u16 { + assert!( + state + .prepare_member_packet_to_tun( + member_id, + ipv4_udp_packet_with_ports(remote, native, 5678, port), + ) + .is_some() + ); + } + let new_port = FLOW_OWNER_LIMIT as u16 + 1; + assert!( + state + .prepare_member_packet_to_tun( + member_id, + ipv4_udp_packet_with_ports(remote, native, 5678, new_port), + ) + .is_none() + ); + assert_eq!(state.nat_table.entries.len(), FLOW_OWNER_LIMIT); + + for entry in state.nat_table.entries.values_mut() { + entry.last_active = Instant::now() - FLOW_IDLE_TIMEOUT - Duration::from_secs(1); + } + // Refresh a receive-only socket before pressure reclaims the idle entries. + for port in [1, new_port] { + let incoming = ipv4_udp_packet_with_ports(remote, native, 5678, port); + let unchanged = incoming.payload().to_vec(); + assert_eq!( + state + .prepare_member_packet_to_tun(member_id, incoming) + .unwrap() + .payload(), + unchanged + ); + } + assert_eq!(state.nat_table.entries.len(), 2); + let active = SharedVirtualNicFlowKey::from_packet(&ipv4_udp_packet_with_ports( + native, remote, 1, 5678, + )) + .unwrap(); + assert!(state.nat_table.entries.contains_key(&(member_id, active))); + } + + #[tokio::test] + async fn dispatcher_colliding_native_and_translated_flows_remain_distinct() { + let source_owner = uuid::Uuid::from_u128(1); + let destination_owner = uuid::Uuid::from_u128(2); + let source = Ipv4Addr::new(10, 231, 1, 1); + let native = Ipv4Addr::new(10, 231, 2, 1); + let destination = Ipv4Addr::new(10, 231, 2, 2); + for native_first in [false, true] { + let (source_sender, mut source_receiver) = mpsc::channel(1); + let (sender, mut receiver) = mpsc::channel(1); + let mut state = SharedVirtualNicDispatcherState::default(); + state.register(source_owner, member_entry(source_sender)); + state.register(destination_owner, member_entry(sender)); + state + .source_table + .update_member_sources(source_owner, member_sources(&["10.231.1.1/24"])); + state + .source_table + .update_member_sources(destination_owner, member_sources(&["10.231.2.1/24"])); + let mut wire_keys = Vec::new(); + for src in if native_first { + [native, source] + } else { + [source, native] + } { + let packet = ipv4_udp_packet(src, destination); + let original = SharedVirtualNicFlowKey::from_packet(&packet).unwrap(); + assert!(state.send_packet(packet).await); + let sent = receiver.try_recv().unwrap(); + let wire = SharedVirtualNicFlowKey::from_packet(&sent).unwrap(); + assert_eq!(wire.src, native.into()); + assert_eq!(wire.dst, destination.into()); + let ipv4 = Ipv4Packet::new_checked(sent.payload()).unwrap(); + assert!(ipv4.verify_checksum()); + let udp = UdpPacket::new_checked(ipv4.payload()).unwrap(); + assert_ne!(udp.checksum(), 0); + assert!( + udp.verify_checksum(&IpAddress::Ipv4(native), &IpAddress::Ipv4(destination)) + ); + wire_keys.push((original, wire)); + } + assert!(source_receiver.try_recv().is_err()); + assert_ne!(wire_keys[0].1, wire_keys[1].1); + for (original, wire) in wire_keys { + let source = original.src.as_ipv4().unwrap(); + assert!( + state + .send_packet(ipv4_udp_packet(source, destination)) + .await + ); + assert_eq!( + SharedVirtualNicFlowKey::from_packet(&receiver.try_recv().unwrap()), + Some(wire) + ); + for (member_id, expected) in [ + (source_owner, wire.reversed()), + (destination_owner, original.reversed()), + ] { + let reply = state + .prepare_member_packet_to_tun( + member_id, + ipv4_udp_packet_with_ports( + destination, + native, + 5678, + wire.ports.unwrap().src, + ), + ) + .unwrap(); + assert_eq!(SharedVirtualNicFlowKey::from_packet(&reply), Some(expected)); + let ipv4 = Ipv4Packet::new_checked(reply.payload()).unwrap(); + assert!(ipv4.verify_checksum()); + let udp = UdpPacket::new_checked(ipv4.payload()).unwrap(); + assert_ne!(udp.checksum(), 0); + assert!(udp.verify_checksum( + &IpAddress::Ipv4(destination), + &IpAddress::Ipv4(expected.dst.as_ipv4().unwrap()) + )); + } + } + } + } + + #[tokio::test] + async fn dispatcher_unrelated_claims_and_unregister_preserve_replies() { + let source_owner = uuid::Uuid::from_u128(1); + let destination_owner = uuid::Uuid::from_u128(2); + let unrelated = uuid::Uuid::from_u128(3); + let source = Ipv4Addr::new(10, 231, 1, 1); + let native = Ipv4Addr::new(10, 231, 2, 1); + let remote = Ipv4Addr::new(10, 231, 2, 2); + let mut state = SharedVirtualNicDispatcherState::default(); + let (sender, _source_receiver) = mpsc::channel(1); + state.register(source_owner, member_entry(sender)); + let (sender, mut receiver) = mpsc::channel(1); + state.register(destination_owner, member_entry(sender)); + let (sender, _unrelated_receiver) = mpsc::channel(1); + state.register(unrelated, member_entry(sender)); + state + .source_table + .update_member_sources(source_owner, member_sources(&["10.231.1.1/24"])); + state + .source_table + .update_member_sources(destination_owner, member_sources(&["10.231.2.1/24"])); + state + .forward_tun_packet_to_member(ipv4_udp_packet(source, remote)) + .await; + receiver.try_recv().unwrap(); + + for (member_id, sources) in [ + ( + unrelated, + member_sources_with_ipv4_routes(&["10.231.3.1/24"], &["192.0.2.0/24"]), + ), + ( + source_owner, + member_sources_with_ipv4_routes(&["10.231.1.1/24"], &["198.51.100.0/24"]), + ), + ( + destination_owner, + member_sources_with_ipv4_routes( + &["10.231.2.1/24", "10.231.4.1/24"], + &["203.0.113.0/24"], + ), + ), + ] { + let (ack, mut receiver) = oneshot::channel(); + state.handle_control(SharedVirtualNicControl::UpdateSources { + member_id, + sources, + ack, + }); + receiver.try_recv().unwrap(); + let reply = state + .prepare_member_packet_to_tun( + destination_owner, + ipv4_udp_packet_with_ports(remote, native, 5678, 1234), + ) + .unwrap(); + assert_eq!( + Ipv4Packet::new_checked(reply.payload()).unwrap().dst_addr(), + source + ); + } + state.unregister(unrelated, uuid::Uuid::from_u128(1)); + let reply = state + .prepare_member_packet_to_tun( + destination_owner, + ipv4_udp_packet_with_ports(remote, native, 5678, 1234), + ) + .unwrap(); + assert_eq!( + Ipv4Packet::new_checked(reply.payload()).unwrap().dst_addr(), + source + ); + } + + #[tokio::test] + async fn dispatcher_full_member_queue_drops_only_packet_without_reserving_flow() { + let slow = uuid::Uuid::from_u128(1); + let active = uuid::Uuid::from_u128(2); + let slow_ip = Ipv4Addr::new(10, 231, 1, 1); + let active_ip = Ipv4Addr::new(10, 231, 2, 1); + let remote = Ipv4Addr::new(10, 231, 1, 2); + let mut state = SharedVirtualNicDispatcherState::default(); + let (sender, mut slow_receiver) = mpsc::channel(1); + state.register(slow, member_entry(sender)); + let (sender, mut active_receiver) = mpsc::channel(1); + state.register(active, member_entry(sender)); + state + .source_table + .update_member_sources(slow, member_sources(&["10.231.1.1/24"])); + state + .source_table + .update_member_sources(active, member_sources(&["10.231.2.1/24"])); + state + .forward_tun_packet_to_member(ipv4_udp_packet_with_ports(slow_ip, remote, 1000, 5678)) + .await; + tokio::time::timeout( + Duration::from_secs(1), + state.forward_tun_packet_to_member(ipv4_udp_packet(active_ip, remote)), + ) + .await + .unwrap(); + let (ack, mut receiver) = oneshot::channel(); + state.handle_control(SharedVirtualNicControl::UpdateSources { + member_id: active, + sources: member_sources(&["10.231.2.1/24"]), + ack, + }); + receiver.try_recv().unwrap(); + state + .forward_tun_packet_to_member(ipv4_udp_packet(active_ip, Ipv4Addr::new(10, 231, 2, 2))) + .await; + active_receiver.try_recv().unwrap(); + slow_receiver.try_recv().unwrap(); + + // The dropped translated flow must not steal this native flow's port. + state + .forward_tun_packet_to_member(ipv4_udp_packet(slow_ip, remote)) + .await; + let packet = slow_receiver.try_recv().unwrap(); + assert_eq!( + SharedVirtualNicFlowKey::from_packet(&packet) + .unwrap() + .ports + .unwrap() + .src, + 1234 + ); + let reply = state + .prepare_member_packet_to_tun( + slow, + ipv4_udp_packet_with_ports(remote, slow_ip, 5678, 1234), + ) + .unwrap(); + assert_eq!( + Ipv4Packet::new_checked(reply.payload()).unwrap().dst_addr(), + slow_ip + ); + drop(slow_receiver); + state + .forward_tun_packet_to_member(ipv4_udp_packet(slow_ip, remote)) + .await; + assert!(!state.members.contains_key(&slow)); + } + + #[test] + fn nat_pressure_reclaims_idle_flows_but_preserves_active_reservations() { + let member_id = uuid::Uuid::from_u128(1); + let source = Ipv4Addr::new(10, 231, 1, 1); + let native = Ipv4Addr::new(10, 231, 2, 1); + let remote = Ipv4Addr::new(10, 231, 2, 2); + let mut nat = SharedVirtualNicNatTable::default(); + for port in 1..=FLOW_OWNER_LIMIT as u16 { + let mut packet = ipv4_udp_packet_with_ports(source, remote, port, 5678); + assert!(nat.prepare_packet( + member_id, + None, + SharedVirtualNicFlowKey::from_packet(&packet).unwrap(), + &mut packet, + native.into() + )); + } + let mut new_flow = + ipv4_udp_packet_with_ports(source, remote, FLOW_OWNER_LIMIT as u16 + 1, 5678); + let new_key = SharedVirtualNicFlowKey::from_packet(&new_flow).unwrap(); + assert!(!nat.prepare_packet(member_id, None, new_key, &mut new_flow, native.into())); + let mut unsupported = ipv4_udp_packet(native, remote); + { + let mut ipv4 = Ipv4Packet::new_checked(unsupported.mut_payload()).unwrap(); + ipv4.set_next_header(IpProtocol::Unknown(253)); + ipv4.fill_checksum(); + } + let unchanged = unsupported.payload().to_vec(); + assert!(nat.prepare_packet( + member_id, + None, + SharedVirtualNicFlowKey::from_packet(&unsupported).unwrap(), + &mut unsupported, + native.into(), + )); + assert_eq!(unsupported.payload(), unchanged); + for entry in nat.entries.values_mut() { + entry.last_active = Instant::now() - FLOW_IDLE_TIMEOUT - Duration::from_secs(1); + } + // Activity in either direction protects the reservation during lazy reclamation. + let mut active = ipv4_udp_packet_with_ports(source, remote, 1, 5678); + assert!(nat.prepare_packet( + member_id, + None, + SharedVirtualNicFlowKey::from_packet(&active).unwrap(), + &mut active, + native.into() + )); + assert!(nat.translate_reply( + member_id, + &mut ipv4_udp_packet_with_ports(remote, native, 5678, 2) + )); + assert!(nat.prepare_packet(member_id, None, new_key, &mut new_flow, native.into())); + for port in [1, 2] { + let mut native_packet = ipv4_udp_packet_with_ports(native, remote, port, 5678); + assert!(nat.prepare_packet( + member_id, + None, + SharedVirtualNicFlowKey::from_packet(&native_packet).unwrap(), + &mut native_packet, + native.into(), + )); + let wire = SharedVirtualNicFlowKey::from_packet(&native_packet).unwrap(); + assert_ne!(wire.ports.unwrap().src, port); + let mut reply = + ipv4_udp_packet_with_ports(remote, native, 5678, wire.ports.unwrap().src); + assert!(nat.translate_reply(member_id, &mut reply)); + assert_eq!( + SharedVirtualNicFlowKey::from_packet(&reply) + .unwrap() + .ports + .unwrap() + .dst, + port + ); + } + } + + #[test] + fn nat_identical_wire_tuples_are_scoped_to_member() { + let first = uuid::Uuid::from_u128(1); + let second = uuid::Uuid::from_u128(2); + let first_source = Ipv4Addr::new(10, 231, 1, 1); + let second_source = Ipv4Addr::new(10, 231, 2, 1); + let wire_source = Ipv4Addr::new(10, 231, 3, 1); + let remote = Ipv4Addr::new(10, 231, 3, 2); + let mut nat = SharedVirtualNicNatTable::default(); + for (member_id, source) in [(first, first_source), (second, second_source)] { + let mut packet = ipv4_udp_packet(source, remote); + assert!(nat.prepare_packet( + member_id, + None, + SharedVirtualNicFlowKey::from_packet(&packet).unwrap(), + &mut packet, + wire_source.into(), + )); + assert_eq!( + SharedVirtualNicFlowKey::from_packet(&packet) + .unwrap() + .ports + .unwrap() + .src, + 1234 + ); + } + for (member_id, source) in [(first, first_source), (second, second_source)] { + let mut reply = ipv4_udp_packet_with_ports(remote, wire_source, 5678, 1234); + assert!(nat.translate_reply(member_id, &mut reply)); + assert_eq!( + Ipv4Packet::new_checked(reply.payload()).unwrap().dst_addr(), + source + ); + } + } +} diff --git a/easytier/src/instance/test_instance.rs b/easytier/src/instance/test_instance.rs index 3d815950..be52ba55 100644 --- a/easytier/src/instance/test_instance.rs +++ b/easytier/src/instance/test_instance.rs @@ -7,6 +7,8 @@ use easytier_core::{ process_runtime::CoreProcessRuntime, }; +#[cfg(feature = "tun")] +use crate::instance::shared_virtual_nic::{ArcSharedVirtualNicRegistry, SharedVirtualNicRegistry}; use crate::{ common::global_ctx::{ArcGlobalCtx, GlobalCtx}, instance::{ @@ -15,6 +17,8 @@ use crate::{ }, socket::udp::RuntimeUdpSocket, }; +#[cfg(feature = "tun")] +use tokio::sync::Mutex; pub(crate) struct TestInstance { core: Arc, @@ -26,7 +30,27 @@ impl TestInstance { config: TomlConfig, process_runtime: Arc, ) -> Self { - Self::compose(config, process_runtime, |_| {}) + Self::compose( + config, + process_runtime, + #[cfg(feature = "tun")] + Arc::new(Mutex::new(SharedVirtualNicRegistry::new())), + |_| {}, + ) + } + + #[cfg(feature = "tun")] + pub fn new_with_process_runtime_and_shared_virtual_nic_registry( + config: TomlConfig, + process_runtime: Arc, + shared_virtual_nic_registry: ArcSharedVirtualNicRegistry, + ) -> Self { + Self::compose(config, process_runtime, shared_virtual_nic_registry, |_| {}) + } + + #[cfg(feature = "tun")] + pub fn new_shared_virtual_nic_registry() -> ArcSharedVirtualNicRegistry { + Arc::new(Mutex::new(SharedVirtualNicRegistry::new())) } pub fn new_with_process_runtime_and_stun_provider( @@ -35,14 +59,21 @@ impl TestInstance { provider: Box>, ) -> Self { let provider: Arc> = Arc::from(provider); - Self::compose(config, process_runtime, move |adapters| { - adapters.replace_stun_provider(provider); - }) + Self::compose( + config, + process_runtime, + #[cfg(feature = "tun")] + Arc::new(Mutex::new(SharedVirtualNicRegistry::new())), + move |adapters| { + adapters.replace_stun_provider(provider); + }, + ) } fn compose( config: TomlConfig, process_runtime: Arc, + #[cfg(feature = "tun")] shared_virtual_nic_registry: ArcSharedVirtualNicRegistry, customize: impl FnOnce( &mut easytier_core::instance::CoreHostAdapters< crate::instance::host::NativeInstanceHost, @@ -50,7 +81,11 @@ impl TestInstance { ), ) -> Self { let global_ctx = Arc::new(GlobalCtx::new(config.clone())); - let runtime_host = NativeInstanceRuntimeHost::new(global_ctx.clone()); + let runtime_host = NativeInstanceRuntimeHost::new( + global_ctx.clone(), + #[cfg(feature = "tun")] + shared_virtual_nic_registry, + ); let mut adapters = runtime_core_host_adapters_with_packet_egress( global_ctx.clone(), process_runtime, diff --git a/easytier/src/instance/virtual_nic.rs b/easytier/src/instance/virtual_nic.rs index 43c97aa6..0a0a9c2f 100644 --- a/easytier/src/instance/virtual_nic.rs +++ b/easytier/src/instance/virtual_nic.rs @@ -11,6 +11,7 @@ use crate::common::{ error::Error, global_ctx::{ArcGlobalCtx, GlobalCtxEvent}, ifcfg::{IfConfiger, IfConfiguerTrait}, + netns::NetNS, }; use easytier_core::{ @@ -43,6 +44,10 @@ use zerocopy::{NativeEndian, NetworkEndian}; #[cfg(target_os = "windows")] use crate::common::ifcfg::RegistryManager; +use super::shared_virtual_nic::{ + ArcSharedVirtualNicRegistry, SharedVirtualNicMember, SharedVirtualNicMemberId, +}; + pin_project! { pub struct TunStream { #[pin] @@ -240,8 +245,32 @@ impl AsyncWrite for TunAsyncWrite { } } +pub struct VirtualNicConfig { + dev_name: String, + mtu: u32, + net_ns: NetNS, +} + +impl VirtualNicConfig { + pub fn new(dev_name: String, mtu: u32, net_ns: NetNS) -> Self { + Self { + dev_name, + mtu, + net_ns, + } + } + + pub fn mtu(&self) -> u32 { + self.mtu + } + + pub fn net_ns_name(&self) -> Option { + self.net_ns.name() + } +} + pub struct VirtualNic { - global_ctx: ArcGlobalCtx, + config: VirtualNicConfig, ifname: Option, ifcfg: Box, @@ -266,11 +295,11 @@ impl Drop for VirtualNic { } impl VirtualNic { - pub fn new(global_ctx: ArcGlobalCtx) -> Self { + pub fn new(config: VirtualNicConfig) -> Self { Self { - global_ctx, + config, ifname: None, - ifcfg: Box::new(IfConfiger {}), + ifcfg: Box::new(IfConfiger::default()), } } @@ -487,14 +516,14 @@ impl VirtualNic { Ok(()) } - async fn create_tun(&self) -> Result { + async fn create_tun(&mut self) -> Result { let mut config = Configuration::default(); config.layer(Layer::L3); // FreeBSD specific: Check and restore TUN interfaces before creating new one #[cfg(target_os = "freebsd")] { - let dev_name = self.global_ctx.get_flags().dev_name; + let dev_name = self.config.dev_name.clone(); if !dev_name.is_empty() { // Restore TUN interface name if needed, ignoring errors as it's not critical @@ -507,7 +536,7 @@ impl VirtualNic { // Check and create TUN device node if necessary (Linux only) Self::ensure_tun_device_node().await; - let dev_name = self.global_ctx.get_flags().dev_name; + let dev_name = self.config.dev_name.clone(); if !dev_name.is_empty() { config.tun_name(&dev_name); } @@ -521,7 +550,7 @@ impl VirtualNic { #[cfg(target_os = "windows")] { - let dev_name = self.global_ctx.get_flags().dev_name; + let dev_name = self.config.dev_name.clone(); match crate::arch::windows::add_self_to_firewall_allowlist() { Ok(_) => tracing::info!("add_self_to_firewall_allowlist successful!"), @@ -553,10 +582,7 @@ impl VirtualNic { let random_dev_name = format!("et_{}_{}", c, s); config.tun_name(random_dev_name.clone()); - - let mut flags = self.global_ctx.get_flags(); - flags.dev_name = random_dev_name.clone(); - self.global_ctx.set_flags(flags); + self.config.dev_name = random_dev_name; } config.platform_config(|config| { @@ -570,11 +596,16 @@ impl VirtualNic { config.up(); - let _g = self.global_ctx.net_ns.guard(); + let _g = self.config.net_ns.guard(); Ok(tun::create(&config)?) } - #[cfg(mobile)] + #[cfg(any(mobile, all(test, unix)))] + pub fn set_mobile_tun_fd_name(&mut self, tun_fd: std::os::fd::RawFd) { + self.ifname = Some(format!("tunfd_{}", tun_fd)); + } + + #[cfg(any(mobile, all(test, unix)))] pub async fn create_dev_for_mobile( &mut self, tun_fd: std::os::fd::RawFd, @@ -609,7 +640,7 @@ impl VirtualNic { None, ); - self.ifname = Some(format!("tunfd_{}", tun_fd)); + self.set_mobile_tun_fd_name(tun_fd); Ok(Box::new(ft)) } @@ -627,7 +658,7 @@ impl VirtualNic { // FreeBSD TUN interface rename functionality #[cfg(target_os = "freebsd")] { - let dev_name = self.global_ctx.get_flags().dev_name; + let dev_name = self.config.dev_name.clone(); if !dev_name.is_empty() && dev_name != ifname { // Use ifconfig to rename the TUN interface @@ -663,15 +694,22 @@ impl VirtualNic { let dev = AsyncDevice::new(dev)?; - let flags = self.global_ctx.get_flags(); - let mut mtu_in_config = flags.mtu; - if flags.enable_encryption { - mtu_in_config -= 20; - } { // set mtu by ourselves, rust-tun does not handle it correctly on windows - let _g = self.global_ctx.net_ns.guard(); - self.ifcfg.set_mtu(ifname.as_str(), mtu_in_config).await?; + let _g = self.config.net_ns.guard(); + self.ifcfg.set_mtu(ifname.as_str(), self.config.mtu).await?; + } + + #[cfg(any( + all(target_os = "macos", not(feature = "macos-ne")), + target_os = "freebsd" + ))] + { + // remove the 10.0.0.0/24 route (which is added by rust-tun by default) + let _ = self + .ifcfg + .remove_ipv4_route(&ifname, "10.0.0.0".parse().unwrap(), 24) + .await; } let has_packet_info = cfg!(all(target_os = "macos", not(feature = "macos-ne"))); @@ -718,15 +756,63 @@ impl VirtualNic { } pub async fn link_up(&self) -> Result<(), Error> { - let _g = self.global_ctx.net_ns.guard(); + let _g = self.config.net_ns.guard(); self.ifcfg.set_link_status(self.ifname(), true).await?; Ok(()) } pub async fn add_route(&self, address: Ipv4Addr, cidr: u8) -> Result<(), Error> { - let _g = self.global_ctx.net_ns.guard(); + self.add_route_with_cost(address, cidr, None).await + } + + pub async fn add_route_with_cost( + &self, + address: Ipv4Addr, + cidr: u8, + cost: Option, + ) -> Result<(), Error> { + self.add_route_with_cost_and_source_hint(address, cidr, cost, None) + .await + } + + pub async fn add_route_with_cost_and_source_hint( + &self, + address: Ipv4Addr, + cidr: u8, + cost: Option, + source_hint: Option, + ) -> Result<(), Error> { + let _g = self.config.net_ns.guard(); self.ifcfg - .add_ipv4_route(self.ifname(), address, cidr, None) + .add_ipv4_route_with_source_hint(self.ifname(), address, cidr, cost, source_hint) + .await?; + Ok(()) + } + + pub async fn remove_route(&self, address: Ipv4Addr, cidr: u8) -> Result<(), Error> { + let _g = self.config.net_ns.guard(); + self.ifcfg + .remove_ipv4_route(self.ifname(), address, cidr) + .await?; + Ok(()) + } + + pub async fn remove_route_with_cost_and_source_hint( + &self, + address: Ipv4Addr, + cidr: u8, + cost: Option, + source_hint: Option, + ) -> Result<(), Error> { + let _g = self.config.net_ns.guard(); + self.ifcfg + .remove_ipv4_route_with_cost_and_source_hint( + self.ifname(), + address, + cidr, + cost, + source_hint, + ) .await?; Ok(()) } @@ -741,7 +827,7 @@ impl VirtualNic { cidr: u8, cost: Option, ) -> Result<(), Error> { - let _g = self.global_ctx.net_ns.guard(); + let _g = self.config.net_ns.guard(); self.ifcfg .add_ipv6_route(self.ifname(), address, cidr, cost) .await?; @@ -749,7 +835,7 @@ impl VirtualNic { } pub async fn remove_ipv6_route(&self, address: Ipv6Addr, cidr: u8) -> Result<(), Error> { - let _g = self.global_ctx.net_ns.guard(); + let _g = self.config.net_ns.guard(); self.ifcfg .remove_ipv6_route(self.ifname(), address, cidr) .await?; @@ -757,19 +843,19 @@ impl VirtualNic { } pub async fn remove_ip(&self, ip: Option) -> Result<(), Error> { - let _g = self.global_ctx.net_ns.guard(); + let _g = self.config.net_ns.guard(); self.ifcfg.remove_ip(self.ifname(), ip).await?; Ok(()) } pub async fn remove_ipv6(&self, ip: Option) -> Result<(), Error> { - let _g = self.global_ctx.net_ns.guard(); + let _g = self.config.net_ns.guard(); self.ifcfg.remove_ipv6(self.ifname(), ip).await?; Ok(()) } pub async fn add_ip(&self, ip: Ipv4Addr, cidr: i32) -> Result<(), Error> { - let _g = self.global_ctx.net_ns.guard(); + let _g = self.config.net_ns.guard(); self.ifcfg .add_ipv4_ip(self.ifname(), ip, cidr as u8) .await?; @@ -777,15 +863,208 @@ impl VirtualNic { } pub async fn add_ipv6(&self, ip: Ipv6Addr, cidr: i32) -> Result<(), Error> { - let _g = self.global_ctx.net_ns.guard(); + let _g = self.config.net_ns.guard(); self.ifcfg .add_ipv6_ip(self.ifname(), ip, cidr as u8) .await?; Ok(()) } - pub fn get_ifcfg(&self) -> impl IfConfiguerTrait + use<> { - IfConfiger {} + pub async fn set_mtu(&self, mtu: u32) -> Result<(), Error> { + let _g = self.config.net_ns.guard(); + self.ifcfg.set_mtu(self.ifname(), mtu).await?; + Ok(()) + } + + pub fn configured_mtu(&self) -> u32 { + self.config.mtu + } + + #[cfg(test)] + pub(crate) fn set_ifname_for_test(&mut self, ifname: String) { + self.ifname = Some(ifname); + } + + #[cfg(test)] + pub(crate) fn set_ifcfg_for_test( + &mut self, + ifcfg: Box, + ) { + self.ifcfg = ifcfg; + } + + pub fn get_ifcfg(&self) -> IfConfiger { + IfConfiger::default() + } +} + +#[derive(Clone)] +pub enum NicBackend { + Dedicated(Arc>), + Shared(SharedVirtualNicMember), +} + +impl NicBackend { + pub fn dedicated(nic: Arc>) -> Self { + Self::Dedicated(nic) + } + + pub fn shared(member: SharedVirtualNicMember) -> Self { + Self::Shared(member) + } + + pub async fn create_dev(&self) -> Result, Error> { + match self { + Self::Dedicated(nic) => nic.lock().await.create_dev().await, + Self::Shared(member) => member.create_dev().await, + } + } + + #[cfg(mobile)] + pub async fn create_dev_for_mobile( + &self, + tun_fd: std::os::fd::RawFd, + replace_tun_fd: bool, + ) -> Result, Error> { + match self { + Self::Dedicated(nic) => nic.lock().await.create_dev_for_mobile(tun_fd).await, + Self::Shared(member) => member.create_dev_for_mobile(tun_fd, replace_tun_fd).await, + } + } + + pub async fn ifname(&self) -> Option { + match self { + Self::Dedicated(nic) => nic + .lock() + .await + .ifname + .as_ref() + .map(|ifname| ifname.to_owned()), + Self::Shared(member) => { + let shared_nic = member.shared_nic(); + let nic = { + let shared_nic = shared_nic.lock().await; + shared_nic.nic() + }; + nic.lock() + .await + .ifname + .as_ref() + .map(|ifname| ifname.to_owned()) + } + } + } + + #[cfg(not(target_os = "linux"))] + /// Returns a raw ifcfg handle and interface name for platform cleanup. + /// + /// This does not carry `VirtualNic`'s netns guard. Use the typed + /// `NicBackend` methods for normal IP and route configuration. + pub async fn ifcfg_and_ifname(&self) -> Result<(IfConfiger, String), Error> { + match self { + Self::Dedicated(nic) => { + let nic = nic.lock().await; + Ok((nic.get_ifcfg(), nic.ifname().to_owned())) + } + Self::Shared(member) => member.ifcfg_and_ifname().await, + } + } + + pub async fn link_up(&self) -> Result<(), Error> { + match self { + Self::Dedicated(nic) => nic.lock().await.link_up().await, + Self::Shared(member) => member.link_up().await, + } + } + + pub async fn add_route(&self, address: Ipv4Addr, cidr: u8) -> Result<(), Error> { + match self { + Self::Dedicated(nic) => nic.lock().await.add_route(address, cidr).await, + Self::Shared(member) => member.add_route(address, cidr).await, + } + } + + pub async fn add_route_with_cost( + &self, + address: Ipv4Addr, + cidr: u8, + cost: Option, + ) -> Result<(), Error> { + match self { + Self::Dedicated(nic) => { + nic.lock() + .await + .add_route_with_cost(address, cidr, cost) + .await + } + Self::Shared(member) => member.add_route_with_cost(address, cidr, cost).await, + } + } + + pub async fn remove_route(&self, address: Ipv4Addr, cidr: u8) -> Result<(), Error> { + match self { + Self::Dedicated(nic) => nic.lock().await.remove_route(address, cidr).await, + Self::Shared(member) => member.remove_route(address, cidr).await, + } + } + + pub async fn add_ipv6_route(&self, address: Ipv6Addr, cidr: u8) -> Result<(), Error> { + match self { + Self::Dedicated(nic) => nic.lock().await.add_ipv6_route(address, cidr).await, + Self::Shared(member) => member.add_ipv6_route(address, cidr).await, + } + } + + pub async fn add_ipv6_route_with_cost( + &self, + address: Ipv6Addr, + cidr: u8, + cost: Option, + ) -> Result<(), Error> { + match self { + Self::Dedicated(nic) => { + nic.lock() + .await + .add_ipv6_route_with_cost(address, cidr, cost) + .await + } + Self::Shared(member) => member.add_ipv6_route_with_cost(address, cidr, cost).await, + } + } + + pub async fn remove_ipv6_route(&self, address: Ipv6Addr, cidr: u8) -> Result<(), Error> { + match self { + Self::Dedicated(nic) => nic.lock().await.remove_ipv6_route(address, cidr).await, + Self::Shared(member) => member.remove_ipv6_route(address, cidr).await, + } + } + + pub async fn remove_ip(&self, ip: Option) -> Result<(), Error> { + match self { + Self::Dedicated(nic) => nic.lock().await.remove_ip(ip).await, + Self::Shared(member) => member.remove_ip(ip).await, + } + } + + pub async fn remove_ipv6(&self, ip: Option) -> Result<(), Error> { + match self { + Self::Dedicated(nic) => nic.lock().await.remove_ipv6(ip).await, + Self::Shared(member) => member.remove_ipv6(ip).await, + } + } + + pub async fn add_ip(&self, ip: Ipv4Addr, cidr: i32) -> Result<(), Error> { + match self { + Self::Dedicated(nic) => nic.lock().await.add_ip(ip, cidr).await, + Self::Shared(member) => member.add_ip(ip, cidr).await, + } + } + + pub async fn add_ipv6(&self, ip: Ipv6Addr, cidr: i32) -> Result<(), Error> { + match self { + Self::Dedicated(nic) => nic.lock().await.add_ipv6(ip, cidr).await, + Self::Shared(member) => member.add_ipv6(ip, cidr).await, + } } } @@ -796,7 +1075,7 @@ pub struct NicCtx { close_notifier: Arc, - nic: Arc>, + backend: NicBackend, tasks: JoinSet<()>, #[cfg(target_os = "windows")] @@ -804,11 +1083,47 @@ pub struct NicCtx { } impl NicCtx { - pub(crate) fn new( + fn virtual_nic_config_from_parts( + dev_name: String, + mut mtu: u32, + enable_encryption: bool, + net_ns: NetNS, + ) -> VirtualNicConfig { + if enable_encryption { + mtu -= 20; + } + + VirtualNicConfig::new(dev_name, mtu, net_ns) + } + + fn virtual_nic_config(global_ctx: &ArcGlobalCtx) -> VirtualNicConfig { + let flags = global_ctx.get_flags(); + Self::virtual_nic_config_from_parts( + flags.dev_name, + flags.mtu, + flags.enable_encryption, + global_ctx.net_ns.clone(), + ) + } + + pub(crate) fn shared_route_backend_for_dns(&self) -> Option { + match self.backend { + NicBackend::Dedicated(_) => None, + NicBackend::Shared(_) => Some(self.backend.clone()), + } + } + + fn dedicated_backend(global_ctx: &ArcGlobalCtx) -> NicBackend { + let nic_config = Self::virtual_nic_config(global_ctx); + NicBackend::dedicated(Arc::new(Mutex::new(VirtualNic::new(nic_config)))) + } + + fn new_with_backend( global_ctx: ArcGlobalCtx, packet_plane: Arc, peer_packet_receiver: Arc>, close_notifier: Arc, + backend: NicBackend, ) -> Self { NicCtx { global_ctx: global_ctx.clone(), @@ -817,7 +1132,7 @@ impl NicCtx { close_notifier, - nic: Arc::new(Mutex::new(VirtualNic::new(global_ctx))), + backend, tasks: JoinSet::new(), #[cfg(target_os = "windows")] @@ -825,40 +1140,110 @@ impl NicCtx { } } + pub(crate) fn new( + global_ctx: ArcGlobalCtx, + packet_plane: Arc, + peer_packet_receiver: Arc>, + close_notifier: Arc, + ) -> Self { + let backend = Self::dedicated_backend(&global_ctx); + + Self::new_with_backend( + global_ctx, + packet_plane, + peer_packet_receiver, + close_notifier, + backend, + ) + } + + pub(crate) async fn new_shared( + global_ctx: ArcGlobalCtx, + packet_plane: Arc, + peer_packet_receiver: Arc>, + close_notifier: Arc, + registry: ArcSharedVirtualNicRegistry, + member_id: SharedVirtualNicMemberId, + ) -> Result { + let flags = global_ctx.get_flags(); + #[cfg(mobile)] + let dev_name = String::new(); + #[cfg(not(mobile))] + let dev_name = flags.dev_name.clone(); + #[cfg(not(mobile))] + if dev_name.is_empty() { + return Err(anyhow::anyhow!("shared virtual nic requires dev_name").into()); + } + #[cfg(mobile)] + let net_ns = NetNS::new(None); + #[cfg(not(mobile))] + let net_ns = global_ctx.net_ns.clone(); + let nic_config = Self::virtual_nic_config_from_parts( + dev_name.clone(), + flags.mtu, + flags.enable_encryption, + net_ns, + ); + + let member = registry.lock().await.create_member( + dev_name, + nic_config, + member_id, + close_notifier.clone(), + ); + let backend = NicBackend::shared(member); + + Ok(Self::new_with_backend( + global_ctx, + packet_plane, + peer_packet_receiver, + close_notifier, + backend, + )) + } + pub async fn ifname(&self) -> Option { - let nic = self.nic.lock().await; - nic.ifname.as_ref().map(|s| s.to_owned()) + self.backend.ifname().await + } + + async fn tun_ifname(&self) -> Result { + self.backend + .ifname() + .await + .ok_or_else(|| anyhow::anyhow!("tun device has no interface name").into()) } pub async fn assign_ipv4_to_tun_device(&self, ipv4_addr: cidr::Ipv4Inet) -> Result<(), Error> { - let nic = self.nic.lock().await; - nic.link_up().await?; - nic.remove_ip(None).await?; - nic.add_ip(ipv4_addr.address(), ipv4_addr.network_length() as i32) + self.backend.link_up().await?; + self.backend.remove_ip(None).await?; + self.backend + .add_ip(ipv4_addr.address(), ipv4_addr.network_length() as i32) .await?; #[cfg(any( all(target_os = "macos", not(feature = "macos-ne")), target_os = "freebsd" ))] { - nic.add_route(ipv4_addr.first_address(), ipv4_addr.network_length()) + self.backend + .add_route(ipv4_addr.first_address(), ipv4_addr.network_length()) .await?; } Ok(()) } pub async fn assign_ipv6_to_tun_device(&self, ipv6_addr: cidr::Ipv6Inet) -> Result<(), Error> { - let nic = self.nic.lock().await; - nic.link_up().await?; - nic.remove_ipv6(None).await?; - nic.add_ipv6(ipv6_addr.address(), ipv6_addr.network_length() as i32) + self.backend.link_up().await?; + self.backend.remove_ipv6(None).await?; + self.backend + .add_ipv6(ipv6_addr.address(), ipv6_addr.network_length() as i32) .await?; #[cfg(any( all(target_os = "macos", not(feature = "macos-ne")), target_os = "freebsd" ))] { - nic.add_ipv6_route(ipv6_addr.first_address(), ipv6_addr.network_length()) + self.backend + .add_ipv6_route(ipv6_addr.first_address(), ipv6_addr.network_length()) .await?; } Ok(()) @@ -923,6 +1308,15 @@ impl NicCtx { }); } + fn start_tunnel_forwarding(&mut self, tunnel: Box) -> Result<(), Error> { + let (stream, sink) = tunnel.split(); + + self.do_forward_nic_to_peers_task(stream)?; + self.do_forward_peers_to_nic(sink); + + Ok(()) + } + #[cfg(target_os = "windows")] fn start_windows_udp_broadcast_relay(&mut self, virtual_ipv4: Ipv4Inet) { if !self.global_ctx.get_flags().enable_udp_broadcast_relay { @@ -948,13 +1342,11 @@ impl NicCtx { } async fn apply_route_changes( - ifcfg: &impl IfConfiguerTrait, - ifname: &str, - net_ns: &crate::common::netns::NetNS, + backend: &NicBackend, cur_proxy_cidrs: &mut BTreeSet, added: Vec, removed: Vec, - ) { + ) -> Result<(), Error> { tracing::debug!(?added, ?removed, "applying proxy_cidrs route changes"); // Remove routes @@ -962,9 +1354,8 @@ impl NicCtx { if !cur_proxy_cidrs.contains(&cidr) { continue; } - let _g = net_ns.guard(); - let ret = ifcfg - .remove_ipv4_route(ifname, cidr.first_address(), cidr.network_length()) + let ret = backend + .remove_route(cidr.first_address(), cidr.network_length()) .await; if ret.is_err() { @@ -978,30 +1369,31 @@ impl NicCtx { } // Add routes + let mut first_error = None; for cidr in added { if cur_proxy_cidrs.contains(&cidr) { continue; } - let _g = net_ns.guard(); - let ret = ifcfg - .add_ipv4_route(ifname, cidr.first_address(), cidr.network_length(), None) - .await; - - if ret.is_err() { - tracing::trace!( - cidr = ?cidr, - err = ?ret, - "add route failed.", - ); + match backend + .add_route(cidr.first_address(), cidr.network_length()) + .await + { + Ok(()) => { + cur_proxy_cidrs.insert(cidr); + } + Err(err) => { + tracing::error!(?cidr, ?err, "add route failed"); + if first_error.is_none() { + first_error = Some(err); + } + } } - cur_proxy_cidrs.insert(cidr); } + first_error.map_or(Ok(()), Err) } async fn apply_public_ipv6_route_changes( - ifcfg: &impl IfConfiguerTrait, - ifname: &str, - net_ns: &crate::common::netns::NetNS, + backend: &NicBackend, cur_routes: &mut BTreeSet, added: Vec, removed: Vec, @@ -1010,9 +1402,8 @@ impl NicCtx { if !cur_routes.contains(&route) { continue; } - let _g = net_ns.guard(); - let ret = ifcfg - .remove_ipv6_route(ifname, route.address(), route.network_length()) + let ret = backend + .remove_ipv6_route(route.address(), route.network_length()) .await; if ret.is_err() { tracing::trace!(route = ?route, err = ?ret, "remove public ipv6 route failed"); @@ -1024,9 +1415,8 @@ impl NicCtx { if cur_routes.contains(&route) { continue; } - let _g = net_ns.guard(); - let ret = ifcfg - .add_ipv6_route(ifname, route.address(), route.network_length(), None) + let ret = backend + .add_ipv6_route(route.address(), route.network_length()) .await; if ret.is_err() { tracing::trace!(route = ?route, err = ?ret, "add public ipv6 route failed"); @@ -1039,33 +1429,29 @@ impl NicCtx { async fn run_proxy_cidrs_route_updater(&mut self) -> Result<(), Error> { let packet_plane = self.packet_plane.clone(); let global_ctx = self.global_ctx.clone(); - let net_ns = self.global_ctx.net_ns.clone(); - let nic = self.nic.lock().await; - let ifcfg = nic.get_ifcfg(); - let ifname = nic.ifname().to_owned(); + let backend = self.backend.clone(); let mut event_receiver = global_ctx.subscribe(); + let mut cur_proxy_cidrs = BTreeSet::::new(); + + // Initial sync: get current proxy_cidrs state and apply routes + let Some(diff) = packet_plane.proxy_cidr_diff(&cur_proxy_cidrs).await else { + tracing::error!("proxy CIDR monitor host is unavailable"); + return Ok(()); + }; + if let Err(err) = + Self::apply_route_changes(&backend, &mut cur_proxy_cidrs, diff.added, diff.removed) + .await + && matches!(&backend, NicBackend::Shared(_)) + { + return Err(err); + } + self.tasks.spawn(async move { - let mut cur_proxy_cidrs = BTreeSet::::new(); - - // Initial sync: get current proxy_cidrs state and apply routes - let Some(diff) = packet_plane.proxy_cidr_diff(&cur_proxy_cidrs).await else { - tracing::error!("proxy CIDR monitor host is unavailable"); - return; - }; - Self::apply_route_changes( - &ifcfg, - &ifname, - &net_ns, - &mut cur_proxy_cidrs, - diff.added, - diff.removed, - ) - .await; - loop { - let event = match event_receiver.recv().await { - Ok(event) => event, + let should_sync = match event_receiver.recv().await { + Ok(GlobalCtxEvent::ProxyCidrsUpdated(_, _)) => true, + Ok(_) => false, Err(tokio::sync::broadcast::error::RecvError::Closed) => { tracing::debug!("event bus closed, stopping proxy_cidrs route updater"); break; @@ -1075,31 +1461,29 @@ impl NicCtx { "event bus lagged in proxy_cidrs route updater, doing full sync" ); event_receiver = event_receiver.resubscribe(); - // Full sync after lagged to recover consistent state - let Some(diff) = packet_plane.proxy_cidr_diff(&cur_proxy_cidrs).await - else { - tracing::error!("proxy CIDR monitor host is unavailable"); - return; - }; - GlobalCtxEvent::ProxyCidrsUpdated(diff.added, diff.removed) + true } }; + if !should_sync { + continue; + } - // Only handle ProxyCidrsUpdated events - let (added, removed) = match event { - GlobalCtxEvent::ProxyCidrsUpdated(added, removed) => (added, removed), - _ => continue, + // Full sync also retries routes that previously failed to apply. + let Some(diff) = packet_plane.proxy_cidr_diff(&cur_proxy_cidrs).await else { + tracing::error!("proxy CIDR monitor host is unavailable"); + return; }; - Self::apply_route_changes( - &ifcfg, - &ifname, - &net_ns, + if let Err(err) = Self::apply_route_changes( + &backend, &mut cur_proxy_cidrs, - added, - removed, + diff.added, + diff.removed, ) - .await; + .await + { + tracing::error!(?err, "failed to update proxy CIDR routes"); + } } }); @@ -1109,26 +1493,16 @@ impl NicCtx { async fn run_public_ipv6_route_updater(&mut self) -> Result<(), Error> { let packet_plane = self.packet_plane.clone(); let global_ctx = self.global_ctx.clone(); - let net_ns = self.global_ctx.net_ns.clone(); - let nic = self.nic.lock().await; - let ifcfg = nic.get_ifcfg(); - let ifname = nic.ifname().to_owned(); + let backend = self.backend.clone(); let mut event_receiver = global_ctx.subscribe(); - self.tasks.spawn(async move { - let mut cur_routes = BTreeSet::::new(); - let initial_routes = packet_plane.public_ipv6_routes().await; - let initial_added = initial_routes.iter().copied().collect::>(); - Self::apply_public_ipv6_route_changes( - &ifcfg, - &ifname, - &net_ns, - &mut cur_routes, - initial_added, - Vec::new(), - ) + let mut cur_routes = BTreeSet::::new(); + let initial_routes = packet_plane.public_ipv6_routes().await; + let initial_added = initial_routes.iter().copied().collect::>(); + Self::apply_public_ipv6_route_changes(&backend, &mut cur_routes, initial_added, Vec::new()) .await; + self.tasks.spawn(async move { loop { let event = match event_receiver.recv().await { Ok(event) => event, @@ -1147,15 +1521,8 @@ impl NicCtx { _ => continue, }; - Self::apply_public_ipv6_route_changes( - &ifcfg, - &ifname, - &net_ns, - &mut cur_routes, - added, - removed, - ) - .await; + Self::apply_public_ipv6_route_changes(&backend, &mut cur_routes, added, removed) + .await; } }); @@ -1165,27 +1532,29 @@ impl NicCtx { async fn run_public_ipv6_addr_updater(&mut self) -> Result<(), Error> { let packet_plane = self.packet_plane.clone(); let global_ctx = self.global_ctx.clone(); - let nic = self.nic.clone(); + let backend = self.backend.clone(); let mut event_receiver = global_ctx.subscribe(); - self.tasks.spawn(async move { - let mut current_addr = packet_plane.public_ipv6_addr().await; - if let Some(addr) = current_addr { - let nic = nic.lock().await; - if let Err(err) = nic.link_up().await { - tracing::warn!(?err, "failed to bring public ipv6 nic link up"); - } - if let Err(err) = nic.add_ipv6(addr.address(), addr.network_length() as i32).await { - tracing::warn!(addr = ?addr, ?err, "failed to add public ipv6 address"); - } - if let Err(err) = nic - .add_ipv6_route_with_cost(Ipv6Addr::UNSPECIFIED, 0, Some(5)) - .await - { - tracing::warn!(route = %Ipv6Addr::UNSPECIFIED, prefix = 0, ?err, "failed to add default public ipv6 route"); - } + let mut current_addr = packet_plane.public_ipv6_addr().await; + if let Some(addr) = current_addr { + if let Err(err) = backend.link_up().await { + tracing::warn!(?err, "failed to bring public ipv6 nic link up"); } + if let Err(err) = backend + .add_ipv6(addr.address(), addr.network_length() as i32) + .await + { + tracing::warn!(addr = ?addr, ?err, "failed to add public ipv6 address"); + } + if let Err(err) = backend + .add_ipv6_route_with_cost(Ipv6Addr::UNSPECIFIED, 0, Some(5)) + .await + { + tracing::warn!(route = %Ipv6Addr::UNSPECIFIED, prefix = 0, ?err, "failed to add default public ipv6 route"); + } + } + self.tasks.spawn(async move { loop { let event = match event_receiver.recv().await { Ok(event) => event, @@ -1203,24 +1572,28 @@ impl NicCtx { }; current_addr = new; - let nic = nic.lock().await; - if let Err(err) = nic.link_up().await { + if let Err(err) = backend.link_up().await { tracing::warn!(?err, "failed to bring public ipv6 nic link up"); } if let Some(old) = old { - if let Err(err) = nic.remove_ipv6_route(Ipv6Addr::UNSPECIFIED, 0).await { + if let Err(err) = backend + .remove_ipv6_route(Ipv6Addr::UNSPECIFIED, 0) + .await + { tracing::warn!(route = %Ipv6Addr::UNSPECIFIED, prefix = 0, ?err, "failed to remove default public ipv6 route"); } - if let Err(err) = nic.remove_ipv6(Some(old)).await { + if let Err(err) = backend.remove_ipv6(Some(old)).await { tracing::warn!(addr = ?old, ?err, "failed to remove old public ipv6 address"); } } if let Some(new) = new { - if let Err(err) = nic.add_ipv6(new.address(), new.network_length() as i32).await + if let Err(err) = backend + .add_ipv6(new.address(), new.network_length() as i32) + .await { tracing::warn!(addr = ?new, ?err, "failed to add public ipv6 address"); } - if let Err(err) = nic + if let Err(err) = backend .add_ipv6_route_with_cost(Ipv6Addr::UNSPECIFIED, 0, Some(5)) .await { @@ -1238,43 +1611,30 @@ impl NicCtx { ipv4_addr: Option, ipv6_addr: Option, ) -> Result<(), Error> { - let tunnel = { - let mut nic = self.nic.lock().await; - match nic.create_dev().await { - Ok(ret) => { - #[cfg(target_os = "windows")] - { - let dev_name = self.global_ctx.get_flags().dev_name; - let _ = RegistryManager::reg_change_catrgory_in_profile(&dev_name); - } + let tunnel = match self.backend.create_dev().await { + Ok(ret) => { + let ifname = self.tun_ifname().await?; - #[cfg(any( - all(target_os = "macos", not(feature = "macos-ne")), - target_os = "freebsd" - ))] - { - // remove the 10.0.0.0/24 route (which is added by rust-tun by default) - let _ = nic - .ifcfg - .remove_ipv4_route(nic.ifname(), "10.0.0.0".parse().unwrap(), 24) - .await; + #[cfg(target_os = "windows")] + { + let mut flags = self.global_ctx.get_flags(); + if flags.dev_name.is_empty() { + flags.dev_name = ifname.clone(); + self.global_ctx.set_flags(flags); } + let _ = RegistryManager::reg_change_catrgory_in_profile(&ifname); + } - self.global_ctx - .set_tun_device_ready(nic.ifname().to_string()); - ret - } - Err(err) => { - self.global_ctx.set_tun_device_error(err.to_string()); - return Err(err); - } + self.global_ctx.set_tun_device_ready(ifname); + ret + } + Err(err) => { + self.global_ctx.set_tun_device_error(err.to_string()); + return Err(err); } }; - let (stream, sink) = tunnel.split(); - - self.do_forward_nic_to_peers_task(stream)?; - self.do_forward_peers_to_nic(sink); + self.start_tunnel_forwarding(tunnel)?; // Assign IPv4 address if provided if let Some(ipv4_addr) = ipv4_addr { @@ -1297,27 +1657,104 @@ impl NicCtx { Ok(()) } + #[cfg(any(mobile, test))] + pub(crate) async fn mobile_tun_claims( + global_ctx: &ArcGlobalCtx, + packet_plane: &CorePacketPlane, + ) -> super::shared_virtual_nic::SharedIfConfigClaims { + use super::shared_virtual_nic::{SharedIfConfigClaims, SharedIpv4Route, SharedIpv6Route}; + let mut claims = SharedIfConfigClaims::default(); + claims.ipv4_addresses.extend(global_ctx.get_ipv4()); + claims.ipv6_addresses.extend(global_ctx.get_ipv6()); + if let Some(diff) = packet_plane.proxy_cidr_diff(&BTreeSet::new()).await { + claims + .ipv4_routes + .extend(diff.current.into_iter().map(|route| { + SharedIpv4Route::new(route.first_address(), route.network_length(), None) + })); + } + claims + .ipv6_routes + .extend( + packet_plane + .public_ipv6_routes() + .await + .into_iter() + .map(|route| { + SharedIpv6Route::new(route.first_address(), route.network_length(), None) + }), + ); + if let Some(address) = packet_plane.public_ipv6_addr().await { + claims.ipv6_addresses.insert(address); + claims + .ipv6_routes + .insert(SharedIpv6Route::new(Ipv6Addr::UNSPECIFIED, 0, Some(5))); + } + claims + } + #[cfg(mobile)] - pub async fn run_for_mobile(&mut self, tun_fd: std::os::fd::RawFd) -> Result<(), Error> { - let tunnel = { - let mut nic = self.nic.lock().await; - match nic.create_dev_for_mobile(tun_fd).await { - Ok(ret) => { - self.global_ctx - .set_tun_device_ready(nic.ifname().to_string()); - ret - } - Err(err) => { - self.global_ctx.set_tun_device_error(err.to_string()); - return Err(err); - } + pub(crate) async fn current_mobile_tun_claims( + &self, + ) -> Option { + match &self.backend { + NicBackend::Shared(member) => Some(member.claims().await), + NicBackend::Dedicated(_) => None, + } + } + + #[cfg(any(mobile, test))] + async fn apply_mobile_tun_claims( + &self, + claims: super::shared_virtual_nic::SharedIfConfigClaims, + ) -> Result<(), Error> { + let NicBackend::Shared(member) = &self.backend else { + return Err(anyhow::anyhow!("mobile TUN requires a shared NIC").into()); + }; + member.replace_claims(claims).await + } + + #[cfg(test)] + async fn configure_mobile_tun(&mut self) -> Result<(), Error> { + let claims = Self::mobile_tun_claims(&self.global_ctx, &self.packet_plane).await; + self.apply_mobile_tun_claims(claims).await?; + self.run_proxy_cidrs_route_updater().await?; + self.run_public_ipv6_route_updater().await?; + self.run_public_ipv6_addr_updater().await?; + Ok(()) + } + + #[cfg(mobile)] + pub async fn run_for_mobile( + &mut self, + tun_fd: std::os::fd::RawFd, + replace_tun_fd: bool, + claims: super::shared_virtual_nic::SharedIfConfigClaims, + ) -> Result<(), Error> { + // Reject the entire member configuration before touching the shared fd. + self.apply_mobile_tun_claims(claims).await?; + let (tunnel, ifname) = match self + .backend + .create_dev_for_mobile(tun_fd, replace_tun_fd) + .await + { + Ok(ret) => { + let ifname = self.tun_ifname().await?; + (ret, ifname) + } + Err(err) => { + self.global_ctx.set_tun_device_error(err.to_string()); + return Err(err); } }; - let (stream, sink) = tunnel.split(); + self.run_proxy_cidrs_route_updater().await?; + self.run_public_ipv6_route_updater().await?; + self.run_public_ipv6_addr_updater().await?; - self.do_forward_nic_to_peers_task(stream)?; - self.do_forward_peers_to_nic(sink); + self.global_ctx.set_tun_device_ready(ifname); + + self.start_tunnel_forwarding(tunnel)?; Ok(()) } @@ -1327,10 +1764,163 @@ impl NicCtx { mod tests { use crate::common::{error::Error, global_ctx::tests::get_mock_global_ctx}; - use super::VirtualNic; + use super::{NicCtx, VirtualNic}; + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn mobile_configuration_registers_ipv6_and_tracks_public_updates() { + use std::{net::UdpSocket, process::Command, sync::Arc, time::Duration}; + + use easytier_core::{ + host::packet::{HostPacket, HostPacketChannelSink, HostPacketReceiver}, + process_runtime::CoreProcessRuntime, + }; + use futures::{FutureExt, StreamExt}; + use smoltcp::wire::{IpProtocol, Ipv6Packet}; + use tokio::sync::{Mutex, Notify, mpsc}; + + use super::{IfConfiger, NicBackend, VirtualNicConfig}; + use crate::{ + common::{global_ctx::GlobalCtxEvent, ifcfg::IfConfiguerTrait, netns::NetNS}, + instance::{ + composition::{NativeCoreInstance, runtime_core_host_adapters}, + config::test_core_instance_config, + shared_virtual_nic::SharedVirtualNicRegistry, + }, + }; + + let namespace = format!("mobile-{}", uuid::Uuid::new_v4()); + assert!( + Command::new("ip") + .args(["netns", "add", &namespace]) + .status() + .unwrap() + .success() + ); + guarden::defer! { + let _ = Command::new("ip") + .args(["netns", "del", &namespace]) + .status(); + } + let net_ns = NetNS::new(Some(namespace.clone())); + // Avoid DAD probes and use an unowned source to test destination claims. + { + let _guard = net_ns.guard(); + std::fs::write("/proc/sys/net/ipv6/conf/default/accept_dad", "0").unwrap(); + // Keep automatic link-local traffic out of the member packet streams. + std::fs::write("/proc/sys/net/ipv6/conf/default/addr_gen_mode", "1").unwrap(); + let ifcfg = IfConfiger::default(); + ifcfg.set_link_status("lo", true).await.unwrap(); + ifcfg + .add_ipv6_ip("lo", "fdff::1".parse().unwrap(), 128) + .await + .unwrap(); + } + let global_ctx = get_mock_global_ctx(); + global_ctx.set_ipv4(None); + global_ctx.set_ipv6(Some("fd00::1/64".parse().unwrap())); + let (packet_sink, packet_receiver) = mpsc::channel::(1); + let core = NativeCoreInstance::new( + test_core_instance_config(&global_ctx), + runtime_core_host_adapters( + global_ctx.clone(), + CoreProcessRuntime::new(), + Arc::new(HostPacketChannelSink::new(packet_sink)), + ), + ) + .unwrap(); + let mut registry = SharedVirtualNicRegistry::new(); + let members = [1, 2].map(|id| { + registry.create_member( + "mobile-test".to_owned(), + VirtualNicConfig::new("mobile-test".to_owned(), 1500, net_ns.clone()), + uuid::Uuid::from_u128(id), + Arc::new(Notify::new()), + ) + }); + let (mut other_stream, _other_sink) = members[0].create_dev().await.unwrap().split(); + let (mut stream, _sink) = members[1].create_dev().await.unwrap().split(); + let mut nic = NicCtx::new_with_backend( + global_ctx.clone(), + core.packet_plane(), + Arc::new(Mutex::new(HostPacketReceiver::new(packet_receiver))), + Arc::new(Notify::new()), + NicBackend::shared(members[1].clone()), + ); + + nic.configure_mobile_tun().await.unwrap(); + // Claims are installed before start_tunnel_forwarding is called. + let ifname = nic.tun_ifname().await.unwrap(); + assert!( + net_ns + .run(|| IfConfiger::list_addresses(&ifname)) + .unwrap() + .contains(&"fd00::1/64".parse().unwrap()) + ); + // Linux completes IPv6 address activation asynchronously, even without DAD probes. + tokio::time::timeout(Duration::from_secs(5), async { + loop { + match net_ns.run(|| UdpSocket::bind("[fd00::1]:0")) { + Ok(_) => break, + Err(err) if err.kind() == std::io::ErrorKind::AddrNotAvailable => { + tokio::time::sleep(Duration::from_millis(10)).await; + } + Err(err) => panic!("failed to bind the configured IPv6 address: {err}"), + } + } + }) + .await + .unwrap(); + let route = "2001:db8:1::/64".parse().unwrap(); + let public_addr = "2001:db8:2::1/128".parse().unwrap(); + for (event, source, destination) in [ + (None, "fdff::1", "fd00::2"), + ( + Some(GlobalCtxEvent::PublicIpv6RoutesUpdated(vec![route], vec![])), + "fdff::1", + "2001:db8:1::2", + ), + ( + Some(GlobalCtxEvent::PublicIpv6Changed(None, Some(public_addr))), + "2001:db8:2::1", + "2001:db8:3::2", + ), + ] { + if let Some(event) = event { + global_ctx.issue_event(event); + } + let source = source.parse::().unwrap(); + let destination = destination.parse::().unwrap(); + let shared_nic = members[1].shared_nic(); + tokio::time::timeout(Duration::from_secs(5), async { + loop { + // Do not race an updater's OS change and dispatcher publication. + let guard = shared_nic.lock().await; + let sent = net_ns.run(|| { + UdpSocket::bind((source, 0))?.send_to(b"mobile-ipv6", (destination, 12345)) + }); + drop(guard); + if sent.is_ok() { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + let packet = stream.next().await.unwrap().unwrap(); + let ipv6 = Ipv6Packet::new_checked(packet.payload()).unwrap(); + assert_eq!(ipv6.src_addr(), source); + assert_eq!(ipv6.dst_addr(), destination); + assert_eq!(ipv6.next_header(), IpProtocol::Udp); + assert_eq!(&ipv6.payload()[8..], b"mobile-ipv6"); + }) + .await + .unwrap(); + assert!(other_stream.next().now_or_never().is_none()); + } + } async fn run_test_helper() -> Result { - let mut dev = VirtualNic::new(get_mock_global_ctx()); + let global_ctx = get_mock_global_ctx(); + let mut dev = VirtualNic::new(NicCtx::virtual_nic_config(&global_ctx)); let _tunnel = dev.create_dev().await?; tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; diff --git a/easytier/src/tests/mod.rs b/easytier/src/tests/mod.rs index 4f6c45a1..6bf05b08 100644 --- a/easytier/src/tests/mod.rs +++ b/easytier/src/tests/mod.rs @@ -4,6 +4,9 @@ mod three_node; #[cfg(target_os = "linux")] mod network; +#[cfg(all(target_os = "linux", feature = "tun"))] +mod shared_virtual_nic; + mod ipv6_test; #[cfg(target_os = "linux")] diff --git a/easytier/src/tests/shared_virtual_nic.rs b/easytier/src/tests/shared_virtual_nic.rs new file mode 100644 index 00000000..874e3673 --- /dev/null +++ b/easytier/src/tests/shared_virtual_nic.rs @@ -0,0 +1,874 @@ +use std::{net::Ipv4Addr, process::Command, sync::Arc, time::Duration}; + +use easytier_core::{config::PeerId, process_runtime::CoreProcessRuntime}; + +use super::{ + InstanceTestExt as _, create_netns, del_netns, drop_insts, network::TestNetwork, ping_test, +}; +use crate::{ + common::{ + config::{ConfigLoader, NetworkIdentity, TomlConfigLoader}, + netns::{NetNS, ROOT_NETNS_NAME}, + }, + instance::{ + shared_virtual_nic::{ArcSharedVirtualNicRegistry, SharedIpv4Route}, + test_instance::TestInstance as Instance, + }, + tunnel::common::tests::wait_for_condition, +}; + +const PROXY_CIDR: &str = "10.1.2.0/24"; +const WAIT: Duration = Duration::from_secs(10); + +#[derive(Clone)] +struct SharedTestRuntime { + process: Arc, + registry: ArcSharedVirtualNicRegistry, +} + +impl SharedTestRuntime { + fn new() -> Self { + Self { + process: CoreProcessRuntime::new(), + registry: Instance::new_shared_virtual_nic_registry(), + } + } + + fn instance(&self, config: TomlConfigLoader) -> Instance { + Instance::new_with_process_runtime_and_shared_virtual_nic_registry( + config, + self.process.clone(), + self.registry.clone(), + ) + } +} + +fn test_config( + instance_name: &str, + network_name: &str, + network_secret: &str, + netns: Option<&str>, + dev_name: Option<&str>, + ipv4: &str, +) -> TomlConfigLoader { + let config = TomlConfigLoader::default(); + config.set_inst_name(instance_name.to_owned()); + config.set_network_identity(NetworkIdentity::new( + network_name.to_owned(), + network_secret.to_owned(), + )); + config.set_netns(netns.map(str::to_owned)); + config.set_ipv4(Some(ipv4.parse().unwrap())); + config.set_ipv6(None); + config.set_dhcp(false); + config.set_listeners(vec![]); + config.set_socks5_portal(None); + + let mut flags = config.get_flags(); + flags.dev_name = dev_name.unwrap_or_default().to_owned(); + flags.enable_ipv6 = false; + config.set_flags(flags); + config +} + +fn test_dev_name() -> String { + format!("st{:08x}", rand::random::()) +} + +fn short_name(prefix: &str) -> String { + format!("{prefix}{:04x}", rand::random::()) +} + +struct TestNetnsGuard { + name: String, +} + +impl TestNetnsGuard { + fn new(name: String, ipv4: &str, ipv6: &str) -> Self { + let guard = Self { name }; + del_netns(&guard.name); + create_netns(&guard.name, ipv4, ipv6); + guard + } +} + +impl Drop for TestNetnsGuard { + fn drop(&mut self) { + del_netns(&self.name); + } +} + +struct ProxyLab { + network: TestNetwork, +} + +impl ProxyLab { + fn new() -> Self { + let mut network = TestNetwork::new(); + network.create_namespace("source", "10.1.1.1/24", "fd11::1/64"); + network.create_namespace("owner", "10.1.2.3/24", "fd12::3/64"); + network.create_namespace("target", "10.1.2.4/24", "fd12::4/64"); + network.create_bridge("bridge"); + network.add_to_bridge("bridge", "owner"); + network.add_to_bridge("bridge", "target"); + Self { network } + } +} + +async fn wait_tun_ready(instance: &Instance, expected: &str) { + wait_for_condition( + || async { instance.get_global_ctx().get_tun_device_name().as_deref() == Some(expected) }, + WAIT, + ) + .await; +} + +fn proxy_route_exists( + routes: &[easytier_proto::core_peer::peer::Route], + peer_id: PeerId, + proxy_cidr: &str, +) -> bool { + routes + .iter() + .any(|route| route.peer_id == peer_id && route.proxy_cidrs.iter().any(|c| c == proxy_cidr)) +} + +async fn wait_proxy_route(instance: &Instance, peer_id: PeerId, proxy_cidr: &str) { + wait_for_condition( + || async { + proxy_route_exists( + &instance.get_core_instance().route_snapshots().await, + peer_id, + proxy_cidr, + ) + }, + WAIT, + ) + .await; +} + +async fn wait_proxy_route_absent(instance: &Instance, peer_id: PeerId, proxy_cidr: &str) { + wait_for_condition( + || async { + !proxy_route_exists( + &instance.get_core_instance().route_snapshots().await, + peer_id, + proxy_cidr, + ) + }, + WAIT, + ) + .await; +} + +fn ipv4_route_exists_in_ns(ns: &str, needle: &str) -> bool { + let _root = NetNS::new(Some(ROOT_NETNS_NAME.to_owned())).guard(); + let output = Command::new("ip") + .args(["netns", "exec", ns, "ip", "route", "show"]) + .output() + .unwrap(); + assert!( + output.status.success(), + "failed to list IPv4 routes in {ns}: {}", + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8_lossy(&output.stdout) + .lines() + .any(|line| line.contains(needle)) +} + +fn tun_ifindex(ns: &str, dev_name: &str) -> Option { + let output = Command::new("ip") + .args(["-n", ns, "-j", "link", "show", "dev", dev_name]) + .output() + .unwrap(); + if !output.status.success() { + return None; + } + let links: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); + links[0]["ifindex"].as_u64() +} + +#[cfg(feature = "proxy-cidr-monitor")] +async fn patch_proxy_cidr( + instance: &Instance, + action: crate::proto::api::config::ConfigPatchAction, +) { + use crate::proto::api::config::{InstanceConfigPatch, ProxyNetworkPatch}; + + instance + .get_config_patcher() + .apply_patch(InstanceConfigPatch { + proxy_networks: vec![ProxyNetworkPatch { + action: action as i32, + cidr: Some(PROXY_CIDR.parse().unwrap()), + mapped_cidr: None, + }], + ..Default::default() + }) + .await + .unwrap(); +} + +async fn shared_route_owner_count( + registry: &ArcSharedVirtualNicRegistry, + dev_name: &str, + route: &SharedIpv4Route, +) -> usize { + let nic = { + let registry = registry.lock().await; + registry.get_by_dev_name_for_test(dev_name) + }; + let Some(nic) = nic else { + return 0; + }; + nic.lock().await.ifcfg().owners_of_ipv4_route(route).len() +} + +#[tokio::test] +#[serial_test::serial] +async fn failed_claim_update_closes_all_members_and_allows_recreation() { + use crate::instance::{ + shared_virtual_nic::SharedVirtualNicRegistry, virtual_nic::VirtualNicConfig, + }; + use tokio::sync::Notify; + + let netns = TestNetnsGuard::new(short_name("svf"), "10.233.1.2/24", "fd43::2/64"); + let dev_name = test_dev_name(); + let config = + || VirtualNicConfig::new(dev_name.clone(), 1400, NetNS::new(Some(netns.name.clone()))); + let mut registry = SharedVirtualNicRegistry::new(); + let first_closed = Arc::new(Notify::new()); + let second_closed = Arc::new(Notify::new()); + let first_id = uuid::Uuid::new_v4(); + let first = registry.create_member(dev_name.clone(), config(), first_id, first_closed.clone()); + let second = registry.create_member( + dev_name.clone(), + config(), + uuid::Uuid::new_v4(), + second_closed.clone(), + ); + let first_tunnel = first.create_dev().await.unwrap(); + let second_tunnel = second.create_dev().await.unwrap(); + first.link_up().await.unwrap(); + first + .add_ip("10.233.97.1".parse().unwrap(), 24) + .await + .unwrap(); + second + .add_ip("10.233.98.1".parse().unwrap(), 24) + .await + .unwrap(); + let destination = "10.233.90.0".parse().unwrap(); + first.add_route(destination, 24).await.unwrap(); + assert!(ipv4_route_exists_in_ns(&netns.name, "10.233.90.0/24")); + + // An externally configured, better source makes the address add fail only + // after the existing route has been removed to update its preferred source. + let output = Command::new("ip") + .args([ + "netns", + "exec", + &netns.name, + "ip", + "addr", + "add", + "10.233.96.1/24", + "dev", + &dev_name, + ]) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let err = first + .add_ip("10.233.96.1".parse().unwrap(), 24) + .await + .unwrap_err(); + assert!(matches!( + err, + crate::common::error::Error::IOError(err) + if err.kind() == std::io::ErrorKind::AlreadyExists + )); + assert!(!first.shared_nic().lock().await.is_valid()); + tokio::time::timeout(WAIT, first_closed.notified()) + .await + .unwrap(); + tokio::time::timeout(WAIT, second_closed.notified()) + .await + .unwrap(); + assert!(registry.get(&dev_name, &config()).is_none()); + assert!(first.add_route(destination, 24).await.is_err()); + + let replacement = registry.create_member( + dev_name.clone(), + config(), + first_id, + Arc::new(Notify::new()), + ); + let replacement_tunnel = replacement.create_dev().await.unwrap(); + replacement.link_up().await.unwrap(); + replacement + .add_ip("10.233.97.1".parse().unwrap(), 24) + .await + .unwrap(); + replacement.add_route(destination, 24).await.unwrap(); + assert!(ipv4_route_exists_in_ns( + &netns.name, + &format!("10.233.90.0/24 dev {dev_name} proto static src 10.233.97.1") + )); + + drop((first_tunnel, second_tunnel, replacement_tunnel)); + drop((first, second, replacement)); + wait_for_condition( + || async { !ipv4_route_exists_in_ns(&netns.name, "10.233.90.0/24") }, + WAIT, + ) + .await; +} + +#[cfg(feature = "management-rpc")] +#[tokio::test] +#[serial_test::serial] +async fn shared_tun_survives_creator_executor_shutdown() { + use crate::instance::factory::native_instance_manager_with_runtime; + use easytier_core::instance::manager::ConfigFileControl; + + let local_ns = TestNetnsGuard::new(short_name("sve"), "10.234.1.2/24", "fd44::2/64"); + let peer_ns = TestNetnsGuard::new(short_name("svp"), "10.234.2.2/24", "fd45::2/64"); + let dev_name = test_dev_name(); + let peer_dev = test_dev_name(); + let manager = native_instance_manager_with_runtime(tokio::runtime::Handle::current()); + let mut ids = Vec::new(); + for (name, network, ns, dev, ip, multi_thread) in [ + ( + "creator", + "first", + &local_ns.name, + &dev_name, + "10.244.1.1/24", + false, + ), + ( + "survivor", + "second", + &local_ns.name, + &dev_name, + "10.244.2.1/24", + true, + ), + ( + "peer", + "second", + &peer_ns.name, + &peer_dev, + "10.244.2.2/24", + false, + ), + ] { + let config = test_config(name, network, "secret", Some(ns), Some(dev), ip); + let mut flags = config.get_flags(); + flags.multi_thread = multi_thread; + flags.multi_thread_count = 2; + flags.disable_upnp = true; + flags.disable_p2p = true; + flags.disable_tcp_hole_punching = true; + flags.disable_udp_hole_punching = true; + config.set_flags(flags); + config.set_stun_servers(Some(Vec::new())); + config.set_stun_servers_v6(Some(Vec::new())); + config.set_tcp_stun_servers(Some(Vec::new())); + let id = manager + .run_network_instance(config, ConfigFileControl::STATIC_CONFIG) + .unwrap(); + ids.push(id); + // Make the first instance the actual creator, not merely the first to start. + wait_for_condition(|| async { tun_ifindex(ns, dev).is_some() }, WAIT).await; + wait_for_condition(|| async { manager.instance(id).unwrap().is_ready() }, WAIT).await; + } + let creator_runtime = manager.data_plane_runtime_handle(&ids[0]).unwrap(); + let creator_task = creator_runtime.spawn(std::future::pending::<()>()); + let survivor = manager.instance(ids[1]).unwrap(); + let peer = manager.instance(ids[2]).unwrap(); + peer.add_connector(format!("ring://{}", ids[1]).parse().unwrap()) + .unwrap(); + wait_for_condition( + || async { + survivor.connected_peers().await.contains(&peer.peer_id()) + && peer.connected_peers().await.contains(&survivor.peer_id()) + && ipv4_route_exists_in_ns(&local_ns.name, "10.244.2.0/24") + && ipv4_route_exists_in_ns(&peer_ns.name, "10.244.2.0/24") + }, + WAIT, + ) + .await; + + let bind = |ns: &str, address: &str| { + let socket = + NetNS::new(Some(ns.to_owned())).run(|| std::net::UdpSocket::bind(address).unwrap()); + socket.set_nonblocking(true).unwrap(); + tokio::net::UdpSocket::from_std(socket).unwrap() + }; + let client = bind(&local_ns.name, "10.244.2.1:0"); + let server = bind(&peer_ns.name, "10.244.2.2:0"); + client.connect(server.local_addr().unwrap()).await.unwrap(); + let original_index = tun_ifindex(&local_ns.name, &dev_name).unwrap(); + let mut creator_task = Some(creator_task); + for (pass, message) in [b"before shutdown".as_slice(), b"after shutdown".as_slice()] + .into_iter() + .enumerate() + { + if pass == 1 { + manager + .delete_network_instances(vec![ids[0]]) + .await + .unwrap(); + assert!( + tokio::time::timeout(WAIT, creator_task.take().unwrap()) + .await + .unwrap() + .unwrap_err() + .is_cancelled() + ); + assert_eq!(tun_ifindex(&local_ns.name, &dev_name), Some(original_index)); + } + tokio::time::timeout(WAIT, async { + client.send(message).await.unwrap(); + let mut buf = [0; 64]; + let (len, from) = server.recv_from(&mut buf).await.unwrap(); + assert_eq!(&buf[..len], message); + assert_eq!(from, client.local_addr().unwrap()); + server.send_to(&buf[..len], from).await.unwrap(); + let len = client.recv(&mut buf).await.unwrap(); + assert_eq!(&buf[..len], message); + }) + .await + .expect("the existing UDP socket must work across creator shutdown"); + } + assert_eq!(tun_ifindex(&local_ns.name, &dev_name), Some(original_index)); + drop((client, server, survivor, peer)); + manager + .delete_network_instances(ids[1..].to_vec()) + .await + .unwrap(); + wait_for_condition( + || async { + tun_ifindex(&local_ns.name, &dev_name).is_none() + && tun_ifindex(&peer_ns.name, &peer_dev).is_none() + }, + WAIT, + ) + .await; +} + +#[test] +#[serial_test::serial] +fn shared_tun_claim_cleanup_outlives_member_runtime() { + use crate::instance::{ + shared_virtual_nic::SharedVirtualNicRegistry, virtual_nic::VirtualNicConfig, + }; + use tokio::{runtime::Builder, sync::Notify}; + + let netns = TestNetnsGuard::new(short_name("svc"), "10.235.1.2/24", "fd46::2/64"); + let dev_name = test_dev_name(); + let config = + || VirtualNicConfig::new(dev_name.clone(), 1400, NetNS::new(Some(netns.name.clone()))); + let mut registry = SharedVirtualNicRegistry::new(); + let first = registry.create_member( + dev_name.clone(), + config(), + uuid::Uuid::new_v4(), + Arc::new(Notify::new()), + ); + let second = registry.create_member( + dev_name.clone(), + config(), + uuid::Uuid::new_v4(), + Arc::new(Notify::new()), + ); + let member_runtime = Builder::new_current_thread().enable_all().build().unwrap(); + let observer_runtime = Builder::new_current_thread().enable_all().build().unwrap(); + let first_tunnel = member_runtime.block_on(async { + let tunnel = first.create_dev().await.unwrap(); + first.link_up().await.unwrap(); + first + .add_ip("10.245.1.1".parse().unwrap(), 24) + .await + .unwrap(); + first + .add_route("10.245.90.0".parse().unwrap(), 24) + .await + .unwrap(); + tunnel + }); + let second_tunnel = observer_runtime.block_on(async { + let tunnel = second.create_dev().await.unwrap(); + second + .add_ip("10.245.2.1".parse().unwrap(), 24) + .await + .unwrap(); + tunnel + }); + assert!(ipv4_route_exists_in_ns(&netns.name, "10.245.90.0/24")); + let original_index = tun_ifindex(&netns.name, &dev_name).unwrap(); + let shared_nic = first.shared_nic(); + let guard = observer_runtime.block_on(shared_nic.lock()); + member_runtime.block_on(async { + drop((first_tunnel, first)); + // Claim removal cannot acquire the lock until after this runtime exits. + tokio::task::yield_now().await; + }); + drop(member_runtime); + drop(guard); + + observer_runtime.block_on(async { + wait_for_condition( + || async { !ipv4_route_exists_in_ns(&netns.name, "10.245.90.0/24") }, + WAIT, + ) + .await; + assert_eq!(tun_ifindex(&netns.name, &dev_name), Some(original_index)); + // Reusing the departed member's address also proves its claims were released. + second + .add_ip("10.245.1.1".parse().unwrap(), 24) + .await + .unwrap(); + }); + // The final registration can be dropped without any entered Tokio runtime. + drop((second_tunnel, second)); + observer_runtime.block_on(wait_for_condition( + || async { tun_ifindex(&netns.name, &dev_name).is_none() }, + WAIT, + )); +} + +#[tokio::test] +#[serial_test::serial] +async fn same_namespace_members_share_tun_across_independent_networks() { + let dev_name = test_dev_name(); + let first_peer_ns = TestNetnsGuard::new(short_name("sva"), "10.231.1.2/24", "fd31::2/64"); + let second_peer_ns = TestNetnsGuard::new(short_name("svb"), "10.231.2.2/24", "fd32::2/64"); + let runtime = SharedTestRuntime::new(); + + let mut first = runtime.instance(test_config( + "shared_tun_first", + "shared_tun_network_a", + "shared_tun_secret_a", + None, + Some(&dev_name), + "10.144.250.1/24", + )); + let mut second = runtime.instance(test_config( + "shared_tun_second", + "shared_tun_network_b", + "shared_tun_secret_b", + None, + Some(&dev_name), + "10.144.251.1/24", + )); + let mut first_peer = runtime.instance(test_config( + "shared_tun_first_peer", + "shared_tun_network_a", + "shared_tun_secret_a", + Some(&first_peer_ns.name), + None, + "10.144.250.2/24", + )); + let mut second_peer = runtime.instance(test_config( + "shared_tun_second_peer", + "shared_tun_network_b", + "shared_tun_secret_b", + Some(&second_peer_ns.name), + None, + "10.144.251.2/24", + )); + + first.run().await.unwrap(); + second.run().await.unwrap(); + first_peer.run().await.unwrap(); + second_peer.run().await.unwrap(); + + wait_tun_ready(&first, &dev_name).await; + wait_tun_ready(&second, &dev_name).await; + assert_eq!( + first.get_global_ctx().get_tun_device_name(), + second.get_global_ctx().get_tun_device_name() + ); + + first_peer.add_connector_url(first.ring_listener_url()); + second_peer.add_connector_url(second.ring_listener_url()); + + wait_for_condition( + || async { + first + .get_core_instance() + .route_snapshots() + .await + .iter() + .any(|route| route.peer_id == first_peer.peer_id()) + && second + .get_core_instance() + .route_snapshots() + .await + .iter() + .any(|route| route.peer_id == second_peer.peer_id()) + }, + WAIT, + ) + .await; + wait_for_condition( + || async { ping_test(&first_peer_ns.name, "10.144.250.1", None).await }, + WAIT, + ) + .await; + wait_for_condition( + || async { ping_test(&second_peer_ns.name, "10.144.251.1", None).await }, + WAIT, + ) + .await; + + drop_insts(vec![first, second, first_peer, second_peer]).await; +} + +#[cfg(feature = "proxy-cidr-monitor")] +#[tokio::test] +#[serial_test::serial] +async fn runtime_proxy_patch_adds_and_removes_os_route() { + use crate::proto::api::config::ConfigPatchAction; + + let lab = ProxyLab::new(); + let source_dev = test_dev_name(); + let destination_dev = test_dev_name(); + let runtime = SharedTestRuntime::new(); + let mut source = runtime.instance(test_config( + "shared_patch_source", + "shared_patch_network", + "shared_patch_secret", + Some(lab.network.namespace("source")), + Some(&source_dev), + "10.144.244.1/24", + )); + let mut destination = runtime.instance(test_config( + "shared_patch_destination", + "shared_patch_network", + "shared_patch_secret", + Some(lab.network.namespace("owner")), + Some(&destination_dev), + "10.144.244.2/24", + )); + + source.run().await.unwrap(); + destination.run().await.unwrap(); + wait_tun_ready(&source, &source_dev).await; + wait_tun_ready(&destination, &destination_dev).await; + destination.add_connector_url(source.ring_listener_url()); + wait_for_condition( + || async { + source + .get_core_instance() + .route_snapshots() + .await + .iter() + .any(|route| route.peer_id == destination.peer_id()) + }, + WAIT, + ) + .await; + assert!(!ipv4_route_exists_in_ns( + lab.network.namespace("source"), + &format!("{PROXY_CIDR} dev {source_dev}") + )); + + patch_proxy_cidr(&destination, ConfigPatchAction::Add).await; + wait_proxy_route(&source, destination.peer_id(), PROXY_CIDR).await; + wait_for_condition( + || async { + ipv4_route_exists_in_ns( + lab.network.namespace("source"), + &format!("{PROXY_CIDR} dev {source_dev}"), + ) + }, + WAIT, + ) + .await; + + patch_proxy_cidr(&destination, ConfigPatchAction::Remove).await; + wait_proxy_route_absent(&source, destination.peer_id(), PROXY_CIDR).await; + wait_for_condition( + || async { + !ipv4_route_exists_in_ns( + lab.network.namespace("source"), + &format!("{PROXY_CIDR} dev {source_dev}"), + ) + }, + WAIT, + ) + .await; + + drop_insts(vec![source, destination]).await; +} + +#[cfg(feature = "magic-dns")] +async fn wait_shared_magic_dns_record( + ns: &str, + domain: &str, + expected_source: &str, + expected_ip: &str, +) { + use hickory_proto::{ + op::{Message, MessageType, Query, ResponseCode}, + rr::{RData, RecordType}, + }; + + use crate::instance::dns_server::MAGIC_DNS_FAKE_IP; + + let expected_source = expected_source.parse::().unwrap(); + let expected_ip = expected_ip.parse::().unwrap(); + let mut query = Message::new(); + query + .set_id(rand::random()) + .add_query(Query::query(domain.parse().unwrap(), RecordType::A)); + let payload = query.to_vec().unwrap(); + wait_for_condition( + || async { + // Let the kernel select the source from the shared route, rather + // than binding to the server and hiding a client-owned route. + let Ok(socket) = NetNS::new(Some(ns.to_owned())).run(|| { + let socket = std::net::UdpSocket::bind("0.0.0.0:0")?; + socket.connect((MAGIC_DNS_FAKE_IP, 53))?; + socket.set_nonblocking(true)?; + tokio::net::UdpSocket::from_std(socket) + }) else { + return false; + }; + let source = socket.local_addr().unwrap().ip(); + if socket.send(&payload).await.is_err() { + return false; + } + let mut buffer = [0; 512]; + let Ok(Ok(len)) = + tokio::time::timeout(Duration::from_millis(500), socket.recv(&mut buffer)).await + else { + return false; + }; + let response = Message::from_vec(&buffer[..len]).unwrap(); + source == std::net::IpAddr::V4(expected_source) + && response.id() == query.id() + && response.message_type() == MessageType::Response + && response.response_code() == ResponseCode::NoError + && response.answers().len() == 1 + && matches!(response.answers()[0].data(), RData::A(ip) if ip.0 == expected_ip) + }, + WAIT, + ) + .await; +} + +#[cfg(feature = "magic-dns")] +#[tokio::test] +#[serial_test::serial] +async fn magic_dns_route_follows_serving_member_and_takeover() { + use crate::instance::dns_server::MAGIC_DNS_FAKE_IP; + + let netns = TestNetnsGuard::new(short_name("svd"), "10.232.1.2/24", "fd42::2/64"); + let dev_name = test_dev_name(); + let runtime = SharedTestRuntime::new(); + let first_config = test_config( + "shared_dns_first", + "shared_dns_network", + "shared_dns_secret", + Some(&netns.name), + Some(&dev_name), + "10.144.243.1/24", + ); + // The first runner wins the singleton, but the client sorts first when + // shared route source hints are chosen by UUID. + first_config.set_id(uuid::Uuid::from_u128(2)); + first_config.set_hostname(Some("shared-dns-first".to_owned())); + let mut flags = first_config.get_flags(); + flags.accept_dns = true; + first_config.set_flags(flags.clone()); + let second_config = test_config( + "shared_dns_second", + "shared_dns_network", + "shared_dns_secret", + Some(&netns.name), + Some(&dev_name), + "10.144.242.2/24", + ); + second_config.set_id(uuid::Uuid::from_u128(1)); + second_config.set_hostname(Some("shared-dns-second".to_owned())); + second_config.set_flags(flags); + let mut first = runtime.instance(first_config); + let mut second = runtime.instance(second_config); + + first.run().await.unwrap(); + wait_tun_ready(&first, &dev_name).await; + wait_shared_magic_dns_record( + &netns.name, + "shared-dns-first.et.net.", + "10.144.243.1", + "10.144.243.1", + ) + .await; + + second.run().await.unwrap(); + wait_tun_ready(&second, &dev_name).await; + // Resolving the second member also waits for its client to connect and + // publish, ensuring its failed server election has already completed. + wait_shared_magic_dns_record( + &netns.name, + "shared-dns-second.et.net.", + "10.144.243.1", + "10.144.242.2", + ) + .await; + let route = SharedIpv4Route::new(MAGIC_DNS_FAKE_IP.parse::().unwrap(), 32, None); + assert_eq!( + shared_route_owner_count(&runtime.registry, &dev_name, &route).await, + 1 + ); + + drop_insts(vec![first]).await; + wait_shared_magic_dns_record( + &netns.name, + "shared-dns-second.et.net.", + "10.144.242.2", + "10.144.242.2", + ) + .await; + assert_eq!( + shared_route_owner_count(&runtime.registry, &dev_name, &route).await, + 1 + ); + + // Keep the TUN alive after the final DNS runner stops so device teardown + // cannot hide a leaked fake-IP route. + let mut observer = runtime.instance(test_config( + "shared_dns_observer", + "shared_dns_network", + "shared_dns_secret", + Some(&netns.name), + Some(&dev_name), + "10.144.244.3/24", + )); + observer.run().await.unwrap(); + wait_tun_ready(&observer, &dev_name).await; + + drop_insts(vec![second]).await; + wait_for_condition( + || async { + shared_route_owner_count(&runtime.registry, &dev_name, &route).await == 0 + && !ipv4_route_exists_in_ns( + &netns.name, + &format!("{MAGIC_DNS_FAKE_IP} dev {dev_name}"), + ) + }, + WAIT, + ) + .await; + drop_insts(vec![observer]).await; +} diff --git a/easytier/src/tests/three_node.rs b/easytier/src/tests/three_node.rs index e1810d20..244a22b8 100644 --- a/easytier/src/tests/three_node.rs +++ b/easytier/src/tests/three_node.rs @@ -1016,6 +1016,166 @@ pub async fn public_ipv6_auto_addr_reconnect_reuses_same_address() { drop_insts(vec![provider, client]).await; } +#[cfg(feature = "tun")] +#[tokio::test] +#[serial_test::serial] +pub async fn shared_tun_public_ipv6_auto_addr_end_to_end() { + let lab = PublicIpv6Lab::setup_with_topology(PublicIpv6LabTopology::DelegatedPrefix); + let provider_dev = format!("st{:08x}", rand::random::()); + let client_dev = format!("st{:08x}", rand::random::()); + let process_runtime = CoreProcessRuntime::new(); + let shared_virtual_nic_registry = Instance::new_shared_virtual_nic_registry(); + + let provider_cfg = get_public_ipv6_config( + "provider_shared_public_ipv6", + lab.network.namespace(PublicIpv6Lab::PROVIDER_NS), + "10.144.144.1", + &provider_dev, + uuid::Uuid::parse_str("44444444-4444-4444-4444-444444444444").unwrap(), + ); + provider_cfg.set_ipv6_public_addr_provider(true); + + let client_cfg = get_public_ipv6_config( + "client_shared_public_ipv6", + lab.network.namespace(PublicIpv6Lab::CLIENT_NS), + "10.144.144.2", + &client_dev, + uuid::Uuid::parse_str("55555555-5555-5555-5555-555555555555").unwrap(), + ); + client_cfg.set_ipv6_public_addr_auto(true); + + let client_peer_cfg = get_public_ipv6_config( + "client_shared_public_ipv6_peer", + lab.network.namespace(PublicIpv6Lab::CLIENT_NS), + "10.144.145.3", + &client_dev, + uuid::Uuid::parse_str("66666666-6666-6666-6666-666666666666").unwrap(), + ); + client_peer_cfg.set_listeners(vec![]); + + let mut provider = Instance::new_with_process_runtime_and_shared_virtual_nic_registry( + provider_cfg, + process_runtime.clone(), + shared_virtual_nic_registry.clone(), + ); + let mut client = Instance::new_with_process_runtime_and_shared_virtual_nic_registry( + client_cfg, + process_runtime.clone(), + shared_virtual_nic_registry.clone(), + ); + let mut client_peer = Instance::new_with_process_runtime_and_shared_virtual_nic_registry( + client_peer_cfg, + process_runtime, + shared_virtual_nic_registry, + ); + let mut client_events = client.get_global_ctx().subscribe(); + let mut client_peer_events = client_peer.get_global_ctx().subscribe(); + + provider.run().await.unwrap(); + client.run().await.unwrap(); + client_peer.run().await.unwrap(); + + let shared_ifname = wait_for_tun_ready(&mut client_events).await; + assert_eq!( + shared_ifname, + wait_for_tun_ready(&mut client_peer_events).await + ); + assert_eq!(shared_ifname, client_dev); + + provider.add_connector_url("tcp://10.1.1.2:11010".parse().unwrap()); + + wait_for_condition( + || async { + provider.get_core_instance().route_snapshots().await.len() == 1 + && client.get_core_instance().route_snapshots().await.len() == 1 + }, + Duration::from_secs(8), + ) + .await; + + wait_for_condition( + || async { + provider + .get_core_instance() + .node_snapshot() + .await + .ipv6_public_addr_prefix + == Some(PublicIpv6Lab::PROVIDER_PREFIX.parse().unwrap()) + }, + Duration::from_secs(10), + ) + .await; + + let leased = wait_for_public_ipv6_addr(&client).await; + wait_for_public_ipv6_route(&provider, leased).await; + + wait_for_condition( + || async { + addr_exists_in_ns( + lab.network.namespace(PublicIpv6Lab::CLIENT_NS), + &client_dev, + &leased.to_string(), + ) && route_exists_in_ns( + lab.network.namespace(PublicIpv6Lab::CLIENT_NS), + &format!("default dev {client_dev}"), + ) && route_exists_in_ns( + lab.network.namespace(PublicIpv6Lab::PROVIDER_NS), + &format!("{} dev {provider_dev}", leased.address()), + ) + }, + Duration::from_secs(10), + ) + .await; + + wait_for_condition( + || async { + ping6_test( + lab.network.namespace(PublicIpv6Lab::CLIENT_NS), + PublicIpv6Lab::SERVER_IP, + None, + ) + .await + }, + Duration::from_secs(10), + ) + .await; + + wait_for_condition( + || async { + ping6_test( + lab.network.namespace(PublicIpv6Lab::SERVER_NS), + leased.address().to_string().as_str(), + None, + ) + .await + }, + Duration::from_secs(10), + ) + .await; + + drop_insts(vec![provider, client, client_peer]).await; + drop(lab); +} + +#[cfg(feature = "tun")] +async fn wait_for_tun_ready( + receiver: &mut tokio::sync::broadcast::Receiver, +) -> String { + tokio::time::timeout(Duration::from_secs(5), async { + loop { + match receiver.recv().await.unwrap() { + crate::common::global_ctx::GlobalCtxEvent::TunDeviceReady(ifname) => return ifname, + crate::common::global_ctx::GlobalCtxEvent::TunDeviceError(error) => { + panic!("tun device error: {error}") + } + _ => {} + } + } + }) + .await + .expect("timed out waiting for tun ready") +} + #[rstest::rstest] #[tokio::test] #[serial_test::serial] diff --git a/tauri-plugin-vpnservice/android/src/main/java/TauriVpnService.kt b/tauri-plugin-vpnservice/android/src/main/java/TauriVpnService.kt index d8c1a5d4..73f7cbdb 100644 --- a/tauri-plugin-vpnservice/android/src/main/java/TauriVpnService.kt +++ b/tauri-plugin-vpnservice/android/src/main/java/TauriVpnService.kt @@ -10,6 +10,7 @@ import android.os.ParcelFileDescriptor import android.os.Bundle import android.content.pm.ServiceInfo import androidx.core.app.NotificationCompat +import android.system.OsConstants.AF_INET6 import java.net.InetAddress import java.util.Arrays @@ -20,10 +21,12 @@ class TauriVpnService : VpnService() { @JvmField var triggerCallback: (String, JSObject) -> Unit = { _, _ -> } @JvmField var self: TauriVpnService? = null @JvmField var ipv4Addr: String? = null + @JvmField var ipv4Addrs: Array = emptyArray() @JvmField var routes: Array = emptyArray() @JvmField var dns: String? = null const val IPV4_ADDR = "IPV4_ADDR" + const val IPV4_ADDRS = "IPV4_ADDRS" const val ROUTES = "ROUTES" const val DNS = "DNS" const val DISALLOWED_APPLICATIONS = "DISALLOWED_APPLICATIONS" @@ -39,7 +42,8 @@ class TauriVpnService : VpnService() { println("vpn on start command ${intent?.getExtras()} $intent") startVpnForegroundService() var args = intent?.getExtras() - ipv4Addr = args?.getString(IPV4_ADDR) + ipv4Addrs = getIpv4Addrs(args) + ipv4Addr = ipv4Addrs.firstOrNull() routes = args?.getStringArray(ROUTES) ?: emptyArray() dns = args?.getString(DNS) @@ -90,6 +94,7 @@ class TauriVpnService : VpnService() { private fun clearStatus() { ipv4Addr = null + ipv4Addrs = emptyArray() routes = emptyArray() dns = null } @@ -159,25 +164,40 @@ class TauriVpnService : VpnService() { } } + private fun getIpv4Addrs(args: Bundle?): Array { + val ipv4Addrs = args + ?.getStringArray(IPV4_ADDRS) + ?.filter { it.isNotBlank() } + ?.toTypedArray() + ?: emptyArray() + if (ipv4Addrs.isNotEmpty()) { + return ipv4Addrs + } + + return arrayOf(args?.getString(IPV4_ADDR) ?: "10.126.126.1/24") + } + private fun createVpnInterface(args: Bundle?): ParcelFileDescriptor { var builder = Builder() .setSession("TauriVpnService") .setBlocking(false) var mtu = args?.getInt(MTU) ?: 1500 - var ipv4Addr = args?.getString(IPV4_ADDR) ?: "10.126.126.1/24" + var ipv4Addrs = getIpv4Addrs(args) var dns: String? = args?.getString(DNS) var routes = args?.getStringArray(ROUTES) ?: emptyArray() var disallowedApplications = args?.getStringArray(DISALLOWED_APPLICATIONS) ?: emptyArray() - println("vpn create vpn interface. mtu: $mtu, ipv4Addr: $ipv4Addr, dns:" + + println("vpn create vpn interface. mtu: $mtu, ipv4Addrs: ${java.util.Arrays.toString(ipv4Addrs)}, dns:" + "$dns, routes: ${java.util.Arrays.toString(routes)}," + "disallowedApplications: ${java.util.Arrays.toString(disallowedApplications)}") - val ipParts = ipv4Addr.split("/") - if (ipParts.size != 2) throw IllegalArgumentException("Invalid IP addr string") - builder.addAddress(ipParts[0], ipParts[1].toInt()) - builder.addAddress("fd00::1", 128) + for (ipv4Addr in ipv4Addrs) { + val ipParts = ipv4Addr.split("/") + if (ipParts.size != 2) throw IllegalArgumentException("Invalid IP addr string") + builder.addAddress(ipParts[0], ipParts[1].toInt()) + } + builder.allowFamily(AF_INET6) builder.setMtu(mtu) dns?.let { builder.addDnsServer(it) } diff --git a/tauri-plugin-vpnservice/android/src/main/java/VpnServicePlugin.kt b/tauri-plugin-vpnservice/android/src/main/java/VpnServicePlugin.kt index 25fda216..f90c91a0 100644 --- a/tauri-plugin-vpnservice/android/src/main/java/VpnServicePlugin.kt +++ b/tauri-plugin-vpnservice/android/src/main/java/VpnServicePlugin.kt @@ -12,6 +12,7 @@ import app.tauri.plugin.Invoke import app.tauri.plugin.JSObject import app.tauri.plugin.Plugin import android.webkit.WebView +import org.json.JSONArray @InvokeArg class PingArgs { @@ -21,6 +22,7 @@ class PingArgs { @InvokeArg class StartVpnArgs { var ipv4Addr: String? = null + var ipv4Addrs: Array = emptyArray() var routes: Array = emptyArray() var dns: String? = null var disallowedApplications: Array = emptyArray() @@ -106,6 +108,7 @@ class VpnServicePlugin(private val activity: Activity) : Plugin(activity) { } else { val intent = Intent(activity, TauriVpnService::class.java) intent.putExtra(TauriVpnService.IPV4_ADDR, args.ipv4Addr) + intent.putExtra(TauriVpnService.IPV4_ADDRS, args.ipv4Addrs) intent.putExtra(TauriVpnService.ROUTES, args.routes) intent.putExtra(TauriVpnService.DNS, args.dns) intent.putExtra(TauriVpnService.DISALLOWED_APPLICATIONS, args.disallowedApplications) @@ -137,7 +140,8 @@ class VpnServicePlugin(private val activity: Activity) : Plugin(activity) { val ret = JSObject() ret.put("running", TauriVpnService.self != null) ret.put("ipv4Addr", TauriVpnService.ipv4Addr) - ret.put("routes", TauriVpnService.routes) + ret.put("ipv4Addrs", JSONArray(TauriVpnService.ipv4Addrs)) + ret.put("routes", JSONArray(TauriVpnService.routes)) ret.put("dns", TauriVpnService.dns) invoke.resolve(ret) } diff --git a/tauri-plugin-vpnservice/guest-js/index.ts b/tauri-plugin-vpnservice/guest-js/index.ts index 2044ed5e..a736cfca 100644 --- a/tauri-plugin-vpnservice/guest-js/index.ts +++ b/tauri-plugin-vpnservice/guest-js/index.ts @@ -15,6 +15,7 @@ export interface InvokeResponse { export interface StartVpnRequest { ipv4Addr?: string; + ipv4Addrs?: string[]; routes?: string[]; dns?: string; disallowedApplications?: string[]; @@ -24,6 +25,7 @@ export interface StartVpnRequest { export interface VpnStatusResponse { running: boolean; ipv4Addr?: string; + ipv4Addrs?: string[]; routes?: string[]; dns?: string; } diff --git a/tauri-plugin-vpnservice/src/models.rs b/tauri-plugin-vpnservice/src/models.rs index a1b8a869..aaf018af 100644 --- a/tauri-plugin-vpnservice/src/models.rs +++ b/tauri-plugin-vpnservice/src/models.rs @@ -22,6 +22,7 @@ pub struct VoidRequest {} #[serde(rename_all = "camelCase")] pub struct StartVpnRequest { pub ipv4_addr: Option, + pub ipv4_addrs: Option>, pub routes: Option>, pub dns: Option, pub disallowed_applications: Option>, @@ -39,6 +40,7 @@ pub struct Status { pub struct VpnStatus { pub running: bool, pub ipv4_addr: Option, + pub ipv4_addrs: Option>, pub routes: Option>, pub dns: Option, }