diff --git a/src/main.rs b/src/main.rs index f062df9..5102897 100644 --- a/src/main.rs +++ b/src/main.rs @@ -73,6 +73,14 @@ fn superseded_peer_ids(sessions: &HashMap, ice: &str, des sessions.iter().filter(|(id, peer)| id.as_str() == ice || (desktop && peer.desktop)) .map(|(id, _)| id.clone()).collect() } + +/// A peer whose transport is already broken can leave `PeerConnection::close` +/// waiting indefinitely; signaling must keep flowing without it. +async fn close_peer(pc: &::webrtc::peer_connection::RTCPeerConnection) { + if tokio::time::timeout(std::time::Duration::from_secs(5), pc.close()).await.is_err() { + log::warn!("Peer connection close timed out; continuing"); + } +} fn released_peer_ids(sessions: &HashMap, ice: Option<&serde_json::Value>) -> Vec { let ice = ice.map(|id| id.as_str().map(str::to_owned).unwrap_or_else(|| id.to_string())); sessions.keys().filter(|id| ice.as_deref().map_or(true, |ice| id.as_str() == ice)).cloned().collect() @@ -685,7 +693,7 @@ async fn main() -> Result<(), Box> { if let Some(task) = command_task { task.abort(); } // One exit path for commands, signals, transport failures and UI failures. if let Some(session) = assist.as_mut() { session.allowed = false; } - for (_, old) in webrtc_sessions.lock().await.drain() { let _ = old.pc.close().await; } + for (_, old) in webrtc_sessions.lock().await.drain() { close_peer(&old.pc).await; } virtual_audio::disable(); input.shutdown(); terminal::shutdown_all(); // Switching identity (signed in / out) restarts at once and the new identity republishes the same // device share: logout already withdrew it, and after a QR login the cloud moved it to the account @@ -858,7 +866,7 @@ async fn serve(mut capture: Arc, input: Arc, shutd ::webrtc::peer_connection::peer_connection_state::RTCPeerConnectionState::Closed | ::webrtc::peer_connection::peer_connection_state::RTCPeerConnectionState::Failed)).map(|(id, _)| id.clone()).collect(); let mut desktop_ended = false; - for id in ended { if let Some(s) = sessions.remove(&id) { desktop_ended |= s.role.is_desktop(); let _ = s.pc.close().await; } } + for id in ended { if let Some(s) = sessions.remove(&id) { desktop_ended |= s.role.is_desktop(); close_peer(&s.pc).await; } } if desktop_ended && !sessions.values().any(|s| s.role.is_desktop()) { if let Some(s) = assist.as_mut() { s.session_ended()?; } } if let Some(s) = assist.as_mut() { s.tick()?; } } @@ -946,7 +954,7 @@ async fn serve(mut capture: Arc, input: Arc, shutd ::webrtc::peer_connection::peer_connection_state::RTCPeerConnectionState::Closed | ::webrtc::peer_connection::peer_connection_state::RTCPeerConnectionState::Failed)).map(|(k, _)| k.clone()).collect(); let mut desktop_ended = false; - for k in ended { if let Some(old) = sessions.remove(&k) { desktop_ended |= old.role.is_desktop(); let _ = old.pc.close().await; } } + for k in ended { if let Some(old) = sessions.remove(&k) { desktop_ended |= old.role.is_desktop(); close_peer(&old.pc).await; } } if desktop_ended && !sessions.values().any(|s| s.role.is_desktop()) { if let Some(s) = assist.as_mut() { s.session_ended()?; } } if let Some(s) = assist.as_mut() { s.tick()?; } @@ -961,7 +969,7 @@ async fn serve(mut capture: Arc, input: Arc, shutd if desktop { video::warm_auto_probe(); video::check_ffmpeg_usable(); } // Replace only this ICE id or a previous desktop, never tool peers. for id in superseded_peer_ids(&sessions, &ice, desktop) { - if let Some(old) = sessions.remove(&id) { let _ = old.pc.close().await; } + if let Some(old) = sessions.remove(&id) { close_peer(&old.pc).await; } } // A tool peer promoted by capture_desktop stops its desktop but keeps its tools. if desktop { for old in sessions.values() { old.demote(); } } @@ -1014,7 +1022,7 @@ async fn serve(mut capture: Arc, input: Arc, shutd } Err(_) => { log::error!("Failed to handle offer (details omitted)"); - if let Some(session) = sessions.remove(&ice) { let _ = session.pc.close().await; } + if let Some(session) = sessions.remove(&ice) { close_peer(&session.pc).await; } } }}, Err(_) => log::error!("Create peer failed (details omitted)"), @@ -1037,7 +1045,7 @@ async fn serve(mut capture: Arc, input: Arc, shutd for k in targets { if let Some(old) = sessions.remove(&k) { desktop_ended |= old.role.is_desktop(); - if let Err(e) = old.pc.close().await { log::warn!("Close released peer failed: {}", e); } + close_peer(&old.pc).await; } } if desktop_ended && !sessions.values().any(|s| s.role.is_desktop()) { if let Some(s) = assist.as_mut() { s.session_ended()?; } } @@ -1075,7 +1083,7 @@ async fn serve(mut capture: Arc, input: Arc, shutd let superseded: Vec = sessions.iter().filter(|(id, old)| id.as_str() != ice && old.role.is_desktop()) .map(|(id, _)| id.clone()).collect(); for id in superseded { - if sessions[&id].desktop { if let Some(old) = sessions.remove(&id) { let _ = old.pc.close().await; } } + if sessions[&id].desktop { if let Some(old) = sessions.remove(&id) { close_peer(&old.pc).await; } } else { sessions[&id].demote(); } } if let Some(peer) = sessions.get(&ice) { peer.promote(screen); } diff --git a/src/signaling.rs b/src/signaling.rs index 3448a7a..f29911b 100644 --- a/src/signaling.rs +++ b/src/signaling.rs @@ -3,12 +3,20 @@ use futures_util::{SinkExt, StreamExt}; use serde::{Deserialize, Serialize}; use serde_json::Value; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::Arc; +use std::time::{Duration, Instant}; use tokio::sync::mpsc; use tokio_tungstenite::connect_async; use tokio_tungstenite::tungstenite::client::IntoClientRequest; use tokio_tungstenite::tungstenite::Message; +/// A peer that stops reading must not park the heartbeat loop on a full queue. +/// Kept below the Engine.IO window so the protocol deadline still fires. +const SEND_STALL_TIMEOUT: Duration = Duration::from_secs(10); +/// How often the watchdog checks the last inbound frame. +const WATCHDOG_TICK: Duration = Duration::from_secs(1); + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct SoacMessage { #[serde(rename = "type")] @@ -70,6 +78,11 @@ fn refreshed_key(ack:&Value)->Option { (!key.is_empty() && key.len()<=16384).then(||key.to_owned()) } +/// Deliver an event without letting a stalled consumer hide a dead socket. +async fn send_event(sender: &mpsc::Sender, event: SignalingEvent) -> bool { + matches!(tokio::time::timeout(SEND_STALL_TIMEOUT, sender.send(event)).await, Ok(Ok(()))) +} + #[cfg(test)] mod refresh_tests { use super::refreshed_key; @@ -128,6 +141,73 @@ mod refresh_tests { assert!(disconnected.is_ok(),"a socket without pings must be reported dead"); drop(hold_tx); } + #[tokio::test] + async fn blocked_event_consumer_does_not_hide_a_dead_socket() { + use futures_util::{SinkExt,StreamExt}; + use tokio_tungstenite::tungstenite::Message; + let listener=tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address=listener.local_addr().unwrap(); + let (closed_tx,closed_rx)=tokio::sync::oneshot::channel::<()>(); + tokio::spawn(async move { + let (tcp,_)=listener.accept().await.unwrap(); + let socket=tokio_tungstenite::accept_async(tcp).await.unwrap(); + let (mut sink,mut stream)=socket.split(); + // Shortest accepted window (clamped to 5 s); never ping, only events. + sink.send(Message::Text(r#"0{"pingInterval":1000,"pingTimeout":1000}"#.into())).await.unwrap(); + let _=stream.next().await; + sink.send(Message::Text("40{}".into())).await.unwrap(); + loop { + tokio::select! { + r=stream.next()=>match r { + None|Some(Err(_))|Some(Ok(Message::Close(_)))=>{let _=closed_tx.send(());return;} + _=>{} + }, + _=tokio::time::sleep(std::time::Duration::from_millis(50))=>{ + if sink.send(Message::Text(r#"42["bmsg_push",{}]"#.into())).await.is_err() { + let _=closed_tx.send(());return; + } + } + } + } + }); + // Capacity 1 and no reader: the first event fills the queue, so the + // receive loop would park inside `send().await` without the watchdog. + let (sender,_events)=tokio::sync::mpsc::channel(1); + let _client=super::SignalingClient::connect(&format!("ws://{address}"),"synthetic-only",sender).await.unwrap(); + let closed=tokio::time::timeout(std::time::Duration::from_secs(15),closed_rx).await; + assert!(closed.is_ok(),"a blocked consumer must not keep a dead signaling socket open"); + } + #[tokio::test] + async fn stalled_event_delivery_still_reports_disconnected() { + use futures_util::{SinkExt,StreamExt}; + use tokio_tungstenite::tungstenite::Message; + let listener=tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address=listener.local_addr().unwrap(); + tokio::spawn(async move { + let (tcp,_)=listener.accept().await.unwrap(); + let socket=tokio_tungstenite::accept_async(tcp).await.unwrap(); + let (mut sink,mut stream)=socket.split(); + // A long window: the send stall must fire before the heartbeat deadline. + sink.send(Message::Text(r#"0{"pingInterval":30000,"pingTimeout":30000}"#.into())).await.unwrap(); + let _=stream.next().await; + sink.send(Message::Text("40{}".into())).await.unwrap(); + loop { + tokio::select! { + r=stream.next()=>{ if matches!(r,None|Some(Err(_))|Some(Ok(Message::Close(_)))){return} } + _=tokio::time::sleep(std::time::Duration::from_millis(50))=>{ + if sink.send(Message::Text(r#"42["bmsg_push",{}]"#.into())).await.is_err(){return} + } + } + } + }); + let (sender,mut events)=tokio::sync::mpsc::channel(1); + let _client=super::SignalingClient::connect(&format!("ws://{address}"),"synthetic-only",sender).await.unwrap(); + // Let SEND_STALL_TIMEOUT expire while nothing drains the queue. + tokio::time::sleep(std::time::Duration::from_secs(12)).await; + assert!(matches!(events.recv().await,Some(super::SignalingEvent::Connected))); + let disconnected=tokio::time::timeout(std::time::Duration::from_secs(5),events.recv()).await; + assert!(matches!(disconnected,Ok(Some(super::SignalingEvent::Disconnected))),"a stalled delivery must still report Disconnected"); + } } impl SignalingClient { @@ -158,17 +238,25 @@ impl SignalingClient { log::info!("Connected to Engine.IO / Socket.IO"); let (mut write, mut read) = ws_stream.split(); let (tx_outgoing, mut rx_outgoing) = mpsc::channel::(32); + let started = Instant::now(); + let last_rx = Arc::new(AtomicU64::new(0)); + let ping_window_ms = Arc::new(AtomicU64::new(45_000)); + let receiver_done = Arc::new(AtomicBool::new(false)); // 发送协程 let writer=tokio::spawn(async move { while let Some(msg) = rx_outgoing.recv().await { - if let Err(e) = write.send(Message::Text(msg)).await { - let _ = e; - log::warn!("WS send failed (details omitted)"); - break; + match tokio::time::timeout(SEND_STALL_TIMEOUT, write.send(Message::Text(msg))).await { + Ok(Ok(())) => {} + Ok(Err(_)) | Err(_) => { + // A peer that stopped reading must not park every producer. + log::warn!("WS send failed or stalled (details omitted)"); + break; + } } } }); + let writer_abort = writer.abort_handle(); let tx_out_clone = tx_outgoing.clone(); static GENERATION: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1); @@ -177,9 +265,12 @@ impl SignalingClient { let refreshed=reconnect_key.clone(); // 接收与状态机协程 - tokio::spawn(async move { + let watchdog_events = event_sender.clone(); + let watchdog_last_rx = last_rx.clone(); + let watchdog_ping_window_ms = ping_window_ms.clone(); + let watchdog_done = receiver_done.clone(); + let receiver = tokio::spawn(async move { let mut req_counter = 10; - let started=std::time::Instant::now(); let mut last_ping=started; let mut pings=0u64; let mut pong_queued=0u64; @@ -202,7 +293,7 @@ impl SignalingClient { _=event_sender.closed()=>break, _=tokio::time::sleep_until(ping_deadline) => { log::warn!("Signaling heartbeat lost: generation={} age_s={} pings={} last_ping_gap_s={}",generation,started.elapsed().as_secs(),pings,last_ping.elapsed().as_secs()); - let _ = event_sender.send(SignalingEvent::Disconnected).await; + let _ = send_event(&event_sender, SignalingEvent::Disconnected).await; break; }, // Official ControlledSession RefreshSignalTokenLoop: first @@ -213,8 +304,8 @@ impl SignalingClient { refresh_acks.push_back(req_counter); if refresh_acks.len()>16 { refresh_acks.pop_front(); } let packet=format!("42{}[\"refresh_reconnect_key\"]",req_counter); - if tx_out_clone.send(packet).await.is_err() { - let _=event_sender.send(SignalingEvent::Disconnected).await;break; + if tx_out_clone.try_send(packet).is_err() { + let _=send_event(&event_sender, SignalingEvent::Disconnected).await;break; } next_refresh=tokio::time::Instant::now()+std::time::Duration::from_secs(30); log::info!("Signaling reconnect-key refresh requested: generation={} age_s={}",generation,started.elapsed().as_secs()); @@ -222,9 +313,14 @@ impl SignalingClient { }, msg=read.next()=>match msg { Some(msg)=>msg, - None=>{let _=event_sender.send(SignalingEvent::Disconnected).await;break;} + None=>{let _=send_event(&event_sender, SignalingEvent::Disconnected).await;break;} }, }; + // Any inbound frame proves the socket is alive. A reader parked on + // event delivery stops updating this, which the watchdog catches. + if msg_res.is_ok() { + last_rx.store(started.elapsed().as_millis() as u64, Ordering::Relaxed); + } match msg_res { Ok(Message::Text(txt)) => { // Engine.IO 报文处理 @@ -233,16 +329,20 @@ impl SignalingClient { log::info!("Signaling heartbeat config: generation={} ping_interval_ms={:?} ping_timeout_ms={:?}",generation,open.get("pingInterval").and_then(Value::as_u64),open.get("pingTimeout").and_then(Value::as_u64)); if let (Some(interval),Some(timeout))=(open.get("pingInterval").and_then(Value::as_u64),open.get("pingTimeout").and_then(Value::as_u64)) { ping_window=std::time::Duration::from_millis(interval.saturating_add(timeout).clamp(5_000,300_000)); + ping_window_ms.store(ping_window.as_millis() as u64, Ordering::Relaxed); ping_deadline=tokio::time::Instant::now()+ping_window; } } // Open packet -> 发送 Socket.IO connect - let _ = tx_out_clone.send("40".into()).await; + if tx_out_clone.try_send("40".into()).is_err() { + let _=send_event(&event_sender, SignalingEvent::Disconnected).await;break; + } } else if txt == "2" { // Ping -> 回复 Pong let gap=last_ping.elapsed().as_secs();last_ping=std::time::Instant::now();pings+=1; ping_deadline=tokio::time::Instant::now()+ping_window; - if tx_out_clone.send("3".into()).await.is_ok() { pong_queued+=1; } + if tx_out_clone.try_send("3".into()).is_ok() { pong_queued+=1; } + else { let _=send_event(&event_sender, SignalingEvent::Disconnected).await;break; } if pings==1 || pings%12==0 { log::info!("Signaling heartbeat progress: generation={} age_s={} pings={} pong_queued={} gap_s={}",generation,started.elapsed().as_secs(),pings,pong_queued,gap); } @@ -252,8 +352,10 @@ impl SignalingClient { // Socket.IO connected -> 发送 room_info 请求 req_counter += 1; let room_req = format!("42{}[\"room_info\"]", req_counter); - let _ = tx_out_clone.send(room_req).await; - let _ = event_sender.send(SignalingEvent::Connected).await; + if tx_out_clone.try_send(room_req).is_err() { + let _=send_event(&event_sender, SignalingEvent::Disconnected).await;break; + } + if !send_event(&event_sender, SignalingEvent::Connected).await { break; } } else if txt.starts_with("43") { // Socket.IO ack 回调 let begin=txt.find('[').unwrap_or(2); @@ -272,7 +374,7 @@ impl SignalingClient { next_refresh=tokio::time::Instant::now()+std::time::Duration::from_secs(300); } log::info!("Signaling reconnect-key refresh acknowledged: generation={} age_s={} success={}",generation,started.elapsed().as_secs(),success); - } else { let _ = event_sender.send(SignalingEvent::RoomInfoAck(v)).await; } + } else if !send_event(&event_sender, SignalingEvent::RoomInfoAck(v)).await { break; } } } else if txt.starts_with("45") || txt.starts_with("46") { // Socket.IO binary event/ack: header text, attachments follow as binary frames. @@ -296,14 +398,10 @@ impl SignalingClient { match event_name { "bmsg_push" => { - let _ = event_sender - .send(SignalingEvent::DeviceInfoChanged(data.clone())) - .await; + if !send_event(&event_sender, SignalingEvent::DeviceInfoChanged(data.clone())).await { break; } } "be-controlled" => { - let _ = event_sender - .send(SignalingEvent::BeControlled(data.clone())) - .await; + if !send_event(&event_sender, SignalingEvent::BeControlled(data.clone())).await { break; } } "soac" => { // Offers only: structure (keys, types, lengths), never SDP text or ids. @@ -326,9 +424,7 @@ impl SignalingClient { if soac.client_id.is_none() { soac.client_id = client_id_opt; } - let _ = event_sender - .send(SignalingEvent::Soac(soac)) - .await; + if !send_event(&event_sender, SignalingEvent::Soac(soac)).await { break; } } Err(e) => { let _ = e; @@ -338,20 +434,16 @@ impl SignalingClient { } "released" => { log::info!("Cloud peer event: generation={} event=released age_s={} pings={} last_ping_gap_s={} ice_id_present={}",generation,started.elapsed().as_secs(),pings,last_ping.elapsed().as_secs(),data.get("ice_id").is_some()); - let _ = event_sender - .send(SignalingEvent::Released(data.clone())) - .await; + if !send_event(&event_sender, SignalingEvent::Released(data.clone())).await { break; } } "left" | "leave" => { log::info!("Cloud peer event: generation={} event={} age_s={} pings={} last_ping_gap_s={} ice_id_present={}",generation,event_name,started.elapsed().as_secs(),pings,last_ping.elapsed().as_secs(),data.get("ice_id").is_some()); let ice_id = data.get("ice_id").and_then(|v| v.as_str()).unwrap_or(""); if ice_id.is_empty() { log::warn!("Cloud room leave without peer identifier; renewing signaling (expiry cause unknown)"); - let _ = event_sender.send(SignalingEvent::Disconnected).await; + if !send_event(&event_sender, SignalingEvent::Disconnected).await { break; } } else { - let _ = event_sender - .send(SignalingEvent::Left(data.clone())) - .await; + if !send_event(&event_sender, SignalingEvent::Left(data.clone())).await { break; } } } _ => { @@ -401,28 +493,60 @@ impl SignalingClient { let controller_mac = platform.is_some_and(|v| v.as_i64() == Some(4) || v.as_str().is_some_and(|s| s == "4" || s.to_ascii_lowercase().contains("mac") || s.to_ascii_lowercase().contains("darwin"))); log::info!("Controller platform: {} mac={}", platform.map(|v| v.to_string()).unwrap_or_default().chars().take(24).collect::(), controller_mac); - let _ = event_sender.send(SignalingEvent::ConnectRequest { capture_type, ice_id, transport:crate::session_ice::SessionIce::from_header(&header), controller_mac }).await; + if !send_event(&event_sender, SignalingEvent::ConnectRequest { capture_type, ice_id, transport:crate::session_ice::SessionIce::from_header(&header), controller_mac }).await { break; } } } } Ok(Message::Close(frame)) => { log::warn!("Signaling socket closed: generation={} age_s={} close_code={:?} pings={} last_ping_gap_s={}",generation,started.elapsed().as_secs(),frame.map(|f|u16::from(f.code)),pings,last_ping.elapsed().as_secs()); - let _ = event_sender.send(SignalingEvent::Disconnected).await; + let _ = send_event(&event_sender, SignalingEvent::Disconnected).await; break; } Err(_) => { log::warn!("Signaling socket receive failed: generation={} age_s={} pings={} last_ping_gap_s={}",generation,started.elapsed().as_secs(),pings,last_ping.elapsed().as_secs()); - let _ = event_sender.send(SignalingEvent::Disconnected).await; + let _ = send_event(&event_sender, SignalingEvent::Disconnected).await; break; } _ => {} } } + // A stalled event queue or a writer that already stopped can end this + // loop without a delivered Disconnected. Hand one off from a detached + // task so the main loop always learns the room is dead; duplicates are + // harmless because it breaks on the first one. + let events = event_sender.clone(); + tokio::spawn(async move { let _ = events.send(SignalingEvent::Disconnected).await; }); + receiver_done.store(true, Ordering::Relaxed); log::info!("Signaling receiver stopped: generation={} age_s={}",generation,started.elapsed().as_secs()); // Live peer callbacks can retain the old SignalingClient sender. // Explicitly drop its write half too when this room stops reading. writer.abort(); }); + let receiver_abort = receiver.abort_handle(); + + // Independent of the receive loop: a consumer that stops draining events + // parks the loop inside an `.await`, where the ping deadline can no longer + // be observed. The watchdog closes the socket anyway. + tokio::spawn(async move { + let mut tick = tokio::time::interval(WATCHDOG_TICK); + tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + loop { + tick.tick().await; + if watchdog_done.load(Ordering::Relaxed) || watchdog_events.is_closed() { return; } + let window_ms = watchdog_ping_window_ms.load(Ordering::Relaxed); + let idle_ms = (started.elapsed().as_millis() as u64) + .saturating_sub(watchdog_last_rx.load(Ordering::Relaxed)); + if idle_ms <= window_ms { continue; } + log::warn!("Signaling heartbeat lost (watchdog): idle_s={} window_ms={}", idle_ms/1000, window_ms); + // Deliver from a detached task: the receiver may be the one parked + // on a full queue, and the main loop still needs this event. + let events = watchdog_events.clone(); + tokio::spawn(async move { let _ = events.send(SignalingEvent::Disconnected).await; }); + receiver_abort.abort(); + writer_abort.abort(); + return; + } + }); Ok(Self { tx_outgoing, _reconnect_key:reconnect_key }) } @@ -430,7 +554,10 @@ impl SignalingClient { pub async fn emit_event(&self, event: &str, payload: serde_json::Value) -> Result<(), String> { let packet = format!("42{}", serde_json::json!([event, payload])); log::info!("Emitting signaling event: {}", event); - self.tx_outgoing.send(packet).await.map_err(|e| e.to_string()) + tokio::time::timeout(SEND_STALL_TIMEOUT, self.tx_outgoing.send(packet)) + .await + .map_err(|_| "signaling send stalled".to_string())? + .map_err(|e| e.to_string()) } /// 发送 SOAC 消息(Answer 或 Candidate) @@ -445,9 +572,9 @@ impl SignalingClient { }; let msg = format!("42[\"soac\",{}]", payload); log::info!("Emitting soac packet: type={}", soac.msg_type); - self.tx_outgoing - .send(msg) + tokio::time::timeout(SEND_STALL_TIMEOUT, self.tx_outgoing.send(msg)) .await + .map_err(|_| "Send soac timed out".to_string())? .map_err(|e| format!("Send soac error: {}", e)) } }