Report and track WebSocket errors

Author: shepmaster

ui/frontend/actions.ts

@@ -138,6 +138,12 @@ export enum ActionType {
   WSExecuteResponse = 'WS_EXECUTE_RESPONSE',
 }
 
+export const WebSocketError = z.object({
+  type: z.literal(ActionType.WebSocketError),
+  error: z.string(),
+});
+export type WebSocketError = z.infer<typeof WebSocketError>;
+
 const ExecuteExtra = z.object({
   isAutoBuild: z.boolean(),
 });
@@ -851,7 +857,7 @@ export const browserWidthChanged = (isSmall: boolean) =>
 export const splitRatioChanged = () =>
   createAction(ActionType.SplitRatioChanged);
 
-export const websocketError = () => createAction(ActionType.WebSocketError);
+export const websocketError = (error: string): WebSocketError => createAction(ActionType.WebSocketError, { error });
 export const websocketConnected = () => createAction(ActionType.WebSocketConnected);
 export const websocketDisconnected = () => createAction(ActionType.WebSocketDisconnected);
 export const websocketFeatureFlagEnabled = () => createAction(ActionType.WebSocketFeatureFlagEnabled);

ui/frontend/reducers/websocket.ts

@@ -2,6 +2,7 @@ import { Action, ActionType } from '../actions';
 
 export type State = {
   connected: boolean;
+  error?: string;
   featureFlagEnabled: boolean;
 };
 
@@ -13,13 +14,13 @@ const DEFAULT: State = {
 export default function websocket(state = DEFAULT, action: Action): State {
   switch (action.type) {
     case ActionType.WebSocketConnected:
-      return { ...state, connected: true };
+      return { ...state, connected: true, error: undefined };
 
     case ActionType.WebSocketDisconnected:
       return { ...state, connected: false };
 
     case ActionType.WebSocketError:
-      return { ...state };
+      return { ...state, error: action.error };
 
     case ActionType.WebSocketFeatureFlagEnabled:
       return { ...state, featureFlagEnabled: true };

ui/frontend/websocketMiddleware.ts

@@ -1,20 +1,28 @@
 import { Middleware } from 'redux';
 import { z } from 'zod';
 
-import { ActionType, websocketError, websocketConnected, websocketDisconnected, WSExecuteResponse } from './actions';
+import {
+  ActionType,
+  WSExecuteResponse,
+  WebSocketError,
+  websocketConnected,
+  websocketDisconnected,
+  websocketError,
+} from './actions';
 
-const WSMessageResponse = z.discriminatedUnion('type', [WSExecuteResponse]);
+const WSMessageResponse = z.discriminatedUnion('type', [WebSocketError, WSExecuteResponse]);
 
-const reportWebSocketError = async () => {
+const reportWebSocketError = async (error: string) => {
   try {
     await fetch('/nowebsocket', {
       method: 'post',
       headers: {
-        'Content-Length': '0',
+        'Content-Type': 'application/json',
       },
+      body: JSON.stringify({ error }),
     });
   } catch (reportError) {
-    console.log('Unable to report WebSocket error', reportError);
+    console.log('Unable to report WebSocket error', error, reportError);
   }
 }
 
@@ -26,7 +34,8 @@ const openWebSocket = (currentLocation: Location) => {
   } catch (e) {
     // WebSocket URL error or WebSocket is not supported by browser.
     // Assume it's the second case since URL error is easy to notice.
-    reportWebSocketError()
+    const detail = (e instanceof Error) ? e.toString() : 'An unknown error occurred';
+    reportWebSocketError(`Could not create the WebSocket: ${detail}`)
 
     return null;
   }
@@ -45,8 +54,11 @@ export const websocketMiddleware = (window: Window): Middleware => store => {
     });
 
     socket.addEventListener('error', () => {
-      store.dispatch(websocketError());
-      reportWebSocketError();
+      // We cannot get detailed information about the failure
+      // https://stackoverflow.com/a/31003057/155423
+      const error = 'Generic WebSocket Error';
+      store.dispatch(websocketError(error));
+      reportWebSocketError(error);
     });
 
     // TODO: reconnect on error? (if ever connected? if < n failures?)

ui/src/server_axum.rs

@@ -85,6 +85,7 @@ pub(crate) async fn serve(config: Config) {
         .route("/metrics", get(metrics))
         .route("/websocket", get(websocket))
         .route("/nowebsocket", post(nowebsocket))
+        .route("/whynowebsocket", get(whynowebsocket))
         .layer(Extension(Arc::new(SandboxCache::default())))
         .layer(Extension(config.github_token()));
 
@@ -397,10 +398,34 @@ async fn websocket(ws: WebSocketUpgrade) -> impl IntoResponse {
     ws.on_upgrade(websocket::handle)
 }
 
-async fn nowebsocket() {
+#[derive(Debug, serde::Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct NoWebSocketRequest {
+    #[serde(default)]
+    error: String,
+}
+
+async fn nowebsocket(Json(req): Json<NoWebSocketRequest>) {
+    record_websocket_error(req.error);
     UNAVAILABLE_WS.inc();
 }
 
+lazy_static::lazy_static! {
+    static ref WS_ERRORS: std::sync::Mutex<std::collections::HashMap<String, usize>> = Default::default();
+}
+
+fn record_websocket_error(error: String) {
+    *WS_ERRORS
+        .lock()
+        .unwrap_or_else(|e| e.into_inner())
+        .entry(error)
+        .or_default() += 1;
+}
+
+async fn whynowebsocket() -> String {
+    format!("{:#?}", WS_ERRORS.lock().unwrap_or_else(|e| e.into_inner()))
+}
+
 #[derive(Debug)]
 struct MetricsAuthorization;
 

ui/src/server_axum/websocket.rs

@@ -65,10 +65,18 @@ impl TryFrom<WSExecuteRequest> for (sandbox::ExecuteRequest, serde_json::Value)
 #[derive(Debug, serde::Serialize)]
 #[serde(tag = "type")]
 enum WSMessageResponse {
+    #[serde(rename = "WEBSOCKET_ERROR")]
+    Error(WSError),
     #[serde(rename = "WS_EXECUTE_RESPONSE")]
     WSExecuteResponse(WSExecuteResponse),
 }
 
+#[derive(Debug, serde::Serialize)]
+#[serde(rename_all = "camelCase")]
+struct WSError {
+    error: String,
+}
+
 #[derive(Debug, serde::Serialize)]
 #[serde(rename_all = "camelCase")]
 struct WSExecuteResponse {
@@ -115,13 +123,14 @@ pub async fn handle(mut socket: WebSocket) {
                         // unknown message type
                         continue;
                     }
-                    Some(Err(_)) => panic!("Error: {:?}", request),
+                    Some(Err(e)) => super::record_websocket_error(e.to_string()),
                 }
             },
             resp = rx.recv() => {
                 let resp = resp.expect("The rx should never close as we have a tx");
-                let resp = resp.expect("An error occurred and should be reported back to the frontend");
-                let resp = serde_json::to_string(&resp).expect("An error occurred and should be reported back to the frontend");
+                let resp = resp.unwrap_or_else(|e| WSMessageResponse::Error(WSError { error: e.to_string() }));
+                const LAST_CHANCE_ERROR: &str = r#"{ "type": "WEBSOCKET_ERROR", "error": "Unable to serialize JSON" }"#;
+                let resp = serde_json::to_string(&resp).unwrap_or_else(|_| LAST_CHANCE_ERROR.into());
                 let resp = Message::Text(resp);
 
                 if let Err(_) = socket.send(resp).await {