mirror of
https://github.com/cpvalente/ontime.git
synced 2026-09-16 19:43:06 +00:00
fix: apply backoff to watchdog reconnects
This commit is contained in:
@@ -59,6 +59,7 @@ vi.mock('../../stores/logger', () => ({ addLog: vi.fn() }));
|
|||||||
|
|
||||||
describe('socket connection watchdog', () => {
|
describe('socket connection watchdog', () => {
|
||||||
let connectSocket: () => void;
|
let connectSocket: () => void;
|
||||||
|
let getReconnectAttempts: () => number;
|
||||||
|
|
||||||
beforeEach(async () => {
|
beforeEach(async () => {
|
||||||
vi.useFakeTimers();
|
vi.useFakeTimers();
|
||||||
@@ -67,7 +68,9 @@ describe('socket connection watchdog', () => {
|
|||||||
vi.stubGlobal('WebSocket', MockWebSocket);
|
vi.stubGlobal('WebSocket', MockWebSocket);
|
||||||
// the module keeps connection state in module scope, we need a clean one for each test
|
// the module keeps connection state in module scope, we need a clean one for each test
|
||||||
vi.resetModules();
|
vi.resetModules();
|
||||||
connectSocket = (await import('../socket')).connectSocket;
|
const socketModule = await import('../socket');
|
||||||
|
connectSocket = socketModule.connectSocket;
|
||||||
|
getReconnectAttempts = socketModule.getReconnectAttempts;
|
||||||
});
|
});
|
||||||
|
|
||||||
afterEach(() => {
|
afterEach(() => {
|
||||||
@@ -108,10 +111,12 @@ describe('socket connection watchdog', () => {
|
|||||||
|
|
||||||
vi.advanceTimersByTime(watchdogInterval);
|
vi.advanceTimersByTime(watchdogInterval);
|
||||||
expect(socket.readyState).toBe(MockWebSocket.CLOSED);
|
expect(socket.readyState).toBe(MockWebSocket.CLOSED);
|
||||||
|
|
||||||
|
vi.advanceTimersByTime(socketConfig.reconnectBaseInterval * 2);
|
||||||
expect(MockWebSocket.instances).toHaveLength(2);
|
expect(MockWebSocket.instances).toHaveLength(2);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('gives up on a connection attempt which never completes', () => {
|
it('backs off after a connection attempt which never completes', () => {
|
||||||
connectSocket();
|
connectSocket();
|
||||||
const socket = MockWebSocket.instances[0];
|
const socket = MockWebSocket.instances[0];
|
||||||
|
|
||||||
@@ -119,7 +124,12 @@ describe('socket connection watchdog', () => {
|
|||||||
vi.advanceTimersByTime(connectTimeout + watchdogInterval);
|
vi.advanceTimersByTime(connectTimeout + watchdogInterval);
|
||||||
|
|
||||||
expect(socket.readyState).toBe(MockWebSocket.CLOSED);
|
expect(socket.readyState).toBe(MockWebSocket.CLOSED);
|
||||||
|
expect(MockWebSocket.instances).toHaveLength(1);
|
||||||
|
expect(getReconnectAttempts()).toBe(0);
|
||||||
|
|
||||||
|
vi.advanceTimersByTime(socketConfig.reconnectBaseInterval * 2);
|
||||||
expect(MockWebSocket.instances).toHaveLength(2);
|
expect(MockWebSocket.instances).toHaveLength(2);
|
||||||
|
expect(getReconnectAttempts()).toBe(1);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('logs one warning while repeated connection attempts time out', () => {
|
it('logs one warning while repeated connection attempts time out', () => {
|
||||||
@@ -164,6 +174,7 @@ describe('socket connection watchdog', () => {
|
|||||||
const stale = openConnection();
|
const stale = openConnection();
|
||||||
|
|
||||||
vi.advanceTimersByTime(silenceTimeout + watchdogInterval);
|
vi.advanceTimersByTime(silenceTimeout + watchdogInterval);
|
||||||
|
vi.advanceTimersByTime(socketConfig.reconnectBaseInterval * 2);
|
||||||
expect(MockWebSocket.instances).toHaveLength(2);
|
expect(MockWebSocket.instances).toHaveLength(2);
|
||||||
MockWebSocket.instances[1].open();
|
MockWebSocket.instances[1].open();
|
||||||
|
|
||||||
|
|||||||
@@ -276,14 +276,11 @@ function scheduleReconnect() {
|
|||||||
}, delay);
|
}, delay);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/** Drops a socket that is no longer delivering and retries with the normal backoff. */
|
||||||
* Drops the current connection and immediately opens a new one.
|
function reconnectWithBackoff(reason: string) {
|
||||||
* Used when we have reason to believe the socket is no longer delivering data.
|
|
||||||
*/
|
|
||||||
function reconnectNow(reason: string) {
|
|
||||||
logConnectionIssue(reason);
|
logConnectionIssue(reason);
|
||||||
detachSocket();
|
detachSocket();
|
||||||
connectSocket();
|
scheduleReconnect();
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Detaches a socket before closing it so late events cannot affect its replacement. */
|
/** Detaches a socket before closing it so late events cannot affect its replacement. */
|
||||||
@@ -325,23 +322,19 @@ function checkConnection() {
|
|||||||
if (websocket?.readyState === WebSocket.CONNECTING) {
|
if (websocket?.readyState === WebSocket.CONNECTING) {
|
||||||
// A connection attempt can otherwise hang indefinitely.
|
// A connection attempt can otherwise hang indefinitely.
|
||||||
if (silentFor > socketConfig.connectTimeout) {
|
if (silentFor > socketConfig.connectTimeout) {
|
||||||
reconnectNow('WebSocket: connection attempt timed out');
|
reconnectWithBackoff('WebSocket: connection attempt timed out');
|
||||||
}
|
}
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (websocket?.readyState === WebSocket.OPEN) {
|
if (websocket?.readyState === WebSocket.OPEN) {
|
||||||
if (silentFor > socketConfig.silenceTimeout) {
|
if (silentFor > socketConfig.silenceTimeout) {
|
||||||
reconnectNow('WebSocket: no data from server, reconnecting');
|
reconnectWithBackoff('WebSocket: no data from server, reconnecting');
|
||||||
}
|
}
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
// Covers a close event that the browser never reported.
|
||||||
* The socket is closing, closed or was never created.
|
|
||||||
* Closing schedules its own reconnect, this is what covers us if that did not happen,
|
|
||||||
* so that there is no state the client can settle in without a way out.
|
|
||||||
*/
|
|
||||||
if (!reconnectTimeout) {
|
if (!reconnectTimeout) {
|
||||||
scheduleReconnect();
|
scheduleReconnect();
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user