From 29243e3f6caf264536dd19a81c6744306121550c Mon Sep 17 00:00:00 2001 From: Alex Christoffer Rasmussen Date: Sun, 22 Dec 2024 16:23:47 +0100 Subject: [PATCH] detect `EADDRINUSE` on server start (#1348) * detect `EADDRINUSE` on server start * @ts-expect-error * send portError to electron * use escalateErrorFn * dont move in docker * allow errors to be sendt to electron UI * Log after the new port is found * Update comment Co-authored-by: Carlos Valente <34649812+cpvalente@users.noreply.github.com> * update comment Co-authored-by: Carlos Valente <34649812+cpvalente@users.noreply.github.com> * update comment Co-authored-by: Carlos Valente <34649812+cpvalente@users.noreply.github.com> * rename to escalate * extract `serverTryDesiredPort` * type check server.address() * request shutdown on `unrecoverable` `escalateError` * combine network utils * reject promise * not unrecoverable by default * unneeded assignment --------- Co-authored-by: Carlos Valente <34649812+cpvalente@users.noreply.github.com> --- apps/electron/src/main.js | 11 ++- .../src/api-data/session/session.service.ts | 2 +- apps/server/src/app.ts | 42 +++++----- apps/server/src/classes/Logger.ts | 11 ++- apps/server/src/utils/network.ts | 81 +++++++++++++++++++ apps/server/src/utils/networkInterfaces.ts | 28 ------- 6 files changed, 121 insertions(+), 54 deletions(-) create mode 100644 apps/server/src/utils/network.ts delete mode 100644 apps/server/src/utils/networkInterfaces.ts diff --git a/apps/electron/src/main.js b/apps/electron/src/main.js index 8741afdeb..f200ac7e3 100644 --- a/apps/electron/src/main.js +++ b/apps/electron/src/main.js @@ -23,7 +23,7 @@ if (!isProduction) { } /** Flag holds server loading state */ -let loaded = 'Ontime running'; +let loaded = 'Ontime starting'; /** * Flag whether user has requested a quit @@ -109,8 +109,13 @@ function askToQuit() { * Allows processes to escalate errors to be shown in electron * @param {string} error */ -function escalateError(error) { - dialog.showErrorBox('An unrecoverable error occurred', error); +function escalateError(error, unrecoverable = false) { + if (unrecoverable) { + dialog.showErrorBox('An unrecoverable error occurred', error); + appShutdown(); + } else { + dialog.showErrorBox('An error occurred', error); + } } /** diff --git a/apps/server/src/api-data/session/session.service.ts b/apps/server/src/api-data/session/session.service.ts index ab6ae35bb..7fbee53c3 100644 --- a/apps/server/src/api-data/session/session.service.ts +++ b/apps/server/src/api-data/session/session.service.ts @@ -2,11 +2,11 @@ import { GetInfo, SessionStats } from 'ontime-types'; import { getDataProvider } from '../../classes/data-provider/DataProvider.js'; import { publicDir } from '../../setup/index.js'; -import { getNetworkInterfaces } from '../../utils/networkInterfaces.js'; import { socket } from '../../adapters/WebsocketAdapter.js'; import { getLastRequest } from '../../api-integration/integration.controller.js'; import { getLastLoadedProject } from '../../services/app-state-service/AppStateService.js'; import { runtimeService } from '../../services/runtime-service/RuntimeService.js'; +import { getNetworkInterfaces } from '../../utils/network.js'; const startedAt = new Date(); diff --git a/apps/server/src/app.ts b/apps/server/src/app.ts index 576edb2d6..92f53f3c4 100644 --- a/apps/server/src/app.ts +++ b/apps/server/src/app.ts @@ -3,7 +3,7 @@ import { LogOrigin, Playback, SimpleDirection, SimplePlayback } from 'ontime-typ import 'dotenv/config'; import express from 'express'; import expressStaticGzip from 'express-static-gzip'; -import http, { type Server } from 'http'; +import http, { Server } from 'http'; import cors from 'cors'; import serverTiming from 'server-timing'; import { extname } from 'node:path'; @@ -40,8 +40,8 @@ import { initialiseProject } from './services/project-service/ProjectService.js' // Utilities import { clearUploadfolder } from './utils/upload.js'; import { generateCrashReport } from './utils/generateCrashReport.js'; -import { getNetworkInterfaces } from './utils/networkInterfaces.js'; import { timerConfig } from './config/config.js'; +import { serverTryDesiredPort, getNetworkInterfaces } from './utils/network.js'; console.log('\n'); consoleHighlight(`Starting Ontime version ${ONTIME_VERSION}`); @@ -176,12 +176,20 @@ export const initAssets = async () => { * Starts servers */ export const startServer = async ( - escalateErrorFn?: (error: string) => void, + escalateErrorFn?: (error: string, unrecoverable: boolean) => void, ): Promise<{ message: string; serverPort: number }> => { checkStart(OntimeStartOrder.InitServer); - const { serverPort } = getDataProvider().getSettings(); + // initialise logging service, escalateErrorFn only exists in electron + logger.init(escalateErrorFn); + const settings = getDataProvider().getSettings(); + const { serverPort: desiredPort } = settings; expressServer = http.createServer(app); + + // the express server must be started before the socket otherwise the on error event listener will not attach properly + const resultPort = await serverTryDesiredPort(expressServer, desiredPort); + await getDataProvider().setSettings({ ...settings, serverPort: resultPort }); + socket.init(expressServer, prefix); /** @@ -211,9 +219,6 @@ export const startServer = async ( ping: -1, }); - // initialise logging service, escalateErrorFn is only exists in electron - logger.init(escalateErrorFn); - // initialise rundown service const persistedRundown = getDataProvider().getRundown(); const persistedCustomFields = getDataProvider().getCustomFields(); @@ -225,19 +230,20 @@ export const startServer = async ( // TODO: pass event store to rundownservice runtimeService.init(maybeRestorePoint); - expressServer.listen(serverPort, '0.0.0.0', () => { - const nif = getNetworkInterfaces(); - consoleSuccess(`Local: http://localhost:${serverPort}${prefix}/editor`); - for (const key in nif) { - const address = nif[key].address; - consoleSuccess(`Network: http://${address}:${serverPort}${prefix}/editor`); - } - }); + const nif = getNetworkInterfaces(); + consoleSuccess(`Local: http://localhost:${resultPort}${prefix}/editor`); + for (const key in nif) { + const address = nif[key].address; + consoleSuccess(`Network: http://${address}:${resultPort}${prefix}/editor`); + } - const returnMessage = `Ontime is listening on port ${serverPort}`; + const returnMessage = `Ontime is listening on port ${resultPort}`; logger.info(LogOrigin.Server, returnMessage); - return { message: returnMessage, serverPort }; + return { + message: returnMessage, + serverPort: resultPort, + }; }; /** @@ -307,7 +313,7 @@ process.on('unhandledRejection', async (error) => { consoleError(error.stack); } generateCrashReport(error); - logger.crash(LogOrigin.Server, `Uncaught exception | ${error}`); + logger.crash(LogOrigin.Server, `Uncaught rejection | ${error}`); await shutdown(1); }); diff --git a/apps/server/src/classes/Logger.ts b/apps/server/src/classes/Logger.ts index 715b3c9a9..5c21d77e0 100644 --- a/apps/server/src/classes/Logger.ts +++ b/apps/server/src/classes/Logger.ts @@ -8,7 +8,7 @@ import { isProduction } from '../externals.js'; class Logger { private queue: Log[]; - private escalateErrorFn: ((error: string) => void) | null; + private escalateErrorFn: ((error: string, unrecoverable: boolean) => void) | null; private canLog = false; constructor() { @@ -20,7 +20,7 @@ class Logger { /** * Enabling setup logger after init */ - init(escalateErrorFn?: (error: string) => void) { + init(escalateErrorFn?: (error: string, unrecoverable: boolean) => void) { // flush logs from queue this.queue.forEach((log) => { this._push(log); @@ -103,8 +103,11 @@ class Logger { * @param origin * @param text */ - error(origin: string, text: string) { + error(origin: string, text: string, escalate = false) { this.emit(LogLevel.Error, origin, text); + if (escalate) { + this.escalateErrorFn?.(text, false); + } } /** @@ -114,7 +117,7 @@ class Logger { */ crash(origin: string, text: string) { this.emit(LogLevel.Severe, origin, text); - this.escalateErrorFn?.(text); + this.escalateErrorFn?.(text, true); } /** diff --git a/apps/server/src/utils/network.ts b/apps/server/src/utils/network.ts new file mode 100644 index 000000000..d48b509f6 --- /dev/null +++ b/apps/server/src/utils/network.ts @@ -0,0 +1,81 @@ +import { LogOrigin } from 'ontime-types'; +import { logger } from '../classes/Logger.js'; +import { isDocker } from '../externals.js'; +import http from 'http'; + +import { networkInterfaces } from 'os'; + +/** + * @description Gets information on IPV4 non-internal interfaces + * @returns {array} - Array of objects {name: ip} + */ +export function getNetworkInterfaces(): { name: string; address: string }[] { + const nets = networkInterfaces(); + const results: { name: string; address: string }[] = []; + + for (const name of Object.keys(nets)) { + const netObjects = nets[name]; + if (!netObjects) { + continue; + } + for (const net of netObjects) { + // Skip over non-IPv4 and internal (i.e. 127.0.0.1) addresses + if (net.family === 'IPv4' && !net.internal) { + results.push({ + name, + address: net.address, + }); + } + } + } + + return results; +} + +/** + * @description tries to open the server with the desired port, and if getting a `EADDRINUSE` will change to an random port assigned by the OS + * @param {http.Server}server http server object + * @param {number}desiredPort the desired port + * @returns {number} the resulting port number + * @throws any other server errors will result in a throw + */ +export async function serverTryDesiredPort(server: http.Server, desiredPort: number): Promise { + return new Promise((resolve, reject) => { + server.once('error', (e) => { + if (isDocker) reject(e); // we should only move ports if we are in a desktop environment + if (testForPortInUser(e)) { + server.listen(0, '0.0.0.0', () => { + const address = server.address(); + if (typeof address !== 'object') { + reject('unknown port type, can not proceed'); + return; // the return is needed here to let TS know that we wont continue + } + logger.error( + LogOrigin.Server, + `Failed open the desired port: ${desiredPort} \nMoved to an Ephemeral port: ${address.port}`, + true, + ); + + resolve(address.port); + }); + } else { + reject(e); + } + }); + server.listen(desiredPort, '0.0.0.0', () => { + const address = server.address(); + if (typeof address !== 'object') { + reject('unknown port type, can not proceed'); + return; // the return is needed here to let TS know that we wont continue + } + resolve(address.port); + }); + }); +} + +function testForPortInUser(err: unknown) { + if (typeof err === 'object' && 'code' in err && err.code === 'EADDRINUSE') { + return true; + } + return false; +} diff --git a/apps/server/src/utils/networkInterfaces.ts b/apps/server/src/utils/networkInterfaces.ts deleted file mode 100644 index 138db62c9..000000000 --- a/apps/server/src/utils/networkInterfaces.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { networkInterfaces } from 'os'; - -/** - * @description Gets information on IPV4 non-internal interfaces - * @returns {array} - Array of objects {name: ip} - */ -export function getNetworkInterfaces(): { name: string; address: string }[] { - const nets = networkInterfaces(); - const results: { name: string; address: string }[] = []; - - for (const name of Object.keys(nets)) { - const netObjects = nets[name]; - if (!netObjects) { - continue; - } - for (const net of netObjects) { - // Skip over non-IPv4 and internal (i.e. 127.0.0.1) addresses - if (net.family === 'IPv4' && !net.internal) { - results.push({ - name, - address: net.address, - }); - } - } - } - - return results; -}