detect EADDRINUSE on server start (#1348)

* detect `EADDRINUSE` on server start

* @ts-expect-error

* send portError to electron

* use escalateErrorFn

* dont move in docker

* allow errors to be sendt to electron UI

* Log after the new port is found

* Update comment

Co-authored-by: Carlos Valente <34649812+cpvalente@users.noreply.github.com>

* update comment

Co-authored-by: Carlos Valente <34649812+cpvalente@users.noreply.github.com>

* update comment

Co-authored-by: Carlos Valente <34649812+cpvalente@users.noreply.github.com>

* rename to escalate

* extract `serverTryDesiredPort`

* type check server.address()

* request shutdown on `unrecoverable` `escalateError`

* combine network utils

* reject promise

* not unrecoverable by default

* unneeded assignment

---------

Co-authored-by: Carlos Valente <34649812+cpvalente@users.noreply.github.com>

apps/electron/src/main.js

@@ -23,7 +23,7 @@ if (!isProduction) {
 }
 
 /** Flag holds server loading state */
-let loaded = 'Ontime running';
+let loaded = 'Ontime starting';
 
 /**
  * Flag whether user has requested a quit
@@ -109,8 +109,13 @@ function askToQuit() {
  * Allows processes to escalate errors to be shown in electron
  * @param {string} error
  */
-function escalateError(error) {
+function escalateError(error, unrecoverable = false) {
-  dialog.showErrorBox('An unrecoverable error occurred', error);
+  if (unrecoverable) {
+    dialog.showErrorBox('An unrecoverable error occurred', error);
+    appShutdown();
+  } else {
+    dialog.showErrorBox('An error occurred', error);
+  }
 }
 
 /**

apps/server/src/api-data/session/session.service.ts

@@ -2,11 +2,11 @@ import { GetInfo, SessionStats } from 'ontime-types';
 
 import { getDataProvider } from '../../classes/data-provider/DataProvider.js';
 import { publicDir } from '../../setup/index.js';
-import { getNetworkInterfaces } from '../../utils/networkInterfaces.js';
 import { socket } from '../../adapters/WebsocketAdapter.js';
 import { getLastRequest } from '../../api-integration/integration.controller.js';
 import { getLastLoadedProject } from '../../services/app-state-service/AppStateService.js';
 import { runtimeService } from '../../services/runtime-service/RuntimeService.js';
+import { getNetworkInterfaces } from '../../utils/network.js';
 
 const startedAt = new Date();
 

apps/server/src/app.ts

@@ -3,7 +3,7 @@ import { LogOrigin, Playback, SimpleDirection, SimplePlayback } from 'ontime-typ
 import 'dotenv/config';
 import express from 'express';
 import expressStaticGzip from 'express-static-gzip';
-import http, { type Server } from 'http';
+import http, { Server } from 'http';
 import cors from 'cors';
 import serverTiming from 'server-timing';
 import { extname } from 'node:path';
@@ -40,8 +40,8 @@ import { initialiseProject } from './services/project-service/ProjectService.js'
 // Utilities
 import { clearUploadfolder } from './utils/upload.js';
 import { generateCrashReport } from './utils/generateCrashReport.js';
-import { getNetworkInterfaces } from './utils/networkInterfaces.js';
 import { timerConfig } from './config/config.js';
+import { serverTryDesiredPort, getNetworkInterfaces } from './utils/network.js';
 
 console.log('\n');
 consoleHighlight(`Starting Ontime version ${ONTIME_VERSION}`);
@@ -176,12 +176,20 @@ export const initAssets = async () => {
  * Starts servers
  */
 export const startServer = async (
-  escalateErrorFn?: (error: string) => void,
+  escalateErrorFn?: (error: string, unrecoverable: boolean) => void,
 ): Promise<{ message: string; serverPort: number }> => {
   checkStart(OntimeStartOrder.InitServer);
-  const { serverPort } = getDataProvider().getSettings();
+  // initialise logging service, escalateErrorFn only exists in electron
+  logger.init(escalateErrorFn);
+  const settings = getDataProvider().getSettings();
+  const { serverPort: desiredPort } = settings;
 
   expressServer = http.createServer(app);
+
+  // the express server must be started before the socket otherwise the on error event listener will not attach properly
+  const resultPort = await serverTryDesiredPort(expressServer, desiredPort);
+  await getDataProvider().setSettings({ ...settings, serverPort: resultPort });
+
   socket.init(expressServer, prefix);
 
   /**
@@ -211,9 +219,6 @@ export const startServer = async (
     ping: -1,
   });
 
-  // initialise logging service, escalateErrorFn is only exists in electron
-  logger.init(escalateErrorFn);
-
   // initialise rundown service
   const persistedRundown = getDataProvider().getRundown();
   const persistedCustomFields = getDataProvider().getCustomFields();
@@ -225,19 +230,20 @@ export const startServer = async (
   // TODO: pass event store to rundownservice
   runtimeService.init(maybeRestorePoint);
 
-  expressServer.listen(serverPort, '0.0.0.0', () => {
+  const nif = getNetworkInterfaces();
-    const nif = getNetworkInterfaces();
+  consoleSuccess(`Local: http://localhost:${resultPort}${prefix}/editor`);
-    consoleSuccess(`Local: http://localhost:${serverPort}${prefix}/editor`);
+  for (const key in nif) {
-    for (const key in nif) {
+    const address = nif[key].address;
-      const address = nif[key].address;
+    consoleSuccess(`Network: http://${address}:${resultPort}${prefix}/editor`);
-      consoleSuccess(`Network: http://${address}:${serverPort}${prefix}/editor`);
+  }
-    }
-  });
 
-  const returnMessage = `Ontime is listening on port ${serverPort}`;
+  const returnMessage = `Ontime is listening on port ${resultPort}`;
   logger.info(LogOrigin.Server, returnMessage);
 
-  return { message: returnMessage, serverPort };
+  return {
+    message: returnMessage,
+    serverPort: resultPort,
+  };
 };
 
 /**
@@ -307,7 +313,7 @@ process.on('unhandledRejection', async (error) => {
     consoleError(error.stack);
   }
   generateCrashReport(error);
-  logger.crash(LogOrigin.Server, `Uncaught exception | ${error}`);
+  logger.crash(LogOrigin.Server, `Uncaught rejection | ${error}`);
   await shutdown(1);
 });
 

apps/server/src/classes/Logger.ts

@@ -8,7 +8,7 @@ import { isProduction } from '../externals.js';
 
 class Logger {
   private queue: Log[];
-  private escalateErrorFn: ((error: string) => void) | null;
+  private escalateErrorFn: ((error: string, unrecoverable: boolean) => void) | null;
   private canLog = false;
 
   constructor() {
@@ -20,7 +20,7 @@ class Logger {
   /**
    * Enabling setup logger after init
    */
-  init(escalateErrorFn?: (error: string) => void) {
+  init(escalateErrorFn?: (error: string, unrecoverable: boolean) => void) {
     // flush logs from queue
     this.queue.forEach((log) => {
       this._push(log);
@@ -103,8 +103,11 @@ class Logger {
    * @param origin
    * @param text
    */
-  error(origin: string, text: string) {
+  error(origin: string, text: string, escalate = false) {
     this.emit(LogLevel.Error, origin, text);
+    if (escalate) {
+      this.escalateErrorFn?.(text, false);
+    }
   }
 
   /**
@@ -114,7 +117,7 @@ class Logger {
    */
   crash(origin: string, text: string) {
     this.emit(LogLevel.Severe, origin, text);
-    this.escalateErrorFn?.(text);
+    this.escalateErrorFn?.(text, true);
   }
 
   /**

apps/server/src/utils/network.ts

@@ -0,0 +1,81 @@
+import { LogOrigin } from 'ontime-types';
+import { logger } from '../classes/Logger.js';
+import { isDocker } from '../externals.js';
+import http from 'http';
+
+import { networkInterfaces } from 'os';
+
+/**
+ * @description Gets information on IPV4 non-internal interfaces
+ * @returns {array} - Array of objects {name: ip}
+ */
+export function getNetworkInterfaces(): { name: string; address: string }[] {
+  const nets = networkInterfaces();
+  const results: { name: string; address: string }[] = [];
+
+  for (const name of Object.keys(nets)) {
+    const netObjects = nets[name];
+    if (!netObjects) {
+      continue;
+    }
+    for (const net of netObjects) {
+      // Skip over non-IPv4 and internal (i.e. 127.0.0.1) addresses
+      if (net.family === 'IPv4' && !net.internal) {
+        results.push({
+          name,
+          address: net.address,
+        });
+      }
+    }
+  }
+
+  return results;
+}
+
+/**
+ * @description tries to open the server with the desired port, and if getting a `EADDRINUSE` will change to an random port assigned by the OS
+ * @param {http.Server}server http server object
+ * @param {number}desiredPort the desired port
+ * @returns {number} the resulting port number
+ * @throws any other server errors will result in a throw
+ */
+export async function serverTryDesiredPort(server: http.Server, desiredPort: number): Promise<number> {
+  return new Promise((resolve, reject) => {
+    server.once('error', (e) => {
+      if (isDocker) reject(e); // we should only move ports if we are in a desktop environment
+      if (testForPortInUser(e)) {
+        server.listen(0, '0.0.0.0', () => {
+          const address = server.address();
+          if (typeof address !== 'object') {
+            reject('unknown port type, can not proceed');
+            return; // the return is needed here to let TS know that we wont continue
+          }
+          logger.error(
+            LogOrigin.Server,
+            `Failed open the desired port: ${desiredPort} \nMoved to an Ephemeral port: ${address.port}`,
+            true,
+          );
+
+          resolve(address.port);
+        });
+      } else {
+        reject(e);
+      }
+    });
+    server.listen(desiredPort, '0.0.0.0', () => {
+      const address = server.address();
+      if (typeof address !== 'object') {
+        reject('unknown port type, can not proceed');
+        return; // the return is needed here to let TS know that we wont continue
+      }
+      resolve(address.port);
+    });
+  });
+}
+
+function testForPortInUser(err: unknown) {
+  if (typeof err === 'object' && 'code' in err && err.code === 'EADDRINUSE') {
+    return true;
+  }
+  return false;
+}

apps/server/src/utils/networkInterfaces.ts

@@ -1,28 +0,0 @@
-import { networkInterfaces } from 'os';
-
-/**
- * @description Gets information on IPV4 non-internal interfaces
- * @returns {array} - Array of objects {name: ip}
- */
-export function getNetworkInterfaces(): { name: string; address: string }[] {
-  const nets = networkInterfaces();
-  const results: { name: string; address: string }[] = [];
-
-  for (const name of Object.keys(nets)) {
-    const netObjects = nets[name];
-    if (!netObjects) {
-      continue;
-    }
-    for (const net of netObjects) {
-      // Skip over non-IPv4 and internal (i.e. 127.0.0.1) addresses
-      if (net.family === 'IPv4' && !net.internal) {
-        results.push({
-          name,
-          address: net.address,
-        });
-      }
-    }
-  }
-
-  return results;
-}