refactor: initialise data

apps/server/src/services/app-state-service/AppStateService.ts

@@ -1,61 +1,40 @@
 import { Low } from 'lowdb';
 import { JSONFile } from 'lowdb/node';
 
-import { appStatePath, isTest } from '../../setup/index.js';
+import { appStatePath, isProduction, isTest } from '../../setup/index.js';
+import { isPath } from '../../utils/fileManagement.js';
+import { consoleError } from '../../utils/console.js';
 
-interface Config {
+interface AppState {
   lastLoadedProject?: string;
 }
 
-/**
- * Manages Ontime's runtime memory between boots
- */
+const adapter = new JSONFile<AppState>(appStatePath);
+const config = new Low<AppState>(adapter, {});
 
-class AppState {
-  private config: Low<Config>;
-  private pathToFile: string;
-  private didInit = false;
-
-  constructor(appStatePath: string) {
-    this.pathToFile = appStatePath;
-    const adapter = new JSONFile<Config>(this.pathToFile);
-    this.config = new Low<Config>(adapter, {});
-  }
-
-  private async init() {
-    await this.config.read();
-    await this.config.write();
-    this.didInit = true;
-  }
-
-  private async get(): Promise<Config> {
-    if (!this.didInit) {
-      await this.init();
-    }
-    await this.config.read();
-    return this.config.data;
-  }
-
-  async isLastLoadedProject(projectName: string): Promise<boolean> {
-    const lastLoaded = await this.getLastLoadedProject();
-    return lastLoaded === projectName;
-  }
-
-  async getLastLoadedProject(): Promise<string | undefined> {
-    const data = await this.get();
-    return data.lastLoadedProject;
-  }
-
-  async setLastLoadedProject(filename: string): Promise<void> {
-    if (isTest) return;
-
-    if (!this.didInit) {
-      await this.init();
-    }
-
-    this.config.data.lastLoadedProject = filename;
-    await this.config.write();
-  }
+export async function isLastLoadedProject(projectName: string): Promise<boolean> {
+  const lastLoaded = await getLastLoadedProject();
+  return lastLoaded === projectName;
 }
 
-export const appStateProvider = new AppState(appStatePath);
+export async function getLastLoadedProject(): Promise<string | undefined> {
+  // in test environment, we want to start the demo project
+  if (isTest) return;
+
+  await config.read();
+  return config.data.lastLoadedProject;
+}
+
+export async function setLastLoadedProject(filename: string): Promise<void> {
+  if (isTest) return;
+  if (!isProduction) {
+    if (isPath(filename)) {
+      consoleError(filename);
+      consoleError(new Error('setLastLoadedProject should not be called with a path').stack);
+      process.exit(0);
+    }
+  }
+
+  config.data.lastLoadedProject = filename;
+  await config.write();
+}

apps/server/src/services/project-service/ProjectService.ts

@@ -1,28 +1,44 @@
 import { DatabaseModel, GetInfo, LogOrigin, ProjectData, ProjectFileListResponse } from 'ontime-types';
+import { getErrorMessage } from 'ontime-utils';
 
 import { copyFile, rename } from 'fs/promises';
-import { join } from 'path';
 
-import { DataProvider } from '../../classes/data-provider/DataProvider.js';
 import { logger } from '../../classes/Logger.js';
 import { getNetworkInterfaces } from '../../utils/networkInterfaces.js';
 import { resolveCorruptDirectory, resolveProjectsDirectory, resolveStylesPath } from '../../setup/index.js';
-import { appendToName, ensureDirectory, removeFileExtension } from '../../utils/fileManagement.js';
+import {
+  appendToName,
+  ensureDirectory,
+  generateUniqueFileName,
+  getFileNameFromPath,
+  removeFileExtension,
+} from '../../utils/fileManagement.js';
 import { dbModel } from '../../models/dataModel.js';
 import { deleteFile } from '../../utils/parserUtils.js';
-import { switchDb } from '../../setup/loadDb.js';
-import { generateUniqueFileName } from '../../utils/generateUniqueFilename.js';
-import { parseJson } from '../../utils/parser.js';
+import { parseDatabaseModel } from '../../utils/parser.js';
+import { parseRundown } from '../../utils/parserFunctions.js';
+import { demoDb } from '../../models/demoProject.js';
+import { config } from '../../setup/config.js';
+import { getDataProvider, initPersistence } from '../../classes/data-provider/DataProvider.js';
 
 import { initRundown } from '../rundown-service/RundownService.js';
-import { appStateProvider } from '../app-state-service/AppStateService.js';
+import {
+  getLastLoadedProject,
+  isLastLoadedProject,
+  setLastLoadedProject,
+} from '../app-state-service/AppStateService.js';
 import { runtimeService } from '../runtime-service/RuntimeService.js';
 import { oscIntegration } from '../integration-service/OscIntegration.js';
 import { httpIntegration } from '../integration-service/HttpIntegration.js';
 
-import { parseProjectFile } from './projectFileUtils.js';
-import { doesProjectExist, getPathToProject, getProjectFiles } from './projectServiceUtils.js';
-import { parseRundown } from '../../utils/parserFunctions.js';
+import {
+  copyCorruptFile,
+  doesProjectExist,
+  getPathToProject,
+  getProjectFiles,
+  moveCorruptFile,
+  parseJsonFile,
+} from './projectServiceUtils.js';
 
 // init dependencies
 init();
@@ -32,39 +48,123 @@ init();
  */
 function init() {
   ensureDirectory(resolveProjectsDirectory);
+  ensureDirectory(resolveCorruptDirectory);
+}
+
+/**
+ * Private function loads a demo project
+ * to be composed in the loading functions
+ */
+async function loadDemoProject(): Promise<string> {
+  const pathToNewFile = generateUniqueFileName(resolveProjectsDirectory, config.demoProject);
+  await initPersistence(getPathToProject(pathToNewFile), demoDb);
+  const newName = getFileNameFromPath(pathToNewFile);
+  await setLastLoadedProject(newName);
+  return newName;
+}
+
+/**
+ * Private function loads a new, empty project
+ * to be composed in the loading functions
+ */
+async function loadNewProject(): Promise<string> {
+  const pathToNewFile = generateUniqueFileName(resolveProjectsDirectory, config.newProject);
+  await initPersistence(getPathToProject(pathToNewFile), dbModel);
+  const newName = getFileNameFromPath(pathToNewFile);
+  await setLastLoadedProject(newName);
+  return newName;
+}
+
+/**
+ * Private function handles side effects on currupted files
+ * Corrupted files in this context contain data that failed domain validation
+ */
+async function handleCorruptedFile(filePath: string, fileName: string): Promise<string> {
+  // copy file to corrupted folder
+  await copyCorruptFile(filePath, fileName).catch((_) => {
+    /* while we have to catch the error, we dont need to handle it */
+  });
+
+  // and make a new file with the recovered data
+  const newPath = appendToName(filePath, '(recovered)');
+  await rename(filePath, newPath);
+  return getFileNameFromPath(newPath);
+}
+
+/**
+ * Coordinates the initial load of a project on app startup
+ * This is different from the load project since we need to always load something
+ * @returns {Promise<string>} - name of the loaded file
+ */
+export async function initialiseProject(): Promise<string> {
+  // check what was loaded before
+  const previousProject = await getLastLoadedProject();
+
+  if (!previousProject) {
+    return loadDemoProject();
+  }
+
+  // try and load the previous project
+  const filePath = doesProjectExist(previousProject);
+  if (filePath === null) {
+    logger.warning(LogOrigin.Server, `Previous project file ${previousProject} not found`);
+    return loadNewProject();
+  }
+
+  try {
+    const fileData = await parseJsonFile(filePath);
+    const result = parseDatabaseModel(fileData);
+    let parsedFileName = previousProject;
+    let parsedFilePath = filePath;
+
+    if (result.errors.length > 0) {
+      logger.warning(LogOrigin.Server, 'Project loaded with errors');
+      parsedFileName = await handleCorruptedFile(filePath, previousProject);
+      parsedFilePath = getPathToProject(parsedFileName);
+    }
+
+    await initPersistence(parsedFilePath, result.data);
+    await setLastLoadedProject(parsedFileName);
+    return parsedFileName;
+  } catch (error) {
+    logger.warning(LogOrigin.Server, `Unable to load previous project ${previousProject}: ${getErrorMessage(error)}`);
+    await moveCorruptFile(filePath, previousProject).catch((_) => {
+      /* while we have to catch the error, we dont need to handle it */
+    });
+
+    return loadNewProject();
+  }
 }
 
 /**
  * Loads a data from a file into the runtime
  */
 export async function loadProjectFile(name: string) {
-  const filePath = await doesProjectExist(name);
+  const filePath = doesProjectExist(name);
   if (filePath === null) {
     throw new Error('Project file not found');
   }
 
   // when loading a project file, we allow parsing to fail and interrupt the process
-  const fileData = await parseProjectFile(filePath);
-  const result = parseJson(fileData);
+  const fileData = await parseJsonFile(filePath);
+  const result = parseDatabaseModel(fileData);
+  let parsedFileName = name;
+  let parsedFilePath = filePath;
 
   if (result.errors.length > 0) {
     logger.warning(LogOrigin.Server, 'Project loaded with errors');
-
-    // move original file to corrupted
-    ensureDirectory(resolveCorruptDirectory);
-    copyFile(filePath, join(resolveCorruptDirectory, name));
-
-    // rename file to indicate recovery
-    const newName = appendToName(filePath, '(recovered)');
-    await rename(filePath, newName);
+    parsedFileName = await handleCorruptedFile(filePath, name);
+    parsedFilePath = getPathToProject(parsedFileName);
   }
 
   // change LowDB to point to new file
-  await switchDb(filePath, result.data);
-  logger.info(LogOrigin.Server, `Loaded project ${name}`);
+  await initPersistence(parsedFilePath, result.data);
+  logger.info(LogOrigin.Server, `Loaded project ${parsedFileName}`);
 
   // persist the project selection
-  await appStateProvider.setLastLoadedProject(name);
+  await setLastLoadedProject(parsedFileName);
+
+  // since load happens at runtime, we need to update the services that depend on the data
 
   // apply data model
   runtimeService.stop();
@@ -84,7 +184,7 @@ export async function loadProjectFile(name: string) {
  */
 export async function getProjectList(): Promise<ProjectFileListResponse> {
   const files = await getProjectFiles();
-  const lastLoadedProject = await appStateProvider.getLastLoadedProject();
+  const lastLoadedProject = await getLastLoadedProject();
 
   return {
     files,
@@ -96,12 +196,12 @@ export async function getProjectList(): Promise<ProjectFileListResponse> {
  * Duplicates an existing project file
  */
 export async function duplicateProjectFile(originalFile: string, newFilename: string) {
-  const projectFilePath = await doesProjectExist(originalFile);
+  const projectFilePath = doesProjectExist(originalFile);
   if (projectFilePath === null) {
     throw new Error('Project file not found');
   }
 
-  const duplicateProjectFilePath = await doesProjectExist(newFilename);
+  const duplicateProjectFilePath = doesProjectExist(newFilename);
   if (duplicateProjectFilePath !== null) {
     throw new Error(`Project file with name ${newFilename} already exists`);
   }
@@ -114,12 +214,12 @@ export async function duplicateProjectFile(originalFile: string, newFilename: st
  * Renames an existing project file
  */
 export async function renameProjectFile(originalFile: string, newFilename: string) {
-  const projectFilePath = await doesProjectExist(originalFile);
+  const projectFilePath = doesProjectExist(originalFile);
   if (projectFilePath === null) {
     throw new Error('Project file not found');
   }
 
-  const newProjectFilePath = await doesProjectExist(newFilename);
+  const newProjectFilePath = doesProjectExist(newFilename);
   if (newProjectFilePath !== null) {
     throw new Error(`Project file with name ${newFilename} already exists`);
   }
@@ -128,17 +228,17 @@ export async function renameProjectFile(originalFile: string, newFilename: strin
   await rename(projectFilePath, pathToRenamed);
 
   // Update the last loaded project config if current loaded project is the one being renamed
-  const isLoaded = await appStateProvider.isLastLoadedProject(originalFile);
+  const isLoaded = await isLastLoadedProject(originalFile);
   if (isLoaded) {
-    const fileData = await parseProjectFile(pathToRenamed);
-    const result = parseJson(fileData);
+    const fileData = await parseJsonFile(pathToRenamed);
+    const result = parseDatabaseModel(fileData);
 
     // change LowDB to point to new file
-    await switchDb(pathToRenamed, result.data);
+    await initPersistence(pathToRenamed, result.data);
     logger.info(LogOrigin.Server, `Loaded project ${newFilename}`);
 
     // persist the project selection
-    await appStateProvider.setLastLoadedProject(newFilename);
+    await setLastLoadedProject(newFilename);
 
     // apply data model
     runtimeService.stop();
@@ -170,14 +270,14 @@ export async function createProject(filename: string, projectData: ProjectData)
   const newFile = getPathToProject(uniqueFileName);
 
   // change LowDB to point to new file
-  await switchDb(newFile, data);
+  await initPersistence(newFile, data);
 
   // apply data to running services
   // we dont need to parse since we are creating a new file
   await patchCurrentProject(data);
 
   // update app state to point to new value
-  appStateProvider.setLastLoadedProject(uniqueFileName);
+  setLastLoadedProject(uniqueFileName);
 
   return uniqueFileName;
 }
@@ -186,12 +286,12 @@ export async function createProject(filename: string, projectData: ProjectData)
  * Deletes a project file
  */
 export async function deleteProjectFile(filename: string) {
-  const isLastLoadedProject = await appStateProvider.isLastLoadedProject(filename);
-  if (isLastLoadedProject) {
+  const isPreviousProject = await isLastLoadedProject(filename);
+  if (isPreviousProject) {
     throw new Error('Cannot delete currently loaded project');
   }
 
-  const projectFilePath = await doesProjectExist(filename);
+  const projectFilePath = doesProjectExist(filename);
   if (projectFilePath === null) {
     throw new Error('Project file not found');
   }
@@ -203,8 +303,8 @@ export async function deleteProjectFile(filename: string) {
  * Adds business logic to gathering data for the info endpoint
  */
 export async function getInfo(): Promise<GetInfo> {
-  const { version, serverPort } = DataProvider.getSettings();
-  const osc = DataProvider.getOsc();
+  const { version, serverPort } = getDataProvider().getSettings();
+  const osc = getDataProvider().getOsc();
 
   // get nif and inject localhost
   const ni = getNetworkInterfaces();
@@ -226,10 +326,10 @@ export async function getInfo(): Promise<GetInfo> {
 export async function patchCurrentProject(data: Partial<DatabaseModel>) {
   runtimeService.stop();
 
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars  -- we need to remove the fields before meging
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars  -- we need to remove the fields before merging
   const { rundown, customFields, ...rest } = data;
   // we can pass some stuff straight to the data provider
-  const newData = await DataProvider.mergeIntoData(rest);
+  const newData = await getDataProvider().mergeIntoData(rest);
 
   // ... but rundown and custom fields need to be checked
   if (rundown != null) {

apps/server/src/services/project-service/__tests__/ProjectService.test.ts

@@ -1,6 +1,6 @@
 import { Mock } from 'vitest';
 
-import { appStateProvider } from '../../app-state-service/AppStateService.js';
+import { isLastLoadedProject } from '../../app-state-service/AppStateService.js';
 
 import { deleteProjectFile, duplicateProjectFile, renameProjectFile } from '../ProjectService.js';
 import { doesProjectExist } from '../projectServiceUtils.js';
@@ -13,9 +13,7 @@ vi.mock('../../../setup/loadDb.js', () => {
 });
 
 vi.mock('../../app-state-service/AppStateService.js', () => ({
-  appStateProvider: {
-    isLastLoadedProject: vi.fn(),
-  },
+  isLastLoadedProject: vi.fn(),
 }));
 
 vi.mock('../projectServiceUtils.js', () => ({
@@ -29,12 +27,12 @@ vi.mock('../projectServiceUtils.js', () => ({
  */
 describe('deleteProjectFile', () => {
   it('throws an error if trying to delete the currently loaded project', async () => {
-    (appStateProvider.isLastLoadedProject as Mock).mockResolvedValue(true);
+    (isLastLoadedProject as Mock).mockResolvedValue(true);
     await expect(deleteProjectFile('loadedProject')).rejects.toThrow('Cannot delete currently loaded project');
   });
 
   it('throws an error if the project file does not exist', async () => {
-    (appStateProvider.isLastLoadedProject as Mock).mockResolvedValue(false);
+    (isLastLoadedProject as Mock).mockResolvedValue(false);
     (doesProjectExist as Mock).mockReturnValue(null);
     await expect(deleteProjectFile('nonexistentProject')).rejects.toThrow('Project file not found');
   });

apps/server/src/services/project-service/projectFileUtils.ts

@@ -1,34 +0,0 @@
-import { readFile } from 'fs/promises';
-import { DatabaseModel } from 'ontime-types';
-import { extname } from 'path';
-
-// TODO: move to projectServiceUtils
-
-/**
- * Given an array of file names, filters out any files that do not have a '.json' extension.
- * We assume these are project files
- * @param files
- * @returns
- */
-export function filterProjectFiles(files: Array<string>): Array<string> {
-  return files.filter((file) => {
-    const ext = extname(file).toLowerCase();
-    return ext === '.json';
-  });
-}
-
-export async function parseProjectFile(filePath: string): Promise<Partial<DatabaseModel>> {
-  if (!filePath.endsWith('.json')) {
-    throw new Error('Invalid file type');
-  }
-
-  const rawdata = await readFile(filePath, 'utf-8');
-  const uploadedJson = JSON.parse(rawdata);
-
-  // at this point, we think this is a DatabaseModel
-  // verify by looking for the required fields
-  if (uploadedJson?.settings?.app !== 'ontime') {
-    throw new Error('Not an Ontime project file');
-  }
-  return uploadedJson;
-}

apps/server/src/services/project-service/projectServiceUtils.ts

@@ -1,13 +1,12 @@
-import { MaybeString, ProjectFile } from 'ontime-types';
+import { DatabaseModel, MaybeString, ProjectFile } from 'ontime-types';
 
-import { access, rename, stat } from 'fs/promises';
-import { join } from 'path';
+import { existsSync } from 'fs';
+import { copyFile, readFile, rename, stat } from 'fs/promises';
+import { extname, join } from 'path';
 
-import { resolveProjectsDirectory } from '../../setup/index.js';
+import { resolveCorruptDirectory, resolveProjectsDirectory } from '../../setup/index.js';
 import { getFilesFromFolder, removeFileExtension } from '../../utils/fileManagement.js';
 
-import { filterProjectFiles } from './projectFileUtils.js';
-
 /**
  * Handles the upload of a new project file
  * @param filePath
@@ -51,14 +50,12 @@ export async function getProjectFiles(): Promise<ProjectFile[]> {
  * Checks whether a project of a given name exists
  * @param name
  */
-export async function doesProjectExist(name: string): Promise<MaybeString> {
-  try {
-    const projectFilePath = getPathToProject(name);
-    await access(projectFilePath);
+export function doesProjectExist(name: string): MaybeString {
+  const projectFilePath = getPathToProject(name);
+  if (existsSync(projectFilePath)) {
     return projectFilePath;
-  } catch (_) {
-    return null;
   }
+  return null;
 }
 
 /**
@@ -67,3 +64,43 @@ export async function doesProjectExist(name: string): Promise<MaybeString> {
 export function getPathToProject(name: string): string {
   return join(resolveProjectsDirectory, name);
 }
+
+/**
+ * Makes a copy of a given project to the corrupted directory
+ */
+export async function copyCorruptFile(filePath: string, name: string): Promise<void> {
+  const newPath = join(resolveCorruptDirectory, name);
+  return copyFile(filePath, newPath);
+}
+
+/**
+ * Moves a file permanently to the corrupted directory
+ */
+export async function moveCorruptFile(filePath: string, name: string): Promise<void> {
+  const newPath = join(resolveCorruptDirectory, name);
+  return rename(filePath, newPath);
+}
+
+/**
+ * Given an array of file names, filters out any files that do not have a '.json' extension.
+ * We assume these are project files
+ */
+export function filterProjectFiles(files: Array<string>): Array<string> {
+  return files.filter((file) => {
+    const ext = extname(file).toLowerCase();
+    return ext === '.json';
+  });
+}
+
+/**
+ * Parses a project file and returns the JSON object
+ * @throws It will throw an error if it cannot read or parse the file
+ */
+export async function parseJsonFile(filePath: string): Promise<Partial<DatabaseModel>> {
+  if (!filePath.endsWith('.json')) {
+    throw new Error('Invalid file type');
+  }
+
+  const rawdata = await readFile(filePath, 'utf-8');
+  return JSON.parse(rawdata);
+}

apps/server/src/services/rundown-service/__tests__/rundownCache.test.ts

@@ -27,6 +27,19 @@ import {
   customFieldChangelog,
 } from '../rundownCache.js';
 
+beforeAll(() => {
+  vi.mock('../../../classes/data-provider/DataProvider.js', () => {
+    return {
+      getDataProvider: vi.fn().mockImplementation(() => {
+        return {
+          setCustomFields: vi.fn().mockImplementation((newData) => newData),
+          setRundown: vi.fn().mockImplementation((newData) => newData),
+        };
+      }),
+    };
+  });
+});
+
 describe('generate()', () => {
   it('creates normalised versions of a given rundown', () => {
     const testRundown: OntimeRundown = [
@@ -849,23 +862,6 @@ describe('calculateRuntimeDelaysFrom()', () => {
 
 describe('custom fields', () => {
   describe('createCustomField()', () => {
-    beforeEach(() => {
-      vi.mock('../../classes/data-provider/DataProvider.js', () => {
-        return {
-          DataProvider: {
-            ...vi.fn().mockImplementation(() => {
-              return {};
-            }),
-            getCustomFields: vi.fn().mockReturnValue({}),
-            setCustomFields: vi.fn().mockImplementation((newData) => {
-              return newData;
-            }),
-            persist: vi.fn().mockReturnValue({}),
-          },
-        };
-      });
-    });
-
     it('creates a field from given parameters', async () => {
       const expected = {
         lighting: {

apps/server/src/services/rundown-service/rundownCache.ts

@@ -11,7 +11,7 @@ import {
 } from 'ontime-types';
 import { generateId, insertAtIndex, reorderArray, swapEventData, checkIsNextDay } from 'ontime-utils';
 
-import { DataProvider } from '../../classes/data-provider/DataProvider.js';
+import { getDataProvider } from '../../classes/data-provider/DataProvider.js';
 import { createPatch } from '../../utils/parser.js';
 import { getTotalDuration } from '../timerUtils.js';
 import { apply } from './delayUtils.js';
@@ -59,8 +59,8 @@ export async function init(initialRundown: Readonly<OntimeRundown>, customFields
   persistedRundown = structuredClone(initialRundown) as OntimeRundown;
   persistedCustomFields = structuredClone(customFields);
   generate();
-  await DataProvider.setRundown(persistedRundown);
-  await DataProvider.setCustomFields(customFields);
+  await getDataProvider().setRundown(persistedRundown);
+  await getDataProvider().setCustomFields(customFields);
 }
 
 /**
@@ -248,8 +248,8 @@ export function mutateCache<T extends object>(mutation: MutatingFn<T>) {
     });
 
     // defer writing to the database
-    setImmediate(() => {
-      DataProvider.setRundown(persistedRundown);
+    setImmediate(async () => {
+      await getDataProvider().setRundown(persistedRundown);
     });
 
     return { newEvent, newRundown, didMutate };
@@ -413,9 +413,9 @@ function invalidateIfUsed(label: CustomFieldLabel) {
   }
   // ... and schedule a cache update
   // schedule a non priority cache update
-  setImmediate(() => {
+  setImmediate(async () => {
     generate();
-    DataProvider.setRundown(persistedRundown);
+    await getDataProvider().setRundown(persistedRundown);
   });
 }
 
@@ -424,8 +424,8 @@ function invalidateIfUsed(label: CustomFieldLabel) {
  * @param persistedCustomFields
  */
 function scheduleCustomFieldPersist(persistedCustomFields: CustomFields) {
-  setImmediate(() => {
-    DataProvider.setCustomFields(persistedCustomFields);
+  setImmediate(async () => {
+    await getDataProvider().setCustomFields(persistedCustomFields);
   });
 }