refactor: load project

apps/server/src/api-data/db/db.controller.ts

@@ -14,11 +14,7 @@ import type { Request, Response } from 'express';
 
 import { failEmptyObjects } from '../../utils/routerUtils.js';
 import { resolveDbDirectory } from '../../setup/index.js';
-
+import { handleUploaded } from '../../services/project-service/projectServiceUtils.js';
-import { doesProjectExist, upload } from '../../services/project-service/projectServiceUtils.js';
-import { oscIntegration } from '../../services/integration-service/OscIntegration.js';
-import { httpIntegration } from '../../services/integration-service/HttpIntegration.js';
-import { DataProvider } from '../../classes/data-provider/DataProvider.js';
 import * as projectService from '../../services/project-service/ProjectService.js';
 
 export async function patchPartialProjectFile(req: Request, res: Response<DatabaseModel | ErrorResponse>) {
@@ -94,6 +90,7 @@ export async function projectDownload(req: Request, res: Response) {
 
 /**
  * uploads, parses and applies the data from a given file
+ * Pretty much loadProject but with the extra upload step
  */
 export async function postProjectFile(req: Request, res: Response<MessageResponse | ErrorResponse>) {
   if (!req.file) {
@@ -102,24 +99,18 @@ export async function postProjectFile(req: Request, res: Response<MessageRespons
   }
 
   try {
-    const options = req.query;
     const { filename, path } = req.file;
-
+    await handleUploaded(path, filename);
-    // TODO: controller shouldnt consume this directly
+    await projectService.loadProjectFile(filename);
-    await upload(path, filename);
-    await projectService.applyProjectFile(filename, options);
-
-    const oscSettings = await DataProvider.getOsc();
-    const httpSettings = await DataProvider.getHttp();
-
-    oscIntegration.init(oscSettings);
-    httpIntegration.init(httpSettings);
 
     res.status(201).send({
       message: `Loaded project ${filename}`,
     });
   } catch (error) {
     const message = getErrorMessage(error);
+    if (message.startsWith('Project file')) {
+      return res.status(403).send({ message });
+    }
     res.status(400).send({ message });
   }
 }
@@ -143,23 +134,16 @@ export async function listProjects(_req: Request, res: Response<ProjectFileListR
 export async function loadProject(req: Request, res: Response<MessageResponse | ErrorResponse>) {
   try {
     const name = req.body.filename;
-    if (!doesProjectExist(name)) {
+    await projectService.loadProjectFile(name);
-      return res.status(404).send({ message: 'File not found' });
-    }
-
-    await projectService.applyProjectFile(name);
-
-    const oscSettings = await DataProvider.getOsc();
-    const httpSettings = await DataProvider.getHttp();
-
-    oscIntegration.init(oscSettings);
-    httpIntegration.init(httpSettings);
 
     res.status(201).send({
       message: `Loaded project ${name}`,
     });
   } catch (error) {
     const message = getErrorMessage(error);
+    if (message.startsWith('Project file')) {
+      return res.status(403).send({ message });
+    }
     res.status(500).send({ message });
   }
 }

apps/server/src/classes/data-provider/DataProvider.ts

@@ -106,6 +106,7 @@ export class DataProvider {
   }
 
   private static async persist() {
+    // TODO: this is already handled by lowDb
     if (isTest) {
       return;
     }

apps/server/src/services/project-service/ProjectService.ts

@@ -1,20 +1,27 @@
-import { DatabaseModel, GetInfo, ProjectData, ProjectFileListResponse } from 'ontime-types';
+import { DatabaseModel, GetInfo, LogOrigin, ProjectData, ProjectFileListResponse } from 'ontime-types';
 
 import { copyFile, rename } from 'fs/promises';
+import { join } from 'path';
 
-import { initRundown } from '../rundown-service/RundownService.js';
 import { DataProvider } from '../../classes/data-provider/DataProvider.js';
-import { runtimeService } from '../runtime-service/RuntimeService.js';
+import { logger } from '../../classes/Logger.js';
 import { getNetworkInterfaces } from '../../utils/networkInterfaces.js';
-import { resolveProjectsDirectory, resolveStylesPath } from '../../setup/index.js';
+import { resolveCorruptDirectory, resolveProjectsDirectory, resolveStylesPath } from '../../setup/index.js';
-import { parseProjectFile } from './projectFileUtils.js';
+import { appendToName, ensureDirectory, removeFileExtension } from '../../utils/fileManagement.js';
-import { appStateProvider } from '../app-state-service/AppStateService.js';
-import { ensureDirectory, removeFileExtension } from '../../utils/fileManagement.js';
 import { dbModel } from '../../models/dataModel.js';
 import { deleteFile } from '../../utils/parserUtils.js';
 import { switchDb } from '../../setup/loadDb.js';
-import { doesProjectExist, getPathToProject, getProjectFiles } from './projectServiceUtils.js';
 import { generateUniqueFileName } from '../../utils/generateUniqueFilename.js';
+import { parseJson } from '../../utils/parser.js';
+
+import { initRundown } from '../rundown-service/RundownService.js';
+import { appStateProvider } from '../app-state-service/AppStateService.js';
+import { runtimeService } from '../runtime-service/RuntimeService.js';
+import { oscIntegration } from '../integration-service/OscIntegration.js';
+import { httpIntegration } from '../integration-service/HttpIntegration.js';
+
+import { parseProjectFile } from './projectFileUtils.js';
+import { doesProjectExist, getPathToProject, getProjectFiles } from './projectServiceUtils.js';
 
 // init dependencies
 init();
@@ -26,25 +33,50 @@ function init() {
   ensureDirectory(resolveProjectsDirectory);
 }
 
-type Options = {
-  onlyRundown?: 'true' | 'false';
-};
-
 /**
- * Handles a file from the upload folder and applies its data
+ * Loads a data from a file into the runtime
  */
-export async function applyProjectFile(name: string, options?: Options) {
+export async function loadProjectFile(name: string) {
+  if (!(await doesProjectExist(name))) {
+    throw new Error('Project file not found');
+  }
+
   const filePath = getPathToProject(name);
-  const data = parseProjectFile(filePath);
+
+  // when loading a project file, we allow parsing to fail and interrupt the process
+  const fileData = await parseProjectFile(filePath);
+  const result = parseJson(fileData);
+
+  if (result.errors.length > 0) {
+    logger.warning(LogOrigin.Server, 'Project loaded with errors');
+
+    // move original file to corrupted
+    ensureDirectory(resolveCorruptDirectory);
+    copyFile(filePath, join(resolveCorruptDirectory, name));
+
+    // rename file to indicate recovery
+    const newName = appendToName(filePath, '(recovered)');
+    await rename(filePath, newName);
+  }
 
   // change LowDB to point to new file
-  await switchDb(filePath);
+  await switchDb(filePath, result.data);
-
+  logger.info(LogOrigin.Server, `Loaded project ${name}`);
-  // apply data model
-  await applyDataModel(data, options);
 
   // persist the project selection
   await appStateProvider.setLastLoadedProject(name);
+
+  // apply data model
+  runtimeService.stop();
+
+  const { rundown, customFields, osc, http } = result.data;
+
+  // apply the rundown
+  initRundown(rundown, customFields);
+
+  // apply integrations
+  oscIntegration.init(osc);
+  httpIntegration.init(http);
 }
 
 /**
@@ -64,11 +96,11 @@ export async function getProjectList(): Promise<ProjectFileListResponse> {
  * Duplicates an existing project file
  */
 export async function duplicateProjectFile(originalFile: string, newFilename: string) {
-  if (!doesProjectExist(originalFile)) {
+  if (!(await doesProjectExist(originalFile))) {
     throw new Error('Project file not found');
   }
 
-  if (doesProjectExist(newFilename)) {
+  if (await doesProjectExist(newFilename)) {
     throw new Error(`Project file with name ${newFilename} already exists`);
   }
 
@@ -82,11 +114,11 @@ export async function duplicateProjectFile(originalFile: string, newFilename: st
  * Renames an existing project file
  */
 export async function renameProjectFile(originalFile: string, newFilename: string) {
-  if (!doesProjectExist(originalFile)) {
+  if (!(await doesProjectExist(originalFile))) {
     throw new Error('Project file not found');
   }
 
-  if (doesProjectExist(newFilename)) {
+  if (await doesProjectExist(newFilename)) {
     throw new Error(`Project file with name ${newFilename} already exists`);
   }
 
@@ -98,7 +130,27 @@ export async function renameProjectFile(originalFile: string, newFilename: strin
   // Update the last loaded project config if current loaded project is the one being renamed
   const isLoaded = await appStateProvider.isLastLoadedProject(originalFile);
   if (isLoaded) {
-    await applyProjectFile(newFilename);
+    const fileData = await parseProjectFile(newProjectFilePath);
+    const result = parseJson(fileData);
+
+    // change LowDB to point to new file
+    await switchDb(newProjectFilePath, result.data);
+    logger.info(LogOrigin.Server, `Loaded project ${newFilename}`);
+
+    // persist the project selection
+    await appStateProvider.setLastLoadedProject(newFilename);
+
+    // apply data model
+    runtimeService.stop();
+
+    const { rundown, customFields, osc, http } = result.data;
+
+    // apply the rundown
+    initRundown(rundown, customFields);
+
+    // apply integrations
+    oscIntegration.init(osc);
+    httpIntegration.init(http);
   }
 }
 
@@ -139,7 +191,7 @@ export async function deleteProjectFile(filename: string) {
     throw new Error('Cannot delete currently loaded project');
   }
 
-  if (!doesProjectExist(filename)) {
+  if (!(await doesProjectExist(filename))) {
     throw new Error('Project file not found');
   }
 
@@ -172,7 +224,7 @@ export async function getInfo(): Promise<GetInfo> {
  * applies a partial database model
  */
 // TODO: should be private as part of a load
-export async function applyDataModel(data: Partial<DatabaseModel>, _options?: Options) {
+export async function applyDataModel(data: Partial<DatabaseModel>) {
   runtimeService.stop();
 
   // TODO: allow partial project merge from options

apps/server/src/services/project-service/projectFileUtils.ts

@@ -1,6 +1,9 @@
-import { readFileSync } from 'fs';
+import { readFile } from 'fs/promises';
+import { DatabaseModel } from 'ontime-types';
 import { extname } from 'path';
 
+// TODO: move to projectServiceUtils
+
 /**
  * Given an array of file names, filters out any files that do not have a '.json' extension.
  * We assume these are project files
@@ -14,18 +17,18 @@ export function filterProjectFiles(files: Array<string>): Array<string> {
   });
 }
 
-export function parseProjectFile(filePath: string): object {
+export async function parseProjectFile(filePath: string): Promise<Partial<DatabaseModel>> {
   if (!filePath.endsWith('.json')) {
     throw new Error('Invalid file type');
   }
 
-  const rawdata = readFileSync(filePath, 'utf-8');
+  const rawdata = await readFile(filePath, 'utf-8');
   const uploadedJson = JSON.parse(rawdata);
 
   // at this point, we think this is a DatabaseModel
   // verify by looking for the required fields
   if (uploadedJson?.settings?.app !== 'ontime') {
-    throw new Error('Not a ontime project file');
+    throw new Error('Not an Ontime project file');
   }
   return uploadedJson;
 }

apps/server/src/services/project-service/projectServiceUtils.ts

@@ -1,24 +1,21 @@
 import { ProjectFile } from 'ontime-types';
 
-import { stat } from 'fs/promises';
+import { access, rename, stat } from 'fs/promises';
-import { existsSync } from 'fs';
 import { join } from 'path';
 
 import { resolveProjectsDirectory } from '../../setup/index.js';
-import { filterProjectFiles } from './projectFileUtils.js';
 import { getFilesFromFolder, removeFileExtension } from '../../utils/fileManagement.js';
-import { moveUploadedFile } from '../../utils/upload.js';
+
+import { filterProjectFiles } from './projectFileUtils.js';
 
 /**
  * Handles the upload of a new project file
  * @param filePath
  * @param name
- * @returns
  */
-export async function upload(filePath: string, name: string) {
+export async function handleUploaded(filePath: string, name: string) {
   const newFilePath = join(resolveProjectsDirectory, name);
-  await moveUploadedFile(filePath, newFilePath);
+  await rename(filePath, newFilePath);
-  return name;
 }
 
 /**
@@ -54,9 +51,14 @@ export async function getProjectFiles(): Promise<ProjectFile[]> {
  * Checks whether a project of a given name exists
  * @param name
  */
-export function doesProjectExist(name: string): boolean {
+export async function doesProjectExist(name: string): Promise<boolean> {
-  const projectFilePath = join(resolveProjectsDirectory, name);
+  try {
-  return existsSync(projectFilePath);
+    const projectFilePath = join(resolveProjectsDirectory, name);
+    await access(projectFilePath);
+    return true;
+  } catch (_) {
+    return false;
+  }
 }
 
 /**

apps/server/src/setup/config.ts

@@ -1,5 +1,6 @@
 export const config = {
   appState: 'app-state.json',
+  corrupt: 'corrupt files',
   crash: 'crash logs',
   database: {
     testdb: 'test-db',

apps/server/src/setup/index.ts

@@ -56,10 +56,10 @@ const currentDir = dirname(__dirname);
 // locally we are in src/setup, in the production build, this is a single file at src
 export const srcDirectory = isProduction ? currentDir : join(currentDir, '../');
 
-// resolve path to external
+// TODO: simplify logic
+// resolve path to client
 const productionPath = join(srcDirectory, 'client/');
 const devPath = join(srcDirectory, '../../client/build/');
-
 export const resolvedPath = (): string => {
   if (isTest) {
     return devPath;
@@ -136,7 +136,10 @@ export const resolveRestoreFile = join(getAppDataPath(), config.restoreFile);
 export const resolveSheetsDirectory = join(getAppDataPath(), config.sheets.directory);
 
 // path to crash reports
-export const resolveCrashReportDirectory = getAppDataPath();
+export const resolveCrashReportDirectory = join(getAppDataPath(), config.crash);
 
 // path to projects
 export const resolveProjectsDirectory = join(getAppDataPath(), config.projects);
+
+// path to corrupt files
+export const resolveCorruptDirectory = join(getAppDataPath(), config.corrupt);

apps/server/src/setup/loadDb.ts

@@ -56,7 +56,7 @@ async function loadDb(directory: string, filename: string) {
   let newData: DatabaseModel = dbModel;
 
   try {
-    const maybeProjectFile = parseProjectFile(dbInDisk);
+    const maybeProjectFile = await parseProjectFile(dbInDisk);
     const result = parseJson(maybeProjectFile);
 
     await appStateProvider.setLastLoadedProject(filename);

apps/server/src/utils/__tests__/fileManagement.test.ts

@@ -1,5 +1,5 @@
 import { describe, it, expect } from 'vitest';
-import { ensureJsonExtension } from '../fileManagement.js';
+import { appendToName, ensureJsonExtension } from '../fileManagement.js';
 
 describe('ensureJsonExtension', () => {
   it('should add .json to a filename without an extension', () => {
@@ -26,3 +26,26 @@ describe('ensureJsonExtension', () => {
     expect(result).toBe('my.test.file.json');
   });
 });
+
+describe('appendToName', () => {
+  it('appends a given string to a file name', () => {
+    const filename = 'file.json';
+    const append = '(recovered)';
+    const result = appendToName(filename, append);
+    expect(result).toBe('file (recovered).json');
+  });
+
+  it('handles paths', () => {
+    const path = '/Users/carlos/Library/Application Support/Ontime/projects/file.json';
+    const append = '(recovered)';
+    const result = appendToName(path, append);
+    expect(result).toBe('/Users/carlos/Library/Application Support/Ontime/projects/file (recovered).json');
+  });
+
+  it('handles multiple . in string', () => {
+    const path = 'strange.file.name.json';
+    const append = '(recovered)';
+    const result = appendToName(path, append);
+    expect(result).toBe('strange.file.name (recovered).json');
+  });
+});

apps/server/src/utils/fileManagement.ts

@@ -28,7 +28,7 @@ export function ensureJsonExtension(filename: string | undefined): string | unde
  * Lists all files in a directory
  */
 export async function getFilesFromFolder(folderPath: string): Promise<string[]> {
-  return await readdir(folderPath);
+  return readdir(folderPath);
 }
 
 /**
@@ -38,3 +38,12 @@ export async function getFilesFromFolder(folderPath: string): Promise<string[]>
 export const removeFileExtension = (filename: string): string => {
   return parse(filename).name;
 };
+
+/**
+ * Appends a given string to a file name or path
+ * @example appendToName('file.json', '(recovered)') => 'file (recovered).json'
+ */
+export function appendToName(filePath: string, append: string): string {
+  const extension = filePath.split('.').pop();
+  return filePath.replace(`.${extension}`, ` ${append}.${extension}`);
+}

apps/server/src/utils/upload.ts

@@ -1,11 +1,10 @@
 import multer from 'multer';
 import path from 'path';
 import fs from 'fs';
-import { rename, rm } from 'fs/promises';
+import { rm } from 'fs/promises';
 
 import { ensureDirectory } from './fileManagement.js';
 import { getAppDataPath, uploadsFolderPath } from '../setup/index.js';
-import { deleteFile } from './parserUtils.js';
 
 function generateNewFileName(filePath: string, callback: (newName: string) => void) {
   const baseName = path.basename(filePath, path.extname(filePath));
@@ -69,14 +68,3 @@ export async function clearUploadfolder() {
     // we dont care that there was no folder
   }
 }
-
-/**
- * Copies a file from upload folder to a destination
- * @param filePath
- * @param name
- * @returns
- */
-export async function moveUploadedFile(fromUpload: string, toDestination: string) {
-  await rename(fromUpload, toDestination);
-  await deleteFile(fromUpload);
-}