Project files tweaks (#679)

* chore: fixed issue when loading first time; deleting excel file after upload; only json files from project list

* chore: added last loaded project to endpoint response

* chore: incremental file name

---------

Co-authored-by: Name <Email>

apps/server/src/controllers/ontimeController.ts

@@ -5,8 +5,8 @@ import type {
   GetInfo,
   HttpSettings,
   ProjectData,
-  ProjectFileList,
   ErrorResponse,
+  ProjectFileListResponse,
 } from 'ontime-types';
 
 import { RequestHandler, Request, Response } from 'express';
@@ -19,7 +19,7 @@ import { DataProvider } from '../classes/data-provider/DataProvider.js';
 import { failEmptyObjects, failIsNotArray } from '../utils/routerUtils.js';
 import { PlaybackService } from '../services/PlaybackService.js';
 import { eventStore } from '../stores/EventStore.js';
-import { getAppDataPath, isDocker, resolveDbPath, resolveStylesPath } from '../setup.js';
+import { getAppDataPath, isDocker, lastLoadedProjectConfigPath, resolveDbPath, resolveStylesPath } from '../setup.js';
 import { oscIntegration } from '../services/integration-service/OscIntegration.js';
 import { httpIntegration } from '../services/integration-service/HttpIntegration.js';
 import { logger } from '../classes/Logger.js';
@@ -28,7 +28,7 @@ import { deepmerge } from 'ontime-utils';
 import { runtimeCacheStore } from '../stores/cachingStore.js';
 import { delayedRundownCacheKey } from '../services/rundown-service/delayedRundown.utils.js';
 import { integrationService } from '../services/integration-service/IntegrationService.js';
-import { getFileListFromFolder } from '../utils/getFileListFromFolder.js';
+import { getProjectFiles } from '../utils/getFileListFromFolder.js';
 
 // Create controller for GET request to '/ontime/poll'
 // Returns data for current state
@@ -471,11 +471,16 @@ export const postNew: RequestHandler = async (req, res) => {
  * @param req
  * @param res
  */
-export const listProjects: RequestHandler = (_, res: Response<ProjectFileList | ErrorResponse>) => {
+export const listProjects: RequestHandler = async (_, res: Response<ProjectFileListResponse | ErrorResponse>) => {
   try {
-    const uploadsFolderPath = join(getAppDataPath(), 'uploads');
-    const fileList = getFileListFromFolder(uploadsFolderPath);
-    res.status(200).send(fileList);
+    const fileList = await getProjectFiles();
+
+    const lastLoadedProject = JSON.parse(fs.readFileSync(lastLoadedProjectConfigPath, 'utf8')).lastLoadedProject;
+
+    res.status(200).send({
+      files: fileList,
+      lastLoadedProject,
+    });
   } catch (error) {
     res.status(500).send({ message: error.toString() });
   }

apps/server/src/setup.ts

@@ -3,6 +3,7 @@ import path, { dirname, join } from 'path';
 import fs from 'fs';
 
 import { config } from './config/config.js';
+import { ensureDirectory } from './utils/fileManagement.js';
 
 // =================================================
 // resolve public path
@@ -71,7 +72,7 @@ export const currentDirectory = dirname(__dirname);
 const testDbStartDirectory = isTest ? '../' : getAppDataPath();
 export const externalsStartDirectory = isProduction ? getAppDataPath() : join(currentDirectory, 'external');
 
-const lastLoadedProjectConfigPath = join(getAppDataPath(), 'config.json');
+export const lastLoadedProjectConfigPath = join(getAppDataPath(), 'config.json');
 
 let lastLoadedProject;
 
@@ -79,7 +80,8 @@ try {
   lastLoadedProject = JSON.parse(fs.readFileSync(lastLoadedProjectConfigPath, 'utf8')).lastLoadedProject;
 } catch {
   if (!isTest) {
-    fs.writeFileSync(lastLoadedProjectConfigPath, JSON.stringify({ lastLoadedProject: null }));
+    ensureDirectory(getAppDataPath());
+    fs.writeFileSync(lastLoadedProjectConfigPath, JSON.stringify({ lastLoadedProject: 'default.json' }));
   }
 }
 

apps/server/src/utils/__tests__/getFileListFromFolder.test.ts

@@ -0,0 +1,33 @@
+import { expect, vi } from 'vitest';
+import { getProjectFiles } from '../getFileListFromFolder.js';
+
+vi.mock('fs/promises', () => {
+  const mockFiles = ['file1.json', 'file2.json', 'file3.json', 'document.txt', 'image.png'];
+  const mockStats = {
+    birthtime: new Date('2020-01-01'),
+    mtime: new Date('2021-01-01'),
+  };
+
+  return {
+    readdir: vi.fn().mockResolvedValue(mockFiles),
+    stat: vi.fn().mockResolvedValue(mockStats),
+  };
+});
+
+describe('getProjectFiles test', () => {
+  it('should return a list of project .json files', async () => {
+    const { readdir, stat } = await import('fs/promises');
+
+    const result = await getProjectFiles();
+
+    const expectedFiles = ['file1.json', 'file2.json', 'file3.json'].map((file) => ({
+      filename: file,
+      createdAt: new Date('2020-01-01').toISOString(),
+      updatedAt: new Date('2021-01-01').toISOString(),
+    }));
+
+    expect(result).toEqual(expectedFiles);
+    expect(readdir).toHaveBeenCalled();
+    expect(stat).toHaveBeenCalledTimes(expectedFiles.length);
+  });
+});

apps/server/src/utils/getFileListFromFolder.ts

@@ -1,17 +1,55 @@
 import { ProjectFile } from 'ontime-types';
 
-import { readdirSync, statSync } from 'fs';
+import { getAppDataPath } from '../setup.js';
 
-export const getFileListFromFolder = (folderPath: string): Array<ProjectFile> => {
-  const files = readdirSync(folderPath);
-  return files.map((file) => {
-    const filePath = `${folderPath}/${file}`;
-    const stats = statSync(filePath);
+import { extname, join } from 'path';
+import { readdir, stat } from 'fs/promises';
 
-    return {
-      filename: file,
-      createdAt: stats.birthtime.toISOString(),
-      updatedAt: stats.mtime.toISOString(),
-    };
+const getFilesFromFolder = async (folderPath: string) => {
+  return await readdir(folderPath);
+};
+
+const filterProjectFiles = (files: Array<string>): Array<string> => {
+  return files.filter((file) => {
+    const ext = extname(file).toLowerCase();
+    return ext === '.json';
   });
 };
+
+/**
+ * Asynchronously retrieves and returns an array of project files from the 'uploads' folder.
+ * Each file in the 'uploads' folder is checked, and only those with a '.json' extension are processed.
+ * For each qualifying file, its metadata is retrieved, including filename, creation time, and last modification time.
+ *
+ * @returns {Promise<Array<ProjectFile>>} A promise that resolves to an array of ProjectFile objects,
+ *                                        each representing a file in the 'uploads' folder with its metadata.
+ *                                        The metadata includes the filename, creation time (createdAt),
+ *                                        and last modification time (updatedAt) of each file.
+ *
+ * @throws {Error} Throws an error if there is an issue in reading the directory or fetching file statistics.
+ */
+export const getProjectFiles = async (): Promise<ProjectFile[]> => {
+  const uploadsFolderPath = join(getAppDataPath(), 'uploads');
+
+  try {
+    const allFiles = await getFilesFromFolder(uploadsFolderPath);
+    const filteredFiles = filterProjectFiles(allFiles);
+
+    const projectFiles = [];
+    for (const file of filteredFiles) {
+      const filePath = join(uploadsFolderPath, file);
+      const stats = await stat(filePath);
+
+      projectFiles.push({
+        filename: file,
+        createdAt: stats.birthtime.toISOString(),
+        updatedAt: stats.mtime.toISOString(),
+      });
+    }
+
+    return projectFiles;
+  } catch (err) {
+    console.error(err);
+    throw err;
+  }
+};

apps/server/src/utils/parser.ts

@@ -25,7 +25,7 @@ import path from 'path';
 
 import { event as eventDef } from '../models/eventsDefinition.js';
 import { dbModel } from '../models/dataModel.js';
-import { makeString } from './parserUtils.js';
+import { deleteFile, makeString } from './parserUtils.js';
 import {
   parseAliases,
   parseProject,
@@ -379,7 +379,7 @@ export const fileHandler = async (file: string, options: ExcelImportOptions): Pr
     res.data.project = parseProject(dataFromExcel);
     res.data.userFields = parseUserFields(dataFromExcel);
 
-    await configService.updateDatabaseConfig(fileName);
+    await deleteFile(file);
 
     return res;
   }

apps/server/src/utils/upload.js

@@ -1,5 +1,6 @@
 import multer from 'multer';
 import path from 'path';
+import fs from 'fs';
 
 import { EXCEL_MIME, JSON_MIME } from './parser.js';
 import { ensureDirectory } from './fileManagement.js';
@@ -8,17 +9,26 @@ import { getAppDataPath } from '../setup.js';
 // Define multer storage object
 const storage = multer.diskStorage({
   destination: function (req, file, cb) {
-    // get platform path
     const appDataPath = getAppDataPath();
     if (appDataPath === '') {
       throw new Error('Could not resolve public folder for platform');
     }
-    // append uploads folder
-    const newDestination = path.join(appDataPath, 'uploads');
 
-    // Create directory if not exist
-    ensureDirectory(newDestination);
-    cb(null, newDestination);
+    const uploadsPath = path.join(appDataPath, 'uploads');
+    ensureDirectory(uploadsPath);
+
+    const filePath = path.join(uploadsPath, file.originalname);
+
+    // Check if file already exists
+    fs.access(filePath, fs.constants.F_OK, (err) => {
+      if (err) {
+        // File does not exist, can safely proceed to this destination
+        cb(null, uploadsPath);
+      } else {
+        // File already exists, handle error
+        return cb(new Error('File already exists'), false);
+      }
+    });
   },
   filename: function (_, file, cb) {
     cb(null, file.originalname);

apps/test-db/default.json

@@ -0,0 +1,483 @@
+{
+  "rundown": [
+    {
+      "title": "Albania",
+      "subtitle": "Sekret",
+      "presenter": "Ronela Hajati",
+      "note": "SF1.01",
+      "endAction": "none",
+      "timerType": "count-down",
+      "timeStart": 36000000,
+      "timeEnd": 37200000,
+      "duration": 1200000,
+      "isPublic": true,
+      "skip": false,
+      "colour": "",
+      "user0": "",
+      "user1": "",
+      "user2": "",
+      "user3": "",
+      "user4": "",
+      "user5": "",
+      "user6": "",
+      "user7": "",
+      "user8": "",
+      "user9": "",
+      "type": "event",
+      "revision": 0,
+      "cue": "SF1.01",
+      "id": "32d31"
+    },
+    {
+      "title": "Latvia",
+      "subtitle": "Eat Your Salad",
+      "presenter": "Citi Zeni",
+      "note": "SF1.02",
+      "endAction": "none",
+      "timerType": "count-down",
+      "timeStart": 37500000,
+      "timeEnd": 38700000,
+      "duration": 1200000,
+      "isPublic": true,
+      "skip": false,
+      "colour": "",
+      "user0": "",
+      "user1": "",
+      "user2": "",
+      "user3": "",
+      "user4": "",
+      "user5": "",
+      "user6": "",
+      "user7": "",
+      "user8": "",
+      "user9": "",
+      "type": "event",
+      "revision": 0,
+      "cue": "SF1.02",
+      "id": "21cd2"
+    },
+    {
+      "title": "Lithuania",
+      "subtitle": "Sentimentai",
+      "presenter": "Monika Liu",
+      "note": "SF1.03",
+      "endAction": "none",
+      "timerType": "count-down",
+      "timeStart": 39000000,
+      "timeEnd": 40200000,
+      "duration": 1200000,
+      "isPublic": true,
+      "skip": false,
+      "colour": "",
+      "user0": "",
+      "user1": "",
+      "user2": "",
+      "user3": "",
+      "user4": "",
+      "user5": "",
+      "user6": "",
+      "user7": "",
+      "user8": "",
+      "user9": "",
+      "type": "event",
+      "revision": 0,
+      "cue": "SF1.03",
+      "id": "0b371"
+    },
+    {
+      "title": "Switzerland",
+      "subtitle": "Boys Do Cry",
+      "presenter": "Marius Bear",
+      "note": "SF1.04",
+      "endAction": "none",
+      "timerType": "count-down",
+      "timeStart": 40500000,
+      "timeEnd": 41700000,
+      "duration": 1200000,
+      "isPublic": true,
+      "skip": false,
+      "colour": "",
+      "user0": "",
+      "user1": "",
+      "user2": "",
+      "user3": "",
+      "user4": "",
+      "user5": "",
+      "user6": "",
+      "user7": "",
+      "user8": "",
+      "user9": "",
+      "type": "event",
+      "revision": 0,
+      "cue": "SF1.04",
+      "id": "3cd28"
+    },
+    {
+      "title": "Slovenia",
+      "subtitle": "Disko",
+      "presenter": "LPS",
+      "note": "SF1.05",
+      "endAction": "none",
+      "timerType": "count-down",
+      "timeStart": 42000000,
+      "timeEnd": 43200000,
+      "duration": 1200000,
+      "isPublic": true,
+      "skip": false,
+      "colour": "",
+      "user0": "",
+      "user1": "",
+      "user2": "",
+      "user3": "",
+      "user4": "",
+      "user5": "",
+      "user6": "",
+      "user7": "",
+      "user8": "",
+      "user9": "",
+      "type": "event",
+      "revision": 0,
+      "cue": "SF1.05",
+      "id": "e457f"
+    },
+    {
+      "title": "Lunch break",
+      "type": "block",
+      "id": "01e85"
+    },
+    {
+      "title": "Ukraine",
+      "subtitle": "Stefania",
+      "presenter": "Kalush Orchestra",
+      "note": "SF1.06",
+      "endAction": "none",
+      "timerType": "count-down",
+      "timeStart": 47100000,
+      "timeEnd": 48300000,
+      "duration": 1200000,
+      "isPublic": true,
+      "skip": false,
+      "colour": "",
+      "user0": "",
+      "user1": "",
+      "user2": "",
+      "user3": "",
+      "user4": "",
+      "user5": "",
+      "user6": "",
+      "user7": "",
+      "user8": "",
+      "user9": "",
+      "type": "event",
+      "revision": 0,
+      "cue": "SF1.06",
+      "id": "1c420"
+    },
+    {
+      "title": "Bulgaria",
+      "subtitle": "Intention",
+      "presenter": "Intelligent Music Project",
+      "note": "SF1.07",
+      "endAction": "none",
+      "timerType": "count-down",
+      "timeStart": 48600000,
+      "timeEnd": 49800000,
+      "duration": 1200000,
+      "isPublic": true,
+      "skip": false,
+      "colour": "",
+      "user0": "",
+      "user1": "",
+      "user2": "",
+      "user3": "",
+      "user4": "",
+      "user5": "",
+      "user6": "",
+      "user7": "",
+      "user8": "",
+      "user9": "",
+      "type": "event",
+      "revision": 0,
+      "cue": "SF1.07",
+      "id": "b7737"
+    },
+    {
+      "title": "Netherlands",
+      "subtitle": "De Diepte",
+      "presenter": "S10",
+      "note": "SF1.08",
+      "endAction": "none",
+      "timerType": "count-down",
+      "timeStart": 50100000,
+      "timeEnd": 51300000,
+      "duration": 1200000,
+      "isPublic": true,
+      "skip": false,
+      "colour": "",
+      "user0": "",
+      "user1": "",
+      "user2": "",
+      "user3": "",
+      "user4": "",
+      "user5": "",
+      "user6": "",
+      "user7": "",
+      "user8": "",
+      "user9": "",
+      "type": "event",
+      "revision": 0,
+      "cue": "SF1.08",
+      "id": "d3a80"
+    },
+    {
+      "title": "Moldova",
+      "subtitle": "Trenuletul",
+      "presenter": "Zdob si Zdub",
+      "note": "SF1.09",
+      "endAction": "none",
+      "timerType": "count-down",
+      "timeStart": 51600000,
+      "timeEnd": 52800000,
+      "duration": 1200000,
+      "isPublic": true,
+      "skip": false,
+      "colour": "",
+      "user0": "",
+      "user1": "",
+      "user2": "",
+      "user3": "",
+      "user4": "",
+      "user5": "",
+      "user6": "",
+      "user7": "",
+      "user8": "",
+      "user9": "",
+      "type": "event",
+      "revision": 0,
+      "cue": "SF1.09",
+      "id": "8276c"
+    },
+    {
+      "title": "Portugal",
+      "subtitle": "Saudade Saudade",
+      "presenter": "Maro",
+      "note": "SF1.10",
+      "endAction": "none",
+      "timerType": "count-down",
+      "timeStart": 53100000,
+      "timeEnd": 54300000,
+      "duration": 1200000,
+      "isPublic": true,
+      "skip": false,
+      "colour": "",
+      "user0": "",
+      "user1": "",
+      "user2": "",
+      "user3": "",
+      "user4": "",
+      "user5": "",
+      "user6": "",
+      "user7": "",
+      "user8": "",
+      "user9": "",
+      "type": "event",
+      "revision": 0,
+      "cue": "SF1.10",
+      "id": "2340b"
+    },
+    {
+      "title": "Afternoon break",
+      "type": "block",
+      "id": "cb90b"
+    },
+    {
+      "title": "Croatia",
+      "subtitle": "Guilty Pleasure",
+      "presenter": "Mia Dimsic",
+      "note": "SF1.11",
+      "endAction": "none",
+      "timerType": "count-down",
+      "timeStart": 56100000,
+      "timeEnd": 57300000,
+      "duration": 1200000,
+      "isPublic": true,
+      "skip": false,
+      "colour": "",
+      "user0": "",
+      "user1": "",
+      "user2": "",
+      "user3": "",
+      "user4": "",
+      "user5": "",
+      "user6": "",
+      "user7": "",
+      "user8": "",
+      "user9": "",
+      "type": "event",
+      "revision": 0,
+      "cue": "SF1.11",
+      "id": "503c4"
+    },
+    {
+      "title": "Denmark",
+      "subtitle": "The Show",
+      "presenter": "Reddi",
+      "note": "SF1.12",
+      "endAction": "none",
+      "timerType": "count-down",
+      "timeStart": 57600000,
+      "timeEnd": 58800000,
+      "duration": 1200000,
+      "isPublic": true,
+      "skip": false,
+      "colour": "",
+      "user0": "",
+      "user1": "",
+      "user2": "",
+      "user3": "",
+      "user4": "",
+      "user5": "",
+      "user6": "",
+      "user7": "",
+      "user8": "",
+      "user9": "",
+      "type": "event",
+      "revision": 0,
+      "cue": "SF1.12",
+      "id": "5e965"
+    },
+    {
+      "title": "Austria",
+      "subtitle": "Halo",
+      "presenter": "LUM!X & Pia Maria",
+      "note": "SF1.13",
+      "endAction": "none",
+      "timerType": "count-down",
+      "timeStart": 59100000,
+      "timeEnd": 60300000,
+      "duration": 1200000,
+      "isPublic": true,
+      "skip": false,
+      "colour": "",
+      "user0": "",
+      "user1": "",
+      "user2": "",
+      "user3": "",
+      "user4": "",
+      "user5": "",
+      "user6": "",
+      "user7": "",
+      "user8": "",
+      "user9": "",
+      "type": "event",
+      "revision": 0,
+      "cue": "SF1.13",
+      "id": "bab4a"
+    },
+    {
+      "title": "Greece",
+      "subtitle": "Die Together",
+      "presenter": "Amanda Tenfjord",
+      "note": "SF1.14",
+      "endAction": "none",
+      "timerType": "count-down",
+      "timeStart": 60600000,
+      "timeEnd": 61800000,
+      "duration": 1200000,
+      "isPublic": true,
+      "skip": false,
+      "colour": "",
+      "user0": "",
+      "user1": "",
+      "user2": "",
+      "user3": "",
+      "user4": "",
+      "user5": "",
+      "user6": "",
+      "user7": "",
+      "user8": "",
+      "user9": "",
+      "type": "event",
+      "revision": 0,
+      "cue": "SF1.14",
+      "id": "d3eb1"
+    }
+  ],
+  "project": {
+    "title": "Eurovision Song Contest",
+    "description": "Turin 2022",
+    "publicUrl": "www.getontime.no",
+    "publicInfo": "Rehearsal Schedule - Turin 2022",
+    "backstageUrl": "www.github.com/cpvalente/ontime",
+    "backstageInfo": "Rehearsal Schedule - Turin 2022\nAll performers to wear full costumes for 1st rehearsal"
+  },
+  "settings": {
+    "app": "ontime",
+    "version": "3.0.0-alpha",
+    "serverPort": 4001,
+    "editorKey": null,
+    "operatorKey": null,
+    "timeFormat": "24",
+    "language": "en"
+  },
+  "viewSettings": {
+    "overrideStyles": false,
+    "normalColor": "#ffffffcc",
+    "warningColor": "#FFAB33",
+    "warningThreshold": 120000,
+    "dangerColor": "#ED3333",
+    "dangerThreshold": 60000,
+    "endMessage": ""
+  },
+  "aliases": [
+    {
+      "enabled": true,
+      "alias": "test",
+      "pathAndParams": "lower?bg=ff2&text=f00&size=0.6&transition=5"
+    }
+  ],
+  "userFields": {
+    "user0": "user0",
+    "user1": "user1",
+    "user2": "user2",
+    "user3": "user3",
+    "user4": "user4",
+    "user5": "user5",
+    "user6": "user6",
+    "user7": "user7",
+    "user8": "user8",
+    "user9": "user9"
+  },
+  "osc": {
+    "portIn": 8888,
+    "portOut": 9999,
+    "targetIP": "127.0.0.1",
+    "enabledIn": true,
+    "enabledOut": true,
+    "subscriptions": {
+      "onLoad": [],
+      "onStart": [],
+      "onPause": [],
+      "onStop": [],
+      "onUpdate": [
+        {
+          "id": "10eea",
+          "enabled": true,
+          "message": "/ontime/update/{{timer.current}}"
+        }
+      ],
+      "onFinish": []
+    }
+  },
+  "http": {
+    "enabledOut": true,
+    "subscriptions": {
+      "onLoad": [],
+      "onStart": [],
+      "onPause": [],
+      "onStop": [],
+      "onUpdate": [],
+      "onFinish": []
+    }
+  }
+}

packages/types/src/api/ontime-controller/BackendResponse.type.ts

@@ -21,6 +21,11 @@ export type ProjectFile = {
 
 export type ProjectFileList = Array<ProjectFile>;
 
+export type ProjectFileListResponse = {
+  files: ProjectFileList;
+  lastLoadedProject: string;
+};
+
 export type ErrorResponse = {
   message: string;
-}
+};

packages/types/src/index.ts

@@ -38,7 +38,14 @@ export type { OSCSettings, OscSubscription, OscSubscriptionOptions } from './def
 export type { HttpSettings, HttpSubscription, HttpSubscriptionOptions } from './definitions/core/HttpSettings.type.js';
 
 // SERVER RESPONSES
-export type { NetworkInterface, GetInfo, ProjectFileList, ProjectFile, ErrorResponse } from './api/ontime-controller/BackendResponse.type.js';
+export type {
+  NetworkInterface,
+  GetInfo,
+  ProjectFileList,
+  ProjectFile,
+  ErrorResponse,
+  ProjectFileListResponse,
+} from './api/ontime-controller/BackendResponse.type.js';
 export type { GetRundownCached } from './api/rundown-controller/BackendResponse.type.js';
 
 // SERVER RUNTIME