refactor: excel cleanup (#734)

* refactor: remove import of project data from excel

apps/client/src/common/api/ontimeApi.ts

@@ -196,7 +196,6 @@ export async function patchData(patchDb: Partial<DatabaseModel>) {
 
 type PostPreviewExcelResponse = {
   rundown: OntimeRundown;
-  project: ProjectData;
   userFields: UserFields;
 };
 

apps/client/src/features/modals/sheets-modal/SheetsModal.tsx

@@ -17,10 +17,10 @@ import {
   Select,
 } from '@chakra-ui/react';
 import { useQueryClient } from '@tanstack/react-query';
-import { OntimeRundown, ProjectData, UserFields } from 'ontime-types';
+import { OntimeRundown, UserFields } from 'ontime-types';
 import { defaultExcelImportMap, ExcelImportMap } from 'ontime-utils';
 
-import { PROJECT_DATA, RUNDOWN, USERFIELDS } from '../../../common/api/apiConstants';
+import { RUNDOWN, USERFIELDS } from '../../../common/api/apiConstants';
 import { maybeAxiosError } from '../../../common/api/apiUtils';
 import {
   getAuthentication,
@@ -33,7 +33,6 @@ import {
   postWorksheet,
   uploadSheetClientFile,
 } from '../../../common/api/ontimeApi';
-import { projectDataPlaceholder } from '../../../common/models/ProjectData';
 import { userFieldsPlaceholder } from '../../../common/models/UserFields';
 import { openLink } from '../../../common/utils/linkUtils';
 import ModalLink from '../ModalLink';
@@ -54,7 +53,6 @@ export default function SheetsModal(props: SheetsModalProps) {
 
   const [rundown, setRundown] = useState<OntimeRundown | null>(null);
   const [userFields, setUserFields] = useState<UserFields | null>(null);
-  const [project, setProject] = useState<ProjectData | null>(null);
 
   const [id, setSheetId] = useState('');
   const [worksheet, setWorksheet] = useState('');
@@ -84,12 +82,11 @@ export default function SheetsModal(props: SheetsModalProps) {
 
   const handleClose = () => {
     setRundown(null);
-    setProject(null);
     setUserFields(null);
     onClose();
   };
 
-  //SETP-1 Upload Client ID
+  //STEP-1 Upload Client ID
   const handleClick = () => {
     fileInputRef.current?.click();
   };
@@ -139,7 +136,7 @@ export default function SheetsModal(props: SheetsModalProps) {
       });
   };
 
-  //SETP-2 Authenticate
+  //STEP-2 Authenticate
   const handleAuthenticate = () => {
     getSheetsAuthUrl()
       .then((data) => {
@@ -175,7 +172,7 @@ export default function SheetsModal(props: SheetsModalProps) {
       });
   };
 
-  //SETP-3 set sheet ID
+  //STEP-3 set sheet ID
   const testSheetId = () => {
     postId(id)
       .then((data) => {
@@ -194,7 +191,7 @@ export default function SheetsModal(props: SheetsModalProps) {
       });
   };
 
-  //SETP-4 Select Worksheet
+  //STEP-4 Select Worksheet
   const testWorksheet = (value: string) => {
     excelFileOptions.current.worksheet = value;
     setWorksheet(value);
@@ -208,7 +205,7 @@ export default function SheetsModal(props: SheetsModalProps) {
       });
   };
 
-  //SETP-5 Upload / Download
+  //STEP-5 Upload / Download
   const updateExcelFileOptions = <T extends keyof ExcelImportMap>(field: T, value: ExcelImportMap[T]) => {
     if (excelFileOptions.current[field] !== value) {
       excelFileOptions.current = { ...excelFileOptions.current, [field]: value };
@@ -218,7 +215,6 @@ export default function SheetsModal(props: SheetsModalProps) {
   const handlePullData = () => {
     postPreviewSheet(id, excelFileOptions.current)
       .then((data) => {
-        setProject(data.project);
         setRundown(data.rundown);
         setUserFields(data.userFields);
       })
@@ -244,15 +240,14 @@ export default function SheetsModal(props: SheetsModalProps) {
 
   //GET preview
   const handleFinalise = async () => {
-    if (rundown && userFields && project) {
+    if (rundown && userFields) {
       let doClose = false;
       try {
-        await patchData({ rundown, userFields, project });
+        await patchData({ rundown, userFields });
         queryClient.setQueryData(RUNDOWN, rundown);
         queryClient.setQueryData(USERFIELDS, userFields);
-        queryClient.setQueryData(PROJECT_DATA, project);
         await queryClient.invalidateQueries({
-          queryKey: [...RUNDOWN, ...USERFIELDS, ...PROJECT_DATA],
+          queryKey: [...RUNDOWN, ...USERFIELDS],
         });
         doClose = true;
       } catch (error) {
@@ -418,11 +413,7 @@ export default function SheetsModal(props: SheetsModalProps) {
               </>
             )
           ) : (
-            <PreviewExcel
+            <PreviewExcel rundown={rundown ?? []} userFields={userFields ?? userFieldsPlaceholder} />
-              rundown={rundown ?? []}
-              project={project ?? projectDataPlaceholder}
-              userFields={userFields ?? userFieldsPlaceholder}
-            />
           )}
         </ModalBody>
         <ModalFooter>

apps/client/src/features/modals/upload-modal/UploadModal.tsx

@@ -10,10 +10,10 @@ import {
   ModalOverlay,
 } from '@chakra-ui/react';
 import { useQueryClient } from '@tanstack/react-query';
-import { OntimeRundown, ProjectData, UserFields } from 'ontime-types';
+import { OntimeRundown, UserFields } from 'ontime-types';
 import { defaultExcelImportMap, ExcelImportMap } from 'ontime-utils';
 
-import { PROJECT_DATA, RUNDOWN, USERFIELDS } from '../../../common/api/apiConstants';
+import { RUNDOWN, USERFIELDS } from '../../../common/api/apiConstants';
 import { invalidateAllCaches, maybeAxiosError } from '../../../common/api/apiUtils';
 import {
   patchData,
@@ -21,7 +21,6 @@ import {
   ProjectFileImportOptions,
   uploadProjectFile,
 } from '../../../common/api/ontimeApi';
-import { projectDataPlaceholder } from '../../../common/models/ProjectData';
 import { userFieldsPlaceholder } from '../../../common/models/UserFields';
 
 import PreviewExcel from './preview/PreviewExcel';
@@ -50,7 +49,6 @@ export default function UploadModal({ onClose, isOpen }: UploadModalProps) {
   const [submitting, setSubmitting] = useState(false);
   const [rundown, setRundown] = useState<OntimeRundown | null>(null);
   const [userFields, setUserFields] = useState<UserFields | null>(null);
-  const [project, setProject] = useState<ProjectData | null>(null);
 
   const [errors, setErrors] = useState('');
 
@@ -85,7 +83,6 @@ export default function UploadModal({ onClose, isOpen }: UploadModalProps) {
     setSubmitting(false);
     setRundown(null);
     setUserFields(null);
-    setProject(null);
     setErrors('');
   }, [clear, isOpen]);
 
@@ -127,7 +124,6 @@ export default function UploadModal({ onClose, isOpen }: UploadModalProps) {
       if (response.status === 200) {
         setRundown(response.data.rundown);
         setUserFields(response.data.userFields);
-        setProject(response.data.project);
         // in excel imports we have an extra review step
         setUploadStep('review');
       }
@@ -144,22 +140,20 @@ export default function UploadModal({ onClose, isOpen }: UploadModalProps) {
     clear();
     setRundown([]);
     setUserFields(userFieldsPlaceholder);
-    setProject(projectDataPlaceholder);
     onClose();
   };
 
   const handleFinalise = async () => {
     // this step is currently only used for excel files, after preview
-    if (isExcel && rundown && userFields && project) {
+    if (isExcel && rundown && userFields) {
       let doClose = false;
       setSubmitting(true);
       try {
-        await patchData({ rundown, userFields, project });
+        await patchData({ rundown, userFields });
         queryClient.setQueryData(RUNDOWN, { rundown, revision: -1 });
         queryClient.setQueryData(USERFIELDS, userFields);
-        queryClient.setQueryData(PROJECT_DATA, project);
         await queryClient.invalidateQueries({
-          queryKey: [...RUNDOWN, ...USERFIELDS, ...PROJECT_DATA],
+          queryKey: [...RUNDOWN, ...USERFIELDS],
         });
         doClose = true;
       } catch (error) {
@@ -214,11 +208,7 @@ export default function UploadModal({ onClose, isOpen }: UploadModalProps) {
               {isExcel && <ExcelFileOptions optionsRef={excelFileOptions} updateOptions={updateExcelFileOptions} />}
             </>
           ) : (
-            <PreviewExcel
+            <PreviewExcel rundown={rundown ?? []} userFields={userFields ?? userFieldsPlaceholder} />
-              rundown={rundown ?? []}
-              project={project ?? projectDataPlaceholder}
-              userFields={userFields ?? userFieldsPlaceholder}
-            />
           )}
         </ModalBody>
         <ModalFooter>

apps/client/src/features/modals/upload-modal/preview/PreviewColumn.module.scss

@@ -1,22 +0,0 @@
-@use "../../../../theme/_ontimeColours" as *;
-
-@mixin pad-item {
-  padding-left: 0.5rem;
-  padding-right: 1rem;
-}
-
-.previewTable {
-  display: grid;
-  grid-template-columns: auto 1fr;
-  grid-template-rows: repeat(6, auto);
-  font-size: calc(1rem - 2px);
-}
-
-.field {
-  font-weight: 200;
-  @include pad-item;
-}
-
-.value {
-  @include pad-item;
-}

apps/client/src/features/modals/upload-modal/preview/PreviewExcel.tsx

@@ -1,24 +1,19 @@
-import { OntimeRundown, ProjectData, UserFields } from 'ontime-types';
+import { OntimeRundown, UserFields } from 'ontime-types';
 
-import PreviewProjectData from './PreviewProjectData';
 import PreviewRundown from './PreviewRundown';
 
 import style from '../../Modal.module.scss';
 
 interface PreviewExcelProps {
   rundown: OntimeRundown;
-  project: ProjectData;
   userFields: UserFields;
 }
 
 export default function PreviewExcel(props: PreviewExcelProps) {
-  const { rundown, project, userFields } = props;
+  const { rundown, userFields } = props;
 
   return (
     <div className={`${style.column}`}>
-      <div className={style.title}>Review Project Data</div>
-      <PreviewProjectData project={project} />
-      <div className={style.vSpacer} />
       <div className={style.title}>Review Rundown</div>
       <PreviewRundown rundown={rundown} userFields={userFields} />
     </div>

apps/client/src/features/modals/upload-modal/preview/PreviewProjectData.tsx

@@ -1,26 +0,0 @@
-import { ProjectData } from 'ontime-types';
-
-import style from './PreviewColumn.module.scss';
-
-interface PreviewProjectDataProps {
-  project: ProjectData;
-}
-
-export default function PreviewProjectData({ project }: PreviewProjectDataProps) {
-  return (
-    <div className={style.previewTable}>
-      <span className={style.field}>Title</span>
-      <span className={style.value}>{project.title}</span>
-      <span className={style.field}>Description</span>
-      <span className={style.value}>{project.description}</span>
-      <span className={style.field}>Public URL</span>
-      <span className={style.value}>{project.publicUrl}</span>
-      <span className={style.field}>Public info</span>
-      <span className={style.value}>{project.publicInfo}</span>
-      <span className={style.field}>Backstage URL</span>
-      <span className={style.value}>{project.backstageUrl}</span>
-      <span className={style.field}>Backstage info</span>
-      <span className={style.value}>{project.backstageInfo}</span>
-    </div>
-  );
-}

apps/server/src/controllers/ontimeController.ts

@@ -8,6 +8,7 @@ import type {
   ErrorResponse,
   ProjectFileListResponse,
 } from 'ontime-types';
+import { deepmerge } from 'ontime-utils';
 
 import { RequestHandler, Request, Response } from 'express';
 import fs from 'fs';
@@ -32,7 +33,6 @@ import { oscIntegration } from '../services/integration-service/OscIntegration.j
 import { httpIntegration } from '../services/integration-service/HttpIntegration.js';
 import { logger } from '../classes/Logger.js';
 import { deleteAllEvents, notifyChanges } from '../services/rundown-service/RundownService.js';
-import { deepmerge } from 'ontime-utils';
 import { runtimeCacheStore } from '../stores/cachingStore.js';
 import { delayedRundownCacheKey } from '../services/rundown-service/delayedRundown.utils.js';
 import { integrationService } from '../services/integration-service/IntegrationService.js';
@@ -41,7 +41,6 @@ import { configService } from '../services/ConfigService.js';
 import { deleteFile } from '../utils/parserUtils.js';
 import { validateProjectFiles } from './ontimeController.validate.js';
 import { dbModel } from '../models/dataModel.js';
-
 import { sheet } from '../utils/sheetsAuth.js';
 
 // Create controller for GET request to '/ontime/poll'
@@ -453,6 +452,7 @@ export async function previewExcel(req, res) {
     const data = await parseFile(file, req, res, options);
     res.status(200).send(data);
   } catch (error) {
+    console.log(error)
     res.status(500).send({ message: error.toString() });
   }
 }

apps/server/src/services/ConfigService.ts

@@ -2,7 +2,7 @@ import { Low } from 'lowdb';
 import { JSONFile } from 'lowdb/node';
 import { join } from 'path';
 
-import { getAppDataPath } from '../setup.js';
+import { getAppDataPath, isTest } from '../setup.js';
 
 interface Config {
   lastLoadedProject: string;
@@ -35,7 +35,7 @@ class ConfigService {
   }
 
   async updateDatabaseConfig(filename: string): Promise<void> {
-    if (process.env.IS_TEST) return;
+    if (isTest) return;
 
     this.config.data.lastLoadedProject = filename;
     await this.config.write();

apps/server/src/utils/__tests__/parser.test.ts

@@ -569,14 +569,6 @@ describe('test parseExcel function', () => {
     const testdata = [
       ['Ontime ┬À Schedule Template'],
       [],
-      ['Project Name', 'Test Event'],
-      ['Project Description', 'test description'],
-      ['Public URL', 'www.public.com'],
-      ['Backstage URL', 'www.backstage.com'],
-      ['Public Info', 'test public info'],
-      ['Backstage Info', 'test backstage info'],
-      [],
-      [],
       [
         'Time Start',
         'Time End',
@@ -665,15 +657,6 @@ describe('test parseExcel function', () => {
       user9: 'test9',
     };
 
-    const expectedParsedProjectData = {
-      title: 'Test Event',
-      description: 'test description',
-      publicUrl: 'www.public.com',
-      backstageUrl: 'www.backstage.com',
-      publicInfo: 'test public info',
-      backstageInfo: 'test backstage info',
-    };
-
     // TODO: update tests once import is resolved
     const expectedParsedRundown = [
       {
@@ -721,7 +704,6 @@ describe('test parseExcel function', () => {
     ];
 
     const parsedData = parseExcel(testdata, partialOptions);
-    expect(parsedData.project).toStrictEqual(expectedParsedProjectData);
     expect(parsedData.rundown).toBeDefined();
     expect(parsedData.rundown[0]).toMatchObject(expectedParsedRundown[0]);
     expect(parsedData.rundown[1]).toMatchObject(expectedParsedRundown[1]);

apps/server/src/utils/__tests__/sheetUtils.test.ts

@@ -1,7 +1,7 @@
-import { EndAction, OntimeRundownEntry, ProjectData, SupportedEvent, TimerType } from 'ontime-types';
+import { EndAction, OntimeRundownEntry, SupportedEvent, TimerType } from 'ontime-types';
 import { millisToString } from 'ontime-utils';
 
-import { getA1Notation, cellRequestFromEvent, cellRequenstFromProjectData } from '../sheetUtils.js';
+import { getA1Notation, cellRequestFromEvent } from '../sheetUtils.js';
 
 describe('getA1Notation()', () => {
   test('A1', () => {
@@ -18,7 +18,7 @@ describe('getA1Notation()', () => {
   });
 });
 
-describe('cellRequenstFromEvent()', () => {
+describe('cellRequestFromEvent()', () => {
   test('string to string', () => {
     const event: OntimeRundownEntry = {
       type: SupportedEvent.Event,
@@ -347,106 +347,3 @@ describe('cellRequenstFromEvent()', () => {
     expect(result2.updateCells.fields).toStrictEqual('userEnteredValue');
   });
 });
-
-describe('cellRequenstFromProjectData()', () => {
-  test('string to string', () => {
-    const projectData: ProjectData = {
-      title: 'Title',
-      description: 'Description',
-      publicUrl: 'Public Url',
-      backstageUrl: 'Backstage Url',
-      publicInfo: 'Public Info',
-      backstageInfo: 'Backstage Info',
-    };
-    const metadata = {
-      title: { row: 0, col: 1 },
-      description: { row: 1, col: 1 },
-      publicUrl: { row: 2, col: 1 },
-      backstageUrl: { row: 3, col: 1 },
-      publicInfo: { row: 4, col: 1 },
-      backstageInfo: { row: 5, col: 1 },
-    };
-    const result = cellRequenstFromProjectData(projectData, 1234, metadata);
-    expect(result.updateCells.rows[0].values[0].userEnteredValue.stringValue).toStrictEqual(projectData.title);
-    expect(result.updateCells.rows[1].values[0].userEnteredValue.stringValue).toStrictEqual(projectData.description);
-    expect(result.updateCells.rows[2].values[0].userEnteredValue.stringValue).toStrictEqual(projectData.publicUrl);
-    expect(result.updateCells.rows[3].values[0].userEnteredValue.stringValue).toStrictEqual(projectData.backstageUrl);
-    expect(result.updateCells.rows[4].values[0].userEnteredValue.stringValue).toStrictEqual(projectData.publicInfo);
-    expect(result.updateCells.rows[5].values[0].userEnteredValue.stringValue).toStrictEqual(projectData.backstageInfo);
-  });
-
-  test('metadata offset from zero', () => {
-    const projectData: ProjectData = {
-      title: 'Title',
-      description: 'Description',
-      publicUrl: 'Public Url',
-      backstageUrl: 'Backstage Url',
-      publicInfo: 'Public Info',
-      backstageInfo: 'Backstage Info',
-    };
-    const metadata = {
-      title: { row: 5, col: 10 },
-      description: { row: 6, col: 10 },
-      publicUrl: { row: 7, col: 10 },
-      backstageUrl: { row: 9, col: 10 },
-      publicInfo: { row: 10, col: 10 },
-      backstageInfo: { row: 11, col: 10 },
-    };
-    const result = cellRequenstFromProjectData(projectData, 1234, metadata);
-    expect(result.updateCells.rows[0].values[0].userEnteredValue.stringValue).toStrictEqual(projectData.title);
-    expect(result.updateCells.rows[1].values[0].userEnteredValue.stringValue).toStrictEqual(projectData.description);
-    expect(result.updateCells.rows[2].values[0].userEnteredValue.stringValue).toStrictEqual(projectData.publicUrl);
-    expect(result.updateCells.rows[4].values[0].userEnteredValue.stringValue).toStrictEqual(projectData.backstageUrl);
-    expect(result.updateCells.rows[5].values[0].userEnteredValue.stringValue).toStrictEqual(projectData.publicInfo);
-    expect(result.updateCells.rows[6].values[0].userEnteredValue.stringValue).toStrictEqual(projectData.backstageInfo);
-  });
-
-  test('spacing in metadata', () => {
-    const projectData: ProjectData = {
-      title: 'Title',
-      description: 'Description',
-      publicUrl: 'Public Url',
-      backstageUrl: 'Backstage Url',
-      publicInfo: 'Public Info',
-      backstageInfo: 'Backstage Info',
-    };
-    const metadata = {
-      title: { row: 0, col: 1 },
-      description: { row: 1, col: 1 },
-      publicUrl: { row: 2, col: 1 },
-      backstageUrl: { row: 9, col: 1 },
-      publicInfo: { row: 15, col: 1 },
-      backstageInfo: { row: 50, col: 1 },
-    };
-    const result = cellRequenstFromProjectData(projectData, 1234, metadata);
-    expect(result.updateCells.rows[0].values[0].userEnteredValue.stringValue).toStrictEqual(projectData.title);
-    expect(result.updateCells.rows[1].values[0].userEnteredValue.stringValue).toStrictEqual(projectData.description);
-    expect(result.updateCells.rows[2].values[0].userEnteredValue.stringValue).toStrictEqual(projectData.publicUrl);
-    expect(result.updateCells.rows[9].values[0].userEnteredValue.stringValue).toStrictEqual(projectData.backstageUrl);
-    expect(result.updateCells.rows[15].values[0].userEnteredValue.stringValue).toStrictEqual(projectData.publicInfo);
-    expect(result.updateCells.rows[50].values[0].userEnteredValue.stringValue).toStrictEqual(projectData.backstageInfo);
-  });
-
-  test('sheet setup', () => {
-    const projectData: ProjectData = {
-      title: 'Title',
-      description: 'Description',
-      publicUrl: 'Public Url',
-      backstageUrl: 'Backstage Url',
-      publicInfo: 'Public Info',
-      backstageInfo: 'Backstage Info',
-    };
-    const metadata = {
-      title: { row: 0, col: 10 },
-      description: { row: 1, col: 10 },
-      publicUrl: { row: 2, col: 10 },
-      backstageUrl: { row: 3, col: 10 },
-      publicInfo: { row: 4, col: 10 },
-      backstageInfo: { row: 5, col: 10 },
-    };
-    const result = cellRequenstFromProjectData(projectData, 1234, metadata);
-    expect(result.updateCells.start.rowIndex).toStrictEqual(0);
-    expect(result.updateCells.start.columnIndex).toStrictEqual(11);
-    expect(result.updateCells.fields).toStrictEqual('userEnteredValue');
-  });
-});

apps/server/src/utils/parser.ts

@@ -13,7 +13,6 @@ import {
   OntimeEvent,
   OntimeRundown,
   SupportedEvent,
-  ProjectData,
   UserFields,
   EndAction,
   TimerType,
@@ -43,8 +42,7 @@ import { coerceBoolean } from './coerceType.js';
 export const EXCEL_MIME = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet';
 export const JSON_MIME = 'application/json';
 
-type ExcelData = Pick<DatabaseModel, 'rundown' | 'project' | 'userFields'> & {
+type ExcelData = Pick<DatabaseModel, 'rundown' | 'userFields'> & {
-  projectMetadata: Record<string, { row: number; col: number }>;
   rundownMetadata: Record<string, { row: number; col: number }>;
 };
 
@@ -55,20 +53,11 @@ type ExcelData = Pick<DatabaseModel, 'rundown' | 'project' | 'userFields'> & {
  * @returns {object} - parsed object
  */
 export const parseExcel = (excelData: unknown[][], options?: Partial<ExcelImportMap>): ExcelData => {
-  const projectMetadata = {};
   const rundownMetadata = {};
   const importMap: ExcelImportMap = { ...defaultExcelImportMap, ...options };
   for (const [key, value] of Object.entries(importMap)) {
     importMap[key] = value.toLocaleLowerCase();
   }
-  const projectData: ProjectData = {
-    title: '',
-    description: '',
-    publicUrl: '',
-    publicInfo: '',
-    backstageUrl: '',
-    backstageInfo: '',
-  };
   const customUserFields: UserFields = {
     user0: importMap.user0,
     user1: importMap.user1,
@@ -122,41 +111,9 @@ export const parseExcel = (excelData: unknown[][], options?: Partial<ExcelImport
     if (row.length === 0) {
       return;
     }
-    // these fields contain the data to its right
-    let projectTitleNext = false;
-    let projectDescriptionNext = false;
-    let publicUrlNext = false;
-    let publicInfoNext = false;
-    let backstageUrlNext = false;
-    let backstageInfoNext = false;
 
     const event: Partial<OntimeEvent> = {};
     const handlers = {
-      [importMap.projectName]: (row: number, col: number) => {
-        projectTitleNext = true;
-        projectMetadata['title'] = { row, col };
-      },
-      [importMap.projectDescription]: (row: number, col: number) => {
-        projectDescriptionNext = true;
-        projectMetadata['description'] = { row, col };
-      },
-      [importMap.publicUrl]: (row: number, col: number) => {
-        publicUrlNext = true;
-        projectMetadata['publicUrl'] = { row, col };
-      },
-      [importMap.publicInfo]: (row: number, col: number) => {
-        publicInfoNext = true;
-        projectMetadata['publicInfo'] = { row, col };
-      },
-      [importMap.backstageUrl]: (row: number, col: number) => {
-        backstageUrlNext = true;
-        projectMetadata['backstageUrl'] = { row, col };
-      },
-      [importMap.backstageInfo]: (row: number, col: number) => {
-        backstageInfoNext = true;
-        projectMetadata['backstageInfo'] = { row, col };
-      },
-
       [importMap.timeStart]: (row: number, col: number) => {
         timeStartIndex = col;
         rundownMetadata['timeStart'] = { row, col };
@@ -264,25 +221,7 @@ export const parseExcel = (excelData: unknown[][], options?: Partial<ExcelImport
 
     row.forEach((column, j) => {
       // 1. we check if we have set a flag for a known field
-      if (projectTitleNext) {
+      if (j === timeStartIndex) {
-        projectData.title = makeString(column, '');
-        projectTitleNext = false;
-      } else if (projectDescriptionNext) {
-        projectData.description = makeString(column, '');
-        projectDescriptionNext = false;
-      } else if (publicUrlNext) {
-        projectData.publicUrl = makeString(column, '');
-        publicUrlNext = false;
-      } else if (publicInfoNext) {
-        projectData.publicInfo = makeString(column, '');
-        publicInfoNext = false;
-      } else if (backstageUrlNext) {
-        projectData.backstageUrl = makeString(column, '');
-        backstageUrlNext = false;
-      } else if (backstageInfoNext) {
-        projectData.backstageInfo = makeString(column, '');
-        backstageInfoNext = false;
-      } else if (j === timeStartIndex) {
         event.timeStart = parseExcelDate(column);
       } else if (j === timeEndIndex) {
         event.timeEnd = parseExcelDate(column);
@@ -354,9 +293,7 @@ export const parseExcel = (excelData: unknown[][], options?: Partial<ExcelImport
 
   return {
     rundown,
-    project: projectData,
     userFields: customUserFields,
-    projectMetadata,
     rundownMetadata,
   };
 };
@@ -371,27 +308,18 @@ export const parseJson = async (jsonData): Promise<DatabaseModel | null> => {
     return null;
   }
 
-  // object containing the parsed data
+  const returnData: DatabaseModel = {
-  const returnData: Partial<DatabaseModel> = {};
+    rundown: parseRundown(jsonData),
+    project: parseProject(jsonData) ?? dbModel.project,
+    settings: parseSettings(jsonData) ?? dbModel.settings,
+    viewSettings: parseViewSettings(jsonData) ?? dbModel.viewSettings,
+    aliases: parseAliases(jsonData),
+    userFields: parseUserFields(jsonData),
+    osc: parseOsc(jsonData) ?? dbModel.osc,
+    http: parseHttp(jsonData) ?? dbModel.http,
+  };
 
-  // parse Events
+  return returnData;
-  returnData.rundown = parseRundown(jsonData);
-  // parse Event
-  returnData.project = parseProject(jsonData) ?? dbModel.project;
-  // Settings handled partially
-  returnData.settings = parseSettings(jsonData) ?? dbModel.settings;
-  // View settings handled partially
-  returnData.viewSettings = parseViewSettings(jsonData) ?? dbModel.viewSettings;
-  // Import Aliases if any
-  returnData.aliases = parseAliases(jsonData);
-  // Import user fields if any
-  returnData.userFields = parseUserFields(jsonData);
-  // Import OSC settings if any
-  returnData.osc = parseOsc(jsonData) ?? dbModel.osc;
-  // Import HTTP settings if any
-  returnData.http = parseHttp(jsonData) ?? dbModel.http;
-
-  return returnData as DatabaseModel;
 };
 
 export function createPatch(originalEvent: OntimeEvent, patchEvent: Partial<OntimeEvent>): OntimeEvent {
@@ -476,7 +404,7 @@ export const fileHandler = async (file: string, options: ExcelImportOptions): Pr
   if (file.endsWith('.xlsx')) {
     // we need to check that the options are applicable
     if (!isExcelImportMap(options)) {
-      throw new Error('Got incorrect options to excel import', JSON.parse(options));
+      throw new Error('Got incorrect options to excel import');
     }
 
     const excelData = xlsx
@@ -494,15 +422,16 @@ export const fileHandler = async (file: string, options: ExcelImportOptions): Pr
     if (res.data.rundown.length < 1) {
       throw new Error(`Could not find data to import in the worksheet ${options.worksheet}`);
     }
-    res.data.project = parseProject(dataFromExcel);
     res.data.userFields = parseUserFields(dataFromExcel);
 
-    await deleteFile(file);
+    deleteFile(file);
 
     return res;
   }
 
   if (file.endsWith('.json')) {
+    console.log('JSON!');
+
     const rawdata = fs.readFileSync(file).toString();
     let uploadedJson = null;
 
@@ -512,4 +441,5 @@ export const fileHandler = async (file: string, options: ExcelImportOptions): Pr
     await configService.updateDatabaseConfig(fileName);
     return res;
   }
+  console.log('NOTHIGN');
 };

apps/server/src/utils/sheetUtils.ts

@@ -1,6 +1,6 @@
 import { sheets_v4 } from '@googleapis/sheets';
 import { millisToString } from 'ontime-utils';
-import { OntimeRundownEntry, ProjectData, isOntimeEvent } from 'ontime-types';
+import { OntimeRundownEntry, isOntimeEvent } from 'ontime-types';
 
 /**
  *
@@ -102,64 +102,3 @@ export function cellRequestFromEvent(
     },
   };
 }
-
-/**
- * @description - creates updateCells request from ontime event
- * @param {ProjectData} projectData
- * @param {number} worksheetId
- * @param {any} metadata - object with all the cell positions of the title of each attribute
- * @returns {sheets_v4.Schema} - list of update requests
- */
-export function cellRequenstFromProjectData(
-  projectData: ProjectData,
-  worksheetId: number,
-  metadata,
-): sheets_v4.Schema$Request {
-  const returnRows: sheets_v4.Schema$RowData[] = [];
-  const tmp = Object.entries(metadata)
-    .filter(([_, value]) => value !== undefined)
-    .sort(([_a, a], [_b, b]) => a['col'] - b['col']) as [string, { col: number; row: number }][];
-
-  const minRow = Object.values(metadata).reduce(
-    (accumulator: number, val) => Math.min(accumulator, val['row']),
-    Number.MAX_VALUE,
-  ) as number;
-  const minCol = tmp[0][1].col + 1;
-
-  for (const [index, e] of tmp.entries()) {
-    if (index != 0) {
-      const prevRow = tmp[index - 1][1].row;
-      const thisRow = e[1].row;
-      const diff = thisRow - prevRow;
-      if (diff > 1) {
-        const fillArr = new Array<(typeof tmp)[0]>(1).fill(['blank', { row: prevRow + 1, col: e[1].col }]);
-        tmp.splice(index, 0, ...fillArr);
-      }
-    }
-  }
-  tmp.forEach(([key, _]) => {
-    if (key == 'blank') {
-      returnRows.push({});
-    } else {
-      returnRows.push({
-        values: [
-          {
-            userEnteredValue: { stringValue: projectData[key] },
-          },
-        ],
-      });
-    }
-  });
-
-  return {
-    updateCells: {
-      start: {
-        sheetId: worksheetId,
-        rowIndex: minRow,
-        columnIndex: minCol,
-      },
-      fields: 'userEnteredValue',
-      rows: returnRows,
-    },
-  };
-}

apps/server/src/utils/sheetsAuth.ts

@@ -1,19 +1,21 @@
+import { DatabaseModel, LogOrigin } from 'ontime-types';
+import { ExcelImportMap } from 'ontime-utils';
+
 import { sheets, sheets_v4 } from '@googleapis/sheets';
 import { writeFile } from 'fs/promises';
 import { readFileSync } from 'fs';
 import { OAuth2Client } from 'google-auth-library';
 import http from 'http';
-import { DatabaseModel, LogOrigin } from 'ontime-types';
 import { join } from 'path';
 import { URL } from 'url';
+
 import { logger } from '../classes/Logger.js';
 import { DataProvider } from '../classes/data-provider/DataProvider.js';
 import { getAppDataPath } from '../setup.js';
 import { ensureDirectory } from './fileManagement.js';
-import { cellRequestFromEvent, cellRequenstFromProjectData, getA1Notation } from './sheetUtils.js';
+import { cellRequestFromEvent, getA1Notation } from './sheetUtils.js';
 import { parseExcel } from './parser.js';
-import { parseProject, parseRundown, parseUserFields } from './parserFunctions.js';
+import { parseRundown, parseUserFields } from './parserFunctions.js';
-import { ExcelImportMap } from 'ontime-utils';
 
 type ResponseOK = {
   data: Partial<DatabaseModel>;
@@ -40,12 +42,11 @@ class Sheet {
 
   constructor() {
     const appDataPath = getAppDataPath();
-    if (appDataPath === '') {
+
-      throw new Error('Sheet: Could not resolve sheet folser');
-    }
     this.sheetsFolder = join(appDataPath, 'sheets');
     this.clientSecretFile = join(this.sheetsFolder, 'client_secret.json');
     ensureDirectory(this.sheetsFolder);
+
     try {
       const secrets = JSON.parse(readFileSync(this.clientSecretFile, 'utf-8'));
       const isKeyMissing = this.requiredClientKeys.some((key) => !(key in secrets['installed']));
@@ -53,12 +54,12 @@ class Sheet {
         Sheet.clientSecret = secrets;
       }
     } catch (_) {
-      /* empty - it is ok thet there is no clientSecret */
+      /* empty - it is ok that there is no clientSecret */
     }
   }
 
   /**
-   * @description SETP 1 - saves secrets object to appdata path as client_secret.json
+   * @description STEP 1 - saves secrets object to appdata path as client_secret.json
    * @param {object} secrets
    * @throws
    */
@@ -79,14 +80,14 @@ class Sheet {
   }
 
   /**
-   * @description SETP 1 - test that the saved object is pressent
+   * @description STEP 1 - test that the saved object is pressent
    */
   testClientSecret() {
     return Sheet.clientSecret !== null;
   }
 
   /**
-   * @description SETP 2 - create server to interact with th OAuth2 request
+   * @description STEP 2 - create server to interact with th OAuth2 request
    * @returns {Promise<string | null>} - returns url path serve on success
    * @throws
    */
@@ -159,7 +160,7 @@ class Sheet {
     });
     let listenPort = 3000;
     if (keyFile.installed) {
-      // Use emphemeral port if not a web client
+      // Use ephemeral port if not a web client
       listenPort = 0;
     } else if (redirectUri.port !== '') {
       listenPort = Number(redirectUri.port);
@@ -188,7 +189,7 @@ class Sheet {
   }
 
   /**
-   * @description SETP 2 - test that the reciveed OAuth2 is still valid
+   * @description STEP 2 - test that the reciveed OAuth2 is still valid
    * @throws
    */
   async testAuthentication() {
@@ -205,7 +206,7 @@ class Sheet {
   }
 
   /**
-   * @description SETP 3 - test the given sheet id
+   * @description STEP 3 - test the given sheet id
    * @throws
    */
   async testSheetId(id: string) {
@@ -220,7 +221,7 @@ class Sheet {
   }
 
   /**
-   * @description SETP 4 - test the given worksheet
+   * @description STEP 4 - test the given worksheet
    * @throws
    */
   async testWorksheet(id: string, worksheet: string) {
@@ -264,7 +265,7 @@ class Sheet {
   }
 
   /**
-   * @description SETP 5 - Upload the rundown to sheet
+   * @description STEP 5 - Upload the rundown to sheet
    * @param {string} id - id of the sheet https://docs.google.com/spreadsheets/d/[[spreadsheetId]]/edit#gid=0
    * @param {ExcelImportMap} options
    * @throws
@@ -279,14 +280,13 @@ class Sheet {
       range: range,
     });
     if (readResponse.status === 200) {
-      const { rundownMetadata, projectMetadata } = parseExcel(readResponse.data.values, options);
+      const { rundownMetadata } = parseExcel(readResponse.data.values, options);
       const rundown = DataProvider.getRundown();
-      const projectData = DataProvider.getProjectData();
       const titleRow = Object.values(rundownMetadata)[0]['row'];
 
       const updateRundown = Array<sheets_v4.Schema$Request>();
 
-      // we can't delete the last unflozzen row so we create an empty one
+      // we can't delete the last unfrozen row so we create an empty one
       updateRundown.push({
         insertDimension: {
           inheritFromBefore: false,
@@ -320,9 +320,6 @@ class Sheet {
         updateRundown.push(cellRequestFromEvent(entry, index, worksheetId, rundownMetadata)),
       );
 
-      //update project data
-      updateRundown.push(cellRequenstFromProjectData(projectData, worksheetId, projectMetadata));
-
       const writeResponse = await sheets({ version: 'v4', auth: Sheet.client }).spreadsheets.batchUpdate({
         spreadsheetId: id,
         requestBody: {
@@ -361,6 +358,7 @@ class Sheet {
       range,
     });
 
+    // TODO: we need to pass this into a service that can safely merge the datasets
     if (googleResponse.status === 200) {
       res.data = {};
       const dataFromSheet = parseExcel(googleResponse.data.values, options);
@@ -368,7 +366,6 @@ class Sheet {
       if (res.data.rundown.length < 1) {
         throw new Error(`Sheet: Could not find data to import in the worksheet`);
       }
-      res.data.project = parseProject(dataFromSheet);
       res.data.userFields = parseUserFields(dataFromSheet);
       return res;
     } else {

apps/server/src/utils/upload.js

@@ -44,7 +44,7 @@ const filterAllowed = (req, file, cb) => {
   if (file.mimetype.includes(JSON_MIME) || file.mimetype.includes(EXCEL_MIME)) {
     cb(null, true);
   } else {
-    console.log('ERROR: Unrecognised file type');
+    console.error('ERROR: Unrecognised file type');
     cb(null, false);
   }
 };

e2e/tests/utils/uploadTestShowfile.ts

@@ -15,7 +15,7 @@ export async function uploadTestDb(request) {
     multipart: {
       file: {
         fileName: filePath,
-        mimeType: "application/json",
+        mimeType: 'application/json',
         buffer: file,
       },
     },

packages/utils/src/feature/excel-import/__tests__/excelImport.test.ts

@@ -0,0 +1,42 @@
+import { isExcelImportMap } from '../excelImport';
+
+describe('isExcelImportMap', () => {
+  test('migrate v2 map', () => {
+    const v2ImportMap = {
+      worksheet: 'event schedule',
+      projectName: 'project name',
+      projectDescription: 'project description',
+      publicUrl: 'public url',
+      publicInfo: 'public info',
+      backstageUrl: 'backstage url',
+      backstageInfo: 'backstage info',
+      timeStart: 'time start',
+      timeEnd: 'time end',
+      duration: 'duration',
+      cue: 'cue',
+      title: 'title',
+      presenter: 'presenter',
+      subtitle: 'subtitle',
+      isPublic: 'public',
+      skip: 'skip',
+      note: 'notes',
+      colour: 'colour',
+      endAction: 'end action',
+      timerType: 'timer type',
+      user0: 'header',
+      user1: 'user1',
+      user2: 'user2',
+      user3: 'user3',
+      user4: 'user4',
+      user5: 'user5',
+      user6: 'user6',
+      user7: 'user7',
+      user8: 'user8',
+      user9: 'user9',
+      timeWarning: 'warning time',
+      timeDanger: 'danger time',
+    };
+
+    expect(isExcelImportMap(v2ImportMap)).toBe(true);
+  });
+});

packages/utils/src/feature/excel-import/excelImport.ts

@@ -3,12 +3,6 @@ export type ExcelImportMap = typeof defaultExcelImportMap;
 
 export const defaultExcelImportMap = {
   worksheet: 'event schedule',
-  projectName: 'project name',
-  projectDescription: 'project description',
-  publicUrl: 'public url',
-  publicInfo: 'public info',
-  backstageUrl: 'backstage url',
-  backstageInfo: 'backstage info',
   timeStart: 'time start',
   timeEnd: 'time end',
   duration: 'duration',
@@ -41,6 +35,6 @@ export function isExcelImportMap(obj: unknown): obj is ExcelImportMap {
     return false;
   }
 
-  const keys = Object.keys(obj);
+  const keys = Object.keys(defaultExcelImportMap);
-  return keys.every((key) => Object.hasOwn(defaultExcelImportMap, key));
+  return keys.every((key) => Object.hasOwn(obj, key));
 }