diff --git a/apps/server/src/utils/__tests__/parser.test.ts b/apps/server/src/utils/__tests__/parser.test.ts index b868f32b0..835dac841 100644 --- a/apps/server/src/utils/__tests__/parser.test.ts +++ b/apps/server/src/utils/__tests__/parser.test.ts @@ -1,6 +1,6 @@ import { vi } from 'vitest'; -import { EndAction, TimerType } from 'ontime-types'; +import { EndAction, OntimeEvent, TimerType } from 'ontime-types'; import { dbModel } from '../../models/dataModel.js'; import { parseExcel, parseJson, validateEvent } from '../parser.js'; @@ -587,16 +587,16 @@ describe('test parseExcel function', () => { 'Is Public? (x)', 'Skip? (x)', 'Notes', - 'User0:test0', - 'User1:test1', - 'User2:test2', - 'User3:test3', - 'User4:test4', - 'User5:test5', - 'User6:test6', - 'user7:test7', - 'user8:test8', - 'user9:test9', + 'test0', + 'test1', + 'test2', + 'test3', + 'test4', + 'test5', + 'test6', + 'test7', + 'test8', + 'test9', 'Colour', 'cue', ], @@ -651,6 +651,19 @@ describe('test parseExcel function', () => { [], ]; + const partialOptions = { + user0: 'test0', + user1: 'test1', + user2: 'test2', + user3: 'test3', + user4: 'test4', + user5: 'test5', + user6: 'test6', + user7: 'test7', + user8: 'test8', + user9: 'test9', + }; + const expectedParsedProjectData = { title: 'Test Event', description: 'test description', @@ -706,7 +719,7 @@ describe('test parseExcel function', () => { }, ]; - const parsedData = await parseExcel(testdata); + const parsedData = parseExcel(testdata, partialOptions); expect(parsedData.project).toStrictEqual(expectedParsedProjectData); expect(parsedData.rundown).toBeDefined(); expect(parsedData.rundown[0]).toMatchObject(expectedParsedRundown[0]); @@ -835,7 +848,16 @@ describe('test views import', () => { app: 'ontime', version: 2, }, - viewSettings: {}, + viewSettings: { + normalColor: '#ffffffcc', + warningColor: '#FFAB33', + warningThreshold: 120000, + dangerColor: '#ED3333', + dangerThreshold: 60000, + endMessage: '', + overrideStyles: false, + notAthing: true, + }, views: { overrideStyles: true, }, @@ -849,7 +871,7 @@ describe('test views import', () => { endMessage: '', overrideStyles: false, }; - const parsed = parseViewSettings(testData, false); + const parsed = parseViewSettings(testData); expect(parsed).toStrictEqual(expectedParsedViewSettings); }); @@ -861,16 +883,7 @@ describe('test views import', () => { version: 2, }, }; - const expectedParsedViewSettings = { - normalColor: '#ffffffcc', - warningColor: '#FFAB33', - warningThreshold: 120000, - dangerColor: '#ED3333', - dangerThreshold: 60000, - endMessage: '', - overrideStyles: false, - }; - const parsed = parseViewSettings(testData, true); - expect(parsed).toStrictEqual(expectedParsedViewSettings); + const parsed = parseViewSettings(testData); + expect(parsed).toStrictEqual({}); }); }); diff --git a/apps/server/src/utils/__tests__/parserUtils.test.ts b/apps/server/src/utils/__tests__/parserUtils.test.ts index 0b348a083..a69957328 100644 --- a/apps/server/src/utils/__tests__/parserUtils.test.ts +++ b/apps/server/src/utils/__tests__/parserUtils.test.ts @@ -45,7 +45,7 @@ describe('mergeObject()', () => { third: '', }); }); - test.skip('it only merges fields of the first object', () => { + test('it only merges fields of the first object', () => { const a = { first: 'yes', second: 'yes', @@ -64,6 +64,35 @@ describe('mergeObject()', () => { third: '', }); }); + test('merges nested objects', () => { + // Define a sample object with nested properties + const a = { + name: 'John', + address: { + city: 'New York', + postalCode: '10001', + }, + }; + + // Define a partial object with nested properties for merging + const b = { + name: 'Doe', + address: { + city: 'San Francisco', + state: 'CA', + }, + }; + + const merged = mergeObject(a, b); + + expect(merged.name).toBe('Doe'); + expect(merged.address.city).toBe('San Francisco'); + // @ts-expect-error -- its ok, just checking + expect(merged.address.state).toBe('CA'); + expect(merged.address.postalCode).toBe('10001'); + expect(merged.address).not.toBe(a.address); + expect(merged.address).not.toBe(b.address); + }); }); describe('removeUndefined()', () => { diff --git a/apps/server/src/utils/parser.ts b/apps/server/src/utils/parser.ts index 61e944eb2..59998bba2 100644 --- a/apps/server/src/utils/parser.ts +++ b/apps/server/src/utils/parser.ts @@ -1,18 +1,18 @@ -// eslint-disable-next-line @typescript-eslint/ban-ts-comment -// @ts-nocheck -- not ready to fully type +import { + generateId, + calculateDuration, + isExcelImportMap, + type ExcelImportMap, + defaultExcelImportMap, + validateEndAction, + validateTimerType, + type ExcelImportOptions, +} from 'ontime-utils'; +import { DatabaseModel, OntimeEvent, OntimeRundown, SupportedEvent, ProjectData, UserFields } from 'ontime-types'; import fs from 'fs'; import xlsx from 'node-xlsx'; -import { generateId, calculateDuration } from 'ontime-utils'; -import { - DatabaseModel, - EndAction, - OntimeEvent, - OntimeRundown, - SupportedEvent, - TimerType, - UserFields, -} from 'ontime-types'; + import { event as eventDef } from '../models/eventsDefinition.js'; import { dbModel } from '../models/dataModel.js'; import { deleteFile, makeString } from './parserUtils.js'; @@ -33,27 +33,55 @@ export const JSON_MIME = 'application/json'; /** * @description Excel array parser * @param {array} excelData - array with excel sheet + * @param {ExcelImportOptions} options - an object that contains the import map * @returns {object} - parsed object */ -export const parseExcel = async (excelData) => { +export const parseExcel = (excelData: unknown[][], options?: Partial) => { + const importMap: ExcelImportMap = { ...defaultExcelImportMap, ...options }; const projectData: Partial = { title: '', description: '', publicUrl: '', + publicInfo: '', backstageUrl: '', + backstageInfo: '', + }; + const customUserFields: Partial = { + user0: importMap.user0, + user1: importMap.user1, + user2: importMap.user2, + user3: importMap.user3, + user4: importMap.user4, + user5: importMap.user5, + user6: importMap.user6, + user7: importMap.user7, + user8: importMap.user8, + user9: importMap.user9, }; - const customUserFields: Partial = {}; const rundown: OntimeRundown = []; - let timeStartIndex: number | null = null; - let timeEndIndex: number | null = null; + + // title stuff: strings let titleIndex: number | null = null; let cueIndex: number | null = null; let presenterIndex: number | null = null; let subtitleIndex: number | null = null; - let isPublicIndex: number | null = null; - let skipIndex: number | null = null; let notesIndex: number | null = null; let colourIndex: number | null = null; + + // options: booleans + let isPublicIndex: number | null = null; + let skipIndex: number | null = null; + + // times: numbers + // TODO: handle duration + let timeStartIndex: number | null = null; + let timeEndIndex: number | null = null; + + // options: enum properties + let endActionIndex: number | null = null; + let timerTypeIndex: number | null = null; + + // user fields: strings let user0Index: number | null = null; let user1Index: number | null = null; let user2Index: number | null = null; @@ -64,12 +92,11 @@ export const parseExcel = async (excelData) => { let user7Index: number | null = null; let user8Index: number | null = null; let user9Index: number | null = null; - let endActionIndex: number | null = null; - let timerTypeIndex: number | null = null; excelData .filter((e) => e.length > 0) .forEach((row) => { + // these fields contain the data to its right let eventTitleNext = false; let projectTitleNext = false; let publicUrlNext = false; @@ -80,24 +107,24 @@ export const parseExcel = async (excelData) => { const event: Partial = {}; row.forEach((column, j) => { - // check flags + // 1. we check if we have set a flag for a known field if (eventTitleNext) { - projectData.title = column; + projectData.title = makeString(column, ''); eventTitleNext = false; } else if (projectTitleNext) { - projectData.description = column; + projectData.description = makeString(column, ''); projectTitleNext = false; } else if (publicUrlNext) { - projectData.publicUrl = column; + projectData.publicUrl = makeString(column, ''); publicUrlNext = false; } else if (publicInfoNext) { - projectData.publicInfo = column; + projectData.publicInfo = makeString(column, ''); publicInfoNext = false; } else if (backstageUrlNext) { - projectData.backstageUrl = column; + projectData.backstageUrl = makeString(column, ''); backstageUrlNext = false; } else if (backstageInfoNext) { - projectData.backstageInfo = column; + projectData.backstageInfo = makeString(column, ''); backstageInfoNext = false; } else if (j === timeStartIndex) { event.timeStart = parseExcelDate(column); @@ -118,155 +145,127 @@ export const parseExcel = async (excelData) => { } else if (j === notesIndex) { event.note = makeString(column, ''); } else if (j === endActionIndex) { - if (column === '') { - event.endAction = EndAction.None; - } else { - event.endAction = column; - } + event.endAction = validateEndAction(column); } else if (j === timerTypeIndex) { - if (column === '') { - event.timerType = TimerType.CountDown; - } else { - event.timerType = column; - } + event.timerType = validateTimerType(column); } else if (j === colourIndex) { - event.colour = column; + event.colour = makeString(column, ''); } else if (j === user0Index) { - event.user0 = column; + event.user0 = makeString(column, ''); } else if (j === user1Index) { - event.user1 = column; + event.user1 = makeString(column, ''); } else if (j === user2Index) { - event.user2 = column; + event.user2 = makeString(column, ''); } else if (j === user3Index) { - event.user3 = column; + event.user3 = makeString(column, ''); } else if (j === user4Index) { - event.user4 = column; + event.user4 = makeString(column, ''); } else if (j === user5Index) { - event.user5 = column; + event.user5 = makeString(column, ''); } else if (j === user6Index) { - event.user6 = column; + event.user6 = makeString(column, ''); } else if (j === user7Index) { - event.user7 = column; + event.user7 = makeString(column, ''); } else if (j === user8Index) { - event.user8 = column; + event.user8 = makeString(column, ''); } else if (j === user9Index) { - event.user9 = column; + event.user9 = makeString(column, ''); } else { + // 2. if there is no flag, lets see if we know the field type if (typeof column === 'string') { const col = column.toLowerCase(); // look for keywords // need to make sure it is a string first switch (col) { - case 'project name': + case importMap.projectName: eventTitleNext = true; break; - case 'project description': + case importMap.projectDescription: projectTitleNext = true; break; - case 'public url': + case importMap.publicUrl: publicUrlNext = true; break; - case 'public info': + case importMap.publicInfo: publicInfoNext = true; break; - case 'backstage url': + case importMap.backstageUrl: backstageUrlNext = true; break; - case 'backstage info': + case importMap.backstageInfo: backstageInfoNext = true; break; - case 'time start': - case 'start': + case importMap.timeStart: timeStartIndex = j; break; - case 'time end': - case 'end': - case 'finish': + case importMap.timeEnd: timeEndIndex = j; break; - case 'cue': - case 'page': + case importMap.cue: cueIndex = j; break; - case 'event title': - case 'title': + case importMap.title: titleIndex = j; break; - case 'presenter name': - case 'speaker': - case 'presenter': + case importMap.presenter: presenterIndex = j; break; - case 'event subtitle': - case 'subtitle': + case importMap.subtitle: subtitleIndex = j; break; - case 'is public? (x)': - case 'is public': - case 'public': + case importMap.isPublic: isPublicIndex = j; break; - case 'skip? (x)': - case 'skip?': - case 'skip': + case importMap.skip: skipIndex = j; break; - case 'note': - case 'notes': + case importMap.note: notesIndex = j; break; - case 'colour': - case 'color': + case importMap.colour: colourIndex = j; break; - case 'end action': + case importMap.endAction: endActionIndex = j; break; - case 'timer type': + case importMap.timerType: timerTypeIndex = j; break; - default: - // look for user defined - if (col.startsWith('user')) { - const index = column.charAt(4); - // name is the bit after the : - const [, name] = column.split(':'); - if (typeof name !== 'undefined') { - if (index === '0') { - customUserFields.user0 = name; - user0Index = j; - } else if (index === '1') { - customUserFields.user1 = name; - user1Index = j; - } else if (index === '2') { - customUserFields.user2 = name; - user2Index = j; - } else if (index === '3') { - customUserFields.user3 = name; - user3Index = j; - } else if (index === '4') { - customUserFields.user4 = name; - user4Index = j; - } else if (index === '5') { - customUserFields.user5 = name; - user5Index = j; - } else if (index === '6') { - customUserFields.user6 = name; - user6Index = j; - } else if (index === '7') { - customUserFields.user7 = name; - user7Index = j; - } else if (index === '8') { - customUserFields.user8 = name; - user8Index = j; - } else if (index === '9') { - customUserFields.user9 = name; - user9Index = j; - } - } - } + case importMap.user0: + user0Index = j; break; + case importMap.user1: + user1Index = j; + break; + case importMap.user2: + user2Index = j; + break; + case importMap.user3: + user3Index = j; + break; + case importMap.user4: + user4Index = j; + break; + case importMap.user5: + user5Index = j; + break; + case importMap.user6: + user6Index = j; + break; + case importMap.user7: + user7Index = j; + break; + case importMap.user8: + user8Index = j; + break; + case importMap.user9: + user9Index = j; + break; + + default: + // we don't know how to handle this column + // just ignore it } } } @@ -285,17 +284,16 @@ export const parseExcel = async (excelData) => { app: 'ontime', version: 2, }, - userFields: { ...dbModel.userFields, ...customUserFields }, + userFields: customUserFields, }; }; /** - * @description JSON parser function for v1 of data system - * @param {object} jsonData - json data JSON object to be parsed - * @param {boolean} [enforce=false] - flag, tells to create an object anyway + * @description JSON parser function for ontime project file + * @param {object} jsonData - project file to be parsed * @returns {object} - parsed object */ -export const parseJson = async (jsonData, enforce = false): Promise => { +export const parseJson = async (jsonData): Promise => { if (!jsonData || typeof jsonData !== 'object') { return null; } @@ -306,17 +304,17 @@ export const parseJson = async (jsonData, enforce = false): Promise, cueFallback: stri return event; }; -type ResponseOK = { data: Partial; message: 'success' }; -type ResponseError = { error: true; message: string }; +type ResponseOK = { + data: Partial; +}; /** * @description Middleware function that checks file type and calls relevant parser * @param {string} file - reference to file + * @param options - import options * @return {object} - parse result message */ -export const fileHandler = async (file): Promise => { - let res: Partial = {}; +export const fileHandler = async (file: string, options: ExcelImportOptions): Promise> => { + const res: Partial = {}; // check which file type are we dealing with if (file.endsWith('.xlsx')) { - try { - const excelData = xlsx - .parse(file, { cellDates: true }) - .find(({ name }) => name.toLowerCase() === 'ontime' || name.toLowerCase() === 'event schedule'); + // we need to check that the options are applicable + if (!isExcelImportMap(options)) { + throw new Error('Got incorrect options to excel import', JSON.parse(options)); + } - // we only look at worksheets called ontime or event schedule - if (excelData?.data) { - const dataFromExcel = await parseExcel(excelData.data); - res.data = {}; - res.data.rundown = parseRundown(dataFromExcel); - res.data.project = parseProject(dataFromExcel, true); - res.data.userFields = parseUserFields(dataFromExcel); - res.message = 'success'; - } else { - const errorMessage = 'No sheet found named "ontime" or "event schedule"'; - res = { - error: true, - message: errorMessage, - }; - } - } catch (error) { - res = { error: true, message: `Error parsing file: ${error}` }; + const excelData = xlsx + .parse(file, { cellDates: true }) + .find(({ name }) => name.toLowerCase() === options.worksheet); + + if (excelData?.data) { + const dataFromExcel = parseExcel(excelData.data, options); + // we run the parsed data through an extra step to ensure the objects shape + res.data = {}; + res.data.rundown = parseRundown(dataFromExcel); + res.data.project = parseProject(dataFromExcel); + res.data.userFields = parseUserFields(dataFromExcel); + return res; + } else { + throw new Error(`Could not find data to import, maybe the worksheet name is incorrect: ${options.worksheet}`); } } if (file.endsWith('.json')) { // if json check version - const rawdata = fs.readFileSync(file); + const rawdata = fs.readFileSync(file).toString(); let uploadedJson = null; - try { - uploadedJson = JSON.parse(rawdata); - } catch (error) { - return { error: true, message: 'Error parsing JSON file' }; + uploadedJson = JSON.parse(rawdata); + if (uploadedJson.settings.version !== 2) { + throw new Error(`Project version unknown ${uploadedJson.settings.version}`); } + res.data = await parseJson(uploadedJson); - if (uploadedJson.settings.version === 2) { - try { - res.data = await parseJson(uploadedJson); - res.message = 'success'; - } catch (error) { - res = { error: true, message: `Error parsing file: ${error}` }; - } - } else { - res = { error: true, message: 'Error parsing file, version unknown' }; - } + // delete file + await deleteFile(file); + return res; } - - // delete file - await deleteFile(file); - return res; }; diff --git a/apps/server/src/utils/parserFunctions.ts b/apps/server/src/utils/parserFunctions.ts index 2bc0e3b75..8b4287ad2 100644 --- a/apps/server/src/utils/parserFunctions.ts +++ b/apps/server/src/utils/parserFunctions.ts @@ -88,10 +88,9 @@ export const parseRundown = (data): OntimeRundown => { /** * Parse event portion of an entry * @param {object} data - data object - * @param {boolean} enforce - whether to create a definition if one is missing * @returns {object} - event object data */ -export const parseProject = (data, enforce): ProjectData => { +export const parseProject = (data): ProjectData => { let newProjectData: Partial = {}; // we are adding this here to aid transition, should be removed once enough time has past that users have fully migrated // TODO: Remove eventually @@ -109,9 +108,6 @@ export const parseProject = (data, enforce): ProjectData => { backstageUrl: project.backstageUrl || dbModel.project.backstageUrl, backstageInfo: project.backstageInfo || dbModel.project.backstageInfo, }; - } else if (enforce) { - newProjectData = { ...dbModel.project }; - console.log('Created project object in db'); } return newProjectData as ProjectData; }; @@ -119,10 +115,9 @@ export const parseProject = (data, enforce): ProjectData => { /** * Parse settings portion of an entry * @param {object} data - data object - * @param {boolean} enforce - whether to create a definition if one is missing * @returns {object} - event object data */ -export const parseSettings = (data, enforce): Settings => { +export const parseSettings = (data): Settings => { let newSettings: Partial = {}; if ('settings' in data) { console.log('Found settings definition, importing...'); @@ -146,9 +141,6 @@ export const parseSettings = (data, enforce): Settings => { ...settings, }; } - } else if (enforce) { - newSettings = dbModel.settings; - console.log('Created settings object in db'); } return newSettings as Settings; }; @@ -156,10 +148,9 @@ export const parseSettings = (data, enforce): Settings => { /** * Parse settings portion of an entry * @param {object} data - data object - * @param {boolean} enforce - whether to create a definition if one is missing * @returns {object} - event object data */ -export const parseViewSettings = (data, enforce): ViewSettings => { +export const parseViewSettings = (data): ViewSettings => { let newViews: Partial = {}; if ('viewSettings' in data) { console.log('Found view definition, importing...'); @@ -175,13 +166,7 @@ export const parseViewSettings = (data, enforce): ViewSettings => { endMessage: v.endMessage ?? dbModel.viewSettings.endMessage, }; - // write to db - newViews = { - ...viewSettings, - }; - } else if (enforce) { - newViews = dbModel.viewSettings; - console.log('Created viewSettings object in db'); + newViews = { ...viewSettings }; } return newViews as ViewSettings; }; @@ -224,16 +209,11 @@ export const validateOscObject = (data: OscSubscription): boolean => { /** * Parse osc portion of an entry */ -export const parseOsc = ( - data: { - osc?: Partial; - }, - enforce: boolean, -): OSCSettings | Record => { +export const parseOsc = (data: { osc?: Partial }): OSCSettings => { if ('osc' in data) { console.log('Found OSC definition, importing...'); - const loadedConfig = data?.osc || {}; + const loadedConfig = data.osc || {}; const validatedSubscriptions = validateOscObject(loadedConfig.subscriptions) ? loadedConfig.subscriptions : dbModel.osc.subscriptions; @@ -246,10 +226,7 @@ export const parseOsc = ( enabledOut: loadedConfig.enabledOut ?? dbModel.osc.enabledOut, subscriptions: validatedSubscriptions, }; - } else if (enforce) { - console.log('Created OSC object in db'); - return { ...dbModel.osc }; - } else return {}; + } }; /** diff --git a/apps/server/src/utils/parserUtils.ts b/apps/server/src/utils/parserUtils.ts index 4a09058c0..7adf2b7f9 100644 --- a/apps/server/src/utils/parserUtils.ts +++ b/apps/server/src/utils/parserUtils.ts @@ -1,4 +1,5 @@ import fs from 'fs'; +import { deepmerge } from 'ontime-utils'; /** * @description Ensures variable is string, it skips object types @@ -52,16 +53,30 @@ export const isEmptyObject = (obj: object) => { /** * @description Merges two objects, suppressing undefined keys - * @param {object} a - * @param {object} b + * @param {object} a - any object + * @param {object} b - a potential partial object of same time as a */ -export const mergeObject = (a, b) => { - const merged = {}; - Object.keys({ ...a, ...b }).map((key) => { - merged[key] = typeof b[key] === 'undefined' ? a[key] : b[key]; - }); +export function mergeObject>(a: T, b: Partial>): T { + const merged = { ...a }; + + for (const key in b) { + const aValue = a[key]; + const bValue = b[key]; + + // ignore keys that do not exist in original object + if (!Object.hasOwn(merged, key)) { + continue; + } + + if (typeof bValue === 'object' && bValue !== null && typeof aValue === 'object' && aValue !== null) { + // @ts-expect-error -- library side, ignore for now + merged[key] = deepmerge(aValue, bValue); + } else if (bValue !== undefined) { + merged[key] = bValue; + } + } return merged; -}; +} /** * @description Removes undefined diff --git a/packages/utils/index.ts b/packages/utils/index.ts index 3526e290a..b9e92aba4 100644 --- a/packages/utils/index.ts +++ b/packages/utils/index.ts @@ -23,6 +23,12 @@ export { deepmerge } from './src/externals/deepmerge.js'; // generic utilities export { isNumeric } from './src/types/types.js'; + +// model validation +export { validateEndAction, validateTimerType } from './src/validate-events/validateEvent.js'; + +// feature business logic + // feature business logic - excel import export { type ExcelImportMap,