refactor: parse import fields

This commit is contained in:
cv
2023-09-28 08:15:24 +02:00
parent 9fc04f2fd1
commit ba5ef6668d
6 changed files with 263 additions and 237 deletions
+38 -25
View File
@@ -1,6 +1,6 @@
import { vi } from 'vitest';
import { EndAction, TimerType } from 'ontime-types';
import { EndAction, OntimeEvent, TimerType } from 'ontime-types';
import { dbModel } from '../../models/dataModel.js';
import { parseExcel, parseJson, validateEvent } from '../parser.js';
@@ -587,16 +587,16 @@ describe('test parseExcel function', () => {
'Is Public? (x)',
'Skip? (x)',
'Notes',
'User0:test0',
'User1:test1',
'User2:test2',
'User3:test3',
'User4:test4',
'User5:test5',
'User6:test6',
'user7:test7',
'user8:test8',
'user9:test9',
'test0',
'test1',
'test2',
'test3',
'test4',
'test5',
'test6',
'test7',
'test8',
'test9',
'Colour',
'cue',
],
@@ -651,6 +651,19 @@ describe('test parseExcel function', () => {
[],
];
const partialOptions = {
user0: 'test0',
user1: 'test1',
user2: 'test2',
user3: 'test3',
user4: 'test4',
user5: 'test5',
user6: 'test6',
user7: 'test7',
user8: 'test8',
user9: 'test9',
};
const expectedParsedProjectData = {
title: 'Test Event',
description: 'test description',
@@ -706,7 +719,7 @@ describe('test parseExcel function', () => {
},
];
const parsedData = await parseExcel(testdata);
const parsedData = parseExcel(testdata, partialOptions);
expect(parsedData.project).toStrictEqual(expectedParsedProjectData);
expect(parsedData.rundown).toBeDefined();
expect(parsedData.rundown[0]).toMatchObject(expectedParsedRundown[0]);
@@ -835,7 +848,16 @@ describe('test views import', () => {
app: 'ontime',
version: 2,
},
viewSettings: {},
viewSettings: {
normalColor: '#ffffffcc',
warningColor: '#FFAB33',
warningThreshold: 120000,
dangerColor: '#ED3333',
dangerThreshold: 60000,
endMessage: '',
overrideStyles: false,
notAthing: true,
},
views: {
overrideStyles: true,
},
@@ -849,7 +871,7 @@ describe('test views import', () => {
endMessage: '',
overrideStyles: false,
};
const parsed = parseViewSettings(testData, false);
const parsed = parseViewSettings(testData);
expect(parsed).toStrictEqual(expectedParsedViewSettings);
});
@@ -861,16 +883,7 @@ describe('test views import', () => {
version: 2,
},
};
const expectedParsedViewSettings = {
normalColor: '#ffffffcc',
warningColor: '#FFAB33',
warningThreshold: 120000,
dangerColor: '#ED3333',
dangerThreshold: 60000,
endMessage: '',
overrideStyles: false,
};
const parsed = parseViewSettings(testData, true);
expect(parsed).toStrictEqual(expectedParsedViewSettings);
const parsed = parseViewSettings(testData);
expect(parsed).toStrictEqual({});
});
});
@@ -45,7 +45,7 @@ describe('mergeObject()', () => {
third: '',
});
});
test.skip('it only merges fields of the first object', () => {
test('it only merges fields of the first object', () => {
const a = {
first: 'yes',
second: 'yes',
@@ -64,6 +64,35 @@ describe('mergeObject()', () => {
third: '',
});
});
test('merges nested objects', () => {
// Define a sample object with nested properties
const a = {
name: 'John',
address: {
city: 'New York',
postalCode: '10001',
},
};
// Define a partial object with nested properties for merging
const b = {
name: 'Doe',
address: {
city: 'San Francisco',
state: 'CA',
},
};
const merged = mergeObject(a, b);
expect(merged.name).toBe('Doe');
expect(merged.address.city).toBe('San Francisco');
// @ts-expect-error -- its ok, just checking
expect(merged.address.state).toBe('CA');
expect(merged.address.postalCode).toBe('10001');
expect(merged.address).not.toBe(a.address);
expect(merged.address).not.toBe(b.address);
});
});
describe('removeUndefined()', () => {
+159 -173
View File
@@ -1,18 +1,18 @@
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-nocheck -- not ready to fully type
import {
generateId,
calculateDuration,
isExcelImportMap,
type ExcelImportMap,
defaultExcelImportMap,
validateEndAction,
validateTimerType,
type ExcelImportOptions,
} from 'ontime-utils';
import { DatabaseModel, OntimeEvent, OntimeRundown, SupportedEvent, ProjectData, UserFields } from 'ontime-types';
import fs from 'fs';
import xlsx from 'node-xlsx';
import { generateId, calculateDuration } from 'ontime-utils';
import {
DatabaseModel,
EndAction,
OntimeEvent,
OntimeRundown,
SupportedEvent,
TimerType,
UserFields,
} from 'ontime-types';
import { event as eventDef } from '../models/eventsDefinition.js';
import { dbModel } from '../models/dataModel.js';
import { deleteFile, makeString } from './parserUtils.js';
@@ -33,27 +33,55 @@ export const JSON_MIME = 'application/json';
/**
* @description Excel array parser
* @param {array} excelData - array with excel sheet
* @param {ExcelImportOptions} options - an object that contains the import map
* @returns {object} - parsed object
*/
export const parseExcel = async (excelData) => {
export const parseExcel = (excelData: unknown[][], options?: Partial<ExcelImportMap>) => {
const importMap: ExcelImportMap = { ...defaultExcelImportMap, ...options };
const projectData: Partial<ProjectData> = {
title: '',
description: '',
publicUrl: '',
publicInfo: '',
backstageUrl: '',
backstageInfo: '',
};
const customUserFields: Partial<UserFields> = {
user0: importMap.user0,
user1: importMap.user1,
user2: importMap.user2,
user3: importMap.user3,
user4: importMap.user4,
user5: importMap.user5,
user6: importMap.user6,
user7: importMap.user7,
user8: importMap.user8,
user9: importMap.user9,
};
const customUserFields: Partial<UserFields> = {};
const rundown: OntimeRundown = [];
let timeStartIndex: number | null = null;
let timeEndIndex: number | null = null;
// title stuff: strings
let titleIndex: number | null = null;
let cueIndex: number | null = null;
let presenterIndex: number | null = null;
let subtitleIndex: number | null = null;
let isPublicIndex: number | null = null;
let skipIndex: number | null = null;
let notesIndex: number | null = null;
let colourIndex: number | null = null;
// options: booleans
let isPublicIndex: number | null = null;
let skipIndex: number | null = null;
// times: numbers
// TODO: handle duration
let timeStartIndex: number | null = null;
let timeEndIndex: number | null = null;
// options: enum properties
let endActionIndex: number | null = null;
let timerTypeIndex: number | null = null;
// user fields: strings
let user0Index: number | null = null;
let user1Index: number | null = null;
let user2Index: number | null = null;
@@ -64,12 +92,11 @@ export const parseExcel = async (excelData) => {
let user7Index: number | null = null;
let user8Index: number | null = null;
let user9Index: number | null = null;
let endActionIndex: number | null = null;
let timerTypeIndex: number | null = null;
excelData
.filter((e) => e.length > 0)
.forEach((row) => {
// these fields contain the data to its right
let eventTitleNext = false;
let projectTitleNext = false;
let publicUrlNext = false;
@@ -80,24 +107,24 @@ export const parseExcel = async (excelData) => {
const event: Partial<OntimeEvent> = {};
row.forEach((column, j) => {
// check flags
// 1. we check if we have set a flag for a known field
if (eventTitleNext) {
projectData.title = column;
projectData.title = makeString(column, '');
eventTitleNext = false;
} else if (projectTitleNext) {
projectData.description = column;
projectData.description = makeString(column, '');
projectTitleNext = false;
} else if (publicUrlNext) {
projectData.publicUrl = column;
projectData.publicUrl = makeString(column, '');
publicUrlNext = false;
} else if (publicInfoNext) {
projectData.publicInfo = column;
projectData.publicInfo = makeString(column, '');
publicInfoNext = false;
} else if (backstageUrlNext) {
projectData.backstageUrl = column;
projectData.backstageUrl = makeString(column, '');
backstageUrlNext = false;
} else if (backstageInfoNext) {
projectData.backstageInfo = column;
projectData.backstageInfo = makeString(column, '');
backstageInfoNext = false;
} else if (j === timeStartIndex) {
event.timeStart = parseExcelDate(column);
@@ -118,155 +145,127 @@ export const parseExcel = async (excelData) => {
} else if (j === notesIndex) {
event.note = makeString(column, '');
} else if (j === endActionIndex) {
if (column === '') {
event.endAction = EndAction.None;
} else {
event.endAction = column;
}
event.endAction = validateEndAction(column);
} else if (j === timerTypeIndex) {
if (column === '') {
event.timerType = TimerType.CountDown;
} else {
event.timerType = column;
}
event.timerType = validateTimerType(column);
} else if (j === colourIndex) {
event.colour = column;
event.colour = makeString(column, '');
} else if (j === user0Index) {
event.user0 = column;
event.user0 = makeString(column, '');
} else if (j === user1Index) {
event.user1 = column;
event.user1 = makeString(column, '');
} else if (j === user2Index) {
event.user2 = column;
event.user2 = makeString(column, '');
} else if (j === user3Index) {
event.user3 = column;
event.user3 = makeString(column, '');
} else if (j === user4Index) {
event.user4 = column;
event.user4 = makeString(column, '');
} else if (j === user5Index) {
event.user5 = column;
event.user5 = makeString(column, '');
} else if (j === user6Index) {
event.user6 = column;
event.user6 = makeString(column, '');
} else if (j === user7Index) {
event.user7 = column;
event.user7 = makeString(column, '');
} else if (j === user8Index) {
event.user8 = column;
event.user8 = makeString(column, '');
} else if (j === user9Index) {
event.user9 = column;
event.user9 = makeString(column, '');
} else {
// 2. if there is no flag, lets see if we know the field type
if (typeof column === 'string') {
const col = column.toLowerCase();
// look for keywords
// need to make sure it is a string first
switch (col) {
case 'project name':
case importMap.projectName:
eventTitleNext = true;
break;
case 'project description':
case importMap.projectDescription:
projectTitleNext = true;
break;
case 'public url':
case importMap.publicUrl:
publicUrlNext = true;
break;
case 'public info':
case importMap.publicInfo:
publicInfoNext = true;
break;
case 'backstage url':
case importMap.backstageUrl:
backstageUrlNext = true;
break;
case 'backstage info':
case importMap.backstageInfo:
backstageInfoNext = true;
break;
case 'time start':
case 'start':
case importMap.timeStart:
timeStartIndex = j;
break;
case 'time end':
case 'end':
case 'finish':
case importMap.timeEnd:
timeEndIndex = j;
break;
case 'cue':
case 'page':
case importMap.cue:
cueIndex = j;
break;
case 'event title':
case 'title':
case importMap.title:
titleIndex = j;
break;
case 'presenter name':
case 'speaker':
case 'presenter':
case importMap.presenter:
presenterIndex = j;
break;
case 'event subtitle':
case 'subtitle':
case importMap.subtitle:
subtitleIndex = j;
break;
case 'is public? (x)':
case 'is public':
case 'public':
case importMap.isPublic:
isPublicIndex = j;
break;
case 'skip? (x)':
case 'skip?':
case 'skip':
case importMap.skip:
skipIndex = j;
break;
case 'note':
case 'notes':
case importMap.note:
notesIndex = j;
break;
case 'colour':
case 'color':
case importMap.colour:
colourIndex = j;
break;
case 'end action':
case importMap.endAction:
endActionIndex = j;
break;
case 'timer type':
case importMap.timerType:
timerTypeIndex = j;
break;
default:
// look for user defined
if (col.startsWith('user')) {
const index = column.charAt(4);
// name is the bit after the :
const [, name] = column.split(':');
if (typeof name !== 'undefined') {
if (index === '0') {
customUserFields.user0 = name;
user0Index = j;
} else if (index === '1') {
customUserFields.user1 = name;
user1Index = j;
} else if (index === '2') {
customUserFields.user2 = name;
user2Index = j;
} else if (index === '3') {
customUserFields.user3 = name;
user3Index = j;
} else if (index === '4') {
customUserFields.user4 = name;
user4Index = j;
} else if (index === '5') {
customUserFields.user5 = name;
user5Index = j;
} else if (index === '6') {
customUserFields.user6 = name;
user6Index = j;
} else if (index === '7') {
customUserFields.user7 = name;
user7Index = j;
} else if (index === '8') {
customUserFields.user8 = name;
user8Index = j;
} else if (index === '9') {
customUserFields.user9 = name;
user9Index = j;
}
}
}
case importMap.user0:
user0Index = j;
break;
case importMap.user1:
user1Index = j;
break;
case importMap.user2:
user2Index = j;
break;
case importMap.user3:
user3Index = j;
break;
case importMap.user4:
user4Index = j;
break;
case importMap.user5:
user5Index = j;
break;
case importMap.user6:
user6Index = j;
break;
case importMap.user7:
user7Index = j;
break;
case importMap.user8:
user8Index = j;
break;
case importMap.user9:
user9Index = j;
break;
default:
// we don't know how to handle this column
// just ignore it
}
}
}
@@ -285,17 +284,16 @@ export const parseExcel = async (excelData) => {
app: 'ontime',
version: 2,
},
userFields: { ...dbModel.userFields, ...customUserFields },
userFields: customUserFields,
};
};
/**
* @description JSON parser function for v1 of data system
* @param {object} jsonData - json data JSON object to be parsed
* @param {boolean} [enforce=false] - flag, tells to create an object anyway
* @description JSON parser function for ontime project file
* @param {object} jsonData - project file to be parsed
* @returns {object} - parsed object
*/
export const parseJson = async (jsonData, enforce = false): Promise<DatabaseModel | null> => {
export const parseJson = async (jsonData): Promise<DatabaseModel | null> => {
if (!jsonData || typeof jsonData !== 'object') {
return null;
}
@@ -306,17 +304,17 @@ export const parseJson = async (jsonData, enforce = false): Promise<DatabaseMode
// parse Events
returnData.rundown = parseRundown(jsonData);
// parse Event
returnData.project = parseProject(jsonData, enforce);
returnData.project = parseProject(jsonData) ?? dbModel.project;
// Settings handled partially
returnData.settings = parseSettings(jsonData, enforce);
returnData.settings = parseSettings(jsonData) ?? dbModel.settings;
// View settings handled partially
returnData.viewSettings = parseViewSettings(jsonData, enforce);
returnData.viewSettings = parseViewSettings(jsonData) ?? dbModel.viewSettings;
// Import Aliases if any
returnData.aliases = parseAliases(jsonData);
// Import user fields if any
returnData.userFields = parseUserFields(jsonData);
// Import OSC settings if any
returnData.osc = parseOsc(jsonData, enforce);
returnData.osc = parseOsc(jsonData) ?? dbModel.osc;
// Import HTTP settings if any
// returnData.http = parseHttp(jsonData, enforce);
@@ -379,68 +377,56 @@ export const validateEvent = (eventArgs: Partial<OntimeEvent>, cueFallback: stri
return event;
};
type ResponseOK = { data: Partial<DatabaseModel>; message: 'success' };
type ResponseError = { error: true; message: string };
type ResponseOK = {
data: Partial<DatabaseModel>;
};
/**
* @description Middleware function that checks file type and calls relevant parser
* @param {string} file - reference to file
* @param options - import options
* @return {object} - parse result message
*/
export const fileHandler = async (file): Promise<ResponseOK | ResponseError> => {
let res: Partial<ResponseOK | ResponseError> = {};
export const fileHandler = async (file: string, options: ExcelImportOptions): Promise<Partial<ResponseOK>> => {
const res: Partial<ResponseOK> = {};
// check which file type are we dealing with
if (file.endsWith('.xlsx')) {
try {
const excelData = xlsx
.parse(file, { cellDates: true })
.find(({ name }) => name.toLowerCase() === 'ontime' || name.toLowerCase() === 'event schedule');
// we need to check that the options are applicable
if (!isExcelImportMap(options)) {
throw new Error('Got incorrect options to excel import', JSON.parse(options));
}
// we only look at worksheets called ontime or event schedule
if (excelData?.data) {
const dataFromExcel = await parseExcel(excelData.data);
res.data = {};
res.data.rundown = parseRundown(dataFromExcel);
res.data.project = parseProject(dataFromExcel, true);
res.data.userFields = parseUserFields(dataFromExcel);
res.message = 'success';
} else {
const errorMessage = 'No sheet found named "ontime" or "event schedule"';
res = {
error: true,
message: errorMessage,
};
}
} catch (error) {
res = { error: true, message: `Error parsing file: ${error}` };
const excelData = xlsx
.parse(file, { cellDates: true })
.find(({ name }) => name.toLowerCase() === options.worksheet);
if (excelData?.data) {
const dataFromExcel = parseExcel(excelData.data, options);
// we run the parsed data through an extra step to ensure the objects shape
res.data = {};
res.data.rundown = parseRundown(dataFromExcel);
res.data.project = parseProject(dataFromExcel);
res.data.userFields = parseUserFields(dataFromExcel);
return res;
} else {
throw new Error(`Could not find data to import, maybe the worksheet name is incorrect: ${options.worksheet}`);
}
}
if (file.endsWith('.json')) {
// if json check version
const rawdata = fs.readFileSync(file);
const rawdata = fs.readFileSync(file).toString();
let uploadedJson = null;
try {
uploadedJson = JSON.parse(rawdata);
} catch (error) {
return { error: true, message: 'Error parsing JSON file' };
uploadedJson = JSON.parse(rawdata);
if (uploadedJson.settings.version !== 2) {
throw new Error(`Project version unknown ${uploadedJson.settings.version}`);
}
res.data = await parseJson(uploadedJson);
if (uploadedJson.settings.version === 2) {
try {
res.data = await parseJson(uploadedJson);
res.message = 'success';
} catch (error) {
res = { error: true, message: `Error parsing file: ${error}` };
}
} else {
res = { error: true, message: 'Error parsing file, version unknown' };
}
// delete file
await deleteFile(file);
return res;
}
// delete file
await deleteFile(file);
return res;
};
+7 -30
View File
@@ -88,10 +88,9 @@ export const parseRundown = (data): OntimeRundown => {
/**
* Parse event portion of an entry
* @param {object} data - data object
* @param {boolean} enforce - whether to create a definition if one is missing
* @returns {object} - event object data
*/
export const parseProject = (data, enforce): ProjectData => {
export const parseProject = (data): ProjectData => {
let newProjectData: Partial<ProjectData> = {};
// we are adding this here to aid transition, should be removed once enough time has past that users have fully migrated
// TODO: Remove eventually
@@ -109,9 +108,6 @@ export const parseProject = (data, enforce): ProjectData => {
backstageUrl: project.backstageUrl || dbModel.project.backstageUrl,
backstageInfo: project.backstageInfo || dbModel.project.backstageInfo,
};
} else if (enforce) {
newProjectData = { ...dbModel.project };
console.log('Created project object in db');
}
return newProjectData as ProjectData;
};
@@ -119,10 +115,9 @@ export const parseProject = (data, enforce): ProjectData => {
/**
* Parse settings portion of an entry
* @param {object} data - data object
* @param {boolean} enforce - whether to create a definition if one is missing
* @returns {object} - event object data
*/
export const parseSettings = (data, enforce): Settings => {
export const parseSettings = (data): Settings => {
let newSettings: Partial<Settings> = {};
if ('settings' in data) {
console.log('Found settings definition, importing...');
@@ -146,9 +141,6 @@ export const parseSettings = (data, enforce): Settings => {
...settings,
};
}
} else if (enforce) {
newSettings = dbModel.settings;
console.log('Created settings object in db');
}
return newSettings as Settings;
};
@@ -156,10 +148,9 @@ export const parseSettings = (data, enforce): Settings => {
/**
* Parse settings portion of an entry
* @param {object} data - data object
* @param {boolean} enforce - whether to create a definition if one is missing
* @returns {object} - event object data
*/
export const parseViewSettings = (data, enforce): ViewSettings => {
export const parseViewSettings = (data): ViewSettings => {
let newViews: Partial<ViewSettings> = {};
if ('viewSettings' in data) {
console.log('Found view definition, importing...');
@@ -175,13 +166,7 @@ export const parseViewSettings = (data, enforce): ViewSettings => {
endMessage: v.endMessage ?? dbModel.viewSettings.endMessage,
};
// write to db
newViews = {
...viewSettings,
};
} else if (enforce) {
newViews = dbModel.viewSettings;
console.log('Created viewSettings object in db');
newViews = { ...viewSettings };
}
return newViews as ViewSettings;
};
@@ -224,16 +209,11 @@ export const validateOscObject = (data: OscSubscription): boolean => {
/**
* Parse osc portion of an entry
*/
export const parseOsc = (
data: {
osc?: Partial<OSCSettings>;
},
enforce: boolean,
): OSCSettings | Record<string, never> => {
export const parseOsc = (data: { osc?: Partial<OSCSettings> }): OSCSettings => {
if ('osc' in data) {
console.log('Found OSC definition, importing...');
const loadedConfig = data?.osc || {};
const loadedConfig = data.osc || {};
const validatedSubscriptions = validateOscObject(loadedConfig.subscriptions)
? loadedConfig.subscriptions
: dbModel.osc.subscriptions;
@@ -246,10 +226,7 @@ export const parseOsc = (
enabledOut: loadedConfig.enabledOut ?? dbModel.osc.enabledOut,
subscriptions: validatedSubscriptions,
};
} else if (enforce) {
console.log('Created OSC object in db');
return { ...dbModel.osc };
} else return {};
}
};
/**
+23 -8
View File
@@ -1,4 +1,5 @@
import fs from 'fs';
import { deepmerge } from 'ontime-utils';
/**
* @description Ensures variable is string, it skips object types
@@ -52,16 +53,30 @@ export const isEmptyObject = (obj: object) => {
/**
* @description Merges two objects, suppressing undefined keys
* @param {object} a
* @param {object} b
* @param {object} a - any object
* @param {object} b - a potential partial object of same time as a
*/
export const mergeObject = (a, b) => {
const merged = {};
Object.keys({ ...a, ...b }).map((key) => {
merged[key] = typeof b[key] === 'undefined' ? a[key] : b[key];
});
export function mergeObject<T extends Record<string, any>>(a: T, b: Partial<Record<keyof T, any>>): T {
const merged = { ...a };
for (const key in b) {
const aValue = a[key];
const bValue = b[key];
// ignore keys that do not exist in original object
if (!Object.hasOwn(merged, key)) {
continue;
}
if (typeof bValue === 'object' && bValue !== null && typeof aValue === 'object' && aValue !== null) {
// @ts-expect-error -- library side, ignore for now
merged[key] = deepmerge(aValue, bValue);
} else if (bValue !== undefined) {
merged[key] = bValue;
}
}
return merged;
};
}
/**
* @description Removes undefined
+6
View File
@@ -23,6 +23,12 @@ export { deepmerge } from './src/externals/deepmerge.js';
// generic utilities
export { isNumeric } from './src/types/types.js';
// model validation
export { validateEndAction, validateTimerType } from './src/validate-events/validateEvent.js';
// feature business logic
// feature business logic - excel import
export {
type ExcelImportMap,