mirror of
https://github.com/cpvalente/ontime.git
synced 2026-08-12 19:03:47 +00:00
feat: excel import (#527)
* refactor: simplify excel import * chore: add external deepmerge utility * chore: create import map utilities * chore: increase size limits on uploads * style: small presentation tweaks * feat: resolve times on excel import, refs #508
This commit is contained in:
Carlos Valente
GitHub
+167
-220
@@ -1,18 +1,27 @@
|
||||
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
|
||||
// @ts-nocheck -- not ready to fully type
|
||||
|
||||
import fs from 'fs';
|
||||
import xlsx from 'node-xlsx';
|
||||
import { generateId, calculateDuration } from 'ontime-utils';
|
||||
import {
|
||||
generateId,
|
||||
isExcelImportMap,
|
||||
type ExcelImportMap,
|
||||
defaultExcelImportMap,
|
||||
validateEndAction,
|
||||
validateTimerType,
|
||||
type ExcelImportOptions,
|
||||
validateTimes,
|
||||
} from 'ontime-utils';
|
||||
import {
|
||||
DatabaseModel,
|
||||
EndAction,
|
||||
OntimeEvent,
|
||||
OntimeRundown,
|
||||
SupportedEvent,
|
||||
TimerType,
|
||||
ProjectData,
|
||||
UserFields,
|
||||
EndAction,
|
||||
TimerType,
|
||||
} from 'ontime-types';
|
||||
|
||||
import fs from 'fs';
|
||||
import xlsx from 'node-xlsx';
|
||||
|
||||
import { event as eventDef } from '../models/eventsDefinition.js';
|
||||
import { dbModel } from '../models/dataModel.js';
|
||||
import { deleteFile, makeString } from './parserUtils.js';
|
||||
@@ -33,27 +42,55 @@ export const JSON_MIME = 'application/json';
|
||||
/**
|
||||
* @description Excel array parser
|
||||
* @param {array} excelData - array with excel sheet
|
||||
* @param {ExcelImportOptions} options - an object that contains the import map
|
||||
* @returns {object} - parsed object
|
||||
*/
|
||||
export const parseExcel = async (excelData) => {
|
||||
export const parseExcel = (excelData: unknown[][], options?: Partial<ExcelImportMap>) => {
|
||||
const importMap: ExcelImportMap = { ...defaultExcelImportMap, ...options };
|
||||
const projectData: Partial<ProjectData> = {
|
||||
title: '',
|
||||
description: '',
|
||||
publicUrl: '',
|
||||
publicInfo: '',
|
||||
backstageUrl: '',
|
||||
backstageInfo: '',
|
||||
};
|
||||
const customUserFields: Partial<UserFields> = {
|
||||
user0: importMap.user0,
|
||||
user1: importMap.user1,
|
||||
user2: importMap.user2,
|
||||
user3: importMap.user3,
|
||||
user4: importMap.user4,
|
||||
user5: importMap.user5,
|
||||
user6: importMap.user6,
|
||||
user7: importMap.user7,
|
||||
user8: importMap.user8,
|
||||
user9: importMap.user9,
|
||||
};
|
||||
const customUserFields: Partial<UserFields> = {};
|
||||
const rundown: OntimeRundown = [];
|
||||
let timeStartIndex: number | null = null;
|
||||
let timeEndIndex: number | null = null;
|
||||
|
||||
// title stuff: strings
|
||||
let titleIndex: number | null = null;
|
||||
let cueIndex: number | null = null;
|
||||
let presenterIndex: number | null = null;
|
||||
let subtitleIndex: number | null = null;
|
||||
let isPublicIndex: number | null = null;
|
||||
let skipIndex: number | null = null;
|
||||
let notesIndex: number | null = null;
|
||||
let colourIndex: number | null = null;
|
||||
|
||||
// options: booleans
|
||||
let isPublicIndex: number | null = null;
|
||||
let skipIndex: number | null = null;
|
||||
|
||||
// times: numbers
|
||||
let timeStartIndex: number | null = null;
|
||||
let timeEndIndex: number | null = null;
|
||||
let durationIndex: number | null = null;
|
||||
|
||||
// options: enum properties
|
||||
let endActionIndex: number | null = null;
|
||||
let timerTypeIndex: number | null = null;
|
||||
|
||||
// user fields: strings
|
||||
let user0Index: number | null = null;
|
||||
let user1Index: number | null = null;
|
||||
let user2Index: number | null = null;
|
||||
@@ -64,13 +101,11 @@ export const parseExcel = async (excelData) => {
|
||||
let user7Index: number | null = null;
|
||||
let user8Index: number | null = null;
|
||||
let user9Index: number | null = null;
|
||||
let endActionIndex: number | null = null;
|
||||
let timerTypeIndex: number | null = null;
|
||||
|
||||
excelData
|
||||
.filter((e) => e.length > 0)
|
||||
.forEach((row) => {
|
||||
// project data imports are on the column to the right
|
||||
// these fields contain the data to its right
|
||||
let projectTitleNext = false;
|
||||
let projectDescriptionNext = false;
|
||||
let publicUrlNext = false;
|
||||
@@ -79,31 +114,68 @@ export const parseExcel = async (excelData) => {
|
||||
let backstageInfoNext = false;
|
||||
|
||||
const event: Partial<OntimeEvent> = {};
|
||||
const handlers = {
|
||||
[importMap.projectName]: () => (projectTitleNext = true),
|
||||
[importMap.projectDescription]: () => (projectDescriptionNext = true),
|
||||
[importMap.publicUrl]: () => (publicUrlNext = true),
|
||||
[importMap.publicInfo]: () => (publicInfoNext = true),
|
||||
[importMap.backstageUrl]: () => (backstageUrlNext = true),
|
||||
[importMap.backstageInfo]: () => (backstageInfoNext = true),
|
||||
|
||||
[importMap.timeStart]: (index: number) => (timeStartIndex = index),
|
||||
[importMap.timeEnd]: (index: number) => (timeEndIndex = index),
|
||||
[importMap.duration]: (index: number) => (durationIndex = index),
|
||||
|
||||
[importMap.cue]: (index: number) => (cueIndex = index),
|
||||
[importMap.title]: (index: number) => (titleIndex = index),
|
||||
[importMap.presenter]: (index: number) => (presenterIndex = index),
|
||||
[importMap.subtitle]: (index: number) => (subtitleIndex = index),
|
||||
[importMap.isPublic]: (index: number) => (isPublicIndex = index),
|
||||
[importMap.skip]: (index: number) => (skipIndex = index),
|
||||
[importMap.note]: (index: number) => (notesIndex = index),
|
||||
[importMap.colour]: (index: number) => (colourIndex = index),
|
||||
|
||||
[importMap.endAction]: (index: number) => (endActionIndex = index),
|
||||
[importMap.timerType]: (index: number) => (timerTypeIndex = index),
|
||||
|
||||
[importMap.user0]: (index: number) => (user0Index = index),
|
||||
[importMap.user1]: (index: number) => (user1Index = index),
|
||||
[importMap.user2]: (index: number) => (user2Index = index),
|
||||
[importMap.user3]: (index: number) => (user3Index = index),
|
||||
[importMap.user4]: (index: number) => (user4Index = index),
|
||||
[importMap.user5]: (index: number) => (user5Index = index),
|
||||
[importMap.user6]: (index: number) => (user6Index = index),
|
||||
[importMap.user7]: (index: number) => (user7Index = index),
|
||||
[importMap.user8]: (index: number) => (user8Index = index),
|
||||
[importMap.user9]: (index: number) => (user9Index = index),
|
||||
} as const;
|
||||
|
||||
row.forEach((column, j) => {
|
||||
// check flags
|
||||
// 1. we check if we have set a flag for a known field
|
||||
if (projectTitleNext) {
|
||||
projectData.title = column;
|
||||
projectData.title = makeString(column, '');
|
||||
projectTitleNext = false;
|
||||
} else if (projectDescriptionNext) {
|
||||
projectData.description = column;
|
||||
projectData.description = makeString(column, '');
|
||||
projectDescriptionNext = false;
|
||||
} else if (publicUrlNext) {
|
||||
projectData.publicUrl = column;
|
||||
projectData.publicUrl = makeString(column, '');
|
||||
publicUrlNext = false;
|
||||
} else if (publicInfoNext) {
|
||||
projectData.publicInfo = column;
|
||||
projectData.publicInfo = makeString(column, '');
|
||||
publicInfoNext = false;
|
||||
} else if (backstageUrlNext) {
|
||||
projectData.backstageUrl = column;
|
||||
projectData.backstageUrl = makeString(column, '');
|
||||
backstageUrlNext = false;
|
||||
} else if (backstageInfoNext) {
|
||||
projectData.backstageInfo = column;
|
||||
projectData.backstageInfo = makeString(column, '');
|
||||
backstageInfoNext = false;
|
||||
} else if (j === timeStartIndex) {
|
||||
event.timeStart = parseExcelDate(column);
|
||||
} else if (j === timeEndIndex) {
|
||||
event.timeEnd = parseExcelDate(column);
|
||||
} else if (j === durationIndex) {
|
||||
event.duration = parseExcelDate(column);
|
||||
} else if (j === titleIndex) {
|
||||
event.title = makeString(column, '');
|
||||
} else if (j === cueIndex) {
|
||||
@@ -119,166 +191,51 @@ export const parseExcel = async (excelData) => {
|
||||
} else if (j === notesIndex) {
|
||||
event.note = makeString(column, '');
|
||||
} else if (j === endActionIndex) {
|
||||
if (column === '') {
|
||||
event.endAction = EndAction.None;
|
||||
} else {
|
||||
event.endAction = column;
|
||||
}
|
||||
event.endAction = validateEndAction(column);
|
||||
} else if (j === timerTypeIndex) {
|
||||
if (column === '') {
|
||||
event.timerType = TimerType.CountDown;
|
||||
} else {
|
||||
event.timerType = column;
|
||||
}
|
||||
event.timerType = validateTimerType(column);
|
||||
} else if (j === colourIndex) {
|
||||
event.colour = column;
|
||||
event.colour = makeString(column, '');
|
||||
} else if (j === user0Index) {
|
||||
event.user0 = column;
|
||||
event.user0 = makeString(column, '');
|
||||
} else if (j === user1Index) {
|
||||
event.user1 = column;
|
||||
event.user1 = makeString(column, '');
|
||||
} else if (j === user2Index) {
|
||||
event.user2 = column;
|
||||
event.user2 = makeString(column, '');
|
||||
} else if (j === user3Index) {
|
||||
event.user3 = column;
|
||||
event.user3 = makeString(column, '');
|
||||
} else if (j === user4Index) {
|
||||
event.user4 = column;
|
||||
event.user4 = makeString(column, '');
|
||||
} else if (j === user5Index) {
|
||||
event.user5 = column;
|
||||
event.user5 = makeString(column, '');
|
||||
} else if (j === user6Index) {
|
||||
event.user6 = column;
|
||||
event.user6 = makeString(column, '');
|
||||
} else if (j === user7Index) {
|
||||
event.user7 = column;
|
||||
event.user7 = makeString(column, '');
|
||||
} else if (j === user8Index) {
|
||||
event.user8 = column;
|
||||
event.user8 = makeString(column, '');
|
||||
} else if (j === user9Index) {
|
||||
event.user9 = column;
|
||||
event.user9 = makeString(column, '');
|
||||
} else {
|
||||
// 2. if there is no flag, lets see if we know the field type
|
||||
if (typeof column === 'string') {
|
||||
const col = column.toLowerCase();
|
||||
|
||||
// look for keywords
|
||||
// need to make sure it is a string first
|
||||
switch (col) {
|
||||
case 'project name':
|
||||
projectTitleNext = true;
|
||||
break;
|
||||
case 'project description':
|
||||
projectDescriptionNext = true;
|
||||
break;
|
||||
case 'public url':
|
||||
publicUrlNext = true;
|
||||
break;
|
||||
case 'public info':
|
||||
publicInfoNext = true;
|
||||
break;
|
||||
case 'backstage url':
|
||||
backstageUrlNext = true;
|
||||
break;
|
||||
case 'backstage info':
|
||||
backstageInfoNext = true;
|
||||
break;
|
||||
case 'time start':
|
||||
case 'start':
|
||||
timeStartIndex = j;
|
||||
break;
|
||||
case 'time end':
|
||||
case 'end':
|
||||
case 'finish':
|
||||
timeEndIndex = j;
|
||||
break;
|
||||
case 'cue':
|
||||
case 'page':
|
||||
cueIndex = j;
|
||||
break;
|
||||
case 'event title':
|
||||
case 'title':
|
||||
titleIndex = j;
|
||||
break;
|
||||
case 'presenter name':
|
||||
case 'speaker':
|
||||
case 'presenter':
|
||||
presenterIndex = j;
|
||||
break;
|
||||
case 'event subtitle':
|
||||
case 'subtitle':
|
||||
subtitleIndex = j;
|
||||
break;
|
||||
case 'is public? (x)':
|
||||
case 'is public':
|
||||
case 'public':
|
||||
isPublicIndex = j;
|
||||
break;
|
||||
case 'skip? (x)':
|
||||
case 'skip?':
|
||||
case 'skip':
|
||||
skipIndex = j;
|
||||
break;
|
||||
case 'note':
|
||||
case 'notes':
|
||||
notesIndex = j;
|
||||
break;
|
||||
case 'colour':
|
||||
case 'color':
|
||||
colourIndex = j;
|
||||
break;
|
||||
case 'end action':
|
||||
endActionIndex = j;
|
||||
break;
|
||||
case 'timer type':
|
||||
timerTypeIndex = j;
|
||||
break;
|
||||
default:
|
||||
// look for user defined
|
||||
if (col.startsWith('user')) {
|
||||
const index = column.charAt(4);
|
||||
// name is the bit after the :
|
||||
const [, name] = column.split(':');
|
||||
if (typeof name !== 'undefined') {
|
||||
if (index === '0') {
|
||||
customUserFields.user0 = name;
|
||||
user0Index = j;
|
||||
} else if (index === '1') {
|
||||
customUserFields.user1 = name;
|
||||
user1Index = j;
|
||||
} else if (index === '2') {
|
||||
customUserFields.user2 = name;
|
||||
user2Index = j;
|
||||
} else if (index === '3') {
|
||||
customUserFields.user3 = name;
|
||||
user3Index = j;
|
||||
} else if (index === '4') {
|
||||
customUserFields.user4 = name;
|
||||
user4Index = j;
|
||||
} else if (index === '5') {
|
||||
customUserFields.user5 = name;
|
||||
user5Index = j;
|
||||
} else if (index === '6') {
|
||||
customUserFields.user6 = name;
|
||||
user6Index = j;
|
||||
} else if (index === '7') {
|
||||
customUserFields.user7 = name;
|
||||
user7Index = j;
|
||||
} else if (index === '8') {
|
||||
customUserFields.user8 = name;
|
||||
user8Index = j;
|
||||
} else if (index === '9') {
|
||||
customUserFields.user9 = name;
|
||||
user9Index = j;
|
||||
}
|
||||
}
|
||||
}
|
||||
break;
|
||||
if (handlers[col]) {
|
||||
handlers[col](j);
|
||||
}
|
||||
// else. we don't know how to handle this column
|
||||
// just ignore it
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
if (Object.keys(event).length > 0) {
|
||||
// if any data was found, push to array
|
||||
// take care of it in the next step
|
||||
rundown.push({ ...event, type: SupportedEvent.Event } as OntimeEvent);
|
||||
}
|
||||
});
|
||||
|
||||
return {
|
||||
rundown,
|
||||
project: projectData,
|
||||
@@ -286,17 +243,16 @@ export const parseExcel = async (excelData) => {
|
||||
app: 'ontime',
|
||||
version: 2,
|
||||
},
|
||||
userFields: { ...dbModel.userFields, ...customUserFields },
|
||||
userFields: customUserFields,
|
||||
};
|
||||
};
|
||||
|
||||
/**
|
||||
* @description JSON parser function for v1 of data system
|
||||
* @param {object} jsonData - json data JSON object to be parsed
|
||||
* @param {boolean} [enforce=false] - flag, tells to create an object anyway
|
||||
* @description JSON parser function for ontime project file
|
||||
* @param {object} jsonData - project file to be parsed
|
||||
* @returns {object} - parsed object
|
||||
*/
|
||||
export const parseJson = async (jsonData, enforce = false): Promise<DatabaseModel | null> => {
|
||||
export const parseJson = async (jsonData): Promise<DatabaseModel | null> => {
|
||||
if (!jsonData || typeof jsonData !== 'object') {
|
||||
return null;
|
||||
}
|
||||
@@ -307,17 +263,17 @@ export const parseJson = async (jsonData, enforce = false): Promise<DatabaseMode
|
||||
// parse Events
|
||||
returnData.rundown = parseRundown(jsonData);
|
||||
// parse Event
|
||||
returnData.project = parseProject(jsonData, enforce);
|
||||
returnData.project = parseProject(jsonData) ?? dbModel.project;
|
||||
// Settings handled partially
|
||||
returnData.settings = parseSettings(jsonData, enforce);
|
||||
returnData.settings = parseSettings(jsonData) ?? dbModel.settings;
|
||||
// View settings handled partially
|
||||
returnData.viewSettings = parseViewSettings(jsonData, enforce);
|
||||
returnData.viewSettings = parseViewSettings(jsonData) ?? dbModel.viewSettings;
|
||||
// Import Aliases if any
|
||||
returnData.aliases = parseAliases(jsonData);
|
||||
// Import user fields if any
|
||||
returnData.userFields = parseUserFields(jsonData);
|
||||
// Import OSC settings if any
|
||||
returnData.osc = parseOsc(jsonData, enforce);
|
||||
returnData.osc = parseOsc(jsonData) ?? dbModel.osc;
|
||||
// Import HTTP settings if any
|
||||
// returnData.http = parseHttp(jsonData, enforce);
|
||||
|
||||
@@ -344,19 +300,19 @@ export const validateEvent = (eventArgs: Partial<OntimeEvent>, cueFallback: stri
|
||||
|
||||
const e = eventArgs;
|
||||
const d = eventDef;
|
||||
const start = e.timeStart != null && typeof e.timeStart === 'number' ? e.timeStart : d.timeStart;
|
||||
const end = e.timeEnd != null && typeof e.timeEnd === 'number' ? e.timeEnd : d.timeEnd;
|
||||
|
||||
const { timeStart, timeEnd, duration } = validateTimes(e.timeStart, e.timeEnd, e.duration);
|
||||
|
||||
event = {
|
||||
...d,
|
||||
title: makeString(e.title, d.title),
|
||||
subtitle: makeString(e.subtitle, d.subtitle),
|
||||
presenter: makeString(e.presenter, d.presenter),
|
||||
timeStart: start,
|
||||
timeEnd: end,
|
||||
endAction: makeString(e.endAction, d.endAction),
|
||||
timerType: makeString(e.timerType, d.timerType),
|
||||
duration: calculateDuration(start, end),
|
||||
timeStart,
|
||||
timeEnd,
|
||||
duration,
|
||||
endAction: validateEndAction(e.endAction, EndAction.None),
|
||||
timerType: validateTimerType(e.timerType, TimerType.CountDown),
|
||||
isPublic: typeof e.isPublic === 'boolean' ? e.isPublic : d.isPublic,
|
||||
skip: typeof e.skip === 'boolean' ? e.skip : d.skip,
|
||||
note: makeString(e.note, d.note),
|
||||
@@ -371,8 +327,8 @@ export const validateEvent = (eventArgs: Partial<OntimeEvent>, cueFallback: stri
|
||||
user8: makeString(e.user8, d.user8),
|
||||
user9: makeString(e.user9, d.user9),
|
||||
colour: makeString(e.colour, d.colour),
|
||||
id,
|
||||
cue: makeString(e.cue, cueFallback),
|
||||
id,
|
||||
type: 'event',
|
||||
};
|
||||
}
|
||||
@@ -380,68 +336,59 @@ export const validateEvent = (eventArgs: Partial<OntimeEvent>, cueFallback: stri
|
||||
return event;
|
||||
};
|
||||
|
||||
type ResponseOK = { data: Partial<DatabaseModel>; message: 'success' };
|
||||
type ResponseError = { error: true; message: string };
|
||||
type ResponseOK = {
|
||||
data: Partial<DatabaseModel>;
|
||||
};
|
||||
|
||||
/**
|
||||
* @description Middleware function that checks file type and calls relevant parser
|
||||
* @param {string} file - reference to file
|
||||
* @param options - import options
|
||||
* @return {object} - parse result message
|
||||
*/
|
||||
export const fileHandler = async (file): Promise<ResponseOK | ResponseError> => {
|
||||
let res: Partial<ResponseOK | ResponseError> = {};
|
||||
export const fileHandler = async (file: string, options: ExcelImportOptions): Promise<Partial<ResponseOK>> => {
|
||||
const res: Partial<ResponseOK> = {};
|
||||
|
||||
// check which file type are we dealing with
|
||||
if (file.endsWith('.xlsx')) {
|
||||
try {
|
||||
const excelData = xlsx
|
||||
.parse(file, { cellDates: true })
|
||||
.find(({ name }) => name.toLowerCase() === 'ontime' || name.toLowerCase() === 'event schedule');
|
||||
|
||||
// we only look at worksheets called ontime or event schedule
|
||||
if (excelData?.data) {
|
||||
const dataFromExcel = await parseExcel(excelData.data);
|
||||
res.data = {};
|
||||
res.data.rundown = parseRundown(dataFromExcel);
|
||||
res.data.project = parseProject(dataFromExcel, true);
|
||||
res.data.userFields = parseUserFields(dataFromExcel);
|
||||
res.message = 'success';
|
||||
} else {
|
||||
const errorMessage = 'No sheet found named "ontime" or "event schedule"';
|
||||
res = {
|
||||
error: true,
|
||||
message: errorMessage,
|
||||
};
|
||||
}
|
||||
} catch (error) {
|
||||
res = { error: true, message: `Error parsing file: ${error}` };
|
||||
// we need to check that the options are applicable
|
||||
if (!isExcelImportMap(options)) {
|
||||
throw new Error('Got incorrect options to excel import', JSON.parse(options));
|
||||
}
|
||||
|
||||
const excelData = xlsx
|
||||
.parse(file, { cellDates: true })
|
||||
.find(({ name }) => name.toLowerCase() === options.worksheet.toLowerCase());
|
||||
|
||||
if (!excelData?.data) {
|
||||
throw new Error(`Could not find data to import, maybe the worksheet name is incorrect: ${options.worksheet}`);
|
||||
}
|
||||
|
||||
const dataFromExcel = parseExcel(excelData.data, options);
|
||||
// we run the parsed data through an extra step to ensure the objects shape
|
||||
res.data = {};
|
||||
res.data.rundown = parseRundown(dataFromExcel);
|
||||
if (res.data.rundown.length < 1) {
|
||||
throw new Error(`Could not find data to import in the worksheet ${options.worksheet}`);
|
||||
}
|
||||
res.data.project = parseProject(dataFromExcel);
|
||||
res.data.userFields = parseUserFields(dataFromExcel);
|
||||
return res;
|
||||
}
|
||||
|
||||
if (file.endsWith('.json')) {
|
||||
// if json check version
|
||||
const rawdata = fs.readFileSync(file);
|
||||
const rawdata = fs.readFileSync(file).toString();
|
||||
let uploadedJson = null;
|
||||
|
||||
try {
|
||||
uploadedJson = JSON.parse(rawdata);
|
||||
} catch (error) {
|
||||
return { error: true, message: 'Error parsing JSON file' };
|
||||
uploadedJson = JSON.parse(rawdata);
|
||||
if (uploadedJson.settings.version !== 2) {
|
||||
throw new Error(`Project version unknown ${uploadedJson.settings.version}`);
|
||||
}
|
||||
res.data = await parseJson(uploadedJson);
|
||||
|
||||
if (uploadedJson.settings.version === 2) {
|
||||
try {
|
||||
res.data = await parseJson(uploadedJson);
|
||||
res.message = 'success';
|
||||
} catch (error) {
|
||||
res = { error: true, message: `Error parsing file: ${error}` };
|
||||
}
|
||||
} else {
|
||||
res = { error: true, message: 'Error parsing file, version unknown' };
|
||||
}
|
||||
// delete file
|
||||
await deleteFile(file);
|
||||
return res;
|
||||
}
|
||||
|
||||
// delete file
|
||||
await deleteFile(file);
|
||||
return res;
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user