mirror of
https://github.com/cpvalente/ontime.git
synced 2026-08-14 03:43:50 +00:00
refactor: migrate custom fields to transactions
refactor: extract functions to api domain refactor: strict custom field parsing refactor: remove rundown cache utilities refactor: directory restructure
This commit is contained in:
Carlos Valente
arc-alex
@@ -0,0 +1,582 @@
|
||||
import { CustomFields, OntimeEvent, SupportedEntry, TimerType } from 'ontime-types';
|
||||
import { defaultImportMap, ImportMap, MILLIS_PER_MINUTE } from 'ontime-utils';
|
||||
|
||||
import { getCustomFieldData, parseExcel } from '../excel.parser.js';
|
||||
|
||||
import { dataFromExcelTemplate } from './mockData.js';
|
||||
|
||||
describe('parseExcel()', () => {
|
||||
it('parses the example file', () => {
|
||||
// partial import map with only custom fields
|
||||
const importMap = {
|
||||
custom: {
|
||||
user0: 't0',
|
||||
user1: 'Test1',
|
||||
user2: 'test2',
|
||||
user3: 'test3',
|
||||
},
|
||||
};
|
||||
|
||||
const existingCustomFields: CustomFields = {
|
||||
user0: { type: 'string', colour: 'red', label: 'user0' },
|
||||
user1: { type: 'string', colour: 'green', label: 'user1' },
|
||||
user2: { type: 'string', colour: 'blue', label: 'user2' },
|
||||
};
|
||||
|
||||
const parsedData = parseExcel(dataFromExcelTemplate, existingCustomFields, 'testSheet', importMap);
|
||||
expect(parsedData.customFields).toStrictEqual({
|
||||
user0: {
|
||||
type: 'string',
|
||||
colour: 'red',
|
||||
label: 'user0',
|
||||
},
|
||||
user1: {
|
||||
type: 'string',
|
||||
colour: 'green',
|
||||
label: 'user1',
|
||||
},
|
||||
user2: {
|
||||
type: 'string',
|
||||
colour: 'blue',
|
||||
label: 'user2',
|
||||
},
|
||||
user3: {
|
||||
type: 'string',
|
||||
colour: '',
|
||||
label: 'user3',
|
||||
},
|
||||
});
|
||||
expect(parsedData.rundown.order.length).toBe(2);
|
||||
// TODO: why dont we parse the date in UTC?
|
||||
expect(parsedData.rundown.entries).toMatchObject({
|
||||
'event-a': {
|
||||
id: 'event-a',
|
||||
//timeStart: 28800000,
|
||||
//timeEnd: 32410000,
|
||||
title: 'Guest Welcome',
|
||||
timerType: 'count-down',
|
||||
endAction: 'none',
|
||||
isPublic: true,
|
||||
skip: false,
|
||||
note: 'Ballyhoo',
|
||||
custom: {
|
||||
user0: 'a0',
|
||||
user1: 'a1',
|
||||
user2: 'a2',
|
||||
user3: 'a3',
|
||||
},
|
||||
colour: 'red',
|
||||
type: 'event',
|
||||
cue: '101',
|
||||
},
|
||||
'event-b': {
|
||||
id: 'event-b',
|
||||
//timeStart: 32400000,
|
||||
//timeEnd: 34200000,
|
||||
title: 'A song from the hearth',
|
||||
timerType: 'clock',
|
||||
endAction: 'load-next',
|
||||
isPublic: false,
|
||||
skip: true,
|
||||
note: 'Rainbow chase',
|
||||
custom: {},
|
||||
colour: '#F00',
|
||||
type: 'event',
|
||||
cue: '102',
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('parses a file without custom fields', () => {
|
||||
// partial import map with only custom fields
|
||||
const importMap = {
|
||||
custom: {
|
||||
niu1: 'niu1',
|
||||
niu2: 'niu2',
|
||||
},
|
||||
};
|
||||
|
||||
const parsedData = parseExcel(dataFromExcelTemplate, {}, 'testSheet', importMap);
|
||||
expect(parsedData.customFields).toStrictEqual({
|
||||
niu1: {
|
||||
type: 'string',
|
||||
colour: '',
|
||||
label: 'niu1',
|
||||
},
|
||||
niu2: {
|
||||
type: 'string',
|
||||
colour: '',
|
||||
label: 'niu2',
|
||||
},
|
||||
});
|
||||
expect(parsedData.rundown.order).toMatchObject(['event-a', 'event-b']);
|
||||
expect(parsedData.rundown.entries['event-a']).toMatchObject({
|
||||
//timeStart: 28800000,
|
||||
//timeEnd: 32410000,
|
||||
id: 'event-a',
|
||||
title: 'Guest Welcome',
|
||||
timerType: 'count-down',
|
||||
endAction: 'none',
|
||||
isPublic: true,
|
||||
skip: false,
|
||||
note: 'Ballyhoo',
|
||||
custom: {},
|
||||
colour: 'red',
|
||||
type: 'event',
|
||||
cue: '101',
|
||||
});
|
||||
expect(parsedData.rundown.entries['event-b']).toMatchObject({
|
||||
//timeStart: 32400000,
|
||||
//timeEnd: 34200000,
|
||||
id: 'event-b',
|
||||
title: 'A song from the hearth',
|
||||
timerType: 'clock',
|
||||
endAction: 'load-next',
|
||||
isPublic: false,
|
||||
skip: true,
|
||||
note: 'Rainbow chase',
|
||||
custom: {},
|
||||
colour: '#F00',
|
||||
type: 'event',
|
||||
cue: '102',
|
||||
});
|
||||
});
|
||||
|
||||
it('ignores unknown event types', () => {
|
||||
const testdata = [
|
||||
['Title', 'Timer type'],
|
||||
['Guest Welcome', 'x'],
|
||||
['A song from the hearth', 'clock'],
|
||||
];
|
||||
|
||||
const importMap = {
|
||||
title: 'title',
|
||||
timerType: 'timer type',
|
||||
};
|
||||
const result = parseExcel(testdata, {}, 'testSheet', importMap);
|
||||
const firstEvent = result.rundown.entries[result.rundown.order[0]];
|
||||
|
||||
expect(result.rundown.order.length).toBe(1);
|
||||
expect((firstEvent as OntimeEvent).title).toBe('A song from the hearth');
|
||||
});
|
||||
|
||||
it('imports blocks', () => {
|
||||
const testdata = [
|
||||
['Title', 'Timer type'],
|
||||
['a block', 'block'],
|
||||
['an event', 'clock'],
|
||||
];
|
||||
|
||||
const importMap = {
|
||||
title: 'title',
|
||||
timerType: 'timer type',
|
||||
};
|
||||
const result = parseExcel(testdata, {}, 'testSheet', importMap);
|
||||
const firstEvent = result.rundown.entries[result.rundown.order[0]];
|
||||
|
||||
expect(result.rundown.order.length).toBe(2);
|
||||
expect((firstEvent as OntimeEvent).type).toBe(SupportedEntry.Block);
|
||||
});
|
||||
|
||||
it('imports as events if there is no timer type column', () => {
|
||||
const testdata = [['Title'], ['no timer type'], ['also no timer type']];
|
||||
|
||||
const importMap = {
|
||||
title: 'title',
|
||||
};
|
||||
|
||||
const result = parseExcel(testdata, {}, 'testSheet', importMap);
|
||||
const firstEvent = result.rundown.entries[result.rundown.order[0]];
|
||||
const secondEvent = result.rundown.entries[result.rundown.order[1]];
|
||||
|
||||
expect(result.rundown.order.length).toBe(2);
|
||||
expect(firstEvent).toMatchObject({
|
||||
type: SupportedEntry.Event,
|
||||
timerType: TimerType.CountDown,
|
||||
});
|
||||
|
||||
expect(secondEvent).toMatchObject({
|
||||
type: SupportedEntry.Event,
|
||||
timerType: TimerType.CountDown,
|
||||
});
|
||||
});
|
||||
|
||||
it('imports as events if timer type is empty or has whitespace', () => {
|
||||
const testdata = [
|
||||
['Title', 'Timer type'],
|
||||
['first', ' '],
|
||||
['second', undefined],
|
||||
['third', ' count-up '],
|
||||
];
|
||||
|
||||
const importMap = {
|
||||
title: 'title',
|
||||
timerType: 'timer type',
|
||||
};
|
||||
const result = parseExcel(testdata, {}, 'testSheet', importMap);
|
||||
const firstEvent = result.rundown.entries[result.rundown.order[0]];
|
||||
const secondEvent = result.rundown.entries[result.rundown.order[1]];
|
||||
const thirdEvent = result.rundown.entries[result.rundown.order[2]];
|
||||
expect(result.rundown.order.length).toBe(3);
|
||||
expect(firstEvent).toMatchObject({ title: 'first', type: SupportedEntry.Event, timerType: TimerType.CountDown });
|
||||
expect(secondEvent).toMatchObject({ title: 'second', type: SupportedEntry.Event, timerType: TimerType.CountDown });
|
||||
expect(thirdEvent).toMatchObject({ title: 'third', type: SupportedEntry.Event, timerType: TimerType.CountUp });
|
||||
});
|
||||
|
||||
it('am/pm conversion to 24h', () => {
|
||||
const testData = [
|
||||
['Time Start', 'Time End', 'ID'],
|
||||
['4:30:00', '4:36:00', 'event-1'],
|
||||
['9:45:00', '10:56:00', 'event-2'],
|
||||
['16:30:00', '16:36:00', 'event-3'],
|
||||
['21:45:00', '22:56:00', 'event-4'],
|
||||
['4:30:00AM', '4:36:00AM', 'event-5'],
|
||||
['9:45:00AM', '10:56:00AM', 'event-6'],
|
||||
['4:30:00PM', '4:36:00PM', 'event-7'],
|
||||
['9:45:00PM', '10:56:00PM', 'event-8'],
|
||||
];
|
||||
|
||||
const importMap = {
|
||||
timeStart: 'time start',
|
||||
timeEnd: 'time end',
|
||||
id: 'id',
|
||||
};
|
||||
const result = parseExcel(testData, {}, 'testSheet', importMap);
|
||||
expect(result.rundown.order.length).toBe(8);
|
||||
expect(result.rundown.entries['event-1']).toMatchObject({
|
||||
timeStart: 16200000,
|
||||
timeEnd: 16560000,
|
||||
});
|
||||
expect(result.rundown.entries['event-2']).toMatchObject({
|
||||
timeStart: 35100000,
|
||||
timeEnd: 39360000,
|
||||
});
|
||||
expect(result.rundown.entries['event-3']).toMatchObject({
|
||||
timeStart: 59400000,
|
||||
timeEnd: 59760000,
|
||||
});
|
||||
expect(result.rundown.entries['event-4']).toMatchObject({
|
||||
timeStart: 78300000,
|
||||
timeEnd: 82560000,
|
||||
});
|
||||
expect(result.rundown.entries['event-5']).toMatchObject({
|
||||
timeStart: 16200000,
|
||||
timeEnd: 16560000,
|
||||
});
|
||||
expect(result.rundown.entries['event-6']).toMatchObject({
|
||||
timeStart: 35100000,
|
||||
timeEnd: 39360000,
|
||||
});
|
||||
expect(result.rundown.entries['event-7']).toMatchObject({
|
||||
timeStart: 59400000,
|
||||
timeEnd: 59760000,
|
||||
});
|
||||
expect(result.rundown.entries['event-8']).toMatchObject({
|
||||
timeStart: 78300000,
|
||||
timeEnd: 82560000,
|
||||
});
|
||||
});
|
||||
|
||||
it('handle leading and trailing whitespace', () => {
|
||||
const testData = [
|
||||
[' ID', ' title ', 'Colour '], // <--- leading and trailing white space
|
||||
['event-a', 'title', '#F00'],
|
||||
];
|
||||
|
||||
const importMap = {
|
||||
id: 'id',
|
||||
title: ' title', // <--- leading white space
|
||||
colour: 'colour ', // <--- trailing white space
|
||||
};
|
||||
|
||||
const result = parseExcel(testData, {}, 'testSheet', importMap);
|
||||
expect(result.rundown.order.length).toBe(1);
|
||||
expect(result.rundown.entries['event-a']).toMatchObject({
|
||||
colour: '#F00',
|
||||
id: 'event-a',
|
||||
title: 'title',
|
||||
});
|
||||
});
|
||||
|
||||
it('parses link start', () => {
|
||||
const testData = [
|
||||
['Time Start', 'Time End', 'ID', 'Link Start', 'Timer type'],
|
||||
['4:30:00', '9:45:00', 'A', '', 'count-down'],
|
||||
['9:45:00', '10:56:00', 'B', 'x', 'count-down'],
|
||||
['10:00:00', '16:36:00', 'C', 'x', 'count-down'],
|
||||
['21:45:00', '22:56:00', 'D', '', 'count-down'],
|
||||
['', '', 'BLOCK', 'x', 'block'], // <-- block with link
|
||||
['00:0:00', '23:56:00', 'E', 'x', 'count-down'], // <-- link past blocks
|
||||
];
|
||||
|
||||
const importMap = {
|
||||
timeStart: 'time start',
|
||||
timeEnd: 'time end',
|
||||
linkStart: 'link start',
|
||||
id: 'id',
|
||||
timerType: 'timer type',
|
||||
};
|
||||
|
||||
const result = parseExcel(testData, {}, 'testSheet', importMap);
|
||||
expect(result.rundown.order.length).toBe(6);
|
||||
expect(result.rundown.order).toMatchObject(['A', 'B', 'C', 'D', 'BLOCK', 'E']);
|
||||
|
||||
expect(result.rundown.entries).toMatchObject({
|
||||
A: {
|
||||
linkStart: false,
|
||||
},
|
||||
B: {
|
||||
linkStart: true,
|
||||
},
|
||||
C: {
|
||||
linkStart: true,
|
||||
},
|
||||
D: {
|
||||
linkStart: false,
|
||||
},
|
||||
BLOCK: {
|
||||
type: SupportedEntry.Block,
|
||||
},
|
||||
E: {
|
||||
linkStart: true,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('#971 BUG: parses time fields and booleans', () => {
|
||||
const testData = [
|
||||
[
|
||||
'ID',
|
||||
'Time Start',
|
||||
'Time End',
|
||||
'Duration',
|
||||
'Link Start',
|
||||
'Timer Type',
|
||||
'End Action',
|
||||
'Warning time',
|
||||
'Danger time',
|
||||
],
|
||||
[
|
||||
'SETUP',
|
||||
'1899-12-30T07:15:00.000Z',
|
||||
'1899-12-30T08:30:00.000Z',
|
||||
'',
|
||||
'false',
|
||||
'count-down',
|
||||
'none',
|
||||
'15',
|
||||
'00:05:00',
|
||||
],
|
||||
[
|
||||
'MEET1',
|
||||
'1899-12-30T08:30:00.000Z',
|
||||
'1899-12-30T10:00:00.000Z',
|
||||
'',
|
||||
'false',
|
||||
'count-down',
|
||||
'none',
|
||||
15,
|
||||
'00:05:00',
|
||||
],
|
||||
['MEET2', '1899-12-30T10:00:00.000Z', '', '60', 'false', 'count-down', 'none', '13', '5'],
|
||||
['lunch', '', '1899-12-30T11:30:00.000Z', '', 'true', 'count-down', 'none', 13, 5],
|
||||
['MEET3', '1899-12-30T11:30:00.000Z', '', 90, false, 'count-up', 'none', '11', 5],
|
||||
['MEET4', '', '', 30, true, 'count-up', 'none', 11, '00:05:00'],
|
||||
];
|
||||
|
||||
const parsedData = parseExcel(testData, {}, 'bug-report');
|
||||
|
||||
// '15' as a string is parsed by smart time entry as minutes
|
||||
expect(parsedData.rundown.entries['SETUP']).toMatchObject({
|
||||
timeWarning: 15 * MILLIS_PER_MINUTE,
|
||||
});
|
||||
|
||||
// elements in bug report
|
||||
// 15 is a number, in which case we parse it as a minutes value
|
||||
expect(parsedData.rundown.entries['MEET1']).toMatchObject({
|
||||
timeWarning: 15 * MILLIS_PER_MINUTE,
|
||||
});
|
||||
|
||||
// in the case where a string is passed, we need to check whether it is an ISO 8601 date
|
||||
expect(parsedData.rundown.entries['MEET2']).toMatchObject({
|
||||
duration: 60 * MILLIS_PER_MINUTE,
|
||||
timeDanger: 5 * MILLIS_PER_MINUTE,
|
||||
});
|
||||
|
||||
expect(parsedData.rundown.entries['lunch']).toMatchObject({
|
||||
timeWarning: 13 * MILLIS_PER_MINUTE,
|
||||
timeDanger: 5 * MILLIS_PER_MINUTE,
|
||||
});
|
||||
|
||||
expect(parsedData.rundown.entries['MEET3']).toMatchObject({
|
||||
duration: 90 * MILLIS_PER_MINUTE,
|
||||
linkStart: false,
|
||||
timeWarning: 11 * MILLIS_PER_MINUTE,
|
||||
timeDanger: 5 * MILLIS_PER_MINUTE,
|
||||
});
|
||||
|
||||
expect(parsedData.rundown.entries['MEET4']).toMatchObject({
|
||||
duration: 30 * MILLIS_PER_MINUTE,
|
||||
timeWarning: 11 * MILLIS_PER_MINUTE,
|
||||
linkStart: true,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('getCustomFieldData()', () => {
|
||||
it('generates a list of keys from the given import map', () => {
|
||||
const importMap = {
|
||||
worksheet: 'event schedule',
|
||||
timeStart: 'time start',
|
||||
linkStart: 'link start',
|
||||
timeEnd: 'time end',
|
||||
duration: 'duration',
|
||||
cue: 'cue',
|
||||
title: 'title',
|
||||
countToEnd: 'count to end',
|
||||
isPublic: 'public',
|
||||
skip: 'skip',
|
||||
note: 'notes',
|
||||
colour: 'colour',
|
||||
endAction: 'end action',
|
||||
timerType: 'timer type',
|
||||
timeWarning: 'warning time',
|
||||
timeDanger: 'danger time',
|
||||
custom: {
|
||||
lighting: 'lx',
|
||||
sound: 'sound',
|
||||
video: 'av',
|
||||
},
|
||||
entryId: 'id',
|
||||
} as ImportMap;
|
||||
|
||||
const result = getCustomFieldData(importMap, {});
|
||||
expect(result.mergedCustomFields).toStrictEqual({
|
||||
lighting: {
|
||||
type: 'string',
|
||||
colour: '',
|
||||
label: 'lighting',
|
||||
},
|
||||
sound: {
|
||||
type: 'string',
|
||||
colour: '',
|
||||
label: 'sound',
|
||||
},
|
||||
video: {
|
||||
type: 'string',
|
||||
colour: '',
|
||||
label: 'video',
|
||||
},
|
||||
});
|
||||
|
||||
// it is an inverted record of <importKey, ontimeKey>
|
||||
expect(result.customFieldImportKeys).toStrictEqual({
|
||||
lx: 'lighting',
|
||||
sound: 'sound',
|
||||
av: 'video',
|
||||
});
|
||||
});
|
||||
|
||||
it('keeps colour information from existing fields', () => {
|
||||
const importMap = {
|
||||
worksheet: 'event schedule',
|
||||
timeStart: 'time start',
|
||||
linkStart: 'link start',
|
||||
timeEnd: 'time end',
|
||||
duration: 'duration',
|
||||
cue: 'cue',
|
||||
title: 'title',
|
||||
countToEnd: 'count to end',
|
||||
isPublic: 'public',
|
||||
skip: 'skip',
|
||||
note: 'notes',
|
||||
colour: 'colour',
|
||||
endAction: 'end action',
|
||||
timerType: 'timer type',
|
||||
timeWarning: 'warning time',
|
||||
timeDanger: 'danger time',
|
||||
custom: {
|
||||
lighting: 'lx',
|
||||
sound: 'sound',
|
||||
video: 'av',
|
||||
'ontime key': 'excel label',
|
||||
},
|
||||
entryId: 'id',
|
||||
} as ImportMap;
|
||||
|
||||
const existingCustomFields: CustomFields = {
|
||||
lighting: { label: 'lighting', type: 'string', colour: 'red' },
|
||||
sound: { label: 'sound', type: 'string', colour: 'green' },
|
||||
ontime_key: { label: 'ontime key', type: 'string', colour: 'blue' },
|
||||
};
|
||||
|
||||
const result = getCustomFieldData(importMap, existingCustomFields);
|
||||
expect(result.mergedCustomFields).toStrictEqual({
|
||||
lighting: {
|
||||
type: 'string',
|
||||
colour: 'red',
|
||||
label: 'lighting',
|
||||
},
|
||||
sound: {
|
||||
type: 'string',
|
||||
colour: 'green',
|
||||
label: 'sound',
|
||||
},
|
||||
video: {
|
||||
type: 'string',
|
||||
colour: '',
|
||||
label: 'video',
|
||||
},
|
||||
ontime_key: {
|
||||
type: 'string',
|
||||
colour: 'blue',
|
||||
label: 'ontime key',
|
||||
},
|
||||
});
|
||||
|
||||
// it is an inverted record of <importKey, ontimeKey>
|
||||
expect(result.customFieldImportKeys).toStrictEqual({
|
||||
lx: 'lighting',
|
||||
sound: 'sound',
|
||||
av: 'video',
|
||||
'excel label': 'ontime_key',
|
||||
});
|
||||
});
|
||||
|
||||
it('lowercases the keys in the import map', () => {
|
||||
const importMap: ImportMap = {
|
||||
...defaultImportMap,
|
||||
custom: {
|
||||
Lighting: 'Lx',
|
||||
Sound: 'sound',
|
||||
video: 'av',
|
||||
},
|
||||
};
|
||||
|
||||
const result = getCustomFieldData(importMap, {});
|
||||
expect(result.mergedCustomFields).toStrictEqual({
|
||||
Lighting: {
|
||||
type: 'string',
|
||||
colour: '',
|
||||
label: 'Lighting',
|
||||
},
|
||||
Sound: {
|
||||
type: 'string',
|
||||
colour: '',
|
||||
label: 'Sound',
|
||||
},
|
||||
video: {
|
||||
type: 'string',
|
||||
colour: '',
|
||||
label: 'video',
|
||||
},
|
||||
});
|
||||
|
||||
// notice that the keys excel keys are lowercased
|
||||
expect(result.customFieldImportKeys).toStrictEqual({
|
||||
lx: 'Lighting',
|
||||
sound: 'Sound',
|
||||
av: 'video',
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,59 @@
|
||||
export const dataFromExcelTemplate = [
|
||||
['Ontime ┬À Schedule Template'],
|
||||
[],
|
||||
[
|
||||
'id',
|
||||
'Time Start',
|
||||
'Time End',
|
||||
'Title',
|
||||
'End Action',
|
||||
'Timer type',
|
||||
'Count to end',
|
||||
'Public',
|
||||
'Skip',
|
||||
'Notes',
|
||||
't0',
|
||||
'Test1',
|
||||
'test2',
|
||||
'test3',
|
||||
'Colour',
|
||||
'cue',
|
||||
],
|
||||
[
|
||||
'event-a', // <-- eventId
|
||||
'07:00:00', // <-- timeStart
|
||||
'08:00:10', // <-- timeEnd
|
||||
'Guest Welcome', // <-- title
|
||||
'', // <-- endAction
|
||||
'', // <-- timerType
|
||||
'x', // <-- count to end
|
||||
'x', // <-- public
|
||||
'', // <-- skip
|
||||
'Ballyhoo', // <-- notes
|
||||
'a0', // <-- t0
|
||||
'a1', // <-- test1
|
||||
'a2', // <-- test2
|
||||
'a3', // <-- test3
|
||||
'red', // <-- colour
|
||||
101, // <-- cue
|
||||
],
|
||||
[
|
||||
'event-b', // <-- eventId
|
||||
'08:00:00', // <-- timeStart
|
||||
'08:30:00', // <-- timeEnd
|
||||
'A song from the hearth', // <-- title
|
||||
'load-next', // <-- endAction
|
||||
'clock', // timerType
|
||||
'x', // <-- count to end
|
||||
'', // <-- public
|
||||
'x', // <-- skip
|
||||
'Rainbow chase', // <-- notes
|
||||
'b0', // <-- t0
|
||||
'', // <-- test1
|
||||
'', // <-- test2
|
||||
'', // <-- test3
|
||||
'#F00', // <-- colour
|
||||
102, // <-- cue
|
||||
],
|
||||
[],
|
||||
];
|
||||
@@ -1,9 +1,10 @@
|
||||
import type { Request } from 'express';
|
||||
import multer, { type FileFilterCallback } from 'multer';
|
||||
|
||||
import { EXCEL_MIME } from '../../utils/parser.js';
|
||||
import { storage } from '../../utils/upload.js';
|
||||
|
||||
export const EXCEL_MIME = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet';
|
||||
|
||||
const filterExcel = (_req: Request, file: Express.Multer.File, cb: FileFilterCallback) => {
|
||||
if (file.mimetype.includes(EXCEL_MIME)) {
|
||||
cb(null, true);
|
||||
|
||||
@@ -0,0 +1,348 @@
|
||||
import {
|
||||
CustomFields,
|
||||
Rundown,
|
||||
OntimeEvent,
|
||||
OntimeBlock,
|
||||
EntryCustomFields,
|
||||
SupportedEntry,
|
||||
isOntimeBlock,
|
||||
TimerType,
|
||||
CustomFieldKey,
|
||||
} from 'ontime-types';
|
||||
import {
|
||||
ImportMap,
|
||||
defaultImportMap,
|
||||
generateId,
|
||||
isKnownTimerType,
|
||||
validateTimerType,
|
||||
validateEndAction,
|
||||
customFieldLabelToKey,
|
||||
isAlphanumericWithSpace,
|
||||
} from 'ontime-utils';
|
||||
|
||||
import { Merge } from 'ts-essentials';
|
||||
|
||||
import { is } from '../../utils/is.js';
|
||||
import { makeString } from '../../utils/parserUtils.js';
|
||||
import { parseExcelDate } from '../../utils/time.js';
|
||||
|
||||
/**
|
||||
* @description Excel array parser
|
||||
* @param {array} excelData - array with excel sheet
|
||||
* @param {ImportOptions} options - an object that contains the import map
|
||||
* @returns {object} - parsed object
|
||||
*/
|
||||
export const parseExcel = (
|
||||
excelData: unknown[][],
|
||||
existingCustomFields: CustomFields,
|
||||
sheetName: string = 'Rundown from excel',
|
||||
options?: Partial<ImportMap>,
|
||||
): {
|
||||
rundown: Rundown;
|
||||
customFields: CustomFields;
|
||||
rundownMetadata: Record<string, { row: number; col: number }>;
|
||||
} => {
|
||||
const rundownMetadata: Record<string, { row: number; col: number }> = {};
|
||||
const importMap: ImportMap = { ...defaultImportMap, ...options };
|
||||
|
||||
for (const [key, value] of Object.entries(importMap)) {
|
||||
if (is.string(value)) {
|
||||
// @ts-expect-error -- we are sure that the key exists
|
||||
importMap[key] = value.toLowerCase().trim();
|
||||
}
|
||||
}
|
||||
|
||||
const { mergedCustomFields, customFieldImportKeys } = getCustomFieldData(importMap, existingCustomFields);
|
||||
const rundown: Rundown = {
|
||||
id: generateId(),
|
||||
title: sheetName,
|
||||
order: [],
|
||||
flatOrder: [],
|
||||
entries: {},
|
||||
revision: 0,
|
||||
};
|
||||
|
||||
// title stuff: strings
|
||||
let titleIndex: number | null = null;
|
||||
let cueIndex: number | null = null;
|
||||
let notesIndex: number | null = null;
|
||||
let colourIndex: number | null = null;
|
||||
|
||||
// options: booleans
|
||||
let isPublicIndex: number | null = null;
|
||||
let skipIndex: number | null = null;
|
||||
let countToEndIndex: number | null = null;
|
||||
|
||||
let linkStartIndex: number | null = null;
|
||||
|
||||
// times: numbers
|
||||
let timeStartIndex: number | null = null;
|
||||
let timeEndIndex: number | null = null;
|
||||
let durationIndex: number | null = null;
|
||||
let timeWarningIndex: number | null = null;
|
||||
let timeDangerIndex: number | null = null;
|
||||
|
||||
// options: enum properties
|
||||
let endActionIndex: number | null = null;
|
||||
let timerTypeIndex: number | null = null;
|
||||
|
||||
//ID
|
||||
let entryIdIndex: number | null = null;
|
||||
|
||||
// record of column index and the name of the field
|
||||
const customFieldIndexes: Record<number, string> = {};
|
||||
|
||||
excelData.forEach((row, rowIndex) => {
|
||||
if (row.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
// TODO: extract generating handlers from importMap
|
||||
const handlers = {
|
||||
[importMap.timeStart]: (row: number, col: number) => {
|
||||
timeStartIndex = col;
|
||||
rundownMetadata['timeStart'] = { row, col };
|
||||
},
|
||||
[importMap.linkStart]: (row: number, col: number) => {
|
||||
linkStartIndex = col;
|
||||
rundownMetadata['linkStart'] = { row, col };
|
||||
},
|
||||
[importMap.timeEnd]: (row: number, col: number) => {
|
||||
timeEndIndex = col;
|
||||
rundownMetadata['timeEnd'] = { row, col };
|
||||
},
|
||||
[importMap.duration]: (row: number, col: number) => {
|
||||
durationIndex = col;
|
||||
rundownMetadata['duration'] = { row, col };
|
||||
},
|
||||
|
||||
[importMap.cue]: (row: number, col: number) => {
|
||||
cueIndex = col;
|
||||
rundownMetadata['cue'] = { row, col };
|
||||
},
|
||||
[importMap.title]: (row: number, col: number) => {
|
||||
titleIndex = col;
|
||||
rundownMetadata['title'] = { row, col };
|
||||
},
|
||||
[importMap.countToEnd]: (row: number, col: number) => {
|
||||
countToEndIndex = col;
|
||||
rundownMetadata['countToEnd'] = { row, col };
|
||||
},
|
||||
[importMap.isPublic]: (row: number, col: number) => {
|
||||
isPublicIndex = col;
|
||||
rundownMetadata['isPublic'] = { row, col };
|
||||
},
|
||||
[importMap.skip]: (row: number, col: number) => {
|
||||
skipIndex = col;
|
||||
rundownMetadata['skip'] = { row, col };
|
||||
},
|
||||
[importMap.note]: (row: number, col: number) => {
|
||||
notesIndex = col;
|
||||
rundownMetadata['note'] = { row, col };
|
||||
},
|
||||
[importMap.colour]: (row: number, col: number) => {
|
||||
colourIndex = col;
|
||||
rundownMetadata['colour'] = { row, col };
|
||||
},
|
||||
[importMap.endAction]: (row: number, col: number) => {
|
||||
endActionIndex = col;
|
||||
rundownMetadata['endAction'] = { row, col };
|
||||
},
|
||||
[importMap.timerType]: (row: number, col: number) => {
|
||||
timerTypeIndex = col;
|
||||
rundownMetadata['timerType'] = { row, col };
|
||||
},
|
||||
[importMap.timeWarning]: (row: number, col: number) => {
|
||||
timeWarningIndex = col;
|
||||
rundownMetadata['timeWarning'] = { row, col };
|
||||
},
|
||||
[importMap.timeDanger]: (row: number, col: number) => {
|
||||
timeDangerIndex = col;
|
||||
rundownMetadata['timeDanger'] = { row, col };
|
||||
},
|
||||
[importMap.entryId]: (row: number, col: number) => {
|
||||
entryIdIndex = col;
|
||||
rundownMetadata['id'] = { row, col };
|
||||
},
|
||||
custom: (row: number, col: number, columnText: string, ontimeKey: string) => {
|
||||
customFieldIndexes[col] = columnText;
|
||||
rundownMetadata[`custom:${ontimeKey}`] = { row, col };
|
||||
},
|
||||
} as const;
|
||||
|
||||
const entry: Partial<Merge<OntimeEvent, OntimeBlock>> = {};
|
||||
const entryCustomFields: EntryCustomFields = {};
|
||||
|
||||
for (let j = 0; j < row.length; j++) {
|
||||
const column = row[j];
|
||||
// 1. we check if we have set a flag for a known field
|
||||
if (j === timerTypeIndex) {
|
||||
const maybeTimeType = makeString(column, '');
|
||||
if (maybeTimeType === 'block') {
|
||||
// we leave this as a clue for the object filtering later on
|
||||
entry.type = SupportedEntry.Block;
|
||||
} else if (maybeTimeType === '' || maybeTimeType === 'event' || isKnownTimerType(maybeTimeType)) {
|
||||
// @ts-expect-error -- we leave this as a clue for the object filtering later on
|
||||
entry.type = SupportedEntry.Event;
|
||||
entry.timerType = validateTimerType(maybeTimeType);
|
||||
} else {
|
||||
// if it is not a block or a known type, we dont import it
|
||||
return;
|
||||
}
|
||||
} else if (j === titleIndex) {
|
||||
entry.title = makeString(column, '');
|
||||
} else if (j === timeStartIndex) {
|
||||
entry.timeStart = parseExcelDate(column);
|
||||
} else if (j === linkStartIndex) {
|
||||
entry.linkStart = parseBooleanString(column);
|
||||
} else if (j === timeEndIndex) {
|
||||
entry.timeEnd = parseExcelDate(column);
|
||||
} else if (j === durationIndex) {
|
||||
entry.duration = parseExcelDate(column);
|
||||
} else if (j === cueIndex) {
|
||||
entry.cue = makeString(column, '');
|
||||
} else if (j === countToEndIndex) {
|
||||
entry.countToEnd = parseBooleanString(column);
|
||||
} else if (j === isPublicIndex) {
|
||||
entry.isPublic = parseBooleanString(column);
|
||||
} else if (j === skipIndex) {
|
||||
entry.skip = parseBooleanString(column);
|
||||
} else if (j === notesIndex) {
|
||||
entry.note = makeString(column, '');
|
||||
} else if (j === endActionIndex) {
|
||||
entry.endAction = validateEndAction(column);
|
||||
} else if (j === timeWarningIndex) {
|
||||
entry.timeWarning = parseExcelDate(column);
|
||||
} else if (j === timeDangerIndex) {
|
||||
entry.timeDanger = parseExcelDate(column);
|
||||
} else if (j === colourIndex) {
|
||||
entry.colour = makeString(column, '');
|
||||
} else if (j === entryIdIndex) {
|
||||
entry.id = encodeURIComponent(makeString(column, undefined));
|
||||
} else if (j in customFieldIndexes) {
|
||||
const importKey = customFieldIndexes[j];
|
||||
const ontimeKey = customFieldImportKeys[importKey];
|
||||
entryCustomFields[ontimeKey] = makeString(column, '');
|
||||
} else {
|
||||
// 2. if there is no flag, lets see if we know the field type
|
||||
if (typeof column === 'string') {
|
||||
// we cant deal with empty content
|
||||
if (column.length === 0) {
|
||||
continue;
|
||||
}
|
||||
const columnText = column.toLowerCase().trim();
|
||||
|
||||
// check if it is an ontime column
|
||||
if (handlers[columnText]) {
|
||||
// @ts-expect-error -- its ok
|
||||
handlers[columnText](rowIndex, j, undefined, undefined);
|
||||
}
|
||||
|
||||
// check if it is a custom field
|
||||
if (columnText in customFieldImportKeys) {
|
||||
const ontimeKey = customFieldImportKeys[columnText];
|
||||
handlers.custom(rowIndex, j, columnText, ontimeKey);
|
||||
}
|
||||
|
||||
// else. we don't know how to handle this column
|
||||
// just ignore it
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// if we didnt find any keys (empty row, or some other data), skip making an event
|
||||
const keysFound = Object.keys(entry).length + Object.keys(entryCustomFields).length;
|
||||
if (keysFound === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const id = entry.id || generateId();
|
||||
// from excel, we can only get blocks and events
|
||||
if (isOntimeBlock(entry)) {
|
||||
const block: OntimeBlock = { ...entry, custom: { ...entryCustomFields } };
|
||||
rundown.order.push(id);
|
||||
rundown.entries[id] = block;
|
||||
return;
|
||||
}
|
||||
|
||||
const event = {
|
||||
...entry,
|
||||
custom: { ...entryCustomFields },
|
||||
type: SupportedEntry.Event,
|
||||
} as OntimeEvent;
|
||||
|
||||
if (timerTypeIndex === null) {
|
||||
event.timerType = TimerType.CountDown;
|
||||
}
|
||||
rundown.order.push(id);
|
||||
rundown.flatOrder.push(id);
|
||||
rundown.entries[id] = event;
|
||||
});
|
||||
|
||||
return {
|
||||
rundown,
|
||||
customFields: mergedCustomFields,
|
||||
rundownMetadata,
|
||||
};
|
||||
};
|
||||
|
||||
/**
|
||||
* Utility function infers a boolean from a string value
|
||||
*/
|
||||
function parseBooleanString(value: unknown): boolean {
|
||||
if (typeof value === 'boolean') {
|
||||
return value;
|
||||
}
|
||||
|
||||
// falsy values would be nullish or empty string
|
||||
if (!value || typeof value !== 'string') {
|
||||
return false;
|
||||
}
|
||||
return value.toLowerCase() !== 'false';
|
||||
}
|
||||
|
||||
/**
|
||||
* Receives an import map which contains custom field labels and a custom fields object
|
||||
* the result importkeys is an inverted record of <importKey, ontimeKey>
|
||||
* We need this function since, when importing from sheets, the user gives us custom field labels, not keys
|
||||
* @returns the new custom fields, and a map of excel column names to ontime keys
|
||||
* @private exported for testing
|
||||
*/
|
||||
export function getCustomFieldData(
|
||||
importMap: ImportMap,
|
||||
existingCustomFields: CustomFields,
|
||||
): {
|
||||
mergedCustomFields: CustomFields;
|
||||
customFieldImportKeys: Record<keyof CustomFields, string>;
|
||||
} {
|
||||
const mergedCustomFields: CustomFields = {};
|
||||
/**
|
||||
* A map of import keys to ontime keys
|
||||
* Map<excel column name, ontime key>
|
||||
*/
|
||||
const customFieldImportKeys: Record<string, CustomFieldKey> = {};
|
||||
|
||||
for (const ontimeLabel in importMap.custom) {
|
||||
// if the label is not valid, we skip the import
|
||||
if (!isAlphanumericWithSpace(ontimeLabel)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// generate a key for the custom field
|
||||
const keyInCustomFields = customFieldLabelToKey(ontimeLabel);
|
||||
// we lower case the excel key to make it easier to match
|
||||
const columnNameInExcel = importMap.custom[ontimeLabel].toLowerCase();
|
||||
const maybeExistingColour = existingCustomFields[keyInCustomFields]?.colour ?? '';
|
||||
|
||||
// 1. add the custom field to the merged custom fields
|
||||
mergedCustomFields[keyInCustomFields] = {
|
||||
type: 'string', // we currently only support string custom fields
|
||||
colour: maybeExistingColour,
|
||||
label: ontimeLabel,
|
||||
};
|
||||
|
||||
// 2. add the column to the import keys
|
||||
customFieldImportKeys[columnNameInExcel] = keyInCustomFields;
|
||||
}
|
||||
return { mergedCustomFields, customFieldImportKeys };
|
||||
}
|
||||
@@ -11,12 +11,13 @@ import { existsSync } from 'fs';
|
||||
import xlsx from 'xlsx';
|
||||
import type { WorkBook } from 'xlsx';
|
||||
|
||||
import { parseExcel } from '../../utils/parser.js';
|
||||
import { parseCustomFields } from '../../utils/parserFunctions.js';
|
||||
import { deleteFile } from '../../utils/parserUtils.js';
|
||||
import { deleteFile } from '../../utils/fileManagement.js';
|
||||
|
||||
import { parseRundown } from '../rundown/rundown.parser.js';
|
||||
import { getProjectCustomFields } from '../rundown/rundown.dao.js';
|
||||
import { parseCustomFields } from '../custom-fields/customFields.parser.js';
|
||||
|
||||
import { parseExcel } from './excel.parser.js';
|
||||
|
||||
let excelData: WorkBook = xlsx.utils.book_new();
|
||||
|
||||
|
||||
Reference in New Issue
Block a user