From 429df21557833687d75a4952cbfc08cab98fa665 Mon Sep 17 00:00:00 2001 From: Carlos Valente <34649812+cpvalente@users.noreply.github.com> Date: Sun, 25 Feb 2024 17:21:59 +0100 Subject: [PATCH] Custom fields caching (#790) * refactor: process custom fields on cache generate --- .../__tests__/rundownCacheUtils.test.ts | 189 ++++++++++++++++++ .../services/rundown-service/rundownCache.ts | 114 ++++++----- .../rundown-service/rundownCacheUtils.ts | 92 +++++++++ 3 files changed, 341 insertions(+), 54 deletions(-) create mode 100644 apps/server/src/services/rundown-service/__tests__/rundownCacheUtils.test.ts create mode 100644 apps/server/src/services/rundown-service/rundownCacheUtils.ts diff --git a/apps/server/src/services/rundown-service/__tests__/rundownCacheUtils.test.ts b/apps/server/src/services/rundown-service/__tests__/rundownCacheUtils.test.ts new file mode 100644 index 000000000..2366dc39c --- /dev/null +++ b/apps/server/src/services/rundown-service/__tests__/rundownCacheUtils.test.ts @@ -0,0 +1,189 @@ +import { CustomFields, OntimeEvent, OntimeRundown, SupportedEvent } from 'ontime-types'; +import { addToCustomAssignment, getLink, handleCustomField, handleLink } from '../rundownCacheUtils.js'; + +describe('getLink()', () => { + it('should return null if there is no link', () => { + const rundown = [ + { type: SupportedEvent.Block, id: 'block' }, + { type: SupportedEvent.Event, id: '1' }, + ] as OntimeRundown; + + const result = getLink(1, rundown); + expect(result).toBeNull(); + }); + + it('returns previous event', () => { + const rundown = [ + { type: SupportedEvent.Event, id: '1', timeEnd: 100 }, + { type: SupportedEvent.Event, id: '2', timeStart: 0, linkStart: '1' }, + ] as OntimeRundown; + + const result = getLink(1, rundown); + expect(result.id).toBe('1'); + }); +}); + +describe('handleLink()', () => { + it('populates data in object and updates link map', () => { + const rundown = [ + { type: SupportedEvent.Event, id: '1', timeEnd: 100 }, + { type: SupportedEvent.Event, id: '2', timeStart: 0, linkStart: '1' }, + ] as OntimeRundown; + const mutableEvent = { ...rundown[1] } as OntimeEvent; + const links = {}; + + const result = handleLink(1, rundown, mutableEvent, links); + expect(result).toBeUndefined(); + expect(mutableEvent.timeStart).toBe(100); + expect(mutableEvent.linkStart).toBe('1'); + expect(links).toStrictEqual({ '1': '2' }); + }); + + it('removes link if linked event is not found', () => { + const rundown = [ + { type: SupportedEvent.Block, id: '1' }, + { type: SupportedEvent.Event, id: '2', timeStart: 0, linkStart: '1' }, + ] as OntimeRundown; + const mutableEvent = { ...rundown[1] } as OntimeEvent; + const links = {}; + + const result = handleLink(1, rundown, mutableEvent, links); + expect(result).toBeUndefined(); + expect(mutableEvent.timeStart).toBe(0); + expect(mutableEvent.linkStart).toBe(null); + expect(links).toStrictEqual({}); + }); +}); + +describe('addToCustomAssignment()', () => { + it('adds given entry to assignedCustomFields', () => { + const assignedCustomFields = {}; + + addToCustomAssignment('label1', 'eventId 1', assignedCustomFields); + expect(assignedCustomFields).toStrictEqual({ label1: ['eventId 1'] }); + + addToCustomAssignment('label1', 'eventId 2', assignedCustomFields); + expect(assignedCustomFields).toStrictEqual({ label1: ['eventId 1', 'eventId 2'] }); + }); +}); + +describe('handleCustomField()', () => { + it('creates a map of where custom fields are used', () => { + const customFields = { + lighting: { + type: 'string', + colour: 'red', + label: 'lighting', + }, + sound: { + type: 'string', + colour: 'red', + label: 'sound', + }, + } as CustomFields; + const customFieldChangelog = {}; + + // @ts-expect-error -- partial event for testing + const event: OntimeEvent = { + type: SupportedEvent.Event, + id: '2', + timeStart: 0, + linkStart: '1', + custom: { + lighting: { value: 'on' }, + }, + }; + const assignedCustomFields = {}; + + const result = handleCustomField(customFields, customFieldChangelog, event, assignedCustomFields); + expect(result).toBeUndefined(); + expect(assignedCustomFields).toStrictEqual({ lighting: ['2'] }); + expect(event.custom).toStrictEqual({ + lighting: { value: 'on' }, + }); + }); + + it('renames a field if in changelog', () => { + const customFields = { + lighting: { + type: 'string', + colour: 'red', + label: 'lighting', + }, + video: { + type: 'string', + colour: 'red', + label: 'video', + }, + } as CustomFields; + + const customFieldChangelog = { + sound: 'video', + }; + + // @ts-expect-error -- partial event for testing + const event: OntimeEvent = { + type: SupportedEvent.Event, + id: '2', + timeStart: 0, + linkStart: '1', + custom: { + sound: { value: 'on' }, + }, + }; + const assignedCustomFields = {}; + + const result = handleCustomField(customFields, customFieldChangelog, event, assignedCustomFields); + expect(result).toBeUndefined(); + expect(assignedCustomFields).toStrictEqual({ video: ['2'] }); + expect(event.custom).toStrictEqual({ + video: { value: 'on' }, + }); + }); + + it('processes all fields', () => { + const customFields = { + field1: { + type: 'string', + colour: 'red', + label: 'field1', + }, + field2: { + type: 'string', + colour: 'red', + label: 'field2', + }, + } as CustomFields; + + const customFieldChangelog = { + field1: 'newField1', + }; + + // @ts-expect-error -- partial event for testing + const mutableEvent: OntimeEvent = { + type: SupportedEvent.Event, + id: 'event1', + custom: { + field1: { value: 'value1' }, + field2: { value: 'value2' }, + }, + }; + + const assignedCustomFields = {}; + + handleCustomField(customFields, customFieldChangelog, mutableEvent, assignedCustomFields); + + // Check that field1 has been renamed to newField1 and the value reassigned + expect(mutableEvent.custom['newField1']).toStrictEqual({ value: 'value1' }); + expect(mutableEvent.custom['field1']).toBeUndefined(); + + // Check that field2 has been processed + expect(mutableEvent.custom['field2']).toStrictEqual({ value: 'value2' }); + + // Check that assignedCustomFields has been updated correctly + expect(assignedCustomFields).toStrictEqual({ + newField1: ['event1'], + field2: ['event1'], + }); + }); +}); diff --git a/apps/server/src/services/rundown-service/rundownCache.ts b/apps/server/src/services/rundown-service/rundownCache.ts index cecb125fa..76cd1d370 100644 --- a/apps/server/src/services/rundown-service/rundownCache.ts +++ b/apps/server/src/services/rundown-service/rundownCache.ts @@ -8,11 +8,12 @@ import { OntimeRundown, OntimeRundownEntry, } from 'ontime-types'; -import { generateId, deleteAtIndex, insertAtIndex, reorderArray, swapEventData, getLinkedTimes } from 'ontime-utils'; +import { generateId, deleteAtIndex, insertAtIndex, reorderArray, swapEventData } from 'ontime-utils'; import { DataProvider } from '../../classes/data-provider/DataProvider.js'; import { createPatch } from '../../utils/parser.js'; import { apply } from './delayUtils.js'; +import { handleCustomField, handleLink } from './rundownCacheUtils.js'; type EventID = string; type NormalisedRundown = Record; @@ -42,7 +43,12 @@ let links: Record = {}; * } */ const customFieldChangelog = {}; -const assignedCustomFields: Record = {}; + +/** + * Keep track of which custom fields are used. + * This will be handy for when we delete custom fields + */ +let assignedCustomFields: Record = {}; export async function init(initialRundown: OntimeRundown, customFields: CustomFields) { persistedRundown = structuredClone(initialRundown); @@ -62,22 +68,12 @@ export async function setRundown(initialRundown: OntimeRundown) { */ export function generate( initialRundown: OntimeRundown = persistedRundown, - customProperties: CustomFields = persistedCustomFields, + customFields: CustomFields = persistedCustomFields, ) { // we decided to re-write this dataset for every change // instead of maintaining logic to update it - function getLink(currentIndex: number): OntimeEvent | null { - // currently the link is the previous event - for (let i = currentIndex - 1; i >= 0; i--) { - const event = initialRundown[i]; - if (isOntimeEvent(event)) { - return event; - } - } - return null; - } - + assignedCustomFields = {}; rundown = {}; order = []; links = {}; @@ -87,42 +83,25 @@ export function generate( for (let i = 0; i < initialRundown.length; i++) { const currentEvent = initialRundown[i]; - let updatedEvent = { ...currentEvent }; + const updatedEvent = { ...currentEvent }; - // handle links if (isOntimeEvent(updatedEvent)) { - if (updatedEvent.linkStart) { - const linkedEvent = getLink(i); - // link is always the previous event for now - if (linkedEvent) { - links[linkedEvent.id] = currentEvent.id; + // 1. handle links + handleLink(i, initialRundown, updatedEvent, links); - const timePatch = getLinkedTimes(updatedEvent, linkedEvent); - updatedEvent = { ...updatedEvent, ...timePatch }; - } else { - updatedEvent.linkStart = null; - } - // update the persisted event - initialRundown[i] = updatedEvent; - } - if (updatedEvent.custom) { - for (const property in updatedEvent.custom) { - const isValid = property in customProperties; - if (!isValid) { - delete updatedEvent.custom[property]; - return; - } - if (!Array.isArray(assignedCustomFields[property])) { - assignedCustomFields[property] = []; - } - assignedCustomFields[property].push(updatedEvent.id); - } - // update the persisted event - initialRundown[i] = updatedEvent; - } + // TODO: wait until the next thing? + // update the persisted event + initialRundown[i] = updatedEvent; + + // 2. handle custom fields + handleCustomField(customFields, customFieldChangelog, updatedEvent, assignedCustomFields); + + // update the persisted event + initialRundown[i] = updatedEvent; } // calculate delays + // !!! this must happen after handling the links if (isOntimeDelay(updatedEvent)) { accumulatedDelay += updatedEvent.duration; } else if (isOntimeEvent(updatedEvent)) { @@ -220,7 +199,7 @@ export function mutateCache(mutation: MutatingFn) { // TODO: should we trottle this? // defer writing to the database setImmediate(() => { - console.log('writing to database', persistedRundown.length) + console.log('writing to database', persistedRundown.length); DataProvider.setRundown(persistedRundown); }); @@ -360,6 +339,37 @@ export function swap({ persistedRundown, fromId, toId }: SwapArgs): MutatingRetu return { newRundown }; } +/** + * Invalidates service cache if a custom field is used + * @param label + */ +function invalidateIfUsed(label: CustomFieldLabel) { + if (label in assignedCustomFields) { + isStale = true; + } + // if the field was in use, we mark the cache as stale + if (label in assignedCustomFields) { + isStale = true; + } + // ... and schedule a cache update + // schedule a non priority cache update + setImmediate(() => { + console.time('rundownCache__init'); + generate(); + console.timeEnd('rundownCache__init'); + }); +} + +/** + * Scheduløes a non priority custom field persist + * @param persistedCustomFields + */ +function scheduleCustomFieldPersist(persistedCustomFields: CustomFields) { + setImmediate(() => { + DataProvider.setCustomFields(persistedCustomFields); + }); +} + /** * Sanitises and creates a custom field in the database * @param field @@ -378,9 +388,7 @@ export const createCustomField = async (field: CustomField) => { // update object and persist persistedCustomFields[label] = { label, type, colour }; - setImmediate(() => { - DataProvider.setCustomFields(persistedCustomFields); - }); + scheduleCustomFieldPersist(persistedCustomFields); return persistedCustomFields; }; @@ -407,9 +415,8 @@ export const editCustomField = async (label: string, newField: Partial { - DataProvider.setCustomFields(persistedCustomFields); - }); + scheduleCustomFieldPersist(persistedCustomFields); + invalidateIfUsed(label); return persistedCustomFields; }; @@ -423,9 +430,8 @@ export const removeCustomField = async (label: string) => { delete persistedCustomFields[label]; } - setImmediate(() => { - DataProvider.setCustomFields(persistedCustomFields); - }); + scheduleCustomFieldPersist(persistedCustomFields); + invalidateIfUsed(label); return persistedCustomFields; }; diff --git a/apps/server/src/services/rundown-service/rundownCacheUtils.ts b/apps/server/src/services/rundown-service/rundownCacheUtils.ts new file mode 100644 index 000000000..aef0f3ec6 --- /dev/null +++ b/apps/server/src/services/rundown-service/rundownCacheUtils.ts @@ -0,0 +1,92 @@ +import { OntimeEvent, isOntimeEvent, OntimeRundown, CustomFieldLabel, CustomFields } from 'ontime-types'; +import { getLinkedTimes } from 'ontime-utils'; + +/** + * Get linked event + */ +export function getLink(currentIndex: number, rundown: OntimeRundown): OntimeEvent | null { + // currently the link is the previous event + for (let i = currentIndex - 1; i >= 0; i--) { + const event = rundown[i]; + if (isOntimeEvent(event)) { + return event; + } + } + return null; +} + +/** + * Populates data from link, if necessary + * Mutates in place mutableEvent + * Mutates in place links + */ +export function handleLink( + currentIndex: number, + rundown: OntimeRundown, + mutableEvent: OntimeEvent, + links: Record, +): void { + if (!mutableEvent.linkStart) { + return; + } + + const linkedEvent = getLink(currentIndex, rundown); + if (!linkedEvent) { + mutableEvent.linkStart = null; + return; + } + + links[linkedEvent.id] = mutableEvent.id; + + const timePatch = getLinkedTimes(mutableEvent, linkedEvent); + // use object.assign to force mutation + Object.assign(mutableEvent, timePatch); +} + +/** + * Utility function to add an entry, mutates given assignedCustomFields in place + * @param label + * @param eventId + */ +export function addToCustomAssignment( + label: CustomFieldLabel, + eventId: string, + assignedCustomFields: Record, +) { + if (!Array.isArray(assignedCustomFields[label])) { + assignedCustomFields[label] = []; + } + assignedCustomFields[label].push(eventId); +} + +/** + * Sanitises custom fields and updates values if necessary + * Mudates in place mutableEvent and assignedCustomFields + */ +export function handleCustomField( + customFields: CustomFields, + customFieldChangelog: Record, + mutableEvent: OntimeEvent, + assignedCustomFields: Record, +) { + for (const field in mutableEvent.custom) { + // rename the property if it is in the changelog + if (field in customFieldChangelog) { + const oldData = mutableEvent.custom[field]; + const newLabel = customFieldChangelog[field]; + + mutableEvent.custom[newLabel] = { ...oldData }; + delete mutableEvent.custom[field]; + addToCustomAssignment(newLabel, mutableEvent.id, assignedCustomFields); + continue; + } + + if (field in customFields) { + // add field to assignment map + addToCustomAssignment(field, mutableEvent.id, assignedCustomFields); + } else { + // delete data if it is not declared in project level custom fields + delete mutableEvent.custom[field]; + } + } +}