Custom fields caching (#790)

* refactor: process custom fields on cache generate

apps/server/src/services/rundown-service/__tests__/rundownCacheUtils.test.ts

@@ -0,0 +1,189 @@
+import { CustomFields, OntimeEvent, OntimeRundown, SupportedEvent } from 'ontime-types';
+import { addToCustomAssignment, getLink, handleCustomField, handleLink } from '../rundownCacheUtils.js';
+
+describe('getLink()', () => {
+  it('should return null if there is no link', () => {
+    const rundown = [
+      { type: SupportedEvent.Block, id: 'block' },
+      { type: SupportedEvent.Event, id: '1' },
+    ] as OntimeRundown;
+
+    const result = getLink(1, rundown);
+    expect(result).toBeNull();
+  });
+
+  it('returns previous event', () => {
+    const rundown = [
+      { type: SupportedEvent.Event, id: '1', timeEnd: 100 },
+      { type: SupportedEvent.Event, id: '2', timeStart: 0, linkStart: '1' },
+    ] as OntimeRundown;
+
+    const result = getLink(1, rundown);
+    expect(result.id).toBe('1');
+  });
+});
+
+describe('handleLink()', () => {
+  it('populates data in object and updates link map', () => {
+    const rundown = [
+      { type: SupportedEvent.Event, id: '1', timeEnd: 100 },
+      { type: SupportedEvent.Event, id: '2', timeStart: 0, linkStart: '1' },
+    ] as OntimeRundown;
+    const mutableEvent = { ...rundown[1] } as OntimeEvent;
+    const links = {};
+
+    const result = handleLink(1, rundown, mutableEvent, links);
+    expect(result).toBeUndefined();
+    expect(mutableEvent.timeStart).toBe(100);
+    expect(mutableEvent.linkStart).toBe('1');
+    expect(links).toStrictEqual({ '1': '2' });
+  });
+
+  it('removes link if linked event is not found', () => {
+    const rundown = [
+      { type: SupportedEvent.Block, id: '1' },
+      { type: SupportedEvent.Event, id: '2', timeStart: 0, linkStart: '1' },
+    ] as OntimeRundown;
+    const mutableEvent = { ...rundown[1] } as OntimeEvent;
+    const links = {};
+
+    const result = handleLink(1, rundown, mutableEvent, links);
+    expect(result).toBeUndefined();
+    expect(mutableEvent.timeStart).toBe(0);
+    expect(mutableEvent.linkStart).toBe(null);
+    expect(links).toStrictEqual({});
+  });
+});
+
+describe('addToCustomAssignment()', () => {
+  it('adds given entry to assignedCustomFields', () => {
+    const assignedCustomFields = {};
+
+    addToCustomAssignment('label1', 'eventId 1', assignedCustomFields);
+    expect(assignedCustomFields).toStrictEqual({ label1: ['eventId 1'] });
+
+    addToCustomAssignment('label1', 'eventId 2', assignedCustomFields);
+    expect(assignedCustomFields).toStrictEqual({ label1: ['eventId 1', 'eventId 2'] });
+  });
+});
+
+describe('handleCustomField()', () => {
+  it('creates a map of where custom fields are used', () => {
+    const customFields = {
+      lighting: {
+        type: 'string',
+        colour: 'red',
+        label: 'lighting',
+      },
+      sound: {
+        type: 'string',
+        colour: 'red',
+        label: 'sound',
+      },
+    } as CustomFields;
+    const customFieldChangelog = {};
+
+    // @ts-expect-error -- partial event for testing
+    const event: OntimeEvent = {
+      type: SupportedEvent.Event,
+      id: '2',
+      timeStart: 0,
+      linkStart: '1',
+      custom: {
+        lighting: { value: 'on' },
+      },
+    };
+    const assignedCustomFields = {};
+
+    const result = handleCustomField(customFields, customFieldChangelog, event, assignedCustomFields);
+    expect(result).toBeUndefined();
+    expect(assignedCustomFields).toStrictEqual({ lighting: ['2'] });
+    expect(event.custom).toStrictEqual({
+      lighting: { value: 'on' },
+    });
+  });
+
+  it('renames a field if in changelog', () => {
+    const customFields = {
+      lighting: {
+        type: 'string',
+        colour: 'red',
+        label: 'lighting',
+      },
+      video: {
+        type: 'string',
+        colour: 'red',
+        label: 'video',
+      },
+    } as CustomFields;
+
+    const customFieldChangelog = {
+      sound: 'video',
+    };
+
+    // @ts-expect-error -- partial event for testing
+    const event: OntimeEvent = {
+      type: SupportedEvent.Event,
+      id: '2',
+      timeStart: 0,
+      linkStart: '1',
+      custom: {
+        sound: { value: 'on' },
+      },
+    };
+    const assignedCustomFields = {};
+
+    const result = handleCustomField(customFields, customFieldChangelog, event, assignedCustomFields);
+    expect(result).toBeUndefined();
+    expect(assignedCustomFields).toStrictEqual({ video: ['2'] });
+    expect(event.custom).toStrictEqual({
+      video: { value: 'on' },
+    });
+  });
+
+  it('processes all fields', () => {
+    const customFields = {
+      field1: {
+        type: 'string',
+        colour: 'red',
+        label: 'field1',
+      },
+      field2: {
+        type: 'string',
+        colour: 'red',
+        label: 'field2',
+      },
+    } as CustomFields;
+
+    const customFieldChangelog = {
+      field1: 'newField1',
+    };
+
+    // @ts-expect-error -- partial event for testing
+    const mutableEvent: OntimeEvent = {
+      type: SupportedEvent.Event,
+      id: 'event1',
+      custom: {
+        field1: { value: 'value1' },
+        field2: { value: 'value2' },
+      },
+    };
+
+    const assignedCustomFields = {};
+
+    handleCustomField(customFields, customFieldChangelog, mutableEvent, assignedCustomFields);
+
+    // Check that field1 has been renamed to newField1 and the value reassigned
+    expect(mutableEvent.custom['newField1']).toStrictEqual({ value: 'value1' });
+    expect(mutableEvent.custom['field1']).toBeUndefined();
+
+    // Check that field2 has been processed
+    expect(mutableEvent.custom['field2']).toStrictEqual({ value: 'value2' });
+
+    // Check that assignedCustomFields has been updated correctly
+    expect(assignedCustomFields).toStrictEqual({
+      newField1: ['event1'],
+      field2: ['event1'],
+    });
+  });
+});

apps/server/src/services/rundown-service/rundownCache.ts

@@ -8,11 +8,12 @@ import {
   OntimeRundown,
   OntimeRundownEntry,
 } from 'ontime-types';
-import { generateId, deleteAtIndex, insertAtIndex, reorderArray, swapEventData, getLinkedTimes } from 'ontime-utils';
+import { generateId, deleteAtIndex, insertAtIndex, reorderArray, swapEventData } from 'ontime-utils';
 
 import { DataProvider } from '../../classes/data-provider/DataProvider.js';
 import { createPatch } from '../../utils/parser.js';
 import { apply } from './delayUtils.js';
+import { handleCustomField, handleLink } from './rundownCacheUtils.js';
 
 type EventID = string;
 type NormalisedRundown = Record<EventID, OntimeRundownEntry>;
@@ -42,7 +43,12 @@ let links: Record<EventID, EventID> = {};
  * }
  */
 const customFieldChangelog = {};
-const assignedCustomFields: Record<CustomFieldLabel, EventID[]> = {};
+
+/**
+ * Keep track of which custom fields are used.
+ * This will be handy for when we delete custom fields
+ */
+let assignedCustomFields: Record<CustomFieldLabel, EventID[]> = {};
 
 export async function init(initialRundown: OntimeRundown, customFields: CustomFields) {
   persistedRundown = structuredClone(initialRundown);
@@ -62,22 +68,12 @@ export async function setRundown(initialRundown: OntimeRundown) {
  */
 export function generate(
   initialRundown: OntimeRundown = persistedRundown,
-  customProperties: CustomFields = persistedCustomFields,
+  customFields: CustomFields = persistedCustomFields,
 ) {
   // we decided to re-write this dataset for every change
   // instead of maintaining logic to update it
 
-  function getLink(currentIndex: number): OntimeEvent | null {
-    // currently the link is the previous event
-    for (let i = currentIndex - 1; i >= 0; i--) {
-      const event = initialRundown[i];
-      if (isOntimeEvent(event)) {
-        return event;
-      }
-    }
-    return null;
-  }
-
+  assignedCustomFields = {};
   rundown = {};
   order = [];
   links = {};
@@ -87,42 +83,25 @@ export function generate(
 
   for (let i = 0; i < initialRundown.length; i++) {
     const currentEvent = initialRundown[i];
-    let updatedEvent = { ...currentEvent };
+    const updatedEvent = { ...currentEvent };
 
-    // handle links
     if (isOntimeEvent(updatedEvent)) {
-      if (updatedEvent.linkStart) {
-        const linkedEvent = getLink(i);
-        // link is always the previous event for now
-        if (linkedEvent) {
-          links[linkedEvent.id] = currentEvent.id;
+      // 1. handle links
+      handleLink(i, initialRundown, updatedEvent, links);
 
-          const timePatch = getLinkedTimes(updatedEvent, linkedEvent);
-          updatedEvent = { ...updatedEvent, ...timePatch };
-        } else {
-          updatedEvent.linkStart = null;
-        }
-        // update the persisted event
-        initialRundown[i] = updatedEvent;
-      }
-      if (updatedEvent.custom) {
-        for (const property in updatedEvent.custom) {
-          const isValid = property in customProperties;
-          if (!isValid) {
-            delete updatedEvent.custom[property];
-            return;
-          }
-          if (!Array.isArray(assignedCustomFields[property])) {
-            assignedCustomFields[property] = [];
-          }
-          assignedCustomFields[property].push(updatedEvent.id);
-        }
-        // update the persisted event
-        initialRundown[i] = updatedEvent;
-      }
+      // TODO: wait until the next thing?
+      // update the persisted event
+      initialRundown[i] = updatedEvent;
+
+      // 2. handle custom fields
+      handleCustomField(customFields, customFieldChangelog, updatedEvent, assignedCustomFields);
+
+      // update the persisted event
+      initialRundown[i] = updatedEvent;
     }
 
     // calculate delays
+    // !!! this must happen after handling the links
     if (isOntimeDelay(updatedEvent)) {
       accumulatedDelay += updatedEvent.duration;
     } else if (isOntimeEvent(updatedEvent)) {
@@ -220,7 +199,7 @@ export function mutateCache<T extends object>(mutation: MutatingFn<T>) {
     // TODO: should we trottle this?
     // defer writing to the database
     setImmediate(() => {
-      console.log('writing to database', persistedRundown.length)
+      console.log('writing to database', persistedRundown.length);
       DataProvider.setRundown(persistedRundown);
     });
 
@@ -360,6 +339,37 @@ export function swap({ persistedRundown, fromId, toId }: SwapArgs): MutatingRetu
   return { newRundown };
 }
 
+/**
+ * Invalidates service cache if a custom field is used
+ * @param label
+ */
+function invalidateIfUsed(label: CustomFieldLabel) {
+  if (label in assignedCustomFields) {
+    isStale = true;
+  }
+  // if the field was in use, we mark the cache as stale
+  if (label in assignedCustomFields) {
+    isStale = true;
+  }
+  // ... and schedule a cache update
+  // schedule a non priority cache update
+  setImmediate(() => {
+    console.time('rundownCache__init');
+    generate();
+    console.timeEnd('rundownCache__init');
+  });
+}
+
+/**
+ * Scheduløes a non priority custom field persist
+ * @param persistedCustomFields
+ */
+function scheduleCustomFieldPersist(persistedCustomFields: CustomFields) {
+  setImmediate(() => {
+    DataProvider.setCustomFields(persistedCustomFields);
+  });
+}
+
 /**
  * Sanitises and creates a custom field in the database
  * @param field
@@ -378,9 +388,7 @@ export const createCustomField = async (field: CustomField) => {
   // update object and persist
   persistedCustomFields[label] = { label, type, colour };
 
-  setImmediate(() => {
-    DataProvider.setCustomFields(persistedCustomFields);
-  });
+  scheduleCustomFieldPersist(persistedCustomFields);
 
   return persistedCustomFields;
 };
@@ -407,9 +415,8 @@ export const editCustomField = async (label: string, newField: Partial<CustomFie
 
   persistedCustomFields[label] = { ...existingField, ...newField };
 
-  setImmediate(() => {
-    DataProvider.setCustomFields(persistedCustomFields);
-  });
+  scheduleCustomFieldPersist(persistedCustomFields);
+  invalidateIfUsed(label);
 
   return persistedCustomFields;
 };
@@ -423,9 +430,8 @@ export const removeCustomField = async (label: string) => {
     delete persistedCustomFields[label];
   }
 
-  setImmediate(() => {
-    DataProvider.setCustomFields(persistedCustomFields);
-  });
+  scheduleCustomFieldPersist(persistedCustomFields);
+  invalidateIfUsed(label);
 
   return persistedCustomFields;
 };

apps/server/src/services/rundown-service/rundownCacheUtils.ts

@@ -0,0 +1,92 @@
+import { OntimeEvent, isOntimeEvent, OntimeRundown, CustomFieldLabel, CustomFields } from 'ontime-types';
+import { getLinkedTimes } from 'ontime-utils';
+
+/**
+ * Get linked event
+ */
+export function getLink(currentIndex: number, rundown: OntimeRundown): OntimeEvent | null {
+  // currently the link is the previous event
+  for (let i = currentIndex - 1; i >= 0; i--) {
+    const event = rundown[i];
+    if (isOntimeEvent(event)) {
+      return event;
+    }
+  }
+  return null;
+}
+
+/**
+ * Populates data from link, if necessary
+ * Mutates in place mutableEvent
+ * Mutates in place links
+ */
+export function handleLink(
+  currentIndex: number,
+  rundown: OntimeRundown,
+  mutableEvent: OntimeEvent,
+  links: Record<string, string>,
+): void {
+  if (!mutableEvent.linkStart) {
+    return;
+  }
+
+  const linkedEvent = getLink(currentIndex, rundown);
+  if (!linkedEvent) {
+    mutableEvent.linkStart = null;
+    return;
+  }
+
+  links[linkedEvent.id] = mutableEvent.id;
+
+  const timePatch = getLinkedTimes(mutableEvent, linkedEvent);
+  // use object.assign to force mutation
+  Object.assign(mutableEvent, timePatch);
+}
+
+/**
+ * Utility function to add an entry, mutates given assignedCustomFields in place
+ * @param label
+ * @param eventId
+ */
+export function addToCustomAssignment(
+  label: CustomFieldLabel,
+  eventId: string,
+  assignedCustomFields: Record<string, string[]>,
+) {
+  if (!Array.isArray(assignedCustomFields[label])) {
+    assignedCustomFields[label] = [];
+  }
+  assignedCustomFields[label].push(eventId);
+}
+
+/**
+ * Sanitises custom fields and updates values if necessary
+ * Mudates in place mutableEvent and assignedCustomFields
+ */
+export function handleCustomField(
+  customFields: CustomFields,
+  customFieldChangelog: Record<string, string>,
+  mutableEvent: OntimeEvent,
+  assignedCustomFields: Record<string, string[]>,
+) {
+  for (const field in mutableEvent.custom) {
+    // rename the property if it is in the changelog
+    if (field in customFieldChangelog) {
+      const oldData = mutableEvent.custom[field];
+      const newLabel = customFieldChangelog[field];
+
+      mutableEvent.custom[newLabel] = { ...oldData };
+      delete mutableEvent.custom[field];
+      addToCustomAssignment(newLabel, mutableEvent.id, assignedCustomFields);
+      continue;
+    }
+
+    if (field in customFields) {
+      // add field to assignment map
+      addToCustomAssignment(field, mutableEvent.id, assignedCustomFields);
+    } else {
+      // delete data if it is not declared in project level custom fields
+      delete mutableEvent.custom[field];
+    }
+  }
+}