refactor: normalise data (#756)

* chore: remove legal from bundle

* refactor: create normalised dataset

* refactor: cuesheet uses flat rundown

* refactor: multi-selection

* refactor: prevent stale data on server restart

* refactor: increase ID size

* chore: instrument operation

* chore: update csv tests

* fix: resolve directory to test-db (#758)

apps/client/src/features/rundown/Rundown.tsx

@@ -1,8 +1,8 @@
 import { Fragment, lazy, useCallback, useEffect, useRef, useState } from 'react';
 import { closestCenter, DndContext, DragEndEvent, PointerSensor, useSensor, useSensors } from '@dnd-kit/core';
 import { arrayMove, SortableContext, verticalListSortingStrategy } from '@dnd-kit/sortable';
-import { isOntimeBlock, isOntimeDelay, isOntimeEvent, OntimeRundown, Playback, SupportedEvent } from 'ontime-types';
-import { getFirst, getNext, getPrevious } from 'ontime-utils';
+import { isOntimeBlock, isOntimeDelay, isOntimeEvent, Playback, RundownCached, SupportedEvent } from 'ontime-types';
+import { getFirstNormal, getNextNormal, getPreviousNormal } from 'ontime-utils';
 
 import { useEventAction } from '../../common/hooks/useEventAction';
 import useFollowComponent from '../../common/hooks/useFollowComponent';
@@ -19,12 +19,12 @@ import style from './Rundown.module.scss';
 const RundownEntry = lazy(() => import('./RundownEntry'));
 
 interface RundownProps {
-  entries: OntimeRundown;
+  data: RundownCached;
 }
 
-export default function Rundown(props: RundownProps) {
-  const { entries } = props;
-  const [statefulEntries, setStatefulEntries] = useState(entries);
+export default function Rundown({ data }: RundownProps) {
+  const { order, rundown } = data;
+  const [statefulEntries, setStatefulEntries] = useState(order);
 
   const featureData = useRundownEditor();
   const { addEvent, reorderEvent } = useEventAction();
@@ -56,7 +56,7 @@ export default function Rundown(props: RundownProps) {
       }
 
       if (type === 'clone') {
-        const cursorEvent = entries.find((event) => event.id === cursor);
+        const cursorEvent = rundown[cursor];
         if (cursorEvent?.type === SupportedEvent.Event) {
           const newEvent = cloneEvent(cursorEvent, cursorEvent.id);
           addEvent(newEvent);
@@ -76,7 +76,7 @@ export default function Rundown(props: RundownProps) {
         addEvent({ type }, { after: cursor });
       }
     },
-    [addEvent, defaultPublic, entries, startTimeIsLastEnd],
+    [addEvent, rundown, defaultPublic, startTimeIsLastEnd],
   );
 
   // Handle keyboard shortcuts
@@ -91,21 +91,23 @@ export default function Rundown(props: RundownProps) {
       if (modKeysAlt) {
         switch (event.code) {
           case 'ArrowDown': {
-            if (entries.length < 1) {
+            if (order.length < 1) {
               return;
             }
-            const nextEvent = cursor == null ? getFirst(entries) : getNext(entries, cursor)?.nextEvent;
+            const nextEvent =
+              cursor == null ? getFirstNormal(rundown, order) : getNextNormal(rundown, order, cursor)?.nextEvent;
             if (nextEvent) {
               // moveCursorTo(nextEvent.id, nextEvent.type === SupportedEvent.Event);
             }
             break;
           }
           case 'ArrowUp': {
-            if (entries.length < 1) {
+            if (order.length < 1) {
               return;
             }
             // eslint-disable-next-line @typescript-eslint/no-non-null-assertion -- we check for this before
-            const previousEvent = cursor == null ? getFirst(entries) : getPrevious(entries, cursor).previousEvent;
+            const previousEvent =
+              cursor == null ? getFirstNormal(rundown, order) : getPreviousNormal(rundown, order, cursor).previousEvent;
             if (previousEvent) {
               // moveCursorTo(previousEvent.id, previousEvent.type === SupportedEvent.Event);
             }
@@ -133,32 +135,30 @@ export default function Rundown(props: RundownProps) {
           }
         }
       } else if (modKeysCtrlAlt) {
-        if (entries.length < 2 || cursor == null) {
+        if (order.length < 2 || cursor == null) {
           return;
         }
         if (event.code == 'ArrowDown') {
-          const { nextEvent, nextIndex } = getNext(entries, cursor);
+          const { nextEvent, nextIndex } = getNextNormal(rundown, order, cursor);
           if (nextEvent && nextIndex !== null) {
             reorderEvent(cursor, nextIndex - 1, nextIndex);
           }
         } else if (event.code == 'ArrowUp') {
-          const { previousEvent, previousIndex } = getPrevious(entries, cursor);
+          const { previousEvent, previousIndex } = getPreviousNormal(rundown, order, cursor);
           if (previousEvent && previousIndex !== null) {
             reorderEvent(cursor, previousIndex + 1, previousIndex);
           }
         }
       }
     },
-    [cursor, entries, insertAtCursor, reorderEvent],
+    [cursor, insertAtCursor, order, rundown, reorderEvent],
   );
 
   // we copy the state from the store here
   // to workaround async updates on the drag mutations
   useEffect(() => {
-    if (entries) {
-      setStatefulEntries(entries);
-    }
-  }, [entries]);
+    setStatefulEntries(order);
+  }, [order]);
 
   // listen to keys
   useEffect(() => {
@@ -193,7 +193,7 @@ export default function Rundown(props: RundownProps) {
     }
   };
 
-  if (statefulEntries?.length < 1) {
+  if (statefulEntries.length < 1) {
     return <RundownEmpty handleAddNew={() => insertAtCursor(SupportedEvent.Event, null)} />;
   }
 
@@ -208,39 +208,46 @@ export default function Rundown(props: RundownProps) {
       <DndContext onDragEnd={handleOnDragEnd} sensors={sensors} collisionDetection={closestCenter}>
         <SortableContext items={statefulEntries} strategy={verticalListSortingStrategy}>
           <div className={style.list}>
-            {statefulEntries.map((entry, index) => {
+            {statefulEntries.map((eventId, index) => {
+              // we iterate through a stateful copy of order to make the operations smoother
+              // this means that this can be out of sync with order until the useEffect runs
+              // instead of writing all the logic guards, we simply short circuit rendering here
+              const event = rundown[eventId];
+              if (!event) {
+                return null;
+              }
               if (index === 0) {
                 eventIndex = 0;
               }
               let isFirstEvent = false;
-              if (isOntimeEvent(entry)) {
+              if (isOntimeEvent(event)) {
                 isFirstEvent = eventIndex === 0;
                 // event indexes are 1 based in frontend
                 eventIndex++;
                 if (!isFirstEvent) {
                   previousEnd = thisEnd;
                 }
-                thisEnd = entry.timeEnd;
-                previousEventId = entry.id;
+                thisEnd = event.timeEnd;
+                previousEventId = event.id;
               }
-              const isLast = index === entries.length - 1;
-              const isSelected = featureData?.selectedEventId === entry.id;
-              const isNext = featureData?.nextEventId === entry.id;
-              const hasCursor = entry.id === cursor;
+              const isLast = index === order.length - 1;
+              const isSelected = featureData?.selectedEventId === event.id;
+              const isNext = featureData?.nextEventId === event.id;
+              const hasCursor = event.id === cursor;
               if (isSelected) {
                 isPast = false;
               }
 
               return (
-                <Fragment key={entry.id}>
+                <Fragment key={event.id}>
                   <div className={style.entryWrapper} data-testid={`entry-${eventIndex}`}>
-                    {entry.type === SupportedEvent.Event && <div className={style.entryIndex}>{eventIndex}</div>}
-                    <div className={style.entry} key={entry.id} ref={hasCursor ? cursorRef : undefined}>
+                    {isOntimeEvent(event) && <div className={style.entryIndex}>{eventIndex}</div>}
+                    <div className={style.entry} key={event.id} ref={hasCursor ? cursorRef : undefined}>
                       <RundownEntry
-                        type={entry.type}
+                        type={event.type}
                         isPast={isPast}
                         eventIndex={eventIndex}
-                        data={entry}
+                        data={event}
                         selected={isSelected}
                         hasCursor={hasCursor}
                         next={isNext}
@@ -254,10 +261,10 @@ export default function Rundown(props: RundownProps) {
                   {((showQuickEntry && hasCursor) || isLast) && (
                     <QuickAddBlock
                       showKbd={hasCursor}
-                      eventId={entry.id}
+                      eventId={event.id}
                       previousEventId={previousEventId}
-                      disableAddDelay={isOntimeDelay(entry)}
-                      disableAddBlock={isOntimeBlock(entry)}
+                      disableAddDelay={isOntimeDelay(event)}
+                      disableAddBlock={isOntimeBlock(event)}
                     />
                   )}
                 </Fragment>

apps/client/src/features/rundown/RundownEntry.tsx

@@ -1,18 +1,8 @@
 import { useCallback } from 'react';
-import {
-  GetRundownCached,
-  isOntimeEvent,
-  MaybeNumber,
-  OntimeEvent,
-  OntimeRundownEntry,
-  Playback,
-  SupportedEvent,
-} from 'ontime-types';
+import { MaybeNumber, OntimeEvent, OntimeRundownEntry, Playback, SupportedEvent } from 'ontime-types';
 
-import { RUNDOWN } from '../../common/api/apiConstants';
 import { useEventAction } from '../../common/hooks/useEventAction';
 import useMemoisedFn from '../../common/hooks/useMemoisedFn';
-import { ontimeQueryClient } from '../../common/queryClient';
 import { useAppMode } from '../../common/stores/appModeStore';
 import { useEditorSettings } from '../../common/stores/editorSettings';
 import { useEmitLog } from '../../common/stores/logger';
@@ -103,28 +93,20 @@ export default function RundownEntry(props: RundownEntryProps) {
       case 'update': {
         // Handles and filters update requests
         const { field, value } = payload as FieldValue;
+        if (field === undefined || value === undefined) {
+          return;
+        }
         const newData: Partial<OntimeEvent> = { id: data.id };
 
         // if selected events are more than one
         // we need to bulk edit
         if (selectedEvents.size > 1) {
           const changes: Partial<OntimeEvent> = { [field]: value };
-          const rundown = ontimeQueryClient.getQueryData<GetRundownCached>(RUNDOWN)?.rundown ?? [];
-          const idsOfRundownEvents = rundown.filter(isOntimeEvent).map((event) => event.id);
-
-          const eventIds = [...selectedEvents.keys()];
-          // check every selected event id to see if they match rundown event ids
-          const areIdsValid = eventIds.every((eventId) => idsOfRundownEvents.includes(eventId));
-
-          if (!areIdsValid) {
-            return;
-          }
-
-          batchUpdateEvents(changes, eventIds);
+          batchUpdateEvents(changes, Array.from(selectedEvents));
           return clearSelectedEvents();
         }
         if (field in data) {
-          // @ts-expect-error not sure how to type this
+          // @ts-expect-error -- not sure how to type this
           newData[field] = value;
           return updateEvent(newData);
         }

apps/client/src/features/rundown/RundownWrapper.tsx

@@ -10,7 +10,7 @@ export default function RundownWrapper() {
 
   return (
     <div className={styles.rundownWrapper}>
-      {status === 'success' && data ? <Rundown entries={data} /> : <Empty text='Connecting to server' />}
+      {status === 'success' && data ? <Rundown data={data} /> : <Empty text='Connecting to server' />}
     </div>
   );
 }

apps/client/src/features/rundown/delay-block/DelayBlock.module.scss

@@ -22,3 +22,10 @@
   @include drag-style;
   grid-area: drag;
 }
+
+.actionButtons {
+  grid-area: btns;
+  display: flex;
+  align-items: center;
+  gap: 0.5rem;
+}

apps/client/src/features/rundown/delay-block/DelayBlock.tsx

@@ -1,5 +1,5 @@
 import { useEffect, useRef } from 'react';
-import { Button, HStack } from '@chakra-ui/react';
+import { Button } from '@chakra-ui/react';
 import { useSortable } from '@dnd-kit/sortable';
 import { CSS } from '@dnd-kit/utilities';
 import { IoCheckmark } from '@react-icons/all-files/io5/IoCheckmark';
@@ -72,7 +72,7 @@ export default function DelayBlock(props: DelayBlockProps) {
         <IoReorderTwo />
       </span>
       <DelayInput eventId={data.id} duration={data.duration} />
-      <HStack spacing='8px' className={style.actionOverlay}>
+      <div className={style.actionButtons}>
         <Button onClick={applyDelayHandler} size='sm' leftIcon={<IoCheckmark />} variant='ontime-subtle-white'>
           Apply
         </Button>
@@ -80,7 +80,7 @@ export default function DelayBlock(props: DelayBlockProps) {
           Cancel
         </Button>
         <BlockActionMenu enableDelete actionHandler={actionHandler} />
-      </HStack>
+      </div>
     </div>
   );
 }

apps/client/src/features/rundown/event-block/EventBlock.tsx

@@ -10,31 +10,17 @@ import { IoSwapVertical } from '@react-icons/all-files/io5/IoSwapVertical';
 import { EndAction, MaybeNumber, OntimeEvent, Playback, TimerType } from 'ontime-types';
 
 import { useContextMenu } from '../../../common/hooks/useContextMenu';
-import useRundown from '../../../common/hooks-query/useRundown';
 import copyToClipboard from '../../../common/utils/copyToClipboard';
-import { isMacOS } from '../../../common/utils/deviceUtils';
 import { cx, getAccessibleColour } from '../../../common/utils/styleUtils';
 import type { EventItemActions } from '../RundownEntry';
 import { useEventIdSwapping } from '../useEventIdSwapping';
-import { EditMode, useEventSelection } from '../useEventSelection';
+import { getSelectionMode, useEventSelection } from '../useEventSelection';
 
 import EventBlockInner from './EventBlockInner';
 import RundownIndicators from './RundownIndicators';
 
 import style from './EventBlock.module.scss';
 
-const getEditMode = (event: MouseEvent): EditMode => {
-  if ((isMacOS() && event.metaKey) || event.ctrlKey) {
-    return 'ctrl';
-  }
-
-  if (event.shiftKey) {
-    return 'shift';
-  }
-
-  return 'click';
-};
-
 interface EventBlockProps {
   cue: string;
   timeStart: number;
@@ -95,7 +81,6 @@ export default function EventBlock(props: EventBlockProps) {
   } = props;
   const { selectedEventId, setSelectedEventId, clearSelectedEventId } = useEventIdSwapping();
   const { selectedEvents, setSelectedEvents } = useEventSelection();
-  const { data: rundown = [] } = useRundown();
   const handleRef = useRef<null | HTMLSpanElement>(null);
   const [isVisible, setIsVisible] = useState(false);
 
@@ -235,8 +220,10 @@ export default function EventBlock(props: EventBlockProps) {
       return;
     }
 
-    const editMode = getEditMode(event);
-    return setSelectedEvents({ id: eventId, index: eventIndex, rundown, editMode });
+    // UI indexes are 1 based
+    const index = eventIndex - 1;
+    const editMode = getSelectionMode(event);
+    return setSelectedEvents({ id: eventId, index, selectMode: editMode });
 
     // moveCursorTo(eventId, true);
   };

apps/client/src/features/rundown/event-editor/EventEditor.tsx

@@ -36,24 +36,30 @@ export type EditorUpdateFields =
 export default function EventEditor() {
   const selectedEvents = useEventSelection((state) => state.selectedEvents);
   const { data } = useRundown();
+  const { order, rundown } = data;
   const { updateEvent } = useEventAction();
 
   const [event, setEvent] = useState<OntimeEvent | null>(null);
 
   useEffect(() => {
-    if (!data) {
+    if (order.length === 0) {
       setEvent(null);
       return;
     }
 
-    const event = data.find((event) => selectedEvents.has(event.id));
+    const selectedEventId = order.find((eventId) => selectedEvents.has(eventId));
+    if (!selectedEventId) {
+      setEvent(null);
+      return;
+    }
+    const event = rundown[selectedEventId];
 
     if (event && isOntimeEvent(event)) {
       setEvent(event);
     } else {
       setEvent(null);
     }
-  }, [data, selectedEvents]);
+  }, [order, rundown, selectedEvents]);
 
   const handleSubmit = useCallback(
     (field: EditorUpdateFields, value: string) => {

apps/client/src/features/rundown/useEventSelection.ts

@@ -1,12 +1,17 @@
-import { isOntimeEvent, OntimeRundown } from 'ontime-types';
+import { MouseEvent } from 'react';
+import { isOntimeEvent, OntimeEvent, RundownCached } from 'ontime-types';
 import { create } from 'zustand';
 
-export type EditMode = 'shift' | 'click' | 'ctrl';
+import { RUNDOWN } from '../../common/api/apiConstants';
+import { ontimeQueryClient } from '../../common/queryClient';
+import { isMacOS } from '../../common/utils/deviceUtils';
+
+export type SelectionMode = 'shift' | 'click' | 'ctrl';
 
 interface EventSelectionStore {
   selectedEvents: Set<string>;
   anchoredIndex: number | null;
-  setSelectedEvents: (selectionArgs: { id: string; index: number; rundown: OntimeRundown; editMode: EditMode }) => void;
+  setSelectedEvents: (selectionArgs: { id: string; index: number; selectMode: SelectionMode }) => void;
   clearSelectedEvents: () => void;
 }
 
@@ -14,72 +19,79 @@ export const useEventSelection = create<EventSelectionStore>()((set, get) => ({
   selectedEvents: new Set(),
   anchoredIndex: null,
   setSelectedEvents: (selectionArgs) => {
-    const { id, index: eventIndex, rundown, editMode } = selectionArgs;
-    // event indexes are not 0 based
-    const index = eventIndex - 1;
-
+    const { id, index, selectMode } = selectionArgs;
     const { selectedEvents, anchoredIndex } = get();
 
-    if (editMode === 'click') {
+    // on click, we replace selection with event
+    if (selectMode === 'click') {
       return set({ selectedEvents: new Set([id]), anchoredIndex: index });
     }
 
-    if (editMode === 'ctrl') {
-      if (selectedEvents.has(id)) {
-        const eventIds = rundown.reduce(
-          (newRundown, event, i) => {
-            if (isOntimeEvent(event) && selectedEvents.has(id)) {
-              return newRundown.concat({ id: event.id, index: i });
-            }
-
-            return newRundown;
-          },
-          [] as { id: string; index: number }[],
-        );
-
-        // find the next available higher index
-        // if unavailable, then grab the last index of events
-        const newAnchoredIndex = eventIds.find(({ index: eventIndex }) => eventIndex > index) ?? eventIds.at(-1);
-
-        selectedEvents.delete(id);
+    // on ctrl + click, we toggle the selection of that event
+    if (selectMode === 'ctrl') {
+      const rundownData = ontimeQueryClient.getQueryData<RundownCached>(RUNDOWN);
+      if (!rundownData) return;
 
+      // if it doesnt exist, simply add to the list and set an anchor
+      if (!selectedEvents.has(id)) {
         return set({
-          selectedEvents,
-          anchoredIndex: newAnchoredIndex?.index ?? 0,
-        });
-      }
-
-      return set({
-        selectedEvents: selectedEvents.add(id),
-        anchoredIndex: index,
-      });
-    }
-
-    if (editMode === 'shift') {
-      const eventIds = rundown.filter(isOntimeEvent);
-
-      if (anchoredIndex === null) {
-        const eventsUntilIndex = eventIds.slice(0, eventIndex).map((event) => event.id);
-
-        return set({ selectedEvents: new Set(eventsUntilIndex), anchoredIndex: index });
-      }
-
-      if (anchoredIndex > index) {
-        const eventsFromIndex = eventIds.slice(index, anchoredIndex + 1).map((event) => event.id);
-
-        return set({
-          selectedEvents: new Set([...selectedEvents, ...eventsFromIndex]),
+          selectedEvents: selectedEvents.add(id),
           anchoredIndex: index,
         });
       }
 
-      const eventsUntilIndex = eventIds.slice(anchoredIndex, eventIndex).map((event) => event.id);
+      // if event is already selected, we remove it from selection
+      // and set the anchor to the event after
+      selectedEvents.delete(id);
+
+      const nextIndex = rundownData.order.findIndex(
+        (eventId, i) => i > index && isOntimeEvent(rundownData.rundown[eventId]) && selectedEvents.has(eventId),
+      );
+
+      // if we didnt find anything after, set the anchor to the last event
+      return set({
+        selectedEvents,
+        anchoredIndex: nextIndex < 0 ? rundownData.order.length - 1 : nextIndex,
+      });
+    }
+
+    // on shift + click, we select a range of events up to the clicked event
+    if (selectMode === 'shift') {
+      const rundownData = ontimeQueryClient.getQueryData<RundownCached>(RUNDOWN);
+      if (!rundownData) return;
+
+      // get list of rundown with only ontime events
+      const events: OntimeEvent[] = [];
+      rundownData.order.forEach((eventId) => {
+        const event = rundownData.rundown[eventId];
+        if (isOntimeEvent(event)) {
+          events.push(event);
+        }
+      });
+
+      const start = anchoredIndex === null ? 0 : Math.min(anchoredIndex, index);
+      const end = anchoredIndex === null ? index : Math.max(anchoredIndex, index + 1);
+
+      // create new set with range of ids from start to end
+      const selectedEventIds = events.slice(start, end).map((event) => event.id);
 
       return set({
-        selectedEvents: new Set([...selectedEvents, ...eventsUntilIndex]),
+        selectedEvents: new Set([...selectedEvents, ...selectedEventIds]),
         anchoredIndex: index,
       });
     }
   },
   clearSelectedEvents: () => set({ selectedEvents: new Set() }),
 }));
+
+export function getSelectionMode(event: MouseEvent): SelectionMode {
+  if ((isMacOS() && event.metaKey) || event.ctrlKey) {
+    return 'ctrl';
+  }
+
+  if (event.shiftKey) {
+    return 'shift';
+  }
+
+  return 'click';
+}