feat(import): add merge strategy and new-rundown destination to spreadsheet import

Co-authored-by: Carlos Valente <34649812+cpvalente@users.noreply.github.com>
This commit is contained in:
Claude
2026-07-06 11:14:35 +00:00
committed by Carlos Valente
parent 5e6debcce0
commit 34360a5b8f
20 changed files with 1064 additions and 80 deletions
@@ -1,4 +1,6 @@
import type {
ImportedFields,
RundownImportMode,
SpreadsheetPreviewResponse,
SpreadsheetWorksheetMetadata,
SpreadsheetWorksheetOptions,
@@ -22,7 +24,7 @@ import Info from '../../../../../common/components/info/Info';
import ExternalLink from '../../../../../common/components/link/external-link/ExternalLink';
import Modal from '../../../../../common/components/modal/Modal';
import useRundown from '../../../../../common/hooks-query/useRundown';
import { validateExcelImport } from '../../../../../common/utils/uploadUtils';
import { removeFileExtension, validateExcelImport } from '../../../../../common/utils/uploadUtils';
import * as Panel from '../../../panel-utils/PanelUtils';
import GSheetSetup from './GSheetSetup';
import SheetImportEditor from './sheet-import/SheetImportEditor';
@@ -35,6 +37,7 @@ const googleSheetDocsUrl = 'https://docs.getontime.no/features/import-spreadshee
type ActiveSource =
| {
kind: 'excel';
fileName: string;
worksheetNames: string[];
initialWorksheetMetadata: SpreadsheetWorksheetMetadata | null;
closedByUser: boolean;
@@ -55,7 +58,7 @@ export default function SourcesPanel() {
const [activeSource, setActiveSource] = useState<ActiveSource | null>(null);
const { data: currentRundown } = useRundown();
const { importRundown } = useSpreadsheetImport();
const { applyImport } = useSpreadsheetImport();
const fileInputRef = useRef<HTMLInputElement>(null);
@@ -74,6 +77,7 @@ export default function SourcesPanel() {
const worksheetOptions = await uploadExcel(fileToUpload);
setActiveSource({
kind: 'excel',
fileName: fileToUpload.name,
worksheetNames: worksheetOptions.worksheets,
initialWorksheetMetadata: worksheetOptions.metadata,
closedByUser: false,
@@ -126,21 +130,32 @@ export default function SourcesPanel() {
setError('');
};
const handleApplyImport = async (preview: SpreadsheetPreviewResponse) => {
const handleApplyImport = async (
preview: SpreadsheetPreviewResponse,
mode: RundownImportMode,
newRundownTitle: string,
providedFields: ImportedFields,
) => {
if (mode === 'new') {
const title = newRundownTitle.trim() || preview.rundown.title;
await applyImport({ mode: 'new', rundown: { ...preview.rundown, title }, customFields: preview.customFields });
handleFinished();
return;
}
if (!currentRundown) {
throw new Error('No current rundown loaded');
}
await importRundown(
{
[currentRundown.id]: {
...preview.rundown,
id: currentRundown.id,
title: currentRundown.title,
},
},
preview.customFields,
);
// override or merge into the current rundown; merge uses providedFields to know which columns
// the sheet supplied so matched events keep the rest (e.g. automations)
await applyImport({
mode,
targetRundownId: currentRundown.id,
rundown: preview.rundown,
customFields: preview.customFields,
providedFields,
});
handleFinished();
};
@@ -208,6 +223,13 @@ export default function SourcesPanel() {
if (activeSource.kind === 'excel') return 'excel';
return `gsheet:${activeSource.sheetId}`;
})();
// suggested name when importing into a new rundown: the spreadsheet file name (without extension)
// for Excel, or the document title for Google Sheets
const spreadsheetName = (() => {
if (!activeSource) return '';
if (activeSource.kind === 'excel') return removeFileExtension(activeSource.fileName);
return activeSource.title;
})();
return (
<Panel.Section>
@@ -287,6 +309,7 @@ export default function SourcesPanel() {
bodyElements={
<SheetImportEditor
sourceKey={sourceKey ?? 'spreadsheet'}
defaultRundownName={spreadsheetName}
worksheetNames={activeSource?.worksheetNames ?? []}
initialMetadata={activeSource?.initialWorksheetMetadata ?? null}
loadMetadata={loadWorksheetMetadata}
@@ -0,0 +1,55 @@
import type { RundownImportMode, SpreadsheetPreviewResponse } from 'ontime-types';
import { isOntimeEvent, isPlayableEvent, Playback } from 'ontime-types';
import { useEffect, useState } from 'react';
import Button from '../../../../../../common/components/buttons/Button';
import { usePlayback, useSelectedEventId } from '../../../../../../common/hooks/useSocket';
interface ApplyImportButtonProps {
preview: SpreadsheetPreviewResponse | null;
mode: RundownImportMode;
disabled: boolean;
loading: boolean;
onApply: () => void;
}
/**
* Apply action for the spreadsheet import.
* Subscribes to playback state on its own so playback updates do not re-render the whole editor.
* Requires a second click to confirm when applying would stop a running playback.
*/
export default function ApplyImportButton({ preview, mode, disabled, loading, onApply }: ApplyImportButtonProps) {
const playback = usePlayback();
const loadedEventId = useSelectedEventId();
// the loaded (playing) event loses its playback unless it still exists as a playable event after the import
const loadedEntry = loadedEventId ? preview?.rundown.entries[loadedEventId] : undefined;
const willLoadedEventBeOverriden = !(
loadedEntry !== undefined &&
isOntimeEvent(loadedEntry) &&
isPlayableEvent(loadedEntry)
);
// applying stops playback when creating a new rundown, or when the playing event does not survive
const willStopPlayback = playback !== Playback.Stop && (mode === 'new' || willLoadedEventBeOverriden);
// two-step confirmation before applying an import that stops playback
const [confirmStop, setConfirmStop] = useState(false);
useEffect(() => {
setConfirmStop(false);
}, [mode, preview]);
const handleClick = () => {
if (willStopPlayback && !confirmStop) {
setConfirmStop(true);
return;
}
onApply();
};
return (
<Button variant='primary' onClick={handleClick} disabled={disabled} loading={loading}>
{willStopPlayback && confirmStop ? 'Confirm — stop playback & apply import' : 'Apply import'}
</Button>
);
}
@@ -33,6 +33,11 @@
white-space: nowrap;
}
.importModeTrigger {
min-width: 12rem;
justify-content: space-between;
}
.addColumnTrigger {
justify-content: center;
white-space: nowrap;
@@ -1,10 +1,20 @@
import type { SpreadsheetPreviewResponse, SpreadsheetWorksheetMetadata } from 'ontime-types';
import type {
ImportedFields,
RundownImportMode,
SpreadsheetPreviewResponse,
SpreadsheetWorksheetMetadata,
} from 'ontime-types';
import type { ImportMap } from 'ontime-utils';
import { IoArrowUpOutline, IoEye } from 'react-icons/io5';
import { useMemo } from 'react';
import { IoArrowUpOutline, IoCheckmark, IoChevronDown, IoEye, IoWarningOutline } from 'react-icons/io5';
import Button from '../../../../../../common/components/buttons/Button';
import { DropdownMenu, DropdownMenuOption } from '../../../../../../common/components/dropdown-menu/DropdownMenu';
import Input from '../../../../../../common/components/input/input/Input';
import Select from '../../../../../../common/components/select/Select';
import * as Panel from '../../../../panel-utils/PanelUtils';
import ApplyImportButton from './ApplyImportButton';
import { isIdColumnMapped } from './importMapUtils';
import PreviewTable from './preview/PreviewTable';
import SheetImportMappingPane from './SheetImportMappingPane';
import { useSheetImportForm } from './useSheetImportForm';
@@ -13,17 +23,46 @@ import style from './SheetImportEditor.module.scss';
interface SheetImportEditorProps {
sourceKey: string;
defaultRundownName: string;
worksheetNames: string[];
initialMetadata: SpreadsheetWorksheetMetadata | null;
loadMetadata: (worksheet: string) => Promise<SpreadsheetWorksheetMetadata>;
previewImport: (importMap: ImportMap) => Promise<SpreadsheetPreviewResponse>;
onApply: (preview: SpreadsheetPreviewResponse) => Promise<void>;
onApply: (
preview: SpreadsheetPreviewResponse,
mode: RundownImportMode,
newRundownTitle: string,
providedFields: ImportedFields,
) => Promise<void>;
onCancel: () => void;
onExport?: (importMap: ImportMap) => Promise<void>;
}
const importModeOptions: Array<{
value: RundownImportMode;
label: string;
description: string;
}> = [
{
value: 'override',
label: 'Replace current rundown',
description: 'Spreadsheet data completely replaces current rundown',
},
{
value: 'merge',
label: 'Merge with current rundown',
description: 'Merge entries referencing their IDs, entries not present in spreadsheet rundown are deleted',
},
{
value: 'new',
label: 'New rundown',
description: 'Create a new rundown to import the data into. Loads the new rundown',
},
];
export default function SheetImportEditor({
sourceKey,
defaultRundownName,
worksheetNames,
initialMetadata,
loadMetadata,
@@ -49,11 +88,16 @@ export default function SheetImportEditor({
isBusy,
canPreview,
displayError,
importMode,
setImportMode,
newRundownTitle,
setNewRundownTitle,
handlePreviewSubmit,
handleExportSubmit,
handleApply,
} = useSheetImportForm({
sourceKey,
defaultRundownName,
worksheetNames,
initialMetadata,
loadMetadata,
@@ -62,6 +106,19 @@ export default function SheetImportEditor({
onExport,
});
const selectedImportMode = importModeOptions.find((option) => option.value === importMode) ?? importModeOptions[0];
const importModeItems = useMemo<DropdownMenuOption[]>(
() =>
importModeOptions.map((option) => ({
type: 'item',
label: option.label,
description: option.description,
icon: importMode === option.value ? IoCheckmark : undefined,
onClick: () => setImportMode(option.value),
})),
[importMode, setImportMode],
);
return (
<Panel.Section as='form' id='spreadsheet-import-workspace' className={style.editor} onSubmit={handlePreviewSubmit}>
<Panel.InlineElements align='apart' wrap='wrap' className={style.editorToolbar}>
@@ -107,33 +164,63 @@ export default function SheetImportEditor({
</div>
{displayError && <Panel.Error>{displayError}</Panel.Error>}
<Panel.InlineElements align='end' wrap='wrap' className={style.editorFooter}>
<Button onClick={onCancel} disabled={isBusy}>
Cancel
</Button>
{onExport && (
<Button onClick={handleExportSubmit} disabled={!canPreview} loading={state.loading === 'export'}>
<IoArrowUpOutline />
Export
{importMode === 'merge' && !isIdColumnMapped(values) && (
<Panel.Description tone='warning'>
<IoWarningOutline /> No ID column mapped merge matches entries by ID, so it will behave like Replace. Export
your rundown to a spreadsheet first to keep its IDs.
</Panel.Description>
)}
<Panel.InlineElements align='apart' wrap='wrap' className={style.editorFooter}>
<Panel.InlineElements wrap='wrap'>
<label className={style.worksheetControl}>
<span className={style.worksheetLabel}>Import mode</span>
<DropdownMenu
render={<Button className={style.importModeTrigger} variant='subtle-white' />}
items={importModeItems}
>
{selectedImportMode.label}
<IoChevronDown />
</DropdownMenu>
</label>
{importMode === 'new' && (
<label className={style.worksheetControl}>
<span className={style.worksheetLabel}>New rundown name</span>
<Input
value={newRundownTitle}
onChange={(event) => setNewRundownTitle(event.target.value)}
placeholder={state.preview?.rundown.title || 'Imported rundown'}
aria-label='New rundown name'
/>
</label>
)}
</Panel.InlineElements>
<Panel.InlineElements wrap='wrap'>
<Button onClick={onCancel} disabled={isBusy}>
Cancel
</Button>
)}
<Button
variant={state.preview ? undefined : 'primary'}
onClick={handlePreviewSubmit}
disabled={!canPreview}
loading={state.loading === 'preview'}
>
<IoEye />
Preview import
</Button>
<Button
variant='primary'
onClick={handleApply}
disabled={!state.preview || isBusy}
loading={state.loading === 'apply'}
>
Apply import
</Button>
{onExport && (
<Button onClick={handleExportSubmit} disabled={!canPreview} loading={state.loading === 'export'}>
<IoArrowUpOutline />
Export
</Button>
)}
<Button
variant={state.preview ? undefined : 'primary'}
onClick={handlePreviewSubmit}
disabled={!canPreview}
loading={state.loading === 'preview'}
>
<IoEye />
Preview import
</Button>
<ApplyImportButton
preview={state.preview}
mode={importMode}
disabled={!state.preview || isBusy}
loading={state.loading === 'apply'}
onApply={handleApply}
/>
</Panel.InlineElements>
</Panel.InlineElements>
</Panel.Section>
);
@@ -1,15 +1,21 @@
import { describe, expect, it } from 'vitest';
import { beforeEach, describe, expect, it } from 'vitest';
import {
builtInFieldDefs,
convertToImportMap,
createDefaultFormValues,
defaultImportMode,
getImportWarnings,
getPersistedImportMode,
getProvidedImportFields,
getResolvedCustomFields,
isIdColumnMapped,
persistImportMode,
} from '../importMapUtils';
const cueIndex = builtInFieldDefs.findIndex((field) => field.label === 'Cue');
const titleIndex = builtInFieldDefs.findIndex((field) => field.label === 'Title');
const idIndex = builtInFieldDefs.findIndex((field) => field.importKey === 'id');
describe('getImportWarnings()', () => {
it('warns when two mappings target the same spreadsheet column', () => {
@@ -128,3 +134,64 @@ describe('convertToImportMap()', () => {
});
});
});
describe('getProvidedImportFields()', () => {
it('reports the mapped built-in and custom fields the sheet supplies', () => {
const values = createDefaultFormValues();
values.builtIn[titleIndex] = { header: 'title', enabled: true };
values.builtIn[cueIndex] = { header: '', enabled: false };
values.custom = [{ ontimeName: 'ignored', importName: 'FOH/Monitor' }];
const provided = getProvidedImportFields(convertToImportMap(values));
// enabled built-in mappings are reported as event fields, disabled ones are not
expect(provided.event).toContain('title');
expect(provided.event).not.toContain('cue');
// the id column is only used for matching, never overwritten
expect(provided.event).not.toContain('id');
// custom fields are reported symmetrically by their resolved Ontime name
expect(provided.custom).toStrictEqual(['FOH Monitor']);
});
});
describe('isIdColumnMapped()', () => {
it('is true when the ID field is enabled with a header', () => {
// the default form maps the ID column
expect(isIdColumnMapped(createDefaultFormValues())).toBe(true);
});
it('is false when the ID field is disabled', () => {
const values = createDefaultFormValues();
values.builtIn[idIndex] = { header: 'id', enabled: false };
expect(isIdColumnMapped(values)).toBe(false);
});
it('is false when the ID field header is blank', () => {
const values = createDefaultFormValues();
values.builtIn[idIndex] = { header: ' ', enabled: true };
expect(isIdColumnMapped(values)).toBe(false);
});
});
describe('import mode persistence', () => {
const sourceKey = 'excel-test';
beforeEach(() => {
localStorage.clear();
});
it('defaults to override when nothing is persisted', () => {
expect(getPersistedImportMode(sourceKey)).toBe('override');
expect(defaultImportMode).toBe('override');
});
it('round-trips a persisted value', () => {
persistImportMode(sourceKey, 'merge');
expect(getPersistedImportMode(sourceKey)).toBe('merge');
});
it('falls back to the default when the persisted value is invalid', () => {
persistImportMode(sourceKey, 'nonsense' as never);
expect(getPersistedImportMode(sourceKey)).toBe('override');
});
});
@@ -1,3 +1,4 @@
import type { ImportedFields, RundownImportMode } from 'ontime-types';
import type { ImportMap } from 'ontime-utils';
import { makeStageKey } from '../../../../../../common/utils/localStorage';
@@ -49,6 +50,16 @@ export function createDefaultFormValues(): ImportFormValues {
};
}
/**
* Whether the mapping supplies an ID column. Merge matches entries by ID, so without one every
* imported entry gets a fresh id and nothing can reconcile with the current rundown.
*/
export function isIdColumnMapped(values: ImportFormValues): boolean {
const idIndex = builtInFieldDefs.findIndex((def) => def.importKey === 'id');
const field = values.builtIn[idIndex];
return Boolean(field?.enabled && field.header.trim());
}
function sanitiseOntimeCustomFieldLabel(importName: string): string {
// Replace punctuation with spaces, then collapse repeated whitespace into single spaces.
const sanitised = importName
@@ -81,6 +92,24 @@ export function getResolvedCustomFields(customFields: ImportFormValues['custom']
});
}
/**
* Returns the fields the import map supplies — the complete description of what the incoming data
* provides, for both built-in and custom fields. A merge uses this to patch exactly these fields
* onto a matched event and keep everything else (e.g. automations) untouched.
* Import-map keys are OntimeEvent field names; `worksheet`/`custom` are meta and `id` is only used
* for matching, not overwritten.
*/
export function getProvidedImportFields(importMap: ImportMap): ImportedFields {
const event: string[] = [];
for (const [key, value] of Object.entries(importMap)) {
if (key === 'worksheet' || key === 'custom' || key === 'id') continue;
if (typeof value === 'string' && value.trim() !== '') {
event.push(key);
}
}
return { event, custom: Object.keys(importMap.custom) };
}
export function convertToImportMap(values: ImportFormValues): ImportMap {
const custom = getResolvedCustomFields(values.custom).reduce<Record<string, string>>(
(accumulator, { ontimeName, importName }) => {
@@ -130,12 +159,12 @@ function isPersistedFormValues(obj: unknown): obj is ImportFormValues {
export function getPersistedImportState(sourceKey: string): { values: ImportFormValues; isPersisted: boolean } {
const storageKey = getImportMapKey(sourceKey);
try {
const raw = localStorage.getItem(storageKey);
if (!raw) {
const persistedData = localStorage.getItem(storageKey);
if (!persistedData) {
return { values: createDefaultFormValues(), isPersisted: false };
}
const parsed: unknown = JSON.parse(raw);
const parsed: unknown = JSON.parse(persistedData);
if (isPersistedFormValues(parsed)) {
return { values: parsed, isPersisted: true };
}
@@ -150,6 +179,32 @@ export function getPersistedImportState(sourceKey: string): { values: ImportForm
}
}
/**
* The import mode (new / merge / override) is persisted separately from the field mapping
* so the mapping schema guard stays untouched.
*/
/** Default import mode: replace matched elements in the current rundown */
export const defaultImportMode: RundownImportMode = 'override';
function getImportModeKey(sourceKey: string) {
return makeStageKey(`import-mode:${sourceKey}`);
}
/** Persists the import mode for a given source */
export function persistImportMode(sourceKey: string, mode: RundownImportMode) {
localStorage.setItem(getImportModeKey(sourceKey), mode);
}
/** Reads the persisted import mode for a source, falling back to the default when absent or invalid */
export function getPersistedImportMode(sourceKey: string): RundownImportMode {
const persisted = localStorage.getItem(getImportModeKey(sourceKey));
if (persisted === 'new' || persisted === 'merge' || persisted === 'override') {
return persisted;
}
return defaultImportMode;
}
/**
* Validates import mappings and generates warnings for duplicate or missing spreadsheet columns.
*/
@@ -1,7 +1,12 @@
import { useQuery, useQueryClient } from '@tanstack/react-query';
import type { SpreadsheetPreviewResponse, SpreadsheetWorksheetMetadata } from 'ontime-types';
import type {
ImportedFields,
RundownImportMode,
SpreadsheetPreviewResponse,
SpreadsheetWorksheetMetadata,
} from 'ontime-types';
import { millisToString } from 'ontime-utils';
import { useCallback, useEffect, useMemo, useReducer, useRef } from 'react';
import { useCallback, useEffect, useMemo, useReducer, useRef, useState } from 'react';
import { useFieldArray, useForm } from 'react-hook-form';
import { maybeAxiosError } from '../../../../../../common/api/utils';
@@ -11,8 +16,11 @@ import {
builtInFieldDefs,
convertToImportMap,
getImportWarnings,
getPersistedImportMode,
getPersistedImportState,
getProvidedImportFields,
getResolvedCustomFields,
persistImportMode,
persistImportState,
} from './importMapUtils';
import { deriveHeaderOptionsState } from './spreadsheetImportUtils';
@@ -100,16 +108,23 @@ function buildColumnLabels(values: ImportFormValues): string[] {
interface UseSheetImportFormProps {
sourceKey: string;
defaultRundownName: string;
worksheetNames: string[];
initialMetadata: SpreadsheetWorksheetMetadata | null;
loadMetadata: (worksheet: string) => Promise<SpreadsheetWorksheetMetadata>;
previewImport: (importMap: ReturnType<typeof convertToImportMap>) => Promise<SpreadsheetPreviewResponse>;
onApply: (preview: SpreadsheetPreviewResponse) => Promise<void>;
onApply: (
preview: SpreadsheetPreviewResponse,
mode: RundownImportMode,
newRundownTitle: string,
providedFields: ImportedFields,
) => Promise<void>;
onExport?: (importMap: ReturnType<typeof convertToImportMap>) => Promise<void>;
}
export function useSheetImportForm({
sourceKey,
defaultRundownName,
worksheetNames,
initialMetadata,
loadMetadata,
@@ -170,6 +185,8 @@ export function useSheetImportForm({
const columnLabels = buildColumnLabels(values);
const [state, dispatch] = useReducer(importReducer, initialImportState);
const [importMode, setImportMode] = useState<RundownImportMode>(() => getPersistedImportMode(sourceKey));
const [newRundownTitle, setNewRundownTitle] = useState(defaultRundownName);
const warnings = getImportWarnings(values, headers);
const warningCount = Object.values(warnings).filter(Boolean).length;
const previewRef = useRef<SpreadsheetPreviewResponse | null>(null);
@@ -182,6 +199,12 @@ export function useSheetImportForm({
dispatch({ type: 'reset' });
}, [initialFormValues, reset]);
// Update import mode and rundown name if the source changes
useEffect(() => {
setImportMode(getPersistedImportMode(sourceKey));
setNewRundownTitle(defaultRundownName);
}, [sourceKey, defaultRundownName]);
// Keep the worksheet selection valid if the available worksheets change underneath the form.
useEffect(() => {
if (worksheetNames.length === 0) return;
@@ -241,13 +264,15 @@ export function useSheetImportForm({
try {
dispatch({ type: 'startApply' });
await onApply(state.preview);
const providedFields = getProvidedImportFields(convertToImportMap(getValues()));
await onApply(state.preview, importMode, newRundownTitle, providedFields);
persistImportState(sourceKey, getValues());
persistImportMode(sourceKey, importMode);
dispatch({ type: 'applySuccess' });
} catch (error) {
dispatch({ type: 'failure', error: maybeAxiosError(error) });
}
}, [getValues, onApply, sourceKey, state.preview]);
}, [getValues, importMode, newRundownTitle, onApply, sourceKey, state.preview]);
const handleExport = useCallback(
async (formValues: ImportFormValues) => {
@@ -300,6 +325,10 @@ export function useSheetImportForm({
isBusy,
canPreview,
displayError,
importMode,
setImportMode,
newRundownTitle,
setNewRundownTitle,
handlePreviewSubmit: handleSubmit(handlePreview),
handleExportSubmit: handleSubmit(handleExport),
handleApply,
@@ -1,30 +1,17 @@
import { useQueryClient } from '@tanstack/react-query';
import { CustomFields, ProjectRundowns } from 'ontime-types';
import { RundownImportPayload } from 'ontime-types';
import { useCallback } from 'react';
import { CUSTOM_FIELDS, RUNDOWN } from '../../../../../common/api/constants';
import { patchData } from '../../../../../common/api/db';
import { importRundownWithOptions } from '../../../../../common/api/rundown';
export default function useSpreadsheetImport() {
const queryClient = useQueryClient();
/** applies rundown and customFields to current project */
const importRundown = useCallback(
async (rundowns: ProjectRundowns, customFields: CustomFields) => {
await patchData({ rundowns, customFields });
// we are unable to optimistically set the rundown since we need
// it to be normalised
await queryClient.invalidateQueries({
queryKey: RUNDOWN,
});
await queryClient.invalidateQueries({
queryKey: CUSTOM_FIELDS,
});
},
[queryClient],
);
/** applies a spreadsheet import: override or merge into the current rundown, or create a new one */
const applyImport = useCallback(async (payload: RundownImportPayload) => {
// the backend broadcasts a refetch once the rundown is parsed and applied, so the caches update
// through that single path rather than racing it with an optimistic write from here
await importRundownWithOptions(payload);
}, []);
return {
importRundown,
applyImport,
};
}