From 9ebb3decdb7d2650b41db4d1e3ff31aaec7086b7 Mon Sep 17 00:00:00 2001 From: Alex Christoffer Rasmussen Date: Thu, 7 Aug 2025 15:48:58 +0200 Subject: [PATCH] V3 project migration (#1652) * migrate settings * migrate viewsettings * migrate url preset * migrate project data * migrate custom fields * migrate automations * migrate rundown * lint * return custom field translation table * check regex * new url type * refactor * migrate old OSC subscriptions * migrate old http subscriptions * update comments * migrate the whole db * remove automation logging unlis there is an error * return a new object * refactor: copy currup is only used in one place * make a copy of original migrated file * refactor: ensure image flder is part of project service init * add entrys to groups * small cleanup * just drop incorrect custom fields in the default data parser --- .../api-data/automation/automation.parser.ts | 31 +- .../automation/clients/http.client.ts | 1 - .../api-data/automation/clients/osc.client.ts | 2 - .../__tests__/customFields.parser.test.ts | 7 +- .../custom-fields/customFields.parser.ts | 35 +- apps/server/src/api-data/db/db.parser.ts | 39 +- .../api-data/db/migration/db.migration.v3.ts | 428 +++++++++++++++ .../api-data/db/migration/migration.test.ts | 492 ++++++++++++++++++ .../project-service/ProjectService.ts | 24 +- .../project-service/projectServiceUtils.ts | 18 +- apps/server/src/setup/config.ts | 1 + apps/server/src/setup/index.ts | 2 + 12 files changed, 993 insertions(+), 87 deletions(-) create mode 100644 apps/server/src/api-data/db/migration/db.migration.v3.ts create mode 100644 apps/server/src/api-data/db/migration/migration.test.ts diff --git a/apps/server/src/api-data/automation/automation.parser.ts b/apps/server/src/api-data/automation/automation.parser.ts index 39cf0e088..23b193493 100644 --- a/apps/server/src/api-data/automation/automation.parser.ts +++ b/apps/server/src/api-data/automation/automation.parser.ts @@ -3,36 +3,7 @@ import { DatabaseModel, AutomationSettings, NormalisedAutomation, Trigger } from import { dbModel } from '../../models/dataModel.js'; import type { ErrorEmitter } from '../../utils/parserUtils.js'; -interface LegacyData extends Partial { - http?: unknown; - osc?: { - enabledIn?: boolean; - portIn?: number; - }; -} - -export function parseAutomationSettings(data: LegacyData, emitError?: ErrorEmitter): AutomationSettings { - // TODO(v4): move to migration script - /** - * Leaving a path for migrating users to the new automations - * This should be removed after a few releases - */ - if (data.http || data.osc) { - emitError?.('Found legacy integrations'); - console.log('Found legacy integrations...'); - if (data.osc) { - return { - enabledAutomations: dbModel.automation.enabledAutomations, - enabledOscIn: data.osc?.enabledIn ?? dbModel.automation.enabledOscIn, - oscPortIn: data.osc?.portIn ?? dbModel.automation.oscPortIn, - triggers: [], - automations: {}, - }; - } else { - return { ...dbModel.automation }; - } - } - +export function parseAutomationSettings(data: Partial, emitError?: ErrorEmitter): AutomationSettings { if (!data.automation) { emitError?.('No data found to import'); return { ...dbModel.automation }; diff --git a/apps/server/src/api-data/automation/clients/http.client.ts b/apps/server/src/api-data/automation/clients/http.client.ts index 4dc752556..865d24ca1 100644 --- a/apps/server/src/api-data/automation/clients/http.client.ts +++ b/apps/server/src/api-data/automation/clients/http.client.ts @@ -21,7 +21,6 @@ function preparePayload(output: HTTPOutput, state: RuntimeState): string { /** Emits message over transport */ async function emit(url: string) { - logger.info(LogOrigin.Rx, `Sending HTTP: ${url}`); try { const response = await fetch(url); if (!response.ok) { diff --git a/apps/server/src/api-data/automation/clients/osc.client.ts b/apps/server/src/api-data/automation/clients/osc.client.ts index 61e32ea0a..2e65eb696 100644 --- a/apps/server/src/api-data/automation/clients/osc.client.ts +++ b/apps/server/src/api-data/automation/clients/osc.client.ts @@ -31,8 +31,6 @@ function preparePayload(output: OSCOutput, state: RuntimeState): OscPacketInput /** Emits message over transport */ function emit(targetIP: string, targetPort: number, packet: OscPacketInput) { - logger.info(LogOrigin.Tx, `Sending OSC: ${targetIP}:${targetPort}`); - const buffer = oscPacketToBuffer(packet); udpClient.send(buffer, 0, buffer.byteLength, targetPort, targetIP, (error) => { if (error) { diff --git a/apps/server/src/api-data/custom-fields/__tests__/customFields.parser.test.ts b/apps/server/src/api-data/custom-fields/__tests__/customFields.parser.test.ts index 066d375c7..de680c234 100644 --- a/apps/server/src/api-data/custom-fields/__tests__/customFields.parser.test.ts +++ b/apps/server/src/api-data/custom-fields/__tests__/customFields.parser.test.ts @@ -14,7 +14,7 @@ describe('parseCustomFields()', () => { const errorEmitter = vi.fn(); // @ts-expect-error -- data is external, we check bad types const customFields = { - 1: { label: 'test', type: 'text', colour: 'red' }, // ok + test: { label: 'test', type: 'text', colour: 'red' }, // ok 2: { label: 'test', type: 'text' }, // duplicate label 3: { label: '', type: 'text' }, // missing colour 4: { type: 'text', colour: '' }, // missing label @@ -88,9 +88,10 @@ describe('sanitiseCustomFields()', () => { expect(sanitationResult).toStrictEqual(expectedCustomFields); }); - it('enforce name cohesion', () => { + it('drop incorrect name/key pairs', () => { const customFields: CustomFields = { - test: { label: 'NewName', type: 'text', colour: 'red' }, + NewName: { label: 'NewName', type: 'text', colour: 'red' }, + willBeGone: { label: 'BadName', type: 'text', colour: 'red' }, }; const expectedCustomFields: CustomFields = { NewName: { label: 'NewName', type: 'text', colour: 'red' }, diff --git a/apps/server/src/api-data/custom-fields/customFields.parser.ts b/apps/server/src/api-data/custom-fields/customFields.parser.ts index be176ffc4..3051ae395 100644 --- a/apps/server/src/api-data/custom-fields/customFields.parser.ts +++ b/apps/server/src/api-data/custom-fields/customFields.parser.ts @@ -23,40 +23,29 @@ export function parseCustomFields(data: Partial, emitError?: Erro export function sanitiseCustomFields(data: object): CustomFields { const newCustomFields: CustomFields = {}; - - for (const [_originalKey, field] of Object.entries(data)) { - if (!isValidField(field)) { - continue; + for (const [key, field] of Object.entries(data)) { + if (isValidField(field, key)) { + newCustomFields[key] = { + type: field.type, + colour: field.colour, + label: field.label, + }; } - - if (!checkRegex.isAlphanumericWithSpace(field.label)) { - continue; - } - - // the key is always made from the label - const key = customFieldLabelToKey(field.label); - - if (key in newCustomFields) { - continue; - } - - newCustomFields[key] = { - type: field.type, - colour: field.colour, - label: field.label, - }; } - function isValidField(data: unknown): data is CustomField { + function isValidField(data: unknown, key: string): data is CustomField { return ( typeof data === 'object' && data !== null && 'label' in data && + typeof data.label === 'string' && data.label !== '' && 'colour' in data && typeof data.colour === 'string' && 'type' in data && - (data.type === 'text' || data.type === 'image') + (data.type === 'text' || data.type === 'image') && + checkRegex.isAlphanumericWithSpace(data.label) && + key === customFieldLabelToKey(data.label) ); } diff --git a/apps/server/src/api-data/db/db.parser.ts b/apps/server/src/api-data/db/db.parser.ts index 18267e516..9c65b0352 100644 --- a/apps/server/src/api-data/db/db.parser.ts +++ b/apps/server/src/api-data/db/db.parser.ts @@ -9,6 +9,7 @@ import { parseSettings } from '../settings/settings.parser.js'; import { parseUrlPresets } from '../url-presets/urlPresets.parser.js'; import { parseViewSettings } from '../view-settings/viewSettings.parser.js'; import { parseCustomFields } from '../custom-fields/customFields.parser.js'; +import * as v3 from './migration/db.migration.v3.js'; type ParsingError = { context: string; @@ -18,12 +19,30 @@ type ParsingError = { /** * @description handles parsing of ontime project file * @param {object} jsonData - project file to be parsed - * @returns {object} - parsed object + * @returns {object} parsed object */ -export function parseDatabaseModel(jsonData: Partial): { data: DatabaseModel; errors: ParsingError[] } { +export function parseDatabaseModel(jsonData: Partial): { + data: DatabaseModel; + errors: ParsingError[]; + migrated: boolean; +} { + //TODO: TEST THIS!!!!!!! + let migrated = false; + let migratedData = jsonData; + if (v3.shouldUseThisMigration(jsonData)) { + try { + migrated = true; + logger.warning(LogOrigin.Server, 'The imported project is from v3, trying to migrate'); + migratedData = v3.migrateAllData(jsonData); + } catch (_error) { + logger.error(LogOrigin.Server, 'Failed to migrate the data'); + migratedData = jsonData; + } + } + // we need to parse settings first to make sure the data is ours // this may throw - const settings = parseSettings(jsonData); + const settings = parseSettings(migratedData); const errors: ParsingError[] = []; const makeEmitError = (context: string) => (message: string) => { @@ -32,18 +51,18 @@ export function parseDatabaseModel(jsonData: Partial): { data: Da }; // we need to parse the custom fields first so they can be used in validating events - const customFields = parseCustomFields(jsonData, makeEmitError('Custom Fields')); - const rundowns = parseRundowns(jsonData, customFields, makeEmitError('Rundowns')); + const customFields = parseCustomFields(migratedData, makeEmitError('Custom Fields')); + const rundowns = parseRundowns(migratedData, customFields, makeEmitError('Rundowns')); const data: DatabaseModel = { rundowns, - project: parseProjectData(jsonData, makeEmitError('Project')), + project: parseProjectData(migratedData, makeEmitError('Project')), settings, - viewSettings: parseViewSettings(jsonData, makeEmitError('View Settings')), - urlPresets: parseUrlPresets(jsonData, makeEmitError('URL Presets')), + viewSettings: parseViewSettings(migratedData, makeEmitError('View Settings')), + urlPresets: parseUrlPresets(migratedData, makeEmitError('URL Presets')), customFields, - automation: parseAutomationSettings(jsonData), + automation: parseAutomationSettings(migratedData), }; - return { data, errors }; + return { data, errors, migrated }; } diff --git a/apps/server/src/api-data/db/migration/db.migration.v3.ts b/apps/server/src/api-data/db/migration/db.migration.v3.ts new file mode 100644 index 000000000..3058698d5 --- /dev/null +++ b/apps/server/src/api-data/db/migration/db.migration.v3.ts @@ -0,0 +1,428 @@ +import { + AutomationSettings, + CustomFields, + DatabaseModel, + EndAction, + EntryCustomFields, + NormalisedAutomation, + OntimeBlock, + OntimeEntry, + ProjectData, + ProjectRundowns, + Rundown, + Settings, + SupportedEntry, + TimerLifeCycle, + Trigger, + URLPreset, + ViewSettings, +} from 'ontime-types'; +import { is } from '../../../utils/is.js'; +import { dbModel } from '../../../models/dataModel.js'; +import { customFieldLabelToKey, checkRegex, isKnownTimerType, validateEndAction } from 'ontime-utils'; +import { event as eventModel } from '../../../models/eventsDefinition.js'; +import { ONTIME_VERSION } from '../../../ONTIME_VERSION.js'; + +// the methodology of the migrations is to just change the necessary keys to match with v4 +// and then let the normal project parser handle ensuring the the file is correct +// we should also avoid relying on the types package as this file should continue to work with old types when things change + +export function shouldUseThisMigration(jsonData: object): boolean { + return ( + is.objectWithKeys(jsonData, ['settings']) && + is.object(jsonData.settings) && + is.objectWithKeys(jsonData.settings, ['version']) && + typeof jsonData.settings.version === 'string' && + jsonData.settings.version.split('.')[0] === '3' + ); +} + +export function migrateAllData(jsonData: object): Partial { + const settings = migrateSettings(jsonData); + const viewSettings = migrateViewSettings(jsonData); + const urlPresets = migrateURLPresets(jsonData); + const project = migrateProjectData(jsonData); + const migratedCustom = migrateCustomFields(jsonData); + const customFields = migratedCustom?.customFields; + const rundowns = migrateRundown(jsonData, migratedCustom?.translationTable); + const automation = migrateAutomations(jsonData); + + return { settings, viewSettings, urlPresets, project, customFields, rundowns, automation }; +} + +type old_Settings = { + version: string; + serverPort: number; + editorKey: null | string; + operatorKey: null | string; + timeFormat: '12' | '24'; + language: string; +}; + +/** + * migrates a settings from v3 to v4 + * - update the version number + */ +export function migrateSettings(jsonData: object): Settings | undefined { + if (is.objectWithKeys(jsonData, ['settings']) && is.object(jsonData.settings)) { + const { serverPort, editorKey, operatorKey, timeFormat, language } = structuredClone( + jsonData.settings, + ) as old_Settings; + return { version: ONTIME_VERSION, serverPort, editorKey, operatorKey, timeFormat, language }; + } +} + +type old_ViewSettings = { + dangerColor: string; + normalColor: string; + overrideStyles: boolean; + warningColor: string; + freezeEnd: boolean; + endMessage: string; +}; + +/** + * migrates a view settings from v3 to v4 + * - drop `freezeEnd` + * - drop `endMessage` + */ +export function migrateViewSettings(jsonData: object): ViewSettings | undefined { + if (is.objectWithKeys(jsonData, ['viewSettings']) && is.object(jsonData.viewSettings)) { + const { dangerColor, normalColor, overrideStyles, warningColor } = structuredClone( + jsonData.viewSettings, + ) as old_ViewSettings; + return { dangerColor, normalColor, overrideStyles, warningColor }; + } +} + +type old_URLPreset = { + enabled: boolean; + alias: string; + pathAndParams: string; +}[]; + +/** + * migrates a url presets from v3 to v4 + * - pathAndParams split into a target and search + */ +export function migrateURLPresets(jsonData: object): URLPreset[] | undefined { + if (is.objectWithKeys(jsonData, ['urlPresets']) && is.array(jsonData.urlPresets)) { + const oldURLPresets = structuredClone(jsonData.urlPresets) as old_URLPreset; + const newURLPreset: URLPreset[] = oldURLPresets.map(({ enabled, alias, pathAndParams }) => { + const [target, search] = pathAndParams.split('?'); + return { enabled, alias, target, search }; + }); + return newURLPreset; + } +} + +type old_ProjectData = { + title: string; + description: string; + backstageUrl: string; + backstageInfo: string; + publicUrl: string; + publicInfo: string; + logo?: string; // is not present in old files + custom?: { title: string; value: string; url: string }[]; // is not present in old files +}; + +/** + * migrates a url presets from v3 to v4 + * - `backstageUrl` -> `url` + * - `backstageInfo` -> `info` + * - drop `publicUrl` + * - drop `publicInfo` + * - ensure `logo` + * - ensure `custom` + */ +export function migrateProjectData(jsonData: object): ProjectData | undefined { + if (is.objectWithKeys(jsonData, ['project']) && is.object(jsonData.project)) { + const { title, description, backstageInfo, backstageUrl, logo, custom } = structuredClone( + jsonData.project, + ) as old_ProjectData; + + return { + title, + description, + url: backstageUrl, + info: backstageInfo, + logo: logo ?? dbModel.project.logo, + custom: custom ?? dbModel.project.custom, + }; + } +} + +// old key -> new key +type CustomFieldsTranslationTable = Map; +type old_CustomFields = Record< + string, + { + type: 'string' | 'image'; + colour: string; + label: string; + } +>; + +/** + * migrates a custom fields from v3 to v4 + * - ensure correct case + * - ensure that the key is derived from the label + * - convert `type` from the string option to the text option + * - create a translation table for the rundown parser + */ +export function migrateCustomFields( + jsonData: object, +): { customFields: CustomFields; translationTable: CustomFieldsTranslationTable } | undefined { + const translationTable: CustomFieldsTranslationTable = new Map(); + + if (is.objectWithKeys(jsonData, ['customFields']) && is.object(jsonData.customFields)) { + // intentionally cast as any so we can extract the values + const oldCustomFields = structuredClone(jsonData.customFields) as old_CustomFields; + const customFields: CustomFields = {}; + + for (const [originalKey, field] of Object.entries(oldCustomFields)) { + if (!checkRegex.isAlphanumericWithSpace(field.label)) { + continue; + } + + // the key is always made from the label + const key = customFieldLabelToKey(field.label); + + if (key in customFields) { + continue; + } + + translationTable.set(originalKey, key); + + customFields[key] = { + type: field.type === 'string' ? 'text' : field.type, + colour: field.colour, + label: field.label, + }; + } + + return { customFields, translationTable }; + } +} + +export type old_OscSubscription = { + id: string; + cycle: TimerLifeCycle; + address: string; + payload: string; + enabled: boolean; +}; + +export type old_OSCSettings = { + portIn: number; + portOut: number; + targetIP: string; + enabledIn: boolean; + enabledOut: boolean; + subscriptions: old_OscSubscription[]; +}; + +export type old_HttpSubscription = { id: string; cycle: TimerLifeCycle; message: string; enabled: boolean }; + +export type old_HttpSettings = { + enabledOut: boolean; + subscriptions: old_HttpSubscription[]; +}; + +/** + * migrates a automations from v3 to v4 + * - in case of a newer v3 project we can just return Automation settings + * - recover older osc and http subscriptions + */ +export function migrateAutomations(jsonData: object): AutomationSettings | undefined { + if (is.objectWithKeys(jsonData, ['automation']) && is.object(jsonData.automation)) { + // For now the automation type used i v3 and in v4 are the same but we will need to update this if it changes at some point + const oldAutomationSettings = structuredClone(jsonData.automation) as AutomationSettings; + return oldAutomationSettings; + } + + let foundOldSetting = false; + const migratedOldStuff = structuredClone(dbModel.automation); + const migratedAutomations: NormalisedAutomation = {}; + const migratedTriggers: Trigger[] = []; + + if (is.objectWithKeys(jsonData, ['osc']) && is.object(jsonData.osc)) { + foundOldSetting = true; + const { subscriptions, portIn, enabledIn, targetIP, portOut } = structuredClone(jsonData.osc) as old_OSCSettings; + migratedOldStuff.enabledOscIn = enabledIn; + migratedOldStuff.oscPortIn = portIn; + + for (const subscription of subscriptions) { + const { id, cycle, address, payload } = subscription; + + migratedTriggers.push({ + id: `${id}-T`, + automationId: `${id}-A`, + title: `Migrated Trigger ${id}`, + trigger: cycle, + }); + + migratedAutomations[`${id}-A`] = { + id: `${id}-A`, + title: `Migrated Automation ${id}`, + filterRule: 'any', + filters: [], + outputs: [{ type: 'osc', address, args: payload, targetIP, targetPort: portOut }], + }; + } + } + + if (is.objectWithKeys(jsonData, ['http']) && is.object(jsonData.http)) { + foundOldSetting = true; + const { subscriptions } = structuredClone(jsonData.http) as old_HttpSettings; + + for (const subscription of subscriptions) { + const { id, cycle, message } = subscription; + migratedTriggers.push({ + id: `${id}-T`, + automationId: `${id}-A`, + title: `Migrated Trigger ${id}`, + trigger: cycle, + }); + + migratedAutomations[`${id}-A`] = { + id: `${id}-A`, + title: `Migrated Automation ${id}`, + filterRule: 'any', + filters: [], + outputs: [{ type: 'http', url: message }], + }; + } + } + + if (foundOldSetting) { + migratedOldStuff.automations = migratedAutomations; + migratedOldStuff.triggers = migratedTriggers; + return migratedOldStuff; + } +} + +/** + * migrates a rundown from v3 to v4 + * - name the rundown default and place it in the multi rundown object + * - generate rundown info placeholders (can be somewhat empty as it will be regenerated by the rundown init) + * + * - events: + * - add flag + * - ensure end action is not stop + * - move the timer type count-to-end to it owen setting + * - ensure triggers + * - add parent + * + * - block: + * - add all the new blocks of the block that is now a group + */ +export function migrateRundown( + jsonData: object, + translationTable: CustomFieldsTranslationTable | undefined, +): ProjectRundowns | undefined { + if (is.objectWithKeys(jsonData, ['rundown']) && is.array(jsonData.rundown)) { + // intentionally cast as any so we can extract the values + const oldRundown = structuredClone(jsonData.rundown) as any[]; + const newRundown: Rundown = { + id: 'default', + title: 'Default', + order: [], + flatOrder: [], + entries: {}, + revision: 0, + }; + + let parent = null; + let children: string[] = []; + + const append = (entry: OntimeEntry) => { + if ('parent' in entry && entry.parent) { + children.push(entry.id); + } else { + newRundown.order.push(entry.id); + } + newRundown.flatOrder.push(entry.id); + newRundown.entries[entry.id] = entry; + }; + + for (const entry of oldRundown) { + if (entry.type === 'event') { + const { custom } = entry as { custom: Record }; + const newCustom: EntryCustomFields = {}; + + if (translationTable) { + Object.entries(custom).map(([key, value]) => { + const newKey = translationTable.get(key); + if (newKey) { + newCustom[newKey] = value; + } + }); + } + + append({ + type: SupportedEntry.Event, + id: entry.id, + flag: false, // new data point + cue: entry.cue, + title: entry.title, + note: entry.note, + endAction: validateEndAction(entry.endAction, EndAction.None), // ensure end action is not stop + timerType: isKnownTimerType(entry.timerType) ? entry.timerType : eventModel.timerType, // ensure the timer type is not count-to-end + countToEnd: entry.timerType === 'count-to-end', // countToEnd was previously a timer type + linkStart: Boolean(entry.linkStart), //this has been null/string + timeStrategy: entry.timeStrategy, + timeStart: entry.timeStart, + timeEnd: entry.timeEnd, + duration: entry.duration, + skip: entry.skip, + colour: entry.colour, + timeWarning: entry.timeWarning, + timeDanger: entry.timeDanger, + custom: newCustom, + triggers: entry.triggers ?? [], // might not be there if the project is a bit older + parent, // new data point + // !==== RUNTIME METADATA ====! // + revision: -1, + delay: 0, + dayOffset: 0, + gap: 0, + }); + } else if (entry.type === 'block') { + if (parent) { + (newRundown.entries[parent] as OntimeBlock).entries = [...children]; + children = []; + } + parent = entry.id; + append({ + id: entry.id, + type: SupportedEntry.Block, + title: entry.title, + note: '', // leave blank + entries: [], // leave empty + targetDuration: null, + colour: '', //leave default colour + custom: {}, // leave empty + // !==== RUNTIME METADATA ====! // + revision: -1, + timeStart: null, + timeEnd: null, + duration: 0, + isFirstLinked: false, + }); + } else if (entry.type === 'delay') { + append({ id: entry.id, type: SupportedEntry.Delay, duration: entry.duration, parent }); + } + } + + if (parent) { + (newRundown.entries[parent] as OntimeBlock).entries = [...children]; + children = []; + } + + return { + default: newRundown, + }; + } +} diff --git a/apps/server/src/api-data/db/migration/migration.test.ts b/apps/server/src/api-data/db/migration/migration.test.ts new file mode 100644 index 000000000..908c68c6c --- /dev/null +++ b/apps/server/src/api-data/db/migration/migration.test.ts @@ -0,0 +1,492 @@ +import { + AutomationSettings, + CustomFields, + EndAction, + ProjectData, + Rundown, + Settings, + SupportedEntry, + TimerLifeCycle, + TimerType, + TimeStrategy, + URLPreset, + ViewSettings, +} from 'ontime-types'; +import * as v3 from './db.migration.v3.js'; +import { dbModel } from '../../../models/dataModel.js'; +import { ONTIME_VERSION } from '../../../ONTIME_VERSION.js'; + +describe('v3 to v4', () => { + const oldDb = { + rundown: [ + { + id: 'event1', + type: 'event', + cue: '123', + title: 'ABC', + note: 'DEF', + endAction: EndAction.None, + timerType: TimerType.CountDown, + countToEnd: false, + linkStart: false, + timeStrategy: TimeStrategy.LockDuration, + timeStart: 0, + timeEnd: 10, + duration: 10, + isPublic: false, + skip: false, + colour: 'blue', + timeWarning: 5, + timeDanger: 2, + custom: { + song: 'Lorem ipsum dolor sit amet, consectetur adipiscing elit', + doseNotExist: 'Lorem ipsum dolor sit amet, consectetur adipiscing elit', + }, + revision: 0, + delay: 0, + dayOffset: 0, + gap: 0, + }, + { id: 'block0', type: 'block', title: 'BLOCK 0' }, + { + id: 'event2', + type: SupportedEntry.Event, + cue: '124', + title: 'ABC', + note: 'DEF', + endAction: EndAction.None, + timerType: TimerType.CountDown, + countToEnd: false, + linkStart: false, + timeStrategy: TimeStrategy.LockDuration, + timeStart: 0, + timeEnd: 10, + duration: 10, + isPublic: false, + skip: false, + colour: 'blue', + timeWarning: 5, + timeDanger: 2, + custom: { + wow: 'http://www.agoodimage.com', + artist: 'Ib Andersen', + }, + triggers: [{ id: 'testTrig', title: 'Test trigger', trigger: TimerLifeCycle.onStart, automationId: '1' }], + revision: 0, + delay: 0, + dayOffset: 0, + gap: 0, + }, + { + id: 'event3', + type: SupportedEntry.Event, + cue: '125', + // title: 'ABC', this has a missing field + note: 'DEF', + endAction: EndAction.None, + timerType: TimerType.CountDown, + countToEnd: false, + linkStart: false, + timeStrategy: TimeStrategy.LockDuration, + timeStart: 0, + timeEnd: 10, + duration: 10, + isPublic: false, + skip: false, + colour: 'blue', + timeWarning: 5, + timeDanger: 2, + custom: { + wow: 'http://www.agoodimage.com', + artist: 'Ib Andersen', + }, + triggers: [{ id: 'testTrig', title: 'Test trigger', trigger: TimerLifeCycle.onStart, automationId: '1' }], + revision: 0, + delay: 0, + dayOffset: 0, + gap: 0, + }, + { id: 'block1', type: 'block', title: 'BLOCK 1' }, + { id: 'delay', type: 'delay', duration: 1000 }, + ], + project: { + title: 'Eurovision Song Contest', + description: 'Turin 2022', + publicUrl: '123', + publicInfo: '456', + backstageUrl: 'www.github.com/cpvalente/ontime', + backstageInfo: 'Rehearsal Schedule - Turin 2022\nAll performers to wear full costumes for 1st rehearsal', + }, + settings: { + version: '3.3.3', + serverPort: 4001, + editorKey: null, + operatorKey: null, + timeFormat: '24', + language: 'en', + }, + viewSettings: { + overrideStyles: false, + normalColor: '#ffffffcc', + warningColor: '#ffa528', + dangerColor: '#ff7300', + freezeEnd: false, + endMessage: '', + }, + urlPresets: [ + { + enabled: true, + alias: 'clock', + pathAndParams: + 'timer?showLeadingZeros=true&timerType=clock&hideClock=true&hideCards=true&hideProgress=true&hideMessage=true&hideSecondary=true&hideLogo=true', + }, + { + enabled: true, + alias: 'minimal', + pathAndParams: + 'timer?showLeadingZeros=true&hideClock=true&hideCards=true&hideProgress=true&hideMessage=true&hideSecondary=true&hideLogo=true', + }, + ], + customFields: { + song: { + label: 'Song and Dance', + type: 'string', + colour: '#339E4E', + }, + artist: { + label: 'Artist and Host', + type: 'string', + colour: '#3E75E8', + }, + wow: { + label: 'WOW 123', + type: 'image', + colour: '#E80000', + }, + }, + automation: { + enabledAutomations: true, + enabledOscIn: true, + oscPortIn: 8888, + triggers: [], + automations: {}, + }, + }; + + test('migrate settings', () => { + const expectSettings: Settings = { + version: ONTIME_VERSION, + serverPort: 4001, + editorKey: null, + operatorKey: null, + timeFormat: '24', + language: 'en', + }; + const newSettings = v3.migrateSettings(oldDb); + expect(newSettings).toEqual(expectSettings); + }); + + test('migrate view settings', () => { + const expectViewSettings: ViewSettings = { + dangerColor: '#ff7300', + normalColor: '#ffffffcc', + overrideStyles: false, + warningColor: '#ffa528', + }; + const newViewSettings = v3.migrateViewSettings(oldDb); + expect(newViewSettings).toEqual(expectViewSettings); + }); + + test('migrate url preset', () => { + const expectUrlPresets: URLPreset[] = [ + { + enabled: true, + alias: 'clock', + target: 'timer', + search: + 'showLeadingZeros=true&timerType=clock&hideClock=true&hideCards=true&hideProgress=true&hideMessage=true&hideSecondary=true&hideLogo=true', + }, + { + enabled: true, + alias: 'minimal', + target: 'timer', + search: + 'showLeadingZeros=true&hideClock=true&hideCards=true&hideProgress=true&hideMessage=true&hideSecondary=true&hideLogo=true', + }, + ]; + const newUrlPreset = v3.migrateURLPresets(oldDb); + expect(newUrlPreset).toEqual(expectUrlPresets); + }); + + test('migrate project data', () => { + const expectProjectData: ProjectData = { + title: 'Eurovision Song Contest', + description: 'Turin 2022', + url: 'www.github.com/cpvalente/ontime', + info: 'Rehearsal Schedule - Turin 2022\nAll performers to wear full costumes for 1st rehearsal', + logo: null, + custom: [], + }; + const newProjectData = v3.migrateProjectData(oldDb); + expect(newProjectData).toEqual(expectProjectData); + }); + + test('migrate custom fields', () => { + const expectCustomFields: CustomFields = { + Song_and_Dance: { + label: 'Song and Dance', + type: 'text', + colour: '#339E4E', + }, + Artist_and_Host: { + label: 'Artist and Host', + type: 'text', + colour: '#3E75E8', + }, + WOW_123: { + label: 'WOW 123', + type: 'image', + colour: '#E80000', + }, + }; + const { customFields, translationTable } = v3.migrateCustomFields(oldDb)!; + expect(customFields).toEqual(expectCustomFields); + + expect(translationTable).toEqual( + new Map([ + ['song', 'Song_and_Dance'], + ['artist', 'Artist_and_Host'], + ['wow', 'WOW_123'], + ]), + ); + }); + + test('migrate rundown', () => { + const expectedRundown: Rundown = { + id: 'default', + title: 'Default', + order: ['event1', 'block0', 'block1'], + flatOrder: ['event1', 'block0', 'event2', 'event3', 'block1', 'delay'], + entries: { + event1: { + id: 'event1', + type: SupportedEntry.Event, + cue: '123', + title: 'ABC', + note: 'DEF', + endAction: EndAction.None, + timerType: TimerType.CountDown, + countToEnd: false, + linkStart: false, + timeStrategy: TimeStrategy.LockDuration, + timeStart: 0, + timeEnd: 10, + duration: 10, + skip: false, + colour: 'blue', + timeWarning: 5, + timeDanger: 2, + custom: { Song_and_Dance: 'Lorem ipsum dolor sit amet, consectetur adipiscing elit' }, + triggers: [], + revision: -1, + flag: false, + parent: null, + delay: 0, + dayOffset: 0, + gap: 0, + }, + block0: { + id: 'block0', + type: SupportedEntry.Block, + title: 'BLOCK 0', + colour: '', + custom: {}, + duration: 0, + entries: ['event2', 'event3'], + isFirstLinked: false, + note: '', + revision: -1, + targetDuration: null, + timeEnd: null, + timeStart: null, + }, + event2: { + id: 'event2', + type: SupportedEntry.Event, + cue: '124', + title: 'ABC', + note: 'DEF', + endAction: EndAction.None, + timerType: TimerType.CountDown, + countToEnd: false, + linkStart: false, + timeStrategy: TimeStrategy.LockDuration, + timeStart: 0, + timeEnd: 10, + duration: 10, + skip: false, + colour: 'blue', + timeWarning: 5, + timeDanger: 2, + custom: { + WOW_123: 'http://www.agoodimage.com', + Artist_and_Host: 'Ib Andersen', + }, + triggers: [{ id: 'testTrig', title: 'Test trigger', trigger: TimerLifeCycle.onStart, automationId: '1' }], + flag: false, + parent: 'block0', + revision: -1, + delay: 0, + dayOffset: 0, + gap: 0, + }, + event3: { + id: 'event3', + type: SupportedEntry.Event, + cue: '125', + //@ts-expect-error - if a field is missing we should pass it through a let our normal import fix it + title: undefined, + note: 'DEF', + endAction: EndAction.None, + timerType: TimerType.CountDown, + countToEnd: false, + linkStart: false, + timeStrategy: TimeStrategy.LockDuration, + timeStart: 0, + timeEnd: 10, + duration: 10, + skip: false, + colour: 'blue', + timeWarning: 5, + timeDanger: 2, + custom: { + WOW_123: 'http://www.agoodimage.com', + Artist_and_Host: 'Ib Andersen', + }, + triggers: [{ id: 'testTrig', title: 'Test trigger', trigger: TimerLifeCycle.onStart, automationId: '1' }], + flag: false, + parent: 'block0', + revision: -1, + delay: 0, + dayOffset: 0, + gap: 0, + }, + block1: { + id: 'block1', + type: SupportedEntry.Block, + title: 'BLOCK 1', + colour: '', + custom: {}, + duration: 0, + entries: ['delay'], + isFirstLinked: false, + note: '', + revision: -1, + targetDuration: null, + timeEnd: null, + timeStart: null, + }, + delay: { + type: SupportedEntry.Delay, + id: 'delay', + duration: 1000, + parent: 'block1', + }, + }, + + revision: 0, + }; + const translationTable = new Map([ + ['song', 'Song_and_Dance'], + ['artist', 'Artist_and_Host'], + ['wow', 'WOW_123'], + ]); + + //@ts-expect-error - we know the default rundown should appear + expect(v3.migrateRundown(oldDb, translationTable)['default']).toStrictEqual(expectedRundown); + }); + + describe('migrate old automation', () => { + test('osc', () => { + const oldData = { + portIn: 8881, + portOut: 55890, + targetIP: '127.0.0.1', + enabledIn: true, + enabledOut: true, + subscriptions: [ + { + id: '23f4d8', + cycle: 'onClock', + address: '/test', + payload: 'bip', + enabled: true, + }, + ], + }; + + const expectedAutomation: AutomationSettings = { + enabledAutomations: true, + enabledOscIn: true, + oscPortIn: 8881, + triggers: [ + { + id: '23f4d8-T', + title: 'Migrated Trigger 23f4d8', + trigger: TimerLifeCycle.onClock, + automationId: '23f4d8-A', + }, + ], + automations: { + '23f4d8-A': { + id: '23f4d8-A', + title: 'Migrated Automation 23f4d8', + filterRule: 'any', + filters: [], + outputs: [{ type: 'osc', targetIP: '127.0.0.1', targetPort: 55890, address: '/test', args: 'bip' }], + }, + }, + }; + + expect(v3.migrateAutomations({ osc: oldData })).toStrictEqual(expectedAutomation); + }); + test('http', () => { + const oldData = { + enabledOut: true, + subscriptions: [ + { + id: '1ge4r8', + cycle: 'onClock', + message: 'http://www.test.com', + enabled: true, + }, + ], + }; + + const expectedAutomation: AutomationSettings = { + enabledAutomations: true, + enabledOscIn: true, + oscPortIn: dbModel.automation.oscPortIn, + triggers: [ + { + id: '1ge4r8-T', + title: 'Migrated Trigger 1ge4r8', + trigger: TimerLifeCycle.onClock, + automationId: '1ge4r8-A', + }, + ], + automations: { + '1ge4r8-A': { + id: '1ge4r8-A', + title: 'Migrated Automation 1ge4r8', + filterRule: 'any', + filters: [], + outputs: [{ type: 'http', url: 'http://www.test.com' }], + }, + }, + }; + + expect(v3.migrateAutomations({ http: oldData })).toStrictEqual(expectedAutomation); + }); + }); +}); diff --git a/apps/server/src/services/project-service/ProjectService.ts b/apps/server/src/services/project-service/ProjectService.ts index de46e3e46..39fe57237 100644 --- a/apps/server/src/services/project-service/ProjectService.ts +++ b/apps/server/src/services/project-service/ProjectService.ts @@ -33,13 +33,13 @@ import { import { runtimeService } from '../runtime-service/RuntimeService.js'; import { - copyCorruptFile, doesProjectExist, getPathToProject, getProjectFiles, moveCorruptFile, parseJsonFile, } from './projectServiceUtils.js'; +import { join } from 'path'; type ProjectState = | { @@ -65,6 +65,8 @@ init(); function init() { ensureDirectory(publicDir.projectsDir); ensureDirectory(publicDir.corruptDir); + ensureDirectory(publicDir.logoDir); + ensureDirectory(publicDir.migrateDir); } export async function getCurrentProject(): Promise<{ filename: string; pathToFile: string }> { @@ -130,7 +132,8 @@ async function loadNewProject(): Promise { */ async function handleCorruptedFile(filePath: string, fileName: string): Promise { // copy file to corrupted folder - await copyCorruptFile(filePath, fileName).catch((_) => { + const copyPath = join(publicDir.corruptDir, fileName); + await copyFile(filePath, copyPath).catch((_) => { /* while we have to catch the error, we dont need to handle it */ }); @@ -140,6 +143,19 @@ async function handleCorruptedFile(filePath: string, fileName: string): Promise< return getFileNameFromPath(newPath); } +async function handleMigratedFile(filePath: string, fileName: string): Promise { + // copy file to migrated folder + const copyPath = join(publicDir.migrateDir, fileName); + await copyFile(filePath, copyPath).catch((_) => { + /* while we have to catch the error, we dont need to handle it */ + }); + + // and make a new file with the recovered data + const newPath = appendToName(filePath, '(migrated)'); + await dockerSafeRename(filePath, newPath); + return getFileNameFromPath(newPath); +} + /** * Coordinates the initial load of a project on app startup * This is different from the load project since we need to always load something @@ -190,6 +206,10 @@ export async function loadProjectFile(fileName: string): Promise { logger.warning(LogOrigin.Server, 'Project loaded with errors'); parsedFileName = await handleCorruptedFile(filePath, fileName); } + if (result.migrated) { + logger.warning(LogOrigin.Server, 'The imported project is migrate, the original file has been backed up'); + parsedFileName = await handleMigratedFile(filePath, fileName); + } const projectName = await loadProject(result.data, parsedFileName); return projectName; diff --git a/apps/server/src/services/project-service/projectServiceUtils.ts b/apps/server/src/services/project-service/projectServiceUtils.ts index 1728c3d8d..d395a8a2e 100644 --- a/apps/server/src/services/project-service/projectServiceUtils.ts +++ b/apps/server/src/services/project-service/projectServiceUtils.ts @@ -1,16 +1,11 @@ import { DatabaseModel, MaybeString, ProjectFile } from 'ontime-types'; import { existsSync } from 'fs'; -import { copyFile, readFile, stat } from 'fs/promises'; +import { readFile, stat } from 'fs/promises'; import { extname, join } from 'path'; import { publicDir } from '../../setup/index.js'; -import { - dockerSafeRename, - ensureDirectory, - getFilesFromFolder, - removeFileExtension, -} from '../../utils/fileManagement.js'; +import { dockerSafeRename, getFilesFromFolder, removeFileExtension } from '../../utils/fileManagement.js'; /** * Handles the upload of a new project file @@ -23,7 +18,6 @@ export async function handleUploaded(filePath: string, name: string) { } export async function handleImageUpload(filePath: string, name: string): Promise { - ensureDirectory(publicDir.logoDir); const newFilePath = join(publicDir.logoDir, name); await dockerSafeRename(filePath, newFilePath); @@ -78,14 +72,6 @@ export function getPathToProject(name: string): string { return join(publicDir.projectsDir, name); } -/** - * Makes a copy of a given project to the corrupted directory - */ -export async function copyCorruptFile(filePath: string, name: string): Promise { - const newPath = join(publicDir.corruptDir, name); - return copyFile(filePath, newPath); -} - /** * Moves a file permanently to the corrupted directory */ diff --git a/apps/server/src/setup/config.ts b/apps/server/src/setup/config.ts index b5583708d..6d845f097 100644 --- a/apps/server/src/setup/config.ts +++ b/apps/server/src/setup/config.ts @@ -11,6 +11,7 @@ export const timerConfig = { export const config = { appState: 'app-state.json', corrupt: 'corrupt files', + migrate: 'migrated files', crash: 'crash logs', demoProject: 'demo project.json', newProject: 'new project.json', diff --git a/apps/server/src/setup/index.ts b/apps/server/src/setup/index.ts index c02f619c1..0b14de7b0 100644 --- a/apps/server/src/setup/index.ts +++ b/apps/server/src/setup/index.ts @@ -116,6 +116,8 @@ export const publicDir = { projectsDir: join(resolvePublicDirectory, config.projects), /** path to corrupt folder */ corruptDir: join(resolvePublicDirectory, config.corrupt), + /** path to migrated folder */ + migrateDir: join(resolvePublicDirectory, config.migrate), /** path to uploads folder */ uploadsDir: join(resolvePublicDirectory, config.uploads), /** path to external folder */