V3 project migration (#1652)

* migrate settings

* migrate viewsettings

* migrate url preset

* migrate project data

* migrate custom fields

* migrate automations

* migrate rundown

* lint

* return custom field translation table

* check regex

* new url type

* refactor

* migrate old OSC subscriptions

* migrate old http subscriptions

* update comments

* migrate the whole db

* remove automation logging unlis there is an error

* return a new object

* refactor: copy currup is only used in one place

* make a copy of original migrated file

* refactor: ensure image flder is part of project service init

* add entrys to groups

* small cleanup

* just drop incorrect custom fields in the default data parser

apps/server/src/api-data/automation/automation.parser.ts

@@ -3,36 +3,7 @@ import { DatabaseModel, AutomationSettings, NormalisedAutomation, Trigger } from
 import { dbModel } from '../../models/dataModel.js';
 import type { ErrorEmitter } from '../../utils/parserUtils.js';
 
-interface LegacyData extends Partial<DatabaseModel> {
+export function parseAutomationSettings(data: Partial<DatabaseModel>, emitError?: ErrorEmitter): AutomationSettings {
-  http?: unknown;
-  osc?: {
-    enabledIn?: boolean;
-    portIn?: number;
-  };
-}
-
-export function parseAutomationSettings(data: LegacyData, emitError?: ErrorEmitter): AutomationSettings {
-  // TODO(v4): move to migration script
-  /**
-   * Leaving a path for migrating users to the new automations
-   * This should be removed after a few releases
-   */
-  if (data.http || data.osc) {
-    emitError?.('Found legacy integrations');
-    console.log('Found legacy integrations...');
-    if (data.osc) {
-      return {
-        enabledAutomations: dbModel.automation.enabledAutomations,
-        enabledOscIn: data.osc?.enabledIn ?? dbModel.automation.enabledOscIn,
-        oscPortIn: data.osc?.portIn ?? dbModel.automation.oscPortIn,
-        triggers: [],
-        automations: {},
-      };
-    } else {
-      return { ...dbModel.automation };
-    }
-  }
-
   if (!data.automation) {
     emitError?.('No data found to import');
     return { ...dbModel.automation };

apps/server/src/api-data/automation/clients/http.client.ts

@@ -21,7 +21,6 @@ function preparePayload(output: HTTPOutput, state: RuntimeState): string {
 
 /** Emits message over transport */
 async function emit(url: string) {
-  logger.info(LogOrigin.Rx, `Sending HTTP: ${url}`);
   try {
     const response = await fetch(url);
     if (!response.ok) {

apps/server/src/api-data/automation/clients/osc.client.ts

@@ -31,8 +31,6 @@ function preparePayload(output: OSCOutput, state: RuntimeState): OscPacketInput
 
 /** Emits message over transport */
 function emit(targetIP: string, targetPort: number, packet: OscPacketInput) {
-  logger.info(LogOrigin.Tx, `Sending OSC: ${targetIP}:${targetPort}`);
-
   const buffer = oscPacketToBuffer(packet);
   udpClient.send(buffer, 0, buffer.byteLength, targetPort, targetIP, (error) => {
     if (error) {

apps/server/src/api-data/custom-fields/__tests__/customFields.parser.test.ts

@@ -14,7 +14,7 @@ describe('parseCustomFields()', () => {
     const errorEmitter = vi.fn();
     // @ts-expect-error -- data is external, we check bad types
     const customFields = {
-      1: { label: 'test', type: 'text', colour: 'red' }, // ok
+      test: { label: 'test', type: 'text', colour: 'red' }, // ok
       2: { label: 'test', type: 'text' }, // duplicate label
       3: { label: '', type: 'text' }, // missing colour
       4: { type: 'text', colour: '' }, // missing label
@@ -88,9 +88,10 @@ describe('sanitiseCustomFields()', () => {
     expect(sanitationResult).toStrictEqual(expectedCustomFields);
   });
 
-  it('enforce name cohesion', () => {
+  it('drop incorrect name/key pairs', () => {
     const customFields: CustomFields = {
-      test: { label: 'NewName', type: 'text', colour: 'red' },
+      NewName: { label: 'NewName', type: 'text', colour: 'red' },
+      willBeGone: { label: 'BadName', type: 'text', colour: 'red' },
     };
     const expectedCustomFields: CustomFields = {
       NewName: { label: 'NewName', type: 'text', colour: 'red' },

apps/server/src/api-data/custom-fields/customFields.parser.ts

@@ -23,40 +23,29 @@ export function parseCustomFields(data: Partial<DatabaseModel>, emitError?: Erro
 
 export function sanitiseCustomFields(data: object): CustomFields {
   const newCustomFields: CustomFields = {};
-
+  for (const [key, field] of Object.entries(data)) {
-  for (const [_originalKey, field] of Object.entries(data)) {
+    if (isValidField(field, key)) {
-    if (!isValidField(field)) {
+      newCustomFields[key] = {
-      continue;
+        type: field.type,
+        colour: field.colour,
+        label: field.label,
+      };
     }
-
-    if (!checkRegex.isAlphanumericWithSpace(field.label)) {
-      continue;
-    }
-
-    // the key is always made from the label
-    const key = customFieldLabelToKey(field.label);
-
-    if (key in newCustomFields) {
-      continue;
-    }
-
-    newCustomFields[key] = {
-      type: field.type,
-      colour: field.colour,
-      label: field.label,
-    };
   }
 
-  function isValidField(data: unknown): data is CustomField {
+  function isValidField(data: unknown, key: string): data is CustomField {
     return (
       typeof data === 'object' &&
       data !== null &&
       'label' in data &&
+      typeof data.label === 'string' &&
       data.label !== '' &&
       'colour' in data &&
       typeof data.colour === 'string' &&
       'type' in data &&
-      (data.type === 'text' || data.type === 'image')
+      (data.type === 'text' || data.type === 'image') &&
+      checkRegex.isAlphanumericWithSpace(data.label) &&
+      key === customFieldLabelToKey(data.label)
     );
   }
 

apps/server/src/api-data/db/db.parser.ts

@@ -9,6 +9,7 @@ import { parseSettings } from '../settings/settings.parser.js';
 import { parseUrlPresets } from '../url-presets/urlPresets.parser.js';
 import { parseViewSettings } from '../view-settings/viewSettings.parser.js';
 import { parseCustomFields } from '../custom-fields/customFields.parser.js';
+import * as v3 from './migration/db.migration.v3.js';
 
 type ParsingError = {
   context: string;
@@ -18,12 +19,30 @@ type ParsingError = {
 /**
  * @description handles parsing of ontime project file
  * @param {object} jsonData - project file to be parsed
- * @returns {object} - parsed object
+ * @returns {object} parsed object
  */
-export function parseDatabaseModel(jsonData: Partial<DatabaseModel>): { data: DatabaseModel; errors: ParsingError[] } {
+export function parseDatabaseModel(jsonData: Partial<DatabaseModel>): {
+  data: DatabaseModel;
+  errors: ParsingError[];
+  migrated: boolean;
+} {
+  //TODO: TEST THIS!!!!!!!
+  let migrated = false;
+  let migratedData = jsonData;
+  if (v3.shouldUseThisMigration(jsonData)) {
+    try {
+      migrated = true;
+      logger.warning(LogOrigin.Server, 'The imported project is from v3, trying to migrate');
+      migratedData = v3.migrateAllData(jsonData);
+    } catch (_error) {
+      logger.error(LogOrigin.Server, 'Failed to migrate the data');
+      migratedData = jsonData;
+    }
+  }
+
   // we need to parse settings first to make sure the data is ours
   // this may throw
-  const settings = parseSettings(jsonData);
+  const settings = parseSettings(migratedData);
 
   const errors: ParsingError[] = [];
   const makeEmitError = (context: string) => (message: string) => {
@@ -32,18 +51,18 @@ export function parseDatabaseModel(jsonData: Partial<DatabaseModel>): { data: Da
   };
 
   // we need to parse the custom fields first so they can be used in validating events
-  const customFields = parseCustomFields(jsonData, makeEmitError('Custom Fields'));
+  const customFields = parseCustomFields(migratedData, makeEmitError('Custom Fields'));
-  const rundowns = parseRundowns(jsonData, customFields, makeEmitError('Rundowns'));
+  const rundowns = parseRundowns(migratedData, customFields, makeEmitError('Rundowns'));
 
   const data: DatabaseModel = {
     rundowns,
-    project: parseProjectData(jsonData, makeEmitError('Project')),
+    project: parseProjectData(migratedData, makeEmitError('Project')),
     settings,
-    viewSettings: parseViewSettings(jsonData, makeEmitError('View Settings')),
+    viewSettings: parseViewSettings(migratedData, makeEmitError('View Settings')),
-    urlPresets: parseUrlPresets(jsonData, makeEmitError('URL Presets')),
+    urlPresets: parseUrlPresets(migratedData, makeEmitError('URL Presets')),
     customFields,
-    automation: parseAutomationSettings(jsonData),
+    automation: parseAutomationSettings(migratedData),
   };
 
-  return { data, errors };
+  return { data, errors, migrated };
 }

apps/server/src/api-data/db/migration/db.migration.v3.ts

@@ -0,0 +1,428 @@
+import {
+  AutomationSettings,
+  CustomFields,
+  DatabaseModel,
+  EndAction,
+  EntryCustomFields,
+  NormalisedAutomation,
+  OntimeBlock,
+  OntimeEntry,
+  ProjectData,
+  ProjectRundowns,
+  Rundown,
+  Settings,
+  SupportedEntry,
+  TimerLifeCycle,
+  Trigger,
+  URLPreset,
+  ViewSettings,
+} from 'ontime-types';
+import { is } from '../../../utils/is.js';
+import { dbModel } from '../../../models/dataModel.js';
+import { customFieldLabelToKey, checkRegex, isKnownTimerType, validateEndAction } from 'ontime-utils';
+import { event as eventModel } from '../../../models/eventsDefinition.js';
+import { ONTIME_VERSION } from '../../../ONTIME_VERSION.js';
+
+// the methodology of the migrations is to just change the necessary keys to match with v4
+// and then let the normal project parser handle ensuring the the file is correct
+// we should also avoid relying on the types package as this file should continue to work with old types when things change
+
+export function shouldUseThisMigration(jsonData: object): boolean {
+  return (
+    is.objectWithKeys(jsonData, ['settings']) &&
+    is.object(jsonData.settings) &&
+    is.objectWithKeys(jsonData.settings, ['version']) &&
+    typeof jsonData.settings.version === 'string' &&
+    jsonData.settings.version.split('.')[0] === '3'
+  );
+}
+
+export function migrateAllData(jsonData: object): Partial<DatabaseModel> {
+  const settings = migrateSettings(jsonData);
+  const viewSettings = migrateViewSettings(jsonData);
+  const urlPresets = migrateURLPresets(jsonData);
+  const project = migrateProjectData(jsonData);
+  const migratedCustom = migrateCustomFields(jsonData);
+  const customFields = migratedCustom?.customFields;
+  const rundowns = migrateRundown(jsonData, migratedCustom?.translationTable);
+  const automation = migrateAutomations(jsonData);
+
+  return { settings, viewSettings, urlPresets, project, customFields, rundowns, automation };
+}
+
+type old_Settings = {
+  version: string;
+  serverPort: number;
+  editorKey: null | string;
+  operatorKey: null | string;
+  timeFormat: '12' | '24';
+  language: string;
+};
+
+/**
+ * migrates a settings from v3 to v4
+ * - update the version number
+ */
+export function migrateSettings(jsonData: object): Settings | undefined {
+  if (is.objectWithKeys(jsonData, ['settings']) && is.object(jsonData.settings)) {
+    const { serverPort, editorKey, operatorKey, timeFormat, language } = structuredClone(
+      jsonData.settings,
+    ) as old_Settings;
+    return { version: ONTIME_VERSION, serverPort, editorKey, operatorKey, timeFormat, language };
+  }
+}
+
+type old_ViewSettings = {
+  dangerColor: string;
+  normalColor: string;
+  overrideStyles: boolean;
+  warningColor: string;
+  freezeEnd: boolean;
+  endMessage: string;
+};
+
+/**
+ * migrates a view settings from v3 to v4
+ * - drop `freezeEnd`
+ * - drop `endMessage`
+ */
+export function migrateViewSettings(jsonData: object): ViewSettings | undefined {
+  if (is.objectWithKeys(jsonData, ['viewSettings']) && is.object(jsonData.viewSettings)) {
+    const { dangerColor, normalColor, overrideStyles, warningColor } = structuredClone(
+      jsonData.viewSettings,
+    ) as old_ViewSettings;
+    return { dangerColor, normalColor, overrideStyles, warningColor };
+  }
+}
+
+type old_URLPreset = {
+  enabled: boolean;
+  alias: string;
+  pathAndParams: string;
+}[];
+
+/**
+ * migrates a url presets from v3 to v4
+ * - pathAndParams split into a target and search
+ */
+export function migrateURLPresets(jsonData: object): URLPreset[] | undefined {
+  if (is.objectWithKeys(jsonData, ['urlPresets']) && is.array(jsonData.urlPresets)) {
+    const oldURLPresets = structuredClone(jsonData.urlPresets) as old_URLPreset;
+    const newURLPreset: URLPreset[] = oldURLPresets.map(({ enabled, alias, pathAndParams }) => {
+      const [target, search] = pathAndParams.split('?');
+      return { enabled, alias, target, search };
+    });
+    return newURLPreset;
+  }
+}
+
+type old_ProjectData = {
+  title: string;
+  description: string;
+  backstageUrl: string;
+  backstageInfo: string;
+  publicUrl: string;
+  publicInfo: string;
+  logo?: string; // is not present in old files
+  custom?: { title: string; value: string; url: string }[]; // is not present in old files
+};
+
+/**
+ * migrates a url presets from v3 to v4
+ * - `backstageUrl` -> `url`
+ * - `backstageInfo` -> `info`
+ * - drop `publicUrl`
+ * - drop `publicInfo`
+ * - ensure `logo`
+ * - ensure `custom`
+ */
+export function migrateProjectData(jsonData: object): ProjectData | undefined {
+  if (is.objectWithKeys(jsonData, ['project']) && is.object(jsonData.project)) {
+    const { title, description, backstageInfo, backstageUrl, logo, custom } = structuredClone(
+      jsonData.project,
+    ) as old_ProjectData;
+
+    return {
+      title,
+      description,
+      url: backstageUrl,
+      info: backstageInfo,
+      logo: logo ?? dbModel.project.logo,
+      custom: custom ?? dbModel.project.custom,
+    };
+  }
+}
+
+// old key -> new key
+type CustomFieldsTranslationTable = Map<string, string>;
+type old_CustomFields = Record<
+  string,
+  {
+    type: 'string' | 'image';
+    colour: string;
+    label: string;
+  }
+>;
+
+/**
+ * migrates a custom fields from v3 to v4
+ * - ensure correct case
+ * - ensure that the key is derived from the label
+ * - convert `type` from the string option to the text option
+ * - create a translation table for the rundown parser
+ */
+export function migrateCustomFields(
+  jsonData: object,
+): { customFields: CustomFields; translationTable: CustomFieldsTranslationTable } | undefined {
+  const translationTable: CustomFieldsTranslationTable = new Map();
+
+  if (is.objectWithKeys(jsonData, ['customFields']) && is.object(jsonData.customFields)) {
+    // intentionally cast as any so we can extract the values
+    const oldCustomFields = structuredClone(jsonData.customFields) as old_CustomFields;
+    const customFields: CustomFields = {};
+
+    for (const [originalKey, field] of Object.entries(oldCustomFields)) {
+      if (!checkRegex.isAlphanumericWithSpace(field.label)) {
+        continue;
+      }
+
+      // the key is always made from the label
+      const key = customFieldLabelToKey(field.label);
+
+      if (key in customFields) {
+        continue;
+      }
+
+      translationTable.set(originalKey, key);
+
+      customFields[key] = {
+        type: field.type === 'string' ? 'text' : field.type,
+        colour: field.colour,
+        label: field.label,
+      };
+    }
+
+    return { customFields, translationTable };
+  }
+}
+
+export type old_OscSubscription = {
+  id: string;
+  cycle: TimerLifeCycle;
+  address: string;
+  payload: string;
+  enabled: boolean;
+};
+
+export type old_OSCSettings = {
+  portIn: number;
+  portOut: number;
+  targetIP: string;
+  enabledIn: boolean;
+  enabledOut: boolean;
+  subscriptions: old_OscSubscription[];
+};
+
+export type old_HttpSubscription = { id: string; cycle: TimerLifeCycle; message: string; enabled: boolean };
+
+export type old_HttpSettings = {
+  enabledOut: boolean;
+  subscriptions: old_HttpSubscription[];
+};
+
+/**
+ * migrates a automations from v3 to v4
+ * - in case of a newer v3 project we can just return Automation settings
+ * - recover older osc and http subscriptions
+ */
+export function migrateAutomations(jsonData: object): AutomationSettings | undefined {
+  if (is.objectWithKeys(jsonData, ['automation']) && is.object(jsonData.automation)) {
+    // For now the automation type used i v3 and in v4 are the same but we will need to update this if it changes at some point
+    const oldAutomationSettings = structuredClone(jsonData.automation) as AutomationSettings;
+    return oldAutomationSettings;
+  }
+
+  let foundOldSetting = false;
+  const migratedOldStuff = structuredClone(dbModel.automation);
+  const migratedAutomations: NormalisedAutomation = {};
+  const migratedTriggers: Trigger[] = [];
+
+  if (is.objectWithKeys(jsonData, ['osc']) && is.object(jsonData.osc)) {
+    foundOldSetting = true;
+    const { subscriptions, portIn, enabledIn, targetIP, portOut } = structuredClone(jsonData.osc) as old_OSCSettings;
+    migratedOldStuff.enabledOscIn = enabledIn;
+    migratedOldStuff.oscPortIn = portIn;
+
+    for (const subscription of subscriptions) {
+      const { id, cycle, address, payload } = subscription;
+
+      migratedTriggers.push({
+        id: `${id}-T`,
+        automationId: `${id}-A`,
+        title: `Migrated Trigger ${id}`,
+        trigger: cycle,
+      });
+
+      migratedAutomations[`${id}-A`] = {
+        id: `${id}-A`,
+        title: `Migrated Automation ${id}`,
+        filterRule: 'any',
+        filters: [],
+        outputs: [{ type: 'osc', address, args: payload, targetIP, targetPort: portOut }],
+      };
+    }
+  }
+
+  if (is.objectWithKeys(jsonData, ['http']) && is.object(jsonData.http)) {
+    foundOldSetting = true;
+    const { subscriptions } = structuredClone(jsonData.http) as old_HttpSettings;
+
+    for (const subscription of subscriptions) {
+      const { id, cycle, message } = subscription;
+      migratedTriggers.push({
+        id: `${id}-T`,
+        automationId: `${id}-A`,
+        title: `Migrated Trigger ${id}`,
+        trigger: cycle,
+      });
+
+      migratedAutomations[`${id}-A`] = {
+        id: `${id}-A`,
+        title: `Migrated Automation ${id}`,
+        filterRule: 'any',
+        filters: [],
+        outputs: [{ type: 'http', url: message }],
+      };
+    }
+  }
+
+  if (foundOldSetting) {
+    migratedOldStuff.automations = migratedAutomations;
+    migratedOldStuff.triggers = migratedTriggers;
+    return migratedOldStuff;
+  }
+}
+
+/**
+ * migrates a rundown from v3 to v4
+ * -  name the rundown default and place it in the multi rundown object
+ * - generate rundown info placeholders (can be somewhat empty as it will be regenerated by the rundown init)
+ *
+ * - events:
+ *  - add flag
+ *  - ensure end action is not stop
+ *  - move the timer type count-to-end to it owen setting
+ *  - ensure triggers
+ *  - add parent
+ *
+ * - block:
+ *  - add all the new blocks of the block that is now a group
+ */
+export function migrateRundown(
+  jsonData: object,
+  translationTable: CustomFieldsTranslationTable | undefined,
+): ProjectRundowns | undefined {
+  if (is.objectWithKeys(jsonData, ['rundown']) && is.array(jsonData.rundown)) {
+    // intentionally cast as any so we can extract the values
+    const oldRundown = structuredClone(jsonData.rundown) as any[];
+    const newRundown: Rundown = {
+      id: 'default',
+      title: 'Default',
+      order: [],
+      flatOrder: [],
+      entries: {},
+      revision: 0,
+    };
+
+    let parent = null;
+    let children: string[] = [];
+
+    const append = (entry: OntimeEntry) => {
+      if ('parent' in entry && entry.parent) {
+        children.push(entry.id);
+      } else {
+        newRundown.order.push(entry.id);
+      }
+      newRundown.flatOrder.push(entry.id);
+      newRundown.entries[entry.id] = entry;
+    };
+
+    for (const entry of oldRundown) {
+      if (entry.type === 'event') {
+        const { custom } = entry as { custom: Record<string, string> };
+        const newCustom: EntryCustomFields = {};
+
+        if (translationTable) {
+          Object.entries(custom).map(([key, value]) => {
+            const newKey = translationTable.get(key);
+            if (newKey) {
+              newCustom[newKey] = value;
+            }
+          });
+        }
+
+        append({
+          type: SupportedEntry.Event,
+          id: entry.id,
+          flag: false, // new data point
+          cue: entry.cue,
+          title: entry.title,
+          note: entry.note,
+          endAction: validateEndAction(entry.endAction, EndAction.None), // ensure end action is not stop
+          timerType: isKnownTimerType(entry.timerType) ? entry.timerType : eventModel.timerType, // ensure the timer type is not count-to-end
+          countToEnd: entry.timerType === 'count-to-end', // countToEnd was previously a timer type
+          linkStart: Boolean(entry.linkStart), //this has been null/string
+          timeStrategy: entry.timeStrategy,
+          timeStart: entry.timeStart,
+          timeEnd: entry.timeEnd,
+          duration: entry.duration,
+          skip: entry.skip,
+          colour: entry.colour,
+          timeWarning: entry.timeWarning,
+          timeDanger: entry.timeDanger,
+          custom: newCustom,
+          triggers: entry.triggers ?? [], // might not be there if the project is a bit older
+          parent, // new data point
+          // !==== RUNTIME METADATA ====! //
+          revision: -1,
+          delay: 0,
+          dayOffset: 0,
+          gap: 0,
+        });
+      } else if (entry.type === 'block') {
+        if (parent) {
+          (newRundown.entries[parent] as OntimeBlock).entries = [...children];
+          children = [];
+        }
+        parent = entry.id;
+        append({
+          id: entry.id,
+          type: SupportedEntry.Block,
+          title: entry.title,
+          note: '', // leave blank
+          entries: [], // leave empty
+          targetDuration: null,
+          colour: '', //leave default colour
+          custom: {}, // leave empty
+          // !==== RUNTIME METADATA ====! //
+          revision: -1,
+          timeStart: null,
+          timeEnd: null,
+          duration: 0,
+          isFirstLinked: false,
+        });
+      } else if (entry.type === 'delay') {
+        append({ id: entry.id, type: SupportedEntry.Delay, duration: entry.duration, parent });
+      }
+    }
+
+    if (parent) {
+      (newRundown.entries[parent] as OntimeBlock).entries = [...children];
+      children = [];
+    }
+
+    return {
+      default: newRundown,
+    };
+  }
+}

apps/server/src/api-data/db/migration/migration.test.ts

@@ -0,0 +1,492 @@
+import {
+  AutomationSettings,
+  CustomFields,
+  EndAction,
+  ProjectData,
+  Rundown,
+  Settings,
+  SupportedEntry,
+  TimerLifeCycle,
+  TimerType,
+  TimeStrategy,
+  URLPreset,
+  ViewSettings,
+} from 'ontime-types';
+import * as v3 from './db.migration.v3.js';
+import { dbModel } from '../../../models/dataModel.js';
+import { ONTIME_VERSION } from '../../../ONTIME_VERSION.js';
+
+describe('v3 to v4', () => {
+  const oldDb = {
+    rundown: [
+      {
+        id: 'event1',
+        type: 'event',
+        cue: '123',
+        title: 'ABC',
+        note: 'DEF',
+        endAction: EndAction.None,
+        timerType: TimerType.CountDown,
+        countToEnd: false,
+        linkStart: false,
+        timeStrategy: TimeStrategy.LockDuration,
+        timeStart: 0,
+        timeEnd: 10,
+        duration: 10,
+        isPublic: false,
+        skip: false,
+        colour: 'blue',
+        timeWarning: 5,
+        timeDanger: 2,
+        custom: {
+          song: 'Lorem ipsum dolor sit amet, consectetur adipiscing elit',
+          doseNotExist: 'Lorem ipsum dolor sit amet, consectetur adipiscing elit',
+        },
+        revision: 0,
+        delay: 0,
+        dayOffset: 0,
+        gap: 0,
+      },
+      { id: 'block0', type: 'block', title: 'BLOCK 0' },
+      {
+        id: 'event2',
+        type: SupportedEntry.Event,
+        cue: '124',
+        title: 'ABC',
+        note: 'DEF',
+        endAction: EndAction.None,
+        timerType: TimerType.CountDown,
+        countToEnd: false,
+        linkStart: false,
+        timeStrategy: TimeStrategy.LockDuration,
+        timeStart: 0,
+        timeEnd: 10,
+        duration: 10,
+        isPublic: false,
+        skip: false,
+        colour: 'blue',
+        timeWarning: 5,
+        timeDanger: 2,
+        custom: {
+          wow: 'http://www.agoodimage.com',
+          artist: 'Ib Andersen',
+        },
+        triggers: [{ id: 'testTrig', title: 'Test trigger', trigger: TimerLifeCycle.onStart, automationId: '1' }],
+        revision: 0,
+        delay: 0,
+        dayOffset: 0,
+        gap: 0,
+      },
+      {
+        id: 'event3',
+        type: SupportedEntry.Event,
+        cue: '125',
+        // title: 'ABC', this has a missing field
+        note: 'DEF',
+        endAction: EndAction.None,
+        timerType: TimerType.CountDown,
+        countToEnd: false,
+        linkStart: false,
+        timeStrategy: TimeStrategy.LockDuration,
+        timeStart: 0,
+        timeEnd: 10,
+        duration: 10,
+        isPublic: false,
+        skip: false,
+        colour: 'blue',
+        timeWarning: 5,
+        timeDanger: 2,
+        custom: {
+          wow: 'http://www.agoodimage.com',
+          artist: 'Ib Andersen',
+        },
+        triggers: [{ id: 'testTrig', title: 'Test trigger', trigger: TimerLifeCycle.onStart, automationId: '1' }],
+        revision: 0,
+        delay: 0,
+        dayOffset: 0,
+        gap: 0,
+      },
+      { id: 'block1', type: 'block', title: 'BLOCK 1' },
+      { id: 'delay', type: 'delay', duration: 1000 },
+    ],
+    project: {
+      title: 'Eurovision Song Contest',
+      description: 'Turin 2022',
+      publicUrl: '123',
+      publicInfo: '456',
+      backstageUrl: 'www.github.com/cpvalente/ontime',
+      backstageInfo: 'Rehearsal Schedule - Turin 2022\nAll performers to wear full costumes for 1st rehearsal',
+    },
+    settings: {
+      version: '3.3.3',
+      serverPort: 4001,
+      editorKey: null,
+      operatorKey: null,
+      timeFormat: '24',
+      language: 'en',
+    },
+    viewSettings: {
+      overrideStyles: false,
+      normalColor: '#ffffffcc',
+      warningColor: '#ffa528',
+      dangerColor: '#ff7300',
+      freezeEnd: false,
+      endMessage: '',
+    },
+    urlPresets: [
+      {
+        enabled: true,
+        alias: 'clock',
+        pathAndParams:
+          'timer?showLeadingZeros=true&timerType=clock&hideClock=true&hideCards=true&hideProgress=true&hideMessage=true&hideSecondary=true&hideLogo=true',
+      },
+      {
+        enabled: true,
+        alias: 'minimal',
+        pathAndParams:
+          'timer?showLeadingZeros=true&hideClock=true&hideCards=true&hideProgress=true&hideMessage=true&hideSecondary=true&hideLogo=true',
+      },
+    ],
+    customFields: {
+      song: {
+        label: 'Song and Dance',
+        type: 'string',
+        colour: '#339E4E',
+      },
+      artist: {
+        label: 'Artist and Host',
+        type: 'string',
+        colour: '#3E75E8',
+      },
+      wow: {
+        label: 'WOW 123',
+        type: 'image',
+        colour: '#E80000',
+      },
+    },
+    automation: {
+      enabledAutomations: true,
+      enabledOscIn: true,
+      oscPortIn: 8888,
+      triggers: [],
+      automations: {},
+    },
+  };
+
+  test('migrate settings', () => {
+    const expectSettings: Settings = {
+      version: ONTIME_VERSION,
+      serverPort: 4001,
+      editorKey: null,
+      operatorKey: null,
+      timeFormat: '24',
+      language: 'en',
+    };
+    const newSettings = v3.migrateSettings(oldDb);
+    expect(newSettings).toEqual(expectSettings);
+  });
+
+  test('migrate view settings', () => {
+    const expectViewSettings: ViewSettings = {
+      dangerColor: '#ff7300',
+      normalColor: '#ffffffcc',
+      overrideStyles: false,
+      warningColor: '#ffa528',
+    };
+    const newViewSettings = v3.migrateViewSettings(oldDb);
+    expect(newViewSettings).toEqual(expectViewSettings);
+  });
+
+  test('migrate url preset', () => {
+    const expectUrlPresets: URLPreset[] = [
+      {
+        enabled: true,
+        alias: 'clock',
+        target: 'timer',
+        search:
+          'showLeadingZeros=true&timerType=clock&hideClock=true&hideCards=true&hideProgress=true&hideMessage=true&hideSecondary=true&hideLogo=true',
+      },
+      {
+        enabled: true,
+        alias: 'minimal',
+        target: 'timer',
+        search:
+          'showLeadingZeros=true&hideClock=true&hideCards=true&hideProgress=true&hideMessage=true&hideSecondary=true&hideLogo=true',
+      },
+    ];
+    const newUrlPreset = v3.migrateURLPresets(oldDb);
+    expect(newUrlPreset).toEqual(expectUrlPresets);
+  });
+
+  test('migrate project data', () => {
+    const expectProjectData: ProjectData = {
+      title: 'Eurovision Song Contest',
+      description: 'Turin 2022',
+      url: 'www.github.com/cpvalente/ontime',
+      info: 'Rehearsal Schedule - Turin 2022\nAll performers to wear full costumes for 1st rehearsal',
+      logo: null,
+      custom: [],
+    };
+    const newProjectData = v3.migrateProjectData(oldDb);
+    expect(newProjectData).toEqual(expectProjectData);
+  });
+
+  test('migrate custom fields', () => {
+    const expectCustomFields: CustomFields = {
+      Song_and_Dance: {
+        label: 'Song and Dance',
+        type: 'text',
+        colour: '#339E4E',
+      },
+      Artist_and_Host: {
+        label: 'Artist and Host',
+        type: 'text',
+        colour: '#3E75E8',
+      },
+      WOW_123: {
+        label: 'WOW 123',
+        type: 'image',
+        colour: '#E80000',
+      },
+    };
+    const { customFields, translationTable } = v3.migrateCustomFields(oldDb)!;
+    expect(customFields).toEqual(expectCustomFields);
+
+    expect(translationTable).toEqual(
+      new Map([
+        ['song', 'Song_and_Dance'],
+        ['artist', 'Artist_and_Host'],
+        ['wow', 'WOW_123'],
+      ]),
+    );
+  });
+
+  test('migrate rundown', () => {
+    const expectedRundown: Rundown = {
+      id: 'default',
+      title: 'Default',
+      order: ['event1', 'block0', 'block1'],
+      flatOrder: ['event1', 'block0', 'event2', 'event3', 'block1', 'delay'],
+      entries: {
+        event1: {
+          id: 'event1',
+          type: SupportedEntry.Event,
+          cue: '123',
+          title: 'ABC',
+          note: 'DEF',
+          endAction: EndAction.None,
+          timerType: TimerType.CountDown,
+          countToEnd: false,
+          linkStart: false,
+          timeStrategy: TimeStrategy.LockDuration,
+          timeStart: 0,
+          timeEnd: 10,
+          duration: 10,
+          skip: false,
+          colour: 'blue',
+          timeWarning: 5,
+          timeDanger: 2,
+          custom: { Song_and_Dance: 'Lorem ipsum dolor sit amet, consectetur adipiscing elit' },
+          triggers: [],
+          revision: -1,
+          flag: false,
+          parent: null,
+          delay: 0,
+          dayOffset: 0,
+          gap: 0,
+        },
+        block0: {
+          id: 'block0',
+          type: SupportedEntry.Block,
+          title: 'BLOCK 0',
+          colour: '',
+          custom: {},
+          duration: 0,
+          entries: ['event2', 'event3'],
+          isFirstLinked: false,
+          note: '',
+          revision: -1,
+          targetDuration: null,
+          timeEnd: null,
+          timeStart: null,
+        },
+        event2: {
+          id: 'event2',
+          type: SupportedEntry.Event,
+          cue: '124',
+          title: 'ABC',
+          note: 'DEF',
+          endAction: EndAction.None,
+          timerType: TimerType.CountDown,
+          countToEnd: false,
+          linkStart: false,
+          timeStrategy: TimeStrategy.LockDuration,
+          timeStart: 0,
+          timeEnd: 10,
+          duration: 10,
+          skip: false,
+          colour: 'blue',
+          timeWarning: 5,
+          timeDanger: 2,
+          custom: {
+            WOW_123: 'http://www.agoodimage.com',
+            Artist_and_Host: 'Ib Andersen',
+          },
+          triggers: [{ id: 'testTrig', title: 'Test trigger', trigger: TimerLifeCycle.onStart, automationId: '1' }],
+          flag: false,
+          parent: 'block0',
+          revision: -1,
+          delay: 0,
+          dayOffset: 0,
+          gap: 0,
+        },
+        event3: {
+          id: 'event3',
+          type: SupportedEntry.Event,
+          cue: '125',
+          //@ts-expect-error - if a field is missing we should pass it through a let our normal import fix it
+          title: undefined,
+          note: 'DEF',
+          endAction: EndAction.None,
+          timerType: TimerType.CountDown,
+          countToEnd: false,
+          linkStart: false,
+          timeStrategy: TimeStrategy.LockDuration,
+          timeStart: 0,
+          timeEnd: 10,
+          duration: 10,
+          skip: false,
+          colour: 'blue',
+          timeWarning: 5,
+          timeDanger: 2,
+          custom: {
+            WOW_123: 'http://www.agoodimage.com',
+            Artist_and_Host: 'Ib Andersen',
+          },
+          triggers: [{ id: 'testTrig', title: 'Test trigger', trigger: TimerLifeCycle.onStart, automationId: '1' }],
+          flag: false,
+          parent: 'block0',
+          revision: -1,
+          delay: 0,
+          dayOffset: 0,
+          gap: 0,
+        },
+        block1: {
+          id: 'block1',
+          type: SupportedEntry.Block,
+          title: 'BLOCK 1',
+          colour: '',
+          custom: {},
+          duration: 0,
+          entries: ['delay'],
+          isFirstLinked: false,
+          note: '',
+          revision: -1,
+          targetDuration: null,
+          timeEnd: null,
+          timeStart: null,
+        },
+        delay: {
+          type: SupportedEntry.Delay,
+          id: 'delay',
+          duration: 1000,
+          parent: 'block1',
+        },
+      },
+
+      revision: 0,
+    };
+    const translationTable = new Map([
+      ['song', 'Song_and_Dance'],
+      ['artist', 'Artist_and_Host'],
+      ['wow', 'WOW_123'],
+    ]);
+
+    //@ts-expect-error - we know  the default rundown  should appear
+    expect(v3.migrateRundown(oldDb, translationTable)['default']).toStrictEqual(expectedRundown);
+  });
+
+  describe('migrate old automation', () => {
+    test('osc', () => {
+      const oldData = {
+        portIn: 8881,
+        portOut: 55890,
+        targetIP: '127.0.0.1',
+        enabledIn: true,
+        enabledOut: true,
+        subscriptions: [
+          {
+            id: '23f4d8',
+            cycle: 'onClock',
+            address: '/test',
+            payload: 'bip',
+            enabled: true,
+          },
+        ],
+      };
+
+      const expectedAutomation: AutomationSettings = {
+        enabledAutomations: true,
+        enabledOscIn: true,
+        oscPortIn: 8881,
+        triggers: [
+          {
+            id: '23f4d8-T',
+            title: 'Migrated Trigger 23f4d8',
+            trigger: TimerLifeCycle.onClock,
+            automationId: '23f4d8-A',
+          },
+        ],
+        automations: {
+          '23f4d8-A': {
+            id: '23f4d8-A',
+            title: 'Migrated Automation 23f4d8',
+            filterRule: 'any',
+            filters: [],
+            outputs: [{ type: 'osc', targetIP: '127.0.0.1', targetPort: 55890, address: '/test', args: 'bip' }],
+          },
+        },
+      };
+
+      expect(v3.migrateAutomations({ osc: oldData })).toStrictEqual(expectedAutomation);
+    });
+    test('http', () => {
+      const oldData = {
+        enabledOut: true,
+        subscriptions: [
+          {
+            id: '1ge4r8',
+            cycle: 'onClock',
+            message: 'http://www.test.com',
+            enabled: true,
+          },
+        ],
+      };
+
+      const expectedAutomation: AutomationSettings = {
+        enabledAutomations: true,
+        enabledOscIn: true,
+        oscPortIn: dbModel.automation.oscPortIn,
+        triggers: [
+          {
+            id: '1ge4r8-T',
+            title: 'Migrated Trigger 1ge4r8',
+            trigger: TimerLifeCycle.onClock,
+            automationId: '1ge4r8-A',
+          },
+        ],
+        automations: {
+          '1ge4r8-A': {
+            id: '1ge4r8-A',
+            title: 'Migrated Automation 1ge4r8',
+            filterRule: 'any',
+            filters: [],
+            outputs: [{ type: 'http', url: 'http://www.test.com' }],
+          },
+        },
+      };
+
+      expect(v3.migrateAutomations({ http: oldData })).toStrictEqual(expectedAutomation);
+    });
+  });
+});

apps/server/src/services/project-service/ProjectService.ts

@@ -33,13 +33,13 @@ import {
 import { runtimeService } from '../runtime-service/RuntimeService.js';
 
 import {
-  copyCorruptFile,
   doesProjectExist,
   getPathToProject,
   getProjectFiles,
   moveCorruptFile,
   parseJsonFile,
 } from './projectServiceUtils.js';
+import { join } from 'path';
 
 type ProjectState =
   | {
@@ -65,6 +65,8 @@ init();
 function init() {
   ensureDirectory(publicDir.projectsDir);
   ensureDirectory(publicDir.corruptDir);
+  ensureDirectory(publicDir.logoDir);
+  ensureDirectory(publicDir.migrateDir);
 }
 
 export async function getCurrentProject(): Promise<{ filename: string; pathToFile: string }> {
@@ -130,7 +132,8 @@ async function loadNewProject(): Promise<string> {
  */
 async function handleCorruptedFile(filePath: string, fileName: string): Promise<string> {
   // copy file to corrupted folder
-  await copyCorruptFile(filePath, fileName).catch((_) => {
+  const copyPath = join(publicDir.corruptDir, fileName);
+  await copyFile(filePath, copyPath).catch((_) => {
     /* while we have to catch the error, we dont need to handle it */
   });
 
@@ -140,6 +143,19 @@ async function handleCorruptedFile(filePath: string, fileName: string): Promise<
   return getFileNameFromPath(newPath);
 }
 
+async function handleMigratedFile(filePath: string, fileName: string): Promise<string> {
+  // copy file to migrated folder
+  const copyPath = join(publicDir.migrateDir, fileName);
+  await copyFile(filePath, copyPath).catch((_) => {
+    /* while we have to catch the error, we dont need to handle it */
+  });
+
+  // and make a new file with the recovered data
+  const newPath = appendToName(filePath, '(migrated)');
+  await dockerSafeRename(filePath, newPath);
+  return getFileNameFromPath(newPath);
+}
+
 /**
  * Coordinates the initial load of a project on app startup
  * This is different from the load project since we need to always load something
@@ -190,6 +206,10 @@ export async function loadProjectFile(fileName: string): Promise<string> {
     logger.warning(LogOrigin.Server, 'Project loaded with errors');
     parsedFileName = await handleCorruptedFile(filePath, fileName);
   }
+  if (result.migrated) {
+    logger.warning(LogOrigin.Server, 'The imported project is migrate, the original file has been backed up');
+    parsedFileName = await handleMigratedFile(filePath, fileName);
+  }
 
   const projectName = await loadProject(result.data, parsedFileName);
   return projectName;

apps/server/src/services/project-service/projectServiceUtils.ts

@@ -1,16 +1,11 @@
 import { DatabaseModel, MaybeString, ProjectFile } from 'ontime-types';
 
 import { existsSync } from 'fs';
-import { copyFile, readFile, stat } from 'fs/promises';
+import { readFile, stat } from 'fs/promises';
 import { extname, join } from 'path';
 
 import { publicDir } from '../../setup/index.js';
-import {
+import { dockerSafeRename, getFilesFromFolder, removeFileExtension } from '../../utils/fileManagement.js';
-  dockerSafeRename,
-  ensureDirectory,
-  getFilesFromFolder,
-  removeFileExtension,
-} from '../../utils/fileManagement.js';
 
 /**
  * Handles the upload of a new project file
@@ -23,7 +18,6 @@ export async function handleUploaded(filePath: string, name: string) {
 }
 
 export async function handleImageUpload(filePath: string, name: string): Promise<string> {
-  ensureDirectory(publicDir.logoDir);
   const newFilePath = join(publicDir.logoDir, name);
   await dockerSafeRename(filePath, newFilePath);
 
@@ -78,14 +72,6 @@ export function getPathToProject(name: string): string {
   return join(publicDir.projectsDir, name);
 }
 
-/**
- * Makes a copy of a given project to the corrupted directory
- */
-export async function copyCorruptFile(filePath: string, name: string): Promise<void> {
-  const newPath = join(publicDir.corruptDir, name);
-  return copyFile(filePath, newPath);
-}
-
 /**
  * Moves a file permanently to the corrupted directory
  */

apps/server/src/setup/config.ts

@@ -11,6 +11,7 @@ export const timerConfig = {
 export const config = {
   appState: 'app-state.json',
   corrupt: 'corrupt files',
+  migrate: 'migrated files',
   crash: 'crash logs',
   demoProject: 'demo project.json',
   newProject: 'new project.json',

apps/server/src/setup/index.ts

@@ -116,6 +116,8 @@ export const publicDir = {
   projectsDir: join(resolvePublicDirectory, config.projects),
   /** path to corrupt folder */
   corruptDir: join(resolvePublicDirectory, config.corrupt),
+  /** path to migrated folder */
+  migrateDir: join(resolvePublicDirectory, config.migrate),
   /** path to uploads folder */
   uploadsDir: join(resolvePublicDirectory, config.uploads),
   /** path to external folder */