diff --git a/apps/client/src/common/api/excel.ts b/apps/client/src/common/api/excel.ts index 38d78d50c..87706029e 100644 --- a/apps/client/src/common/api/excel.ts +++ b/apps/client/src/common/api/excel.ts @@ -1,5 +1,5 @@ import axios, { AxiosResponse } from 'axios'; -import { CustomFields, Rundown, RundownSummary } from 'ontime-types'; +import type { SpreadsheetPreviewResponse, SpreadsheetWorksheetMetadata, SpreadsheetWorksheetOptions } from 'ontime-types'; import { ImportMap } from 'ontime-utils'; import { apiEntryUrl } from './constants'; @@ -11,12 +11,12 @@ const excelPath = `${apiEntryUrl}/excel`; /** * upload Excel file to server - * @return string - file ID op the uploaded file + * Uploads an Excel file and returns worksheet names plus metadata for the initial worksheet. */ -export async function upload(file: File, requestOptions?: RequestOptions): Promise { +export async function upload(file: File, requestOptions?: RequestOptions): Promise { const formData = new FormData(); formData.append('excel', file); - const response = await axios.post(`${excelPath}/upload`, formData, { + const response: AxiosResponse = await axios.post(`${excelPath}/upload`, formData, { signal: requestOptions?.signal, timeout: requestOptions?.timeout ?? axiosConfig.longTimeout, headers: { @@ -26,16 +26,11 @@ export async function upload(file: File, requestOptions?: RequestOptions): Promi return response.data; } -type PreviewSpreadsheetResponse = { - rundown: Rundown; - customFields: CustomFields; - summary: RundownSummary; -}; export async function importRundownPreview( options: ImportMap, requestOptions?: RequestOptions, -): Promise { - const response: AxiosResponse = await axios.post( +): Promise { + const response: AxiosResponse = await axios.post( `${excelPath}/preview`, { options, @@ -48,6 +43,22 @@ export async function importRundownPreview( return response.data; } +export async function getWorksheetMetadata( + worksheet: string, + requestOptions?: RequestOptions, +): Promise { + const response: AxiosResponse = await axios.post( + `${excelPath}/metadata`, + { worksheet }, + { + signal: requestOptions?.signal, + timeout: requestOptions?.timeout ?? axiosConfig.longTimeout, + }, + ); + + return response.data; +} + /** * Downloads a xlsx representation of the rundown from the server */ diff --git a/apps/client/src/common/api/sheets.ts b/apps/client/src/common/api/sheets.ts index 9884e0b3e..1c1654a7f 100644 --- a/apps/client/src/common/api/sheets.ts +++ b/apps/client/src/common/api/sheets.ts @@ -1,5 +1,10 @@ import axios, { AxiosResponse } from 'axios'; -import { AuthenticationStatus, CustomFields, Rundown, RundownSummary } from 'ontime-types'; +import type { + AuthenticationStatus, + SpreadsheetPreviewResponse, + SpreadsheetWorksheetMetadata, + SpreadsheetWorksheetOptions, +} from 'ontime-types'; import { ImportMap } from 'ontime-utils'; import { apiEntryUrl } from './constants'; @@ -59,11 +64,7 @@ export const previewRundown = async ( sheetId: string, options: ImportMap, requestOptions?: RequestOptions, -): Promise<{ - rundown: Rundown; - customFields: CustomFields; - summary: RundownSummary; -}> => { +): Promise => { const response = await axios.post( `${sheetsPath}/${sheetId}/read`, { options }, @@ -75,11 +76,34 @@ export const previewRundown = async ( return response.data; }; -export const getWorksheetNames = async (sheetId: string, requestOptions?: RequestOptions): Promise => { - const response: AxiosResponse = await axios.post(`${sheetsPath}/${sheetId}/worksheets`, undefined, { - signal: requestOptions?.signal, - timeout: requestOptions?.timeout ?? axiosConfig.longTimeout, - }); +export const getWorksheetMetadata = async ( + sheetId: string, + worksheet: string, + requestOptions?: RequestOptions, +): Promise => { + const response: AxiosResponse = await axios.post( + `${sheetsPath}/${sheetId}/metadata`, + { worksheet }, + { + signal: requestOptions?.signal, + timeout: requestOptions?.timeout ?? axiosConfig.longTimeout, + }, + ); + return response.data; +}; + +export const getWorksheetNames = async ( + sheetId: string, + requestOptions?: RequestOptions, +): Promise => { + const response: AxiosResponse = await axios.post( + `${sheetsPath}/${sheetId}/worksheets`, + undefined, + { + signal: requestOptions?.signal, + timeout: requestOptions?.timeout ?? axiosConfig.longTimeout, + }, + ); return response.data; }; diff --git a/apps/server/src/api-data/excel/__tests__/spreadsheetMetadata.utils.test.ts b/apps/server/src/api-data/excel/__tests__/spreadsheetMetadata.utils.test.ts new file mode 100644 index 000000000..7f8c50a81 --- /dev/null +++ b/apps/server/src/api-data/excel/__tests__/spreadsheetMetadata.utils.test.ts @@ -0,0 +1,57 @@ +import { describe, expect, it } from 'vitest'; + +import { getWorksheetMetadataFromRows } from '../spreadsheetMetadata.utils.js'; + +describe('getWorksheetMetadataFromRows()', () => { + it('detects headers from the first meaningful row', () => { + const result = getWorksheetMetadataFromRows('Sheet 1', [ + [''], + ['Cue', 'Title', 'Duration'], + ['1', 'Opening', '00:05:00'], + ['2', 'Intro', '00:02:00'], + ]); + + expect(result.headers).toEqual(['Cue', 'Title', 'Duration']); + }); + + it('throws when no worksheet headers can be found', () => { + expect(() => getWorksheetMetadataFromRows('Empty', [[''], ['']])).toThrow('Could not find any data in worksheet'); + }); + + it('falls back to a single non-empty cell as the header row', () => { + const result = getWorksheetMetadataFromRows('Sheet', [['Title'], ['Opening'], ['Closing']]); + + expect(result.headers).toEqual(['Title']); + }); + + it('normalizes numeric and mixed-type cell values to strings', () => { + const result = getWorksheetMetadataFromRows('Sheet', [ + [42, true, 'Name', 'Value'], + ['a', 'b'], + ]); + + expect(result.headers).toEqual(['42', 'true', 'Name', 'Value']); + }); + + it('filters sparse columns where headers are empty', () => { + const result = getWorksheetMetadataFromRows('Sheet', [ + ['Cue', '', 'Title', '', 'Duration', 'Note'], + ['1', '', 'Opening', '', '', ''], + ['2', '', 'Closing', '', '', ''], + ]); + + expect(result.headers).toEqual(['Cue', 'Title', 'Duration', 'Note']); + }); + + it('uses progressive threshold to find row with most columns', () => { + // First row has 2 columns, second row has 6 - should pick the second row as it's more likely to be a header + const result = getWorksheetMetadataFromRows('Sheet', [ + ['', ''], + ['ID', 'Cue', 'Title', 'Start', 'Duration', 'Note'], + ['1', '1.0', 'Opening', '10:00', '5:00', 'Test'], + ['2', '2.0', 'Closing', '10:05', '3:00', 'Test2'], + ]); + + expect(result.headers).toEqual(['ID', 'Cue', 'Title', 'Start', 'Duration', 'Note']); + }); +}); diff --git a/apps/server/src/api-data/excel/excel.router.ts b/apps/server/src/api-data/excel/excel.router.ts index ffdc19e49..fd08c950b 100644 --- a/apps/server/src/api-data/excel/excel.router.ts +++ b/apps/server/src/api-data/excel/excel.router.ts @@ -1,13 +1,23 @@ import express from 'express'; import type { Request, Response } from 'express'; -import { CustomFields, ErrorResponse, Rundown, RundownSummary } from 'ontime-types'; +import type { + ErrorResponse, + SpreadsheetPreviewResponse, + SpreadsheetWorksheetMetadata, + SpreadsheetWorksheetOptions, +} from 'ontime-types'; import { getDataProvider } from '../../classes/data-provider/DataProvider.js'; import { getProjectCustomFields } from '../rundown/rundown.dao.js'; import { EXCEL_MIME } from './excel.constants.js'; import { uploadExcel } from './excel.middleware.js'; -import { generateExcelFile, generateRundownPreview, readExcelFile } from './excel.service.js'; -import { validateFileExists, validateImportMapOptions, validateRundownExport } from './excel.validation.js'; +import { generateExcelFile, generateRundownPreview, getWorksheetMetadata, readExcelFile } from './excel.service.js'; +import { + validateFileExists, + validateImportMapOptions, + validateRundownExport, + validateWorksheetMetadataRequest, +} from './excel.validation.js'; export const router = express.Router(); @@ -15,12 +25,12 @@ router.post( '/upload', uploadExcel, validateFileExists, - async (req: Request, res: Response) => { + async (req: Request, res: Response) => { try { // file has been validated by middleware const filePath = (req.file as Express.Multer.File).path; - const worksheetNames = await readExcelFile(filePath); - res.status(200).send(worksheetNames); + const worksheetOptions = await readExcelFile(filePath); + res.status(200).send(worksheetOptions); } catch (error) { res.status(500).send({ message: String(error) }); } @@ -30,10 +40,7 @@ router.post( router.post( '/preview', validateImportMapOptions, - ( - req: Request, - res: Response<{ rundown: Rundown; customFields: CustomFields; summary: RundownSummary } | ErrorResponse>, - ) => { + (req: Request, res: Response) => { try { const { options } = req.body; const data = generateRundownPreview(options); @@ -44,6 +51,20 @@ router.post( }, ); +router.post( + '/metadata', + validateWorksheetMetadataRequest, + (req: Request, res: Response) => { + try { + const { worksheet } = req.body; + const data = getWorksheetMetadata(worksheet); + res.status(200).send(data); + } catch (error) { + res.status(500).send({ message: String(error) }); + } + }, +); + router.get('/:rundownId/export', validateRundownExport, (req: Request, res: Response) => { try { const rundown = getDataProvider().getRundown(req.params.rundownId); diff --git a/apps/server/src/api-data/excel/excel.service.ts b/apps/server/src/api-data/excel/excel.service.ts index 4333c738e..517073ca0 100644 --- a/apps/server/src/api-data/excel/excel.service.ts +++ b/apps/server/src/api-data/excel/excel.service.ts @@ -16,6 +16,9 @@ import { parseCustomFields } from '../custom-fields/customFields.parser.js'; import { getProjectCustomFields, processRundown } from '../rundown/rundown.dao.js'; import { parseRundown } from '../rundown/rundown.parser.js'; import { parseExcel } from './excel.parser.js'; +import type { SpreadsheetWorksheetMetadata } from 'ontime-types'; + +import { getWorksheetMetadataFromRows } from './spreadsheetMetadata.utils.js'; import { rundownToTabular } from './excel.utils.js'; // we keep the excel data in memory to allow the flow upload -> preview @@ -33,11 +36,25 @@ function getValidWorksheetName(title: string): string { return truncatedTitle.length > 0 ? truncatedTitle : 'Rundown'; } +function getInitialWorksheetMetadata(worksheets: string[]): SpreadsheetWorksheetMetadata | null { + for (const worksheet of worksheets) { + try { + return getWorksheetMetadata(worksheet); + } catch { + // Continue looking for the first worksheet with usable headers. + } + } + + return null; +} + /** * Receives and parses an excel file * The file is deleted after being read */ -export async function readExcelFile(filePath: string): Promise { +export async function readExcelFile( + filePath: string, +): Promise<{ worksheets: string[]; metadata: SpreadsheetWorksheetMetadata | null }> { if (!existsSync(filePath)) { throw new Error('Upload of excel file failed'); } @@ -50,7 +67,13 @@ export async function readExcelFile(filePath: string): Promise { await deleteFile(filePath); - return excelData.SheetNames; + const worksheets = excelData.SheetNames; + const metadata = getInitialWorksheetMetadata(worksheets); + + return { + worksheets, + metadata, + }; } export function generateRundownPreview(options: ImportMap): { @@ -94,6 +117,17 @@ export function generateRundownPreview(options: ImportMap): { }; } +export function getWorksheetMetadata(worksheet: string) { + const data = excelData.Sheets[worksheet]; + + if (!data) { + throw new Error(`Could not find worksheet: ${worksheet}`); + } + + const arrayOfData: unknown[][] = xlsx.utils.sheet_to_json(data, { header: 1, blankrows: false, raw: false }); + return getWorksheetMetadataFromRows(worksheet, arrayOfData); +} + /** * Creates an xlsx file from a given rundown and custom fields * @throws if the rundown is empty diff --git a/apps/server/src/api-data/excel/excel.validation.ts b/apps/server/src/api-data/excel/excel.validation.ts index fbed298d1..6fd65e976 100644 --- a/apps/server/src/api-data/excel/excel.validation.ts +++ b/apps/server/src/api-data/excel/excel.validation.ts @@ -18,4 +18,9 @@ export const validateImportMapOptions = [ requestValidationFunction, ]; +export const validateWorksheetMetadataRequest = [ + body('worksheet').isString().trim().notEmpty(), + requestValidationFunction, +]; + export const validateRundownExport = [param('rundownId').isString().trim().notEmpty(), requestValidationFunction]; diff --git a/apps/server/src/api-data/excel/spreadsheetMetadata.utils.ts b/apps/server/src/api-data/excel/spreadsheetMetadata.utils.ts new file mode 100644 index 000000000..3c8b0cccc --- /dev/null +++ b/apps/server/src/api-data/excel/spreadsheetMetadata.utils.ts @@ -0,0 +1,75 @@ +import type { SpreadsheetWorksheetMetadata } from 'ontime-types'; + +function normalizeCell(value: unknown): string { + if (value === null || value === undefined) { + return ''; + } + + if (typeof value === 'string') { + return value.trim(); + } + + return String(value).trim(); +} + +function countNonEmpty(row: string[]): number { + return row.filter((cell) => cell.length > 0).length; +} + +/** + * Finds the index of the header row in a spreadsheet. + * Prioritizes rows with >5 non-empty columns (early exit), otherwise returns the row with the most columns. + * This heuristic works because header rows typically have more columns than data rows. + * @param rows - Array of spreadsheet rows, each containing normalized cell values + * @returns Index of the header row, or -1 if no data is found + */ +function findHeaderRowIndex(rows: string[][]): number { + const rowCounts = rows.map(countNonEmpty); + + let bestIndex = -1; + let bestCount = 0; + + for (let index = 0; index < rowCounts.length; index++) { + const count = rowCounts[index]; + + // Early exit if we find a row with >5 columns (likely a header) + if (count > 5) { + return index; + } + + // Otherwise track the row with the most columns + if (count > bestCount) { + bestIndex = index; + bestCount = count; + } + } + + return bestIndex; +} + +/** + * Extracts metadata from a spreadsheet worksheet including detected headers. + * Normalizes cell values and identifies the header row. + * @throws Error if no data is found or headers cannot be detected + */ +export function getWorksheetMetadataFromRows(worksheet: string, sheetRows: unknown[][]): SpreadsheetWorksheetMetadata { + const rows = sheetRows.map((row) => row.map(normalizeCell)); + const headerRowIndex = findHeaderRowIndex(rows); + + if (headerRowIndex === -1) { + throw new Error(`Could not find any data in worksheet: ${worksheet}`); + } + + const headerEntries = rows[headerRowIndex] + .map((header, index) => ({ header, index })) + .filter(({ header }) => header.length > 0); + + if (headerEntries.length === 0) { + throw new Error(`Could not detect worksheet headers in: ${worksheet}`); + } + + return { + worksheet, + headers: headerEntries.map(({ header }) => header), + }; +} diff --git a/apps/server/src/api-data/sheets/sheets.controller.ts b/apps/server/src/api-data/sheets/sheets.controller.ts index e4cfcc27f..253bc053e 100644 --- a/apps/server/src/api-data/sheets/sheets.controller.ts +++ b/apps/server/src/api-data/sheets/sheets.controller.ts @@ -6,12 +6,19 @@ import { readFileSync } from 'fs'; import { Request, Response } from 'express'; -import type { AuthenticationStatus, CustomFields, ErrorResponse, Rundown, RundownSummary } from 'ontime-types'; +import type { + AuthenticationStatus, + ErrorResponse, + SpreadsheetPreviewResponse, + SpreadsheetWorksheetMetadata, + SpreadsheetWorksheetOptions, +} from 'ontime-types'; import { getErrorMessage } from 'ontime-utils'; import { deleteFile } from '../../utils/fileManagement.js'; import { download, + getWorksheetMetadata, getWorksheetOptions, handleClientSecret, handleInitialConnection, @@ -69,10 +76,13 @@ export async function revokeAuthentication( } } -export async function getWorksheetNamesFromSheet(req: Request, res: Response) { +export async function getWorksheetNamesFromSheet( + req: Request, + res: Response, +) { try { const { sheetId } = req.params; - const { worksheetOptions } = await getWorksheetOptions(sheetId); + const worksheetOptions = await getWorksheetOptions(sheetId); res.status(200).send(worksheetOptions); } catch (error) { const message = getErrorMessage(error); @@ -80,16 +90,21 @@ export async function getWorksheetNamesFromSheet(req: Request, res: Response) { + try { + const { sheetId } = req.params; + const { worksheet } = req.body; + const metadata = await getWorksheetMetadata(sheetId, worksheet); + res.status(200).send(metadata); + } catch (error) { + const message = getErrorMessage(error); + res.status(500).send({ message }); + } +} + export async function readFromSheet( req: Request, - res: Response< - | { - rundown: Rundown; - customFields: CustomFields; - summary: RundownSummary; - } - | ErrorResponse - >, + res: Response, ) { try { const { sheetId } = req.params; diff --git a/apps/server/src/api-data/sheets/sheets.router.ts b/apps/server/src/api-data/sheets/sheets.router.ts index ce310580b..5ffea6593 100644 --- a/apps/server/src/api-data/sheets/sheets.router.ts +++ b/apps/server/src/api-data/sheets/sheets.router.ts @@ -5,6 +5,7 @@ import express from 'express'; import { + getWorksheetMetadataFromSheet, getWorksheetNamesFromSheet, readFromSheet, requestConnection, @@ -13,7 +14,12 @@ import { writeToSheet, } from './sheets.controller.js'; import { uploadClientSecret } from './sheets.middleware.js'; -import { validateRequestConnection, validateSheetId, validateSheetOptions } from './sheets.validation.js'; +import { + validateRequestConnection, + validateSheetId, + validateSheetOptions, + validateWorksheetMetadata, +} from './sheets.validation.js'; export const router = express.Router(); @@ -23,6 +29,7 @@ router.post('/:sheetId/connect', uploadClientSecret, validateRequestConnection, router.post('/revoke', revokeAuthentication); router.post('/:sheetId/worksheets', validateSheetId, getWorksheetNamesFromSheet); +router.post('/:sheetId/metadata', validateWorksheetMetadata, getWorksheetMetadataFromSheet); router.post('/:sheetId/read', validateSheetOptions, readFromSheet); router.post('/:sheetId/write', validateSheetOptions, writeToSheet); diff --git a/apps/server/src/api-data/sheets/sheets.service.ts b/apps/server/src/api-data/sheets/sheets.service.ts index ca59a2f7d..8585df504 100644 --- a/apps/server/src/api-data/sheets/sheets.service.ts +++ b/apps/server/src/api-data/sheets/sheets.service.ts @@ -26,6 +26,9 @@ import { logger } from '../../classes/Logger.js'; import { consoleSubdued } from '../../utils/console.js'; import { parseCustomFields } from '../custom-fields/customFields.parser.js'; import { parseExcel } from '../excel/excel.parser.js'; +import type { SpreadsheetWorksheetMetadata } from 'ontime-types'; + +import { getWorksheetMetadataFromRows } from '../excel/spreadsheetMetadata.utils.js'; import { getCurrentRundown, getProjectCustomFields, processRundown } from '../rundown/rundown.dao.js'; import { parseRundowns } from '../rundown/rundown.parser.js'; import { catchCommonImportXlsxError } from './googleApi.utils.js'; @@ -224,7 +227,7 @@ export function hasAuth(): { authenticated: AuthenticationStatus; sheetId: strin async function verifySheet( sheetId = currentSheetId, authClient = currentAuthClient, -): Promise<{ worksheetOptions: string[] }> { +): Promise { if (!sheetId || !authClient) { throw new Error('Missing sheet ID or authentication'); } @@ -245,7 +248,7 @@ async function verifySheet( if (worksheets.length === 0) { throw new Error('No worksheets found'); } - return { worksheetOptions: worksheets }; + return worksheets; } catch (error) { // attempt to catch errors caused by importing xlsx catchCommonImportXlsxError(error); @@ -284,13 +287,61 @@ export async function handleInitialConnection( * Allow calling verification for sheetId * @returns */ -export async function getWorksheetOptions(sheetId: string): ReturnType { +export async function getWorksheetOptions( + sheetId: string, +): Promise<{ worksheets: string[]; metadata: SpreadsheetWorksheetMetadata | null }> { if (!currentAuthClient) { throw new Error('Not authenticated'); } currentSheetId = sheetId; - return verifySheet(sheetId); + const worksheets = await verifySheet(sheetId); + const metadata = await getInitialWorksheetMetadata(sheetId, worksheets); + + return { + worksheets, + metadata, + }; +} + +async function getInitialWorksheetMetadata( + sheetId: string, + worksheets: string[], +): Promise { + for (const worksheet of worksheets) { + try { + return await getWorksheetMetadata(sheetId, worksheet); + } catch { + // Continue looking for the first worksheet with usable headers. + } + } + + return null; +} + +export async function getWorksheetMetadata(sheetId: string, worksheet: string) { + if (!currentAuthClient) { + throw new Error('Not authenticated'); + } + + const { range } = await verifyWorksheet(sheetId, worksheet); + + const googleResponse = await sheets({ version: 'v4', auth: currentAuthClient }).spreadsheets.values.get({ + spreadsheetId: sheetId, + valueRenderOption: 'FORMATTED_VALUE', + majorDimension: 'ROWS', + range, + }); + + if (googleResponse.status !== 200) { + throw new Error(`Sheet read failed: ${googleResponse.statusText}`); + } + + if (!googleResponse.data.values) { + throw new Error('Sheet: No data found in the worksheet'); + } + + return getWorksheetMetadataFromRows(worksheet, googleResponse.data.values); } async function verifyWorksheet(sheetId: string, worksheet: string): Promise<{ worksheetId: number; range: string }> { diff --git a/apps/server/src/api-data/sheets/sheets.validation.ts b/apps/server/src/api-data/sheets/sheets.validation.ts index 62aea3526..5148b7163 100644 --- a/apps/server/src/api-data/sheets/sheets.validation.ts +++ b/apps/server/src/api-data/sheets/sheets.validation.ts @@ -30,6 +30,12 @@ export const validateRequestConnection = [ export const validateSheetId = [param('sheetId').isString().trim().notEmpty(), requestValidationFunction]; +export const validateWorksheetMetadata = [ + param('sheetId').isString().trim().notEmpty(), + body('worksheet').isString().trim().notEmpty(), + requestValidationFunction, +]; + export const validateSheetOptions = [ param('sheetId').isString().trim().notEmpty(), body('options')