From f5e9a2549e0dd0c6037f556c706c5f250310ed9d Mon Sep 17 00:00:00 2001 From: Alex Christoffer Rasmussen Date: Fri, 22 Mar 2024 21:33:11 +0100 Subject: [PATCH] Improve spreadsheet import flow (#839) * feat: add dropdown to worksheet selection * feat: move revoke to separate button * refactor: extract excel flow to separate controller and service * chore: show spinner when uploading sheet and disable the other option * chore: don't show "success" when canceling the import * chore: export boolean types properly to sheets --- apps/client/src/common/api/db.ts | 29 ------ apps/client/src/common/api/excel.ts | 42 ++++++++ apps/client/src/common/api/sheets.ts | 12 ++- apps/client/src/common/utils/uploadUtils.ts | 4 +- .../panel/sources-panel/GSheetSetup.tsx | 7 +- .../panel/sources-panel/ImportReview.tsx | 5 +- .../panel/sources-panel/SourcesPanel.tsx | 84 +++++++++++----- .../import-map/ImportMapForm.tsx | 43 +++++++- .../panel/sources-panel/useGoogleSheet.ts | 2 +- .../panel/sources-panel/useSheetStore.ts | 11 +-- apps/server/src/api-data/db/db.controller.ts | 26 ----- apps/server/src/api-data/db/db.middleware.ts | 15 +-- apps/server/src/api-data/db/db.router.ts | 14 +-- .../src/api-data/excel/excel.controller.ts | 40 ++++++++ .../src/api-data/excel/excel.middleware.ts | 17 ++++ .../server/src/api-data/excel/excel.router.ts | 16 +++ .../src/api-data/excel/excel.service.ts | 53 ++++++++++ .../src/api-data/excel/excel.validation.ts | 28 ++++++ apps/server/src/api-data/index.ts | 2 + .../src/api-data/sheets/sheets.controller.ts | 11 +++ .../src/api-data/sheets/sheets.router.ts | 5 +- .../src/api-data/sheets/sheets.validation.ts | 10 ++ .../services/sheet-service/SheetService.ts | 8 +- .../__tests__/sheetUtils.test.ts | 6 +- .../src/services/sheet-service/sheetUtils.ts | 4 +- .../server/src/utils/__tests__/parser.test.ts | 84 ++++++++++++---- apps/server/src/utils/__tests__/time.test.ts | 2 +- apps/server/src/utils/parser.ts | 93 +++++------------- apps/server/src/utils/time.ts | 32 ++++-- .../features/207-spreadsheet-import.spec.ts | 37 +++++++ e2e/tests/fixtures/test-sheet.xlsx | Bin 0 -> 6621 bytes .../utils/src/date-utils/isTimeString.test.ts | 9 ++ packages/utils/src/date-utils/isTimeString.ts | 12 +-- 33 files changed, 528 insertions(+), 235 deletions(-) create mode 100644 apps/client/src/common/api/excel.ts create mode 100644 apps/server/src/api-data/excel/excel.controller.ts create mode 100644 apps/server/src/api-data/excel/excel.middleware.ts create mode 100644 apps/server/src/api-data/excel/excel.router.ts create mode 100644 apps/server/src/api-data/excel/excel.service.ts create mode 100644 apps/server/src/api-data/excel/excel.validation.ts create mode 100644 e2e/tests/features/207-spreadsheet-import.spec.ts create mode 100644 e2e/tests/fixtures/test-sheet.xlsx diff --git a/apps/client/src/common/api/db.ts b/apps/client/src/common/api/db.ts index 2f6a79615..2ea617707 100644 --- a/apps/client/src/common/api/db.ts +++ b/apps/client/src/common/api/db.ts @@ -1,14 +1,11 @@ import axios, { AxiosResponse } from 'axios'; import { - CustomFields, DatabaseModel, GetInfo, MessageResponse, - OntimeRundown, ProjectData, ProjectFileListResponse, } from 'ontime-types'; -import { ImportMap } from 'ontime-utils'; import { makeCSV, makeTable } from '../../features/cuesheet/cuesheetUtils'; @@ -155,32 +152,6 @@ export async function getInfo(): Promise { return res.data; } -type PreviewSpreadsheetResponse = { - rundown: OntimeRundown; - customFields: CustomFields; -}; - -/** - * Make patch changes to the objects in the db - */ -export async function importSpreadsheetPreview(file: File, options: ImportMap): Promise { - const formData = new FormData(); - formData.append('spreadsheet', file); - formData.append('options', JSON.stringify(options)); - - const response: AxiosResponse = await axios.post( - `${dbPath}/spreadsheet/preview`, - formData, - { - headers: { - 'Content-Type': 'multipart/form-data', - }, - }, - ); - - return response.data; -} - /** * Utility function gets project from db * @param fileName diff --git a/apps/client/src/common/api/excel.ts b/apps/client/src/common/api/excel.ts new file mode 100644 index 000000000..b73aa585f --- /dev/null +++ b/apps/client/src/common/api/excel.ts @@ -0,0 +1,42 @@ +import axios, { AxiosResponse } from 'axios'; +import { CustomFields, OntimeRundown } from 'ontime-types'; +import { ImportMap } from 'ontime-utils'; + +import { apiEntryUrl } from './constants'; + +const excelPath = `${apiEntryUrl}/excel`; + +type PreviewSpreadsheetResponse = { + rundown: OntimeRundown; + customFields: CustomFields; +}; + +/** + * upload Excel file to server + * @return string - file ID op the uploaded file + */ +export async function upload(file: File) { + const formData = new FormData(); + formData.append('excel', file); + await axios.post(`${excelPath}/upload`, formData, { + headers: { + 'Content-Type': 'multipart/form-data', + }, + }); +} + +/** + * Get Worksheet names + * @return string[] - array of available worksheets + */ +export async function getWorksheetNames(): Promise { + const response: AxiosResponse = await axios.get(`${excelPath}/worksheets`); + return response.data; +} + +export async function importRundownPreview(options: ImportMap): Promise { + const response: AxiosResponse = await axios.post(`${excelPath}/preview`, { + options, + }); + return response.data; +} diff --git a/apps/client/src/common/api/sheets.ts b/apps/client/src/common/api/sheets.ts index 4c818b7dd..ebeb9fdce 100644 --- a/apps/client/src/common/api/sheets.ts +++ b/apps/client/src/common/api/sheets.ts @@ -1,4 +1,4 @@ -import axios from 'axios'; +import axios, { AxiosResponse } from 'axios'; import { AuthenticationStatus, CustomFields, OntimeRundown } from 'ontime-types'; import { ImportMap } from 'ontime-utils'; @@ -9,7 +9,10 @@ const sheetsPath = `${apiEntryUrl}/sheets`; /** * HTTP request to verify whether we are authenticated with Google Sheet service */ -export const verifyAuthenticationStatus = async (): Promise<{ authenticated: AuthenticationStatus }> => { +export const verifyAuthenticationStatus = async (): Promise<{ + authenticated: AuthenticationStatus; + sheetId: string; +}> => { const response = await axios.get(`${sheetsPath}/connect`); return response.data; }; @@ -58,6 +61,11 @@ export const previewRundown = async ( return response.data; }; +export const getWorksheetNames = async (sheetId: string): Promise => { + const response: AxiosResponse = await axios.post(`${sheetsPath}/${sheetId}/worksheets`); + return response.data; +}; + /** * HTTP request to upload the rundown to a google sheet */ diff --git a/apps/client/src/common/utils/uploadUtils.ts b/apps/client/src/common/utils/uploadUtils.ts index e76a98fc6..83098e876 100644 --- a/apps/client/src/common/utils/uploadUtils.ts +++ b/apps/client/src/common/utils/uploadUtils.ts @@ -2,7 +2,7 @@ * Collection of rules for pre-validating a spreadsheet * @param file */ -export function validateSpreadsheetImport(file: File) { +export function validateExcelImport(file: File) { if (!isExcelFile(file)) { throw new Error('Unknown file type'); } @@ -12,7 +12,7 @@ export function validateSpreadsheetImport(file: File) { throw new Error('File is empty'); } - // Limit file size of an excel file to around 10MB + // Limit file size of an Excel file to around 10MB if (file.size > 10_000_000) { throw new Error('File size limit (10MB) exceeded'); } diff --git a/apps/client/src/features/app-settings/panel/sources-panel/GSheetSetup.tsx b/apps/client/src/features/app-settings/panel/sources-panel/GSheetSetup.tsx index 7924eb248..66efaef34 100644 --- a/apps/client/src/features/app-settings/panel/sources-panel/GSheetSetup.tsx +++ b/apps/client/src/features/app-settings/panel/sources-panel/GSheetSetup.tsx @@ -3,6 +3,7 @@ import { Button, Input } from '@chakra-ui/react'; import { IoCheckmark } from '@react-icons/all-files/io5/IoCheckmark'; import { IoShieldCheckmarkOutline } from '@react-icons/all-files/io5/IoShieldCheckmarkOutline'; +import { getWorksheetNames } from '../../../../common/api/sheets'; import CopyTag from '../../../../common/components/copy-tag/CopyTag'; import { openLink } from '../../../../common/utils/linkUtils'; import * as Panel from '../PanelUtils'; @@ -27,6 +28,7 @@ export default function GSheetSetup(props: GSheetSetupProps) { const sheetId = useSheetStore((state) => state.sheetId); const setSheetId = useSheetStore((state) => state.setSheetId); + const setWorksheets = useSheetStore((state) => state.setWorksheets); const authenticationStatus = useSheetStore((state) => state.authenticationStatus); const setAuthenticationStatus = useSheetStore((state) => state.setAuthenticationStatus); @@ -53,7 +55,6 @@ export default function GSheetSetup(props: GSheetSetupProps) { }; const handleCancelFlow = async () => { - await handleRevoke(); onCancel(); }; @@ -89,6 +90,10 @@ export default function GSheetSetup(props: GSheetSetupProps) { if (result?.authenticated) { setAuthenticationStatus(result.authenticated); if (result.authenticated !== 'pending') { + if (result.authenticated == 'authenticated') { + const names = await getWorksheetNames(result.sheetId); + setWorksheets(names); + } setLoading(''); return; } diff --git a/apps/client/src/features/app-settings/panel/sources-panel/ImportReview.tsx b/apps/client/src/features/app-settings/panel/sources-panel/ImportReview.tsx index 53ba7c586..71a426c11 100644 --- a/apps/client/src/features/app-settings/panel/sources-panel/ImportReview.tsx +++ b/apps/client/src/features/app-settings/panel/sources-panel/ImportReview.tsx @@ -14,10 +14,11 @@ interface ImportReviewProps { rundown: OntimeRundown; customFields: CustomFields; onFinished: () => void; + onCancel: () => void; } export default function ImportReview(props: ImportReviewProps) { - const { rundown, customFields, onFinished } = props; + const { rundown, customFields, onFinished, onCancel } = props; const [loading, setLoading] = useState(false); const { importRundown } = useGoogleSheet(); @@ -25,7 +26,7 @@ export default function ImportReview(props: ImportReviewProps) { const handleCancel = () => { resetPreview(); - onFinished(); + onCancel(); }; const applyImport = async () => { diff --git a/apps/client/src/features/app-settings/panel/sources-panel/SourcesPanel.tsx b/apps/client/src/features/app-settings/panel/sources-panel/SourcesPanel.tsx index 63fd91a1d..7bb896df4 100644 --- a/apps/client/src/features/app-settings/panel/sources-panel/SourcesPanel.tsx +++ b/apps/client/src/features/app-settings/panel/sources-panel/SourcesPanel.tsx @@ -4,9 +4,14 @@ import { IoCloudOutline } from '@react-icons/all-files/io5/IoCloudOutline'; import { IoDownloadOutline } from '@react-icons/all-files/io5/IoDownloadOutline'; import { ImportMap, unpackError } from 'ontime-utils'; -import { importSpreadsheetPreview } from '../../../../common/api/db'; +import { + getWorksheetNames as getWorksheetNamesExcel, + importRundownPreview as importRundownPreviewExcel, + upload as uploadExcel, +} from '../../../../common/api/excel'; +import { getWorksheetNames } from '../../../../common/api/sheets'; import { maybeAxiosError } from '../../../../common/api/utils'; -import { validateSpreadsheetImport } from '../../../../common/utils/uploadUtils'; +import { validateExcelImport } from '../../../../common/utils/uploadUtils'; import * as Panel from '../PanelUtils'; import ImportMapForm from './import-map/ImportMapForm'; @@ -21,36 +26,43 @@ import style from './SourcesPanel.module.scss'; export default function SourcesPanel() { const [importFlow, setImportFlow] = useState<'none' | 'excel' | 'gsheet' | 'finished'>('none'); const [error, setError] = useState(''); + const [hasFile, setHasFile] = useState<'none' | 'loading' | 'done'>('none'); - const { exportRundown, importRundownPreview, revoke, verifyAuth } = useGoogleSheet(); + const { exportRundown, importRundownPreview, verifyAuth } = useGoogleSheet(); - const spreadsheet = useSheetStore((state) => state.spreadsheet); - const setSpreadsheet = useSheetStore((state) => state.setSpreadsheet); + const setWorksheets = useSheetStore((state) => state.setWorksheets); const authenticationStatus = useSheetStore((state) => state.authenticationStatus); const setAuthenticationStatus = useSheetStore((state) => state.setAuthenticationStatus); const rundown = useSheetStore((state) => state.rundown); const setRundown = useSheetStore((state) => state.setRundown); const customFields = useSheetStore((state) => state.customFields); const setCustomFields = useSheetStore((state) => state.setCustomFields); + const setSheetId = useSheetStore((state) => state.setSheetId); const sheetId = useSheetStore((state) => state.sheetId); const fileInputRef = useRef(null); - const handleFile = (event: ChangeEvent) => { + const handleFile = async (event: ChangeEvent) => { const fileToUpload = event.target.files?.[0]; if (!fileToUpload) { - setSpreadsheet(null); + setWorksheets(null); + setHasFile('none'); return; } try { - validateSpreadsheetImport(fileToUpload); - setSpreadsheet(fileToUpload); + setHasFile('loading'); + validateExcelImport(fileToUpload); + await uploadExcel(fileToUpload); + const names = await getWorksheetNamesExcel(); + setWorksheets(names); setImportFlow('excel'); + setHasFile('done'); } catch (error) { const errorMessage = unpackError(error); setError(`Error uploading file: ${errorMessage}`); - setSpreadsheet(null); + setWorksheets(null); + setHasFile('none'); } }; @@ -58,7 +70,16 @@ export default function SourcesPanel() { fileInputRef.current?.click(); }; - const openGSheetFlow = () => { + const openGSheetFlow = async () => { + const result = await verifyAuth(); + if (result) { + setAuthenticationStatus(result.authenticated); + setSheetId(result.sheetId); + if (result.authenticated === 'authenticated' && result.sheetId) { + const names = await getWorksheetNames(result.sheetId); + setWorksheets(names); + } + } setImportFlow('gsheet'); }; @@ -68,9 +89,8 @@ export default function SourcesPanel() { const handleSubmitImportPreview = async (importMap: ImportMap) => { if (importFlow === 'excel') { - if (!spreadsheet) return; try { - const previewData = await importSpreadsheetPreview(spreadsheet, importMap); + const previewData = await importRundownPreviewExcel(importMap); setRundown(previewData.rundown); setCustomFields(previewData.customFields); } catch (error) { @@ -86,12 +106,9 @@ export default function SourcesPanel() { const cancelImportMap = async () => { setImportFlow('none'); - if (spreadsheet) { - setSpreadsheet(null); - } - + setHasFile('none'); + setWorksheets(null); if (authenticationStatus === 'authenticated') { - await revoke(); const result = await verifyAuth(); if (result) { setAuthenticationStatus(result.authenticated); @@ -102,7 +119,8 @@ export default function SourcesPanel() { const handleFinished = () => { setImportFlow('finished'); setRundown(null); - setSpreadsheet(null); + setHasFile('none'); + setWorksheets(null); setCustomFields(null); }; @@ -113,12 +131,11 @@ export default function SourcesPanel() { const isExcelFlow = importFlow === 'excel'; const isGSheetFlow = importFlow === 'gsheet'; - const hasFile = Boolean(spreadsheet); const isAuthenticated = authenticationStatus === 'authenticated'; const showInput = importFlow === 'none'; const showSuccess = importFlow === 'finished'; const showAuth = isGSheetFlow && !isAuthenticated; - const showImportMap = (isGSheetFlow && isAuthenticated) || (isExcelFlow && hasFile); + const showImportMap = (isGSheetFlow && isAuthenticated) || (isExcelFlow && hasFile === 'done'); const showReview = rundown !== null && customFields !== null; return ( @@ -141,13 +158,25 @@ export default function SourcesPanel() { />
- Accepts .xlsx files
- Start authentication process @@ -172,7 +201,14 @@ export default function SourcesPanel() { onSubmitImport={handleSubmitImportPreview} /> )} - {showReview && } + {showReview && ( + + )} diff --git a/apps/client/src/features/app-settings/panel/sources-panel/import-map/ImportMapForm.tsx b/apps/client/src/features/app-settings/panel/sources-panel/import-map/ImportMapForm.tsx index c0fd7265a..5b3e1ccb2 100644 --- a/apps/client/src/features/app-settings/panel/sources-panel/import-map/ImportMapForm.tsx +++ b/apps/client/src/features/app-settings/panel/sources-panel/import-map/ImportMapForm.tsx @@ -1,12 +1,13 @@ import { useState } from 'react'; import { useFieldArray, useForm } from 'react-hook-form'; -import { Button, IconButton, Input } from '@chakra-ui/react'; +import { Button, IconButton, Input, Select, Tooltip } from '@chakra-ui/react'; import { IoAdd } from '@react-icons/all-files/io5/IoAdd'; import { IoTrash } from '@react-icons/all-files/io5/IoTrash'; import { ImportMap } from 'ontime-utils'; import { isAlphanumeric } from '../../../../../common/utils/regex'; import * as Panel from '../../PanelUtils'; +import useGoogleSheet from '../useGoogleSheet'; import { useSheetStore } from '../useSheetStore'; import { convertToImportMap, getPersistedOptions, NamedImportMap, persistImportMap } from './importMapUtils'; @@ -23,7 +24,7 @@ interface ImportMapFormProps { export default function ImportMapForm(props: ImportMapFormProps) { const { isSpreadsheet, onCancel, onSubmitExport, onSubmitImport } = props; const namedImportMap = getPersistedOptions(); - + const { revoke } = useGoogleSheet(); const { control, handleSubmit, @@ -41,6 +42,7 @@ export default function ImportMapForm(props: ImportMapFormProps) { }); const stepData = useSheetStore((state) => state.stepData); + const worksheetNames = useSheetStore((state) => state.worksheetNames); const [loading, setLoading] = useState<'' | 'export' | 'import'>(''); @@ -52,6 +54,11 @@ export default function ImportMapForm(props: ImportMapFormProps) { setLoading(''); }; + const handleRevoke = async () => { + await revoke(); + onCancel(); + }; + const handleImportPreview = async (values: NamedImportMap) => { setLoading('import'); const importMap = convertToImportMap(values); @@ -78,6 +85,13 @@ export default function ImportMapForm(props: ImportMapFormProps) { Import options
+ {!isSpreadsheet && ( + + + + )} @@ -116,6 +130,31 @@ export default function ImportMapForm(props: ImportMapFormProps) { if (label === 'custom') { return null; } + if (label === 'Worksheet') { + return ( + + {label} + + + + + + ); + } return ( {label} diff --git a/apps/client/src/features/app-settings/panel/sources-panel/useGoogleSheet.ts b/apps/client/src/features/app-settings/panel/sources-panel/useGoogleSheet.ts index f3d19f13d..b5f4c23f6 100644 --- a/apps/client/src/features/app-settings/panel/sources-panel/useGoogleSheet.ts +++ b/apps/client/src/features/app-settings/panel/sources-panel/useGoogleSheet.ts @@ -23,7 +23,7 @@ export default function useGoogleSheet() { const setCustomFields = useSheetStore((state) => state.setCustomFields); /** whether the current session has been authenticated */ - const verifyAuth = async (): Promise<{ authenticated: AuthenticationStatus } | void> => { + const verifyAuth = async (): Promise<{ authenticated: AuthenticationStatus; sheetId: string } | void> => { try { return verifyAuthenticationStatus(); } catch (_error) { diff --git a/apps/client/src/features/app-settings/panel/sources-panel/useSheetStore.ts b/apps/client/src/features/app-settings/panel/sources-panel/useSheetStore.ts index 7f8b0e1f4..efed9ce86 100644 --- a/apps/client/src/features/app-settings/panel/sources-panel/useSheetStore.ts +++ b/apps/client/src/features/app-settings/panel/sources-panel/useSheetStore.ts @@ -5,13 +5,12 @@ import { create } from 'zustand'; type SheetStore = { stepData: typeof initialStepData; patchStepData: (patch: Partial) => void; + setWorksheets: (worksheetNames: string[] | null) => void; + worksheetNames: string[] | null; - spreadsheet: File | null; - setSpreadsheet: (spreadsheet: File | null) => void; - + //gSheet sheetId: string | null; setSheetId: (sheetId: string | null) => void; - authenticationStatus: AuthenticationStatus; setAuthenticationStatus: (status: AuthenticationStatus) => void; @@ -39,7 +38,7 @@ const initialStepData = { const initialState = { stepData: initialStepData, - spreadsheet: null, + worksheetNames: null, sheetId: null, authenticationStatus: 'not_authenticated' as AuthenticationStatus, rundown: null, @@ -55,7 +54,7 @@ export const useSheetStore = create((set, get) => ({ set({ stepData: { ...stepData, ...patch } }); }, - setSpreadsheet: (spreadsheet: File | null) => set({ spreadsheet }), + setWorksheets: (worksheetNames: string[] | null) => set({ worksheetNames }), setSheetId: (sheetId: string | null) => set({ sheetId }), diff --git a/apps/server/src/api-data/db/db.controller.ts b/apps/server/src/api-data/db/db.controller.ts index 496cef013..8cd81c3ff 100644 --- a/apps/server/src/api-data/db/db.controller.ts +++ b/apps/server/src/api-data/db/db.controller.ts @@ -8,7 +8,6 @@ import { } from 'ontime-types'; import type { Request, Response } from 'express'; -import fs from 'fs'; import { failEmptyObjects } from '../../utils/routerUtils.js'; import { resolveDbPath, resolveProjectsDirectory } from '../../setup/index.js'; @@ -17,7 +16,6 @@ import * as projectService from '../../services/project-service/ProjectService.j import { ensureJsonExtension } from '../../utils/fileManagement.js'; import { generateUniqueFileName } from '../../utils/generateUniqueFilename.js'; import { appStateService } from '../../services/app-state-service/AppStateService.js'; -import { handleMaybeExcel } from '../../utils/parser.js'; export async function patchPartialProjectFile(req: Request, res: Response) { // all fields are optional in validation @@ -245,27 +243,3 @@ export async function getInfo(_req: Request, res: Response) { const info = await projectService.getInfo(); res.status(200).send(info); } - -/** - * uploads and parses an excel spreadsheet - * @returns parsed result - */ -export async function previewSpreadsheet(req: Request, res: Response) { - if (!req.file) { - res.status(400).send({ message: 'File not found' }); - return; - } - - try { - const filePath = req.file.path; - if (!fs.existsSync(filePath)) { - throw new Error('Upload failed'); - } - - const options = JSON.parse(req.body.options); - const { data } = handleMaybeExcel(filePath, options); - res.status(200).send(data); - } catch (error) { - res.status(500).send({ message: String(error) }); - } -} diff --git a/apps/server/src/api-data/db/db.middleware.ts b/apps/server/src/api-data/db/db.middleware.ts index a21f7d20e..6caef9070 100644 --- a/apps/server/src/api-data/db/db.middleware.ts +++ b/apps/server/src/api-data/db/db.middleware.ts @@ -1,7 +1,7 @@ import { Request } from 'express'; import multer, { FileFilterCallback } from 'multer'; -import { EXCEL_MIME, JSON_MIME } from '../../utils/parser.js'; +import { JSON_MIME } from '../../utils/parser.js'; import { storage } from '../../utils/upload.js'; const filterProjectFile = (_req: Request, file: Express.Multer.File, cb: FileFilterCallback) => { @@ -12,21 +12,8 @@ const filterProjectFile = (_req: Request, file: Express.Multer.File, cb: FileFil } }; -const filterSpreadsheet = (_req: Request, file: Express.Multer.File, cb: FileFilterCallback) => { - if (file.mimetype.includes(EXCEL_MIME)) { - cb(null, true); - } else { - cb(null, false); - } -}; - // Build multer uploader for a single file export const uploadProjectFile = multer({ storage, fileFilter: filterProjectFile, }).single('project'); - -export const uploadSpreadsheet = multer({ - storage, - fileFilter: filterSpreadsheet, -}).single('spreadsheet'); diff --git a/apps/server/src/api-data/db/db.router.ts b/apps/server/src/api-data/db/db.router.ts index 049f94344..f7fa36c53 100644 --- a/apps/server/src/api-data/db/db.router.ts +++ b/apps/server/src/api-data/db/db.router.ts @@ -2,18 +2,17 @@ import express from 'express'; import { createProjectFile, - projectDownload, deleteProjectFile, + duplicateProjectFile, getInfo, listProjects, - patchPartialProjectFile, - previewSpreadsheet, loadProject, - duplicateProjectFile, - renameProjectFile, + patchPartialProjectFile, postProjectFile, + projectDownload, + renameProjectFile, } from './db.controller.js'; -import { uploadProjectFile, uploadSpreadsheet } from './db.middleware.js'; +import { uploadProjectFile } from './db.middleware.js'; import { projectSanitiser, sanitizeProjectFilename, @@ -39,6 +38,3 @@ router.put('/:filename/rename', validateProjectRename, sanitizeProjectFilename, router.delete('/:filename', sanitizeProjectFilename, deleteProjectFile); router.get('/info', getInfo); - -// TODO: validate import map -router.post('/spreadsheet/preview', uploadSpreadsheet, previewSpreadsheet); diff --git a/apps/server/src/api-data/excel/excel.controller.ts b/apps/server/src/api-data/excel/excel.controller.ts new file mode 100644 index 000000000..1e107e3e1 --- /dev/null +++ b/apps/server/src/api-data/excel/excel.controller.ts @@ -0,0 +1,40 @@ +/** + * This module encapsulates logic related to + * Google Sheets + */ + +import { Request, Response } from 'express'; +import { generateRundownPreview, listWorksheets, saveExcelFile } from './excel.service.js'; + +export async function postExcel(req: Request, res: Response) { + try { + const filePath = req.file.path; + await saveExcelFile(filePath); + res.status(200).send(); + } catch (error) { + res.status(500).send({ message: String(error) }); + } +} + +export async function getWorksheets(req: Request, res: Response) { + try { + const names = listWorksheets(); + res.status(200).send(names); + } catch (error) { + res.status(500).send({ message: String(error) }); + } +} + +/** + * parses an Excel spreadsheet + * @returns parsed result + */ +export async function previewExcel(req: Request, res: Response) { + try { + const { options } = req.body; + const data = generateRundownPreview(options); + res.status(200).send(data); + } catch (error) { + res.status(500).send({ message: String(error) }); + } +} diff --git a/apps/server/src/api-data/excel/excel.middleware.ts b/apps/server/src/api-data/excel/excel.middleware.ts new file mode 100644 index 000000000..914e4a74d --- /dev/null +++ b/apps/server/src/api-data/excel/excel.middleware.ts @@ -0,0 +1,17 @@ +import { Request } from 'express'; +import multer, { FileFilterCallback } from 'multer'; + +import { EXCEL_MIME } from '../../utils/parser.js'; +import { storage } from '../../utils/upload.js'; + +const filterExcel = (_req: Request, file: Express.Multer.File, cb: FileFilterCallback) => { + if (file.mimetype.includes(EXCEL_MIME)) { + cb(null, true); + } else { + cb(null, false); + } +}; +export const uploadExcel = multer({ + storage, + fileFilter: filterExcel, +}).single('excel'); diff --git a/apps/server/src/api-data/excel/excel.router.ts b/apps/server/src/api-data/excel/excel.router.ts new file mode 100644 index 000000000..799396276 --- /dev/null +++ b/apps/server/src/api-data/excel/excel.router.ts @@ -0,0 +1,16 @@ +/** + * This is a feature specific router for integration with Excel + */ + +import express from 'express'; +import { uploadExcel } from './excel.middleware.js'; +import { getWorksheets, postExcel, previewExcel } from './excel.controller.js'; +import { validateFileExists, validateImportMapOptions } from './excel.validation.js'; + +export const router = express.Router(); + +router.post('/upload', uploadExcel, validateFileExists, postExcel); +router.get('/worksheets', getWorksheets); +router.post('/preview', validateImportMapOptions, previewExcel); + +// TODO: validate import map diff --git a/apps/server/src/api-data/excel/excel.service.ts b/apps/server/src/api-data/excel/excel.service.ts new file mode 100644 index 000000000..32930e31c --- /dev/null +++ b/apps/server/src/api-data/excel/excel.service.ts @@ -0,0 +1,53 @@ +/** + * This module encapsulates logic related to + * Google Sheets + */ + +import { extname } from 'path'; +import { existsSync } from 'fs'; +import { ImportMap } from 'ontime-utils'; +import xlsx from 'node-xlsx'; +import { parseExcel } from '../../utils/parser.js'; +import { parseCustomFields, parseRundown } from '../../utils/parserFunctions.js'; +import { deleteFile } from '../../utils/parserUtils.js'; + +let excelData: { name: string; data: unknown[][] }[] = []; + +export async function saveExcelFile(filePath: string) { + if (!existsSync(filePath)) { + throw new Error('Upload of excel file failed'); + } + if (extname(filePath) != '.xlsx') { + throw new Error('Wrong file format'); + } + excelData = xlsx.parse(filePath, { cellDates: true }); + + await deleteFile(filePath); +} + +export function listWorksheets() { + return excelData.map((value) => value.name); +} + +export function generateRundownPreview(options: ImportMap) { + const data = excelData.find(({ name }) => name.toLowerCase() === options.worksheet.toLowerCase())?.data; + + if (!data) { + throw new Error(`Could not find data to import, maybe the worksheet name is incorrect: ${options.worksheet}`); + } + + const dataFromExcel = parseExcel(data, options); + + // we run the parsed data through an extra step to ensure the objects shape + const result = { rundown: [], customFields: {} }; + result.rundown = parseRundown(dataFromExcel); + if (result.rundown.length < 1) { + throw new Error(`Could not find data to import in the worksheet: ${options.worksheet}`); + } + result.customFields = parseCustomFields(dataFromExcel); + + //clear the data + excelData = []; + + return result; +} diff --git a/apps/server/src/api-data/excel/excel.validation.ts b/apps/server/src/api-data/excel/excel.validation.ts new file mode 100644 index 000000000..de6c6cd8b --- /dev/null +++ b/apps/server/src/api-data/excel/excel.validation.ts @@ -0,0 +1,28 @@ +import { isImportMap } from 'ontime-utils'; + +import { body, validationResult } from 'express-validator'; +import { NextFunction, Request, Response } from 'express'; + +export const validateFileExists = [ + (req: Request, res: Response, next: NextFunction) => { + if (!req.file) { + return res.status(422).json({ errors: 'File not found' }); + } + next(); + }, +]; + +export const validateImportMapOptions = [ + body('options') + .exists() + .isObject() + .custom((content) => { + return isImportMap(content); + }), + + (req: Request, res: Response, next: NextFunction) => { + const errors = validationResult(req); + if (!errors.isEmpty()) return res.status(422).json({ errors: errors.array() }); + next(); + }, +]; diff --git a/apps/server/src/api-data/index.ts b/apps/server/src/api-data/index.ts index 12aaee17b..fb8fcde33 100644 --- a/apps/server/src/api-data/index.ts +++ b/apps/server/src/api-data/index.ts @@ -9,6 +9,7 @@ import { router as projectRouter } from './project/project.router.js'; import { router as rundownRouter } from './rundown/rundown.router.js'; import { router as settingsRouter } from './settings/settings.router.js'; import { router as sheetsRouter } from './sheets/sheets.router.js'; +import { router as excelRouter } from './excel/excel.router.js'; import { router as viewSettingsRouter } from './view-settings/viewSettings.router.js'; export const appRouter = express.Router(); @@ -21,5 +22,6 @@ appRouter.use('/project', projectRouter); appRouter.use('/rundown', rundownRouter); appRouter.use('/settings', settingsRouter); appRouter.use('/sheets', sheetsRouter); +appRouter.use('/excel', excelRouter); appRouter.use('/url-presets', urlPresetsRouter); appRouter.use('/view-settings', viewSettingsRouter); diff --git a/apps/server/src/api-data/sheets/sheets.controller.ts b/apps/server/src/api-data/sheets/sheets.controller.ts index 9169a8189..46a70446c 100644 --- a/apps/server/src/api-data/sheets/sheets.controller.ts +++ b/apps/server/src/api-data/sheets/sheets.controller.ts @@ -14,6 +14,7 @@ import { hasAuth, download, upload, + getWorksheetOptions, } from '../../services/sheet-service/SheetService.js'; export async function requestConnection(req: Request, res: Response) { @@ -56,6 +57,16 @@ export async function revokeAuthentication(_req: Request, res: Response) { } } +export async function getWorksheetNamesFromSheet(req: Request, res: Response) { + try { + const { sheetId } = req.params; + const { worksheetOptions } = await getWorksheetOptions(sheetId); + res.status(200).send(worksheetOptions); + } catch (error) { + res.status(500).send({ message: String(error) }); + } +} + export async function readFromSheet(req: Request, res: Response) { try { const { sheetId } = req.params; diff --git a/apps/server/src/api-data/sheets/sheets.router.ts b/apps/server/src/api-data/sheets/sheets.router.ts index 0c096b354..ce310580b 100644 --- a/apps/server/src/api-data/sheets/sheets.router.ts +++ b/apps/server/src/api-data/sheets/sheets.router.ts @@ -5,6 +5,7 @@ import express from 'express'; import { + getWorksheetNamesFromSheet, readFromSheet, requestConnection, revokeAuthentication, @@ -12,7 +13,7 @@ import { writeToSheet, } from './sheets.controller.js'; import { uploadClientSecret } from './sheets.middleware.js'; -import { validateRequestConnection, validateSheetOptions } from './sheets.validation.js'; +import { validateRequestConnection, validateSheetId, validateSheetOptions } from './sheets.validation.js'; export const router = express.Router(); @@ -21,5 +22,7 @@ router.post('/:sheetId/connect', uploadClientSecret, validateRequestConnection, router.post('/revoke', revokeAuthentication); +router.post('/:sheetId/worksheets', validateSheetId, getWorksheetNamesFromSheet); + router.post('/:sheetId/read', validateSheetOptions, readFromSheet); router.post('/:sheetId/write', validateSheetOptions, writeToSheet); diff --git a/apps/server/src/api-data/sheets/sheets.validation.ts b/apps/server/src/api-data/sheets/sheets.validation.ts index 05dda3c34..ab36553ec 100644 --- a/apps/server/src/api-data/sheets/sheets.validation.ts +++ b/apps/server/src/api-data/sheets/sheets.validation.ts @@ -20,6 +20,16 @@ export const validateRequestConnection = [ }, ]; +export const validateSheetId = [ + param('sheetId').exists().isString(), + + (req: Request, res: Response, next: NextFunction) => { + const errors = validationResult(req); + if (!errors.isEmpty()) return res.status(422).json({ errors: errors.array() }); + next(); + }, +]; + export const validateSheetOptions = [ param('sheetId').exists().isString(), body('options') diff --git a/apps/server/src/services/sheet-service/SheetService.ts b/apps/server/src/services/sheet-service/SheetService.ts index 459255ad7..d97a3b38e 100644 --- a/apps/server/src/services/sheet-service/SheetService.ts +++ b/apps/server/src/services/sheet-service/SheetService.ts @@ -12,7 +12,7 @@ import got from 'got'; import { resolveSheetsDirectory } from '../../setup/index.js'; import { ensureDirectory } from '../../utils/fileManagement.js'; -import { type ClientSecret, cellRequestFromEvent, getA1Notation, validateClientSecret } from './sheetUtils.js'; +import { cellRequestFromEvent, type ClientSecret, getA1Notation, validateClientSecret } from './sheetUtils.js'; import { ImportMap } from 'ontime-utils'; import { parseExcel } from '../../utils/parser.js'; import { logger } from '../../classes/Logger.js'; @@ -190,11 +190,11 @@ function verifyConnection( } } -export function hasAuth(): { authenticated: AuthenticationStatus } { +export function hasAuth(): { authenticated: AuthenticationStatus; sheetId: string } { if (cleanupTimeout) { - return { authenticated: 'pending' }; + return { authenticated: 'pending', sheetId: currentSheetId }; } - return { authenticated: currentAuthClient ? 'authenticated' : 'not_authenticated' }; + return { authenticated: currentAuthClient ? 'authenticated' : 'not_authenticated', sheetId: currentSheetId }; } async function verifySheet( diff --git a/apps/server/src/services/sheet-service/__tests__/sheetUtils.test.ts b/apps/server/src/services/sheet-service/__tests__/sheetUtils.test.ts index be6d77233..08b0d68af 100644 --- a/apps/server/src/services/sheet-service/__tests__/sheetUtils.test.ts +++ b/apps/server/src/services/sheet-service/__tests__/sheetUtils.test.ts @@ -108,7 +108,7 @@ describe('cellRequestFromEvent()', () => { expect(result).toStrictEqual(millisToString(event.duration)); }); - test('boolean to x', () => { + test('boolean to TRUE', () => { const event: OntimeEvent = { type: SupportedEvent.Event, cue: '1', @@ -149,8 +149,8 @@ describe('cellRequestFromEvent()', () => { timeDanger: { row: 1, col: 41 }, }; const result = cellRequestFromEvent(event, 1, 1234, metadata); - expect(result.updateCells.rows[0].values[11].userEnteredValue.stringValue).toStrictEqual('x'); - expect(result.updateCells.rows[0].values[12].userEnteredValue.stringValue).toStrictEqual(''); + expect(result.updateCells.rows[0].values[11].userEnteredValue.boolValue).toStrictEqual(true); + expect(result.updateCells.rows[0].values[12].userEnteredValue.boolValue).toStrictEqual(false); }); test('spacing in metadata', () => { diff --git a/apps/server/src/services/sheet-service/sheetUtils.ts b/apps/server/src/services/sheet-service/sheetUtils.ts index c7c5e0fe8..6a71bda5b 100644 --- a/apps/server/src/services/sheet-service/sheetUtils.ts +++ b/apps/server/src/services/sheet-service/sheetUtils.ts @@ -1,4 +1,4 @@ -import { OntimeRundownEntry, isOntimeBlock, isOntimeEvent } from 'ontime-types'; +import { isOntimeBlock, isOntimeEvent, OntimeRundownEntry } from 'ontime-types'; import { millisToString } from 'ontime-utils'; import { sheets_v4 } from '@googleapis/sheets'; @@ -111,7 +111,7 @@ export function cellRequestFromEvent( }); } else if (typeof event[key] === 'boolean') { returnRows.push({ - userEnteredValue: { stringValue: event[key] ? 'x' : '' }, + userEnteredValue: { boolValue: event[key] }, }); } else { returnRows.push({}); diff --git a/apps/server/src/utils/__tests__/parser.test.ts b/apps/server/src/utils/__tests__/parser.test.ts index 2fa338036..1b968141f 100644 --- a/apps/server/src/utils/__tests__/parser.test.ts +++ b/apps/server/src/utils/__tests__/parser.test.ts @@ -5,20 +5,20 @@ import { DatabaseModel, EndAction, OntimeEvent, + OntimeRundown, ProjectData, Settings, SupportedEvent, - TimeStrategy, TimerType, + TimeStrategy, ViewSettings, - OntimeRundown, } from 'ontime-types'; import { dbModel } from '../../models/dataModel.js'; -import { parseExcel, parseJson, createEvent, getCustomFieldData } from '../parser.js'; +import { createEvent, getCustomFieldData, parseExcel, parseJson } from '../parser.js'; import { makeString } from '../parserUtils.js'; -import { parseUrlPresets, parseViewSettings } from '../parserFunctions.js'; +import { parseRundown, parseUrlPresets, parseViewSettings } from '../parserFunctions.js'; describe('test json parser with valid def', () => { const testData: Partial = { @@ -795,8 +795,8 @@ describe('parseExcel()', () => { 'Public', 'Skip', 'Notes', - 'test0', - 'test1', + 't0', + 'UpperCaseFromSheet', 'test2', 'test3', 'test4', @@ -809,8 +809,8 @@ describe('parseExcel()', () => { 'cue', ], [ - '1899-12-30T07:00:00.000Z', - '1899-12-30T08:00:10.000Z', + '07:00:00', + '08:00:10', 'Guest Welcome', '', '', @@ -831,8 +831,8 @@ describe('parseExcel()', () => { 101, ], [ - '1899-12-30T08:00:00.000Z', - '1899-12-30T08:30:00.000Z', + '08:00:00', + '08:30:00', 'A song from the hearth', 'load-next', 'clock', @@ -858,9 +858,9 @@ describe('parseExcel()', () => { // partial import map with only custom fields const importMap = { custom: { - user0: 'test0', - user1: 'test1', - user2: 'test2', + user0: 't0', + user1: 'UpperCaseFromSheet', + UpperCaseFromOntime: 'test2', user3: 'test3', user4: 'test4', user5: 'test5', @@ -874,8 +874,8 @@ describe('parseExcel()', () => { // TODO: update tests once import is resolved const expectedParsedRundown = [ { - //timeStart: 28800000, - //timeEnd: 32410000, + timeStart: 25200000, + timeEnd: 28810000, title: 'Guest Welcome', timerType: 'count-down', endAction: 'none', @@ -885,7 +885,7 @@ describe('parseExcel()', () => { custom: { user0: { value: 'a0' }, user1: { value: 'a1' }, - user2: { value: 'a2' }, + UpperCaseFromOntime: { value: 'a2' }, user3: { value: 'a3' }, user4: { value: 'a4' }, user5: { value: 'a5' }, @@ -899,8 +899,8 @@ describe('parseExcel()', () => { cue: '101', }, { - //timeStart: 32400000, - //timeEnd: 34200000, + timeStart: 28800000, + timeEnd: 30600000, title: 'A song from the hearth', timerType: 'clock', endAction: 'load-next', @@ -929,10 +929,10 @@ describe('parseExcel()', () => { colour: '', label: 'user1', }, - user2: { + UpperCaseFromOntime: { type: 'string', colour: '', - label: 'user2', + label: 'UpperCaseFromOntime', }, user3: { type: 'string', @@ -1363,4 +1363,48 @@ describe('parseExcel()', () => { expect(result.rundown.at(1).type).toBe(SupportedEvent.Event); expect((result.rundown.at(1) as OntimeEvent).timerType).toBe(TimerType.CountDown); }); + + it('am/pm conversion to 24h', () => { + const testData = [ + ['Time Start', 'Time End', 'Title', 'End Action', 'Public', 'Skip', 'Notes', 'Colour', 'cue'], + ['4:30:00', '4:36:00', 'A song from the hearth', 'load-next', 'x', '', 'Rainbow chase', '#F00', 102], + ['9:45:00', '10:56:00', 'Green grass', 'load-next', 'x', '', 'Rainbow chase', '#0F0', 103], + ['16:30:00', '16:36:00', 'A song from the hearth', 'load-next', 'x', '', 'Rainbow chase', '#F00', 102], + ['21:45:00', '22:56:00', 'Green grass', 'load-next', 'x', '', 'Rainbow chase', '#0F0', 103], + ['4:30:00AM', '4:36:00AM', 'A song from the hearth', 'load-next', 'x', '', 'Rainbow chase', '#F00', 102], + ['9:45:00AM', '10:56:00AM', 'Green grass', 'load-next', 'x', '', 'Rainbow chase', '#0F0', 103], + ['4:30:00PM', '4:36:00PM', 'A song from the hearth', 'load-next', 'x', '', 'Rainbow chase', '#F00', 102], + ['9:45:00PM', '10:56:00PM', 'Green grass', 'load-next', 'x', '', 'Rainbow chase', '#0F0', 103], + [], + ]; + + const importMap = { + worksheet: 'event schedule', + timeStart: 'time start', + timeEnd: 'time end', + duration: 'duration', + cue: 'cue', + title: 'title', + isPublic: 'public', + skip: 'skip', + note: 'notes', + colour: 'colour', + endAction: 'end action', + timerType: 'timer type', + timeWarning: 'warning time', + timeDanger: 'danger time', + custom: {}, + }; + const result = parseExcel(testData, importMap); + const rundown = parseRundown(result); + const events = rundown.filter((e) => e.type === SupportedEvent.Event) as OntimeEvent[]; + expect(events.at(0).timeStart).toEqual(16200000); + expect(events.at(1).timeStart).toEqual(35100000); + expect(events.at(2).timeStart).toEqual(59400000); + expect(events.at(3).timeStart).toEqual(78300000); + expect(events.at(4).timeStart).toEqual(16200000); + expect(events.at(5).timeStart).toEqual(35100000); + expect(events.at(6).timeStart).toEqual(59400000); + expect(events.at(7).timeStart).toEqual(78300000); + }); }); diff --git a/apps/server/src/utils/__tests__/time.test.ts b/apps/server/src/utils/__tests__/time.test.ts index a64d105e0..48521a4d3 100644 --- a/apps/server/src/utils/__tests__/time.test.ts +++ b/apps/server/src/utils/__tests__/time.test.ts @@ -37,7 +37,7 @@ describe('parseExcelDate', () => { }); describe('parses a time string that passes validation', () => { - const validFields = ['10:00:00', '10:00']; + const validFields = ['10:00:00', '10:00', '10:00AM', '10:00am', '10:00PM', '10:00pm']; validFields.forEach((field) => { it(`handles ${field}`, () => { const millis = parseExcelDate(field); diff --git a/apps/server/src/utils/parser.ts b/apps/server/src/utils/parser.ts index aac23bcc9..f8bb1b660 100644 --- a/apps/server/src/utils/parser.ts +++ b/apps/server/src/utils/parser.ts @@ -1,40 +1,37 @@ import { - generateId, - isImportMap, - type ImportMap, defaultImportMap, - validateEndAction, - validateTimerType, - type ImportOptions, - validateTimes, + generateId, + type ImportMap, isKnownTimerType, + validateEndAction, validateLinkStart, + validateTimerType, + validateTimes, } from 'ontime-utils'; import { + CustomFields, DatabaseModel, + EventCustomFields, + OntimeBlock, OntimeEvent, OntimeRundown, SupportedEvent, - TimeStrategy, - CustomFields, - EventCustomFields, TimerType, + TimeStrategy, } from 'ontime-types'; -import xlsx from 'node-xlsx'; - import { event as eventDef } from '../models/eventsDefinition.js'; import { dbModel } from '../models/dataModel.js'; -import { deleteFile, makeString } from './parserUtils.js'; +import { makeString } from './parserUtils.js'; import { - parseUrlPresets, - parseProject, - parseOsc, + parseCustomFields, parseHttp, + parseOsc, + parseProject, parseRundown, parseSettings, + parseUrlPresets, parseViewSettings, - parseCustomFields, } from './parserFunctions.js'; import { parseExcelDate } from './time.js'; import { coerceBoolean } from './coerceType.js'; @@ -192,10 +189,6 @@ export const parseExcel = (excelData: unknown[][], options?: Partial) } } else if (j === titleIndex) { event.title = makeString(column, ''); - // if this is a block, we have nothing else to import - if (event.type === SupportedEvent.Block) { - continue; - } } else if (j === timeStartIndex) { event.timeStart = parseExcelDate(column); } else if (j === timeEndIndex) { @@ -237,8 +230,8 @@ export const parseExcel = (excelData: unknown[][], options?: Partial) } // check if it is a custom field - if (columnText in customFieldImportKeys) { - handlers.custom(rowIndex, j, columnText); + if (column in customFieldImportKeys) { + handlers.custom(rowIndex, j, column); } // else. we don't know how to handle this column @@ -250,11 +243,16 @@ export const parseExcel = (excelData: unknown[][], options?: Partial) // if any data was found in row, push to array const keysFound = Object.keys(event).length + Object.keys(eventCustomFields).length; if (keysFound > 0) { - if (timerTypeIndex === null) { - event.timerType = TimerType.CountDown; - event.type = SupportedEvent.Event; + // if it is a Block type drop all other filed + if (event.type === SupportedEvent.Block) { + rundown.push({ type: event.type, id: event.id, title: event.title } as OntimeBlock); + } else { + if (timerTypeIndex === null) { + event.timerType = TimerType.CountDown; + event.type = SupportedEvent.Event; + } + rundown.push({ ...event, custom: { ...eventCustomFields } }); } - rundown.push({ ...event, custom: { ...eventCustomFields } }); } }); @@ -363,44 +361,3 @@ export const createEvent = (eventArgs: Partial, cueFallback: string const event = createPatch(baseEvent, eventArgs); return event; }; - -type ResponseOK = { - data: Partial; -}; - -/** - * Validates and calls parse on an excel file - */ -export function handleMaybeExcel(file: string, options: ImportOptions) { - const res: Partial = {}; - - if (!file.endsWith('.xlsx')) { - throw new Error('unexpected extension for spreadsheet'); - } - - // we need to check that the options are applicable - if (!isImportMap(options)) { - throw new Error('Got incorrect options for spreadsheet import'); - } - - const excelData = xlsx - .parse(file, { cellDates: true }) - .find(({ name }) => name.toLowerCase() === options.worksheet.toLowerCase()); - - if (!excelData?.data) { - throw new Error(`Could not find data to import, maybe the worksheet name is incorrect: ${options.worksheet}`); - } - - const dataFromExcel = parseExcel(excelData.data, options); - // we run the parsed data through an extra step to ensure the objects shape - res.data = {}; - res.data.rundown = parseRundown(dataFromExcel); - if (res.data.rundown.length < 1) { - throw new Error(`Could not find data to import in the worksheet: ${options.worksheet}`); - } - res.data.customFields = parseCustomFields(dataFromExcel); - - deleteFile(file); - - return res; -} diff --git a/apps/server/src/utils/time.ts b/apps/server/src/utils/time.ts index c8b606358..f09d44126 100644 --- a/apps/server/src/utils/time.ts +++ b/apps/server/src/utils/time.ts @@ -35,22 +35,40 @@ const parse = (valueAsString: string): number => { return Math.abs(parsed); }; +const stripAMPM = (value: string) => { + const lowerValue = value.toLowerCase(); + if (lowerValue.endsWith('am')) { + return { sansPostfix: lowerValue.substring(0, lowerValue.length - 2), pastNoon: false }; + } else if (lowerValue.endsWith('pm')) { + return { sansPostfix: lowerValue.substring(0, lowerValue.length - 2), pastNoon: true }; + } else { + return { sansPostfix: lowerValue, pastNoon: false }; + } +}; + /** * @description Parses a time string to millis, copied from client code * @param {string} value - time string * @param {boolean} fillLeft - autofill left = hours / right = seconds * @returns {number} - time string in millis */ -export const forgivingStringToMillis = (value: string, fillLeft = true): number => { +export const forgivingStringToMillis = (value: string, fillLeft: boolean = true): number => { let millis = 0; + // check for AM/PM indicators + const { sansPostfix, pastNoon } = stripAMPM(value); + + //if past noon indicated add 12 hours + if (pastNoon) { + millis = mth * 12; + } // split string at known separators : , . const separatorRegex = /[\s,:.]+/; - const [first, second, third] = value.split(separatorRegex); + const [first, second, third] = sansPostfix.split(separatorRegex); if (first != null && second != null && third != null) { // if string has three sections, treat as [hours] [minutes] [seconds] - millis = parse(first) * mth; + millis += parse(first) * mth; millis += parse(second) * mtm; millis += parse(third) * mts; } else if (first != null && second == null && third == null) { @@ -60,23 +78,23 @@ export const forgivingStringToMillis = (value: string, fillLeft = true): number const hours = first.substring(0, 2); const minutes = first.substring(2, 4); const seconds = first.substring(4); - millis = parse(hours) * mth; + millis += parse(hours) * mth; millis += parse(minutes) * mtm; millis += parse(seconds) * mts; } else { // otherwise lets treat as [minutes] - millis = parse(first) * mtm; + millis += parse(first) * mtm; } } if (first != null && second != null && third == null) { // if string has two sections if (fillLeft) { // treat as [hours] [minutes] - millis = parse(first) * mth; + millis += parse(first) * mth; millis += parse(second) * mtm; } else { // treat as [minutes] [seconds] - millis = parse(first) * mtm; + millis += parse(first) * mtm; millis += parse(second) * mts; } } diff --git a/e2e/tests/features/207-spreadsheet-import.spec.ts b/e2e/tests/features/207-spreadsheet-import.spec.ts new file mode 100644 index 000000000..18e40220b --- /dev/null +++ b/e2e/tests/features/207-spreadsheet-import.spec.ts @@ -0,0 +1,37 @@ +import { expect, test } from '@playwright/test'; + +const fileToUpload = 'e2e/tests/fixtures/test-sheet.xlsx'; + +test('sheet file upload', async ({ page }) => { + await page.goto('http://localhost:4001/editor'); + await page.getByRole('button', { name: 'Rundown menu' }).click(); + await page.getByRole('menuitem', { name: 'Delete all events' }).click(); + + await page.getByRole('button', { name: 'Application settings' }).click(); + await page.getByRole('button', { name: 'Import spreadsheet' }).click(); + + // workaround to upload file on hidden input + // https://playwright.dev/docs/api/class-filechooser + const fileChooserPromise = page.waitForEvent('filechooser'); + await page.getByRole('button', { name: 'Import from spreadsheet', exact: true }).click(); + const fileChooser = await fileChooserPromise; + await fileChooser.setFiles(fileToUpload); + + await page.locator('[id="event\\ schedule"]').selectOption('Sheet2'); + await page.locator('[id="event\\ schedule"]').selectOption('test'); + + await page.getByRole('button', { name: 'Import preview' }).click(); + await page.getByRole('button', { name: 'Apply' }).click(); + await page.getByRole('button', { name: 'Return' }).click(); + await page.getByLabel('close').click(); + + // asset test events + const firstTitle = page.getByTestId('entry-1').getByTestId('block__title'); + await expect(firstTitle).toHaveValue('Attempt light check'); + + const secondTitle = page.getByTestId('entry-2').getByTestId('block__title'); + await expect(secondTitle).toHaveValue('Preset'); + + const thirdTitle = page.getByTestId('entry-3').getByTestId('block__title'); + await expect(thirdTitle).toHaveValue('Albania'); +}); diff --git a/e2e/tests/fixtures/test-sheet.xlsx b/e2e/tests/fixtures/test-sheet.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..aa335bea13947f2caca605f2bb0e5578e28908f5 GIT binary patch literal 6621 zcmbVwby!r7wl+BA&@gm2N)4#AGy>9%bVv;`w4{K7goHzPNDLy~F@SV;H-dnabc*nS z=RQ}D_nhA!-U#rK55LJ|@Y?}0nyfYv+fS~dS2UzgGdwKMR zZ%%Q$NcN()vry8`vI%+wgg6A(Bg??lQ<52}2Rdsakd3B+NVc1@65s<{F3>i<<+>l( z^?4YTw$RcU-ky{Idq z6|k5pM}xU%_`6{sr~64dj)BrsI9lC-Adju3>$kZu^?Vw`YSN*5xTls8jq-F@LY3(e;1 zXAXK-_f!?oG0=w3mND-kAyoj7kW~LGig5mgA{SRrI}4XP4DIUbLsP(HzDp&gSEV(? z&i5OUwI(^oa@-1LHIplMTTlVozNXbY$H(~9(-V9T7 z49;bEjqX^vA?feQWY#){WW>D=#iduJI3QDNaYDi>T(&vy)v3=sf6%52hzaoG5lj zJbbf=`zU-1Ng)xjc!}nrY_91zi+ku+Gd$*9&wK$DE*mR8fS&CbqZ?hIZ+R`^71W!R z(*I&-Oe-QbG|fvWD+8k`eFCIMcRT^8l^O@RvQvkc3a87PZN$($wKi>WiW?=Y$!rjo zDoGv%N!r-BY`QIQ^C!Ra1s3A89g0;2j3B@|6ZF6dw9P$C>2J_pQ#=mV-bP zvh}ma#8j90m?%l_zqaIX8Jp-q%MbY_WAX!ceCUTga=C+^^1T~zn(ns)+ zrri#|;$)Gc9or0z=S`wSKc87%jNv6bHPZdrfGd1F{H6TBh#$$SYt-H;cuBBJrTR%{ zNi@8-!RuP>QbBlZDr5JJy`OCg|MaPBwcttTu1{o5%Jds*P%J4W@|D3N+DN1ezk=lB zFYaGOm{T!3Rf)uA_Xt=hF_%C6Xk_+$5`D&DbDw!#ZNwSZD|NF(piwXQ$zqEO0*iz3 z2v@IHEy|2rZ_+Zkcai&jpr^A(zKK` zCj^nZ;(;;tmZQDz}4B~jV;JfYqdb?dxGdXTN2J9co+#922kAF)5uga$Y= zuyhP~uJ!u#aR_S1laZG}kBVg)NK0CY;%Prl#1Hwe$fRxVv+Nl!$MIHt1D|DM_r0C% z;~E;RVD(*`+svR>vK-%%K!fLT>LxYj^+nG$^dgw;q@L~OL|@bMrE+W#+{~66Jit4p z1o8@Q^dhfeFSU_ey(BuiCMmf;oHw>6VlQem7{nxY@1Z5VG1ErO`^Xtu!bpexeuIFn z$FNVJmz>(z*S5K}u@r1u_7f=3{aVSBpukLWYL-{X_b1T`=Sija2%VQCyqe}-zwUqI zp>C<+DWg@5w}z_~Oh>m7cNhSekechnllFvtqswDZe3aD*j15yN=Ro|31EwNWrWwuZ z25K2XSym60eTc>UXmHGo4$%uga4GT4f1+M7#6X>QDrKuCXn!m+-||+bRX_%BxMFkY z1m6FXW*X*BgPoxdopdtl{`M48{R3jV}#9c9j-hf$dl` zMojBs`C_809=~FG6s{Ij-+YW2@OB9GELDz@y=p1Zqxrdnx%kU++7w?g-XX?EHATUw zdKV$HE>$0Lk=?=gl^2M9BbHIG^Jb@p1WnzJ7K$ikvDCq2o9(CJ@sVLQX# z95>t=?Ka<$ou$PD;0qm}umv4HJFGFzyn6ZyH;>(@d-B8jv}bfFQdwD9i@}(5T)r=$0**28+oUfVDg(QGMlaP@Df)hQjIYeF?q_!nUWM;7xxT0`HJpD_NMhVrDXV1XoxDT9;!f4IK^( z&i5T9dHSt?+?6VzWQaX{W+lH5;~;8JpPbRzPeazgA|$$IAgY!HY~I8_S0HrhFSfy8 z;8sgDC+iyq4)OSPe-&tk*O;&cr;U7`QKG?-yyq3t&G9xt>PM~b@iY+z=DGYn9~7bQ zDP2Pp0;SGIxUxoUga>Lwo0U3OFOu(lb5~r&TE5p?DY(y#Hu?;}_z8m{b%nXaz_~JU zl?W#+fUj1EOl=dXblU>aJzEa%tVBxRb?K_Ck7hpgfLa$A6v@pv>Jk{ zgFe@{$UL%2pLzsxDY{C~l*ZN$!<-s8Q%?;oB**veLfzk&lcbS&u^Yd1F!yTch&@j{ z0gFR7*t7)IcZK+vjZ+xaTH*u*b;R}{R~tiP2ayYTBk!yT%1}k1g*$MnN;x(T3Mv6r ztE3xWNw?y|C&UA;GyGpX$}|oyH_()Oej1>wY@MQjqBz)6PhJPG0>IFO+e0I*_ZT~c zTIXa_w?YO*R}^Jk&j^U9zw9v%x=BN6ow67&e}pJdJC#P&juHg_2%$ym1gXQ(LRz<~ zkUtE2nftRL2C?yjru6~Rrl+(3>8#U1B(>T!N;+WO*{#<1H(9eFcRva`U_%s|+(0oP zrEC_q$hs$|5~WDsW*|OG7)_E=J+aCDA&s2s{-`kpU04D;KpJ)(jQl}PjFT=skVDaa zT~C%RdH}nRDv^2Hr~C!=6W&~iiV!W#&Q8*8uCX&4-#d?cK`pWIku*Ha&Oe`ll0+RP z*?lo)Ur)By615?t&_riQ*`}IxVQIqBE_KA|=cSsQ=)=IQou8q|A9l7}0MZ0kqeyCI zb))nUCB=rl?^`bk_}|KbXspn>DJhq3zLp=HbdN{u-8c}WCR^4zz~{~=qSOVv=nSiH zzTp?)kkLUX!5lf&$TtgonT(6pfJT+c3b+-#Iv`zbsCfINPxeCTA>ZJ)Q@lASpPLlf zt+x0$pNW6Iy?C(ouOsu!fzpH2$s9xbxuvE5aa_3*1$sd^EOv<3)5=W7v6yAEKs2~y z>tQV<$=oPCjIpsa*wsuR-^LysCbx7s1fp+BnG`!2B_vR-!Csy{ zPjj+Ys7>UlQt|FLuK^DEa7mtNu3{E1H(;d|17Y_8M7bmBdKe|t(9bD z>QVjCyUkj>t+dvd+*d;W)Slz!)kRg_)!Y+#M~8*wA~#v0*lXWloo%u*Gu4&Z^sAR5 zju8GeM$gj{0zlQA z>ao07JoGSDw?{n#=xg$www&WJ`j}chu<_t<-8+Q3Y`)Vs{?a;M3EjNbWoE$Rifg;u zt0)=l!FBwzQTt7e7`nz!Pe=Xu}U1)6qiw4I^T`FN>fTe@%B4K-RZhhpq>?OShF;+VjW8%_UDtXlxQobybA;&O< zIsb(~teQ@@6vaic2_ggj!Zgx$R@Lg>a;>HCE~ZbR6FTJA5UI%0;s)!0Am7QAlH$Iv zw7FL}cTh+gFn`~JjD++B#lN6n>}+A@@^7mLuDfp*MUtso+6(&ma zaDm2Hi4y%Yq)xS!;dZ|0Q#e#CDuOde-JMOCyyx`nhd5+%?*H9%;b%A+$%X^B2vv6dopM}M!f+;FNkXb+@cI)+)Y*X2mCZlAh~8z!t3 zXsI1Ocqj36hhhvkC%%6;GU{)({E3Sf{dn{IGWq}t zJKZ5zn_7m)t%*i$rS_kQ^^I@5aNrUmgJL$YW+5#4E;#0vct1U*-8K60D_I=ZN;hY3 z@4x(ZtoZK(HNmaM3#hrOGt|+A+Z^g_@oO=o-lyCK#z!0x!}datkTjp$JV4$GeI{p* zx?;Iw4+){}b5A+lbBbo+90t^+r5)__UF^_(GQsN%(^FC->|jIcfN*IF%4|=Fq-K5= zDiA1wplZ0HytC|38hm706+o(C3s&~WLBPrbH}B61!7)4I?IqMHJ%bLv=J7=f;VFea zwlJPM}k}WW}~p zKQ@TN{hBYT%L7cw3r#MJxx6O2s`fB)&weXe^~uaX>}njt6fGY>;{g~=piG?FIk0&w zD?k*De3MK`)KhujJe|7@2f`Ur zZu&W9H&21)iRQ;AIm48^r(_1q)x-^;#U?b(<$1w@yuFMdE?lXM9n<64WD<+}5}0JB zxqNdZzOLLcXw0IHjGj=O%5NAnoYMeb+M+i+*8B2IWJ@0mF&rYxNQKF2Rm^AF!^&4b zOJR3$RzI^r9@6%~Gr2d2`H_d}9vPr{ovV`W>uRG8`0*SNUlS&T@%MAc%Rvt z>e6r!d&fu|QJc2n8L8V<%k)Npjavg4P%BIkrB9V|M_qAzqu1vrRvIPSymhwEyO^8A zQA9MfIOBQ`aacPnJA+b{v0B-U_{fo1El@2M@?Wobm&F_&9s=XF?uWWNocF2uGzk4b zy^AIu`EfD+Z8Ym|kN=K#tlMb*iwu4dK+zur;JsQI#Pc-Os5i5Qw;p}ij+rO)#>7_t z^z@;5satGmOJuy?>9ZNM5)+LHy6$b0!aZV<^u4K?QmD=VK&hKK8lAF& z?TIKmS>kG1>~p-YU>*8sZZShS+&U2rZpJMF@94SuCfKGhU=}M+fC6>XnvZS}3VYRt zYiRYD`t>XKuNS=`w4sun<3#LxniBm{j$VIa>diXsY6C#+rJySUgkyQ3`K-nHaEqKxQPLthH& z;A-LEYOLw$XyIab2eb}N1(jQZY&v2uFi?xJM-NnOp!T3LC0q@cT&*@8kdo)C_KO-T z!_0Fq|3C=OFKZ57n4Wo&oBRTEP$AoeHwO^y&lUc@zPG1uhrW>|=0Ju|Uwnnm~^>*nj48olgm18VDd-&61CUGe2# zS&-zp&^dDZZ zGI!G`v0qfpzNC}+;ug3|x4n(30`ff&((fJmpL@}}9s0k^UmNwmjsILL-K|c4%hOvt z+%-x6yGZ@p^v}WfZtwJ4q;IeCfByIXY@z=4@@L2P?_L&e3-^EU@@r)N+smKD{;s?G zEfC!Q-0S`A;m=}t*A)B~w_Aq2y^jCVBK+;=Ptv|CoxdgF_VoW0(!b6BWT`u5|1JEt zDfqwU|HJctTmPBMcTDzMf^MVo2e18Y_$R*ZQvJ825&y@;S5-hoyHf(*zJhL%B1w98 G_Wu9{1X(Kp literal 0 HcmV?d00001 diff --git a/packages/utils/src/date-utils/isTimeString.test.ts b/packages/utils/src/date-utils/isTimeString.test.ts index 27a6e565f..dfb829570 100644 --- a/packages/utils/src/date-utils/isTimeString.test.ts +++ b/packages/utils/src/date-utils/isTimeString.test.ts @@ -24,3 +24,12 @@ describe('test isTimeString() function handle different separators', () => { }); } }); + +describe('test isTimeString() function handle AM/PM', () => { + const ts = ['2:10AM', '2:10PM', '2:10']; + for (const s of ts) { + it(`it handles ${s}`, () => { + expect(isTimeString(s)).toBe(true); + }); + } +}); diff --git a/packages/utils/src/date-utils/isTimeString.ts b/packages/utils/src/date-utils/isTimeString.ts index 62a52bf81..52bce2164 100644 --- a/packages/utils/src/date-utils/isTimeString.ts +++ b/packages/utils/src/date-utils/isTimeString.ts @@ -4,16 +4,6 @@ * @returns {boolean} string represents time */ export const isTimeString = (text: string): boolean => { - // ^ # Start of string - // (?: # Try to match... - // (?: # Try to match... - // ([01]?\d|2[0-3]): # HH: - // )? # (optionally). - // ([0-5]?\d): # MM: (required) - // )? # (entire group optional, so either HH:MM:, MM: or nothing) - // ([0-5]?\d) # SS (required) - // $ # End of string - - const regex = /^(?:(?:([01]?\d|2[0-3])[:,.])?([0-5]?\d)[:,.])?([0-5]?\d)$/; + const regex = /^(?:(?:([01]?\d|2[0-3])[:,.])?([0-5]?\d)[:,.])?([0-5]?\d)?(\s)?([APap][Mm])?$/; return regex.test(text); };