cpvalente/ontime
1
0
Fork 0
mirror of https://github.com/cpvalente/ontime.git synced 2026-08-14 20:03:52 +00:00
Code Issues Packages Projects Releases 10 Wiki Activity

Improve spreadsheet import flow (#839)

Browse Source 
* feat: add dropdown to worksheet selection

* feat: move revoke to separate button

* refactor: extract excel flow to separate controller and service

* chore: show spinner when uploading sheet and disable the other option

* chore: don't show "success" when canceling the import

* chore: export boolean types properly to sheets
This commit is contained in:
Alex Christoffer Rasmussen
2024-03-22 21:33:11 +01:00
committed by GitHub
parent f4692db021
commit f5e9a2549e
33 changed files with 528 additions and 235 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/client/src/common/api/db.ts
-29
View File
@@ -1,14 +1,11 @@
import axios, { AxiosResponse } from 'axios';
import {
CustomFields,
DatabaseModel,
GetInfo,
MessageResponse,
OntimeRundown,
ProjectData,
ProjectFileListResponse,
} from 'ontime-types';
import { ImportMap } from 'ontime-utils';
import { makeCSV, makeTable } from '../../features/cuesheet/cuesheetUtils';
@@ -155,32 +152,6 @@ export async function getInfo(): Promise<GetInfo> {
return res.data;
}
type PreviewSpreadsheetResponse = {
rundown: OntimeRundown;
customFields: CustomFields;
};
/**
* Make patch changes to the objects in the db
*/
export async function importSpreadsheetPreview(file: File, options: ImportMap): Promise<PreviewSpreadsheetResponse> {
const formData = new FormData();
formData.append('spreadsheet', file);
formData.append('options', JSON.stringify(options));
const response: AxiosResponse<PreviewSpreadsheetResponse> = await axios.post(
`${dbPath}/spreadsheet/preview`,
formData,
{
headers: {
'Content-Type': 'multipart/form-data',
},
},
);
return response.data;
}
/**
* Utility function gets project from db
* @param fileName
apps/client/src/common/api/excel.ts
+42
View File
@@ -0,0 +1,42 @@
import axios, { AxiosResponse } from 'axios';
import { CustomFields, OntimeRundown } from 'ontime-types';
import { ImportMap } from 'ontime-utils';
import { apiEntryUrl } from './constants';
const excelPath = `${apiEntryUrl}/excel`;
type PreviewSpreadsheetResponse = {
rundown: OntimeRundown;
customFields: CustomFields;
};
/**
* upload Excel file to server
* @return string - file ID op the uploaded file
*/
export async function upload(file: File) {
const formData = new FormData();
formData.append('excel', file);
await axios.post(`${excelPath}/upload`, formData, {
headers: {
'Content-Type': 'multipart/form-data',
},
});
}
/**
* Get Worksheet names
* @return string[] - array of available worksheets
*/
export async function getWorksheetNames(): Promise<string[]> {
const response: AxiosResponse<string[]> = await axios.get(`${excelPath}/worksheets`);
return response.data;
}
export async function importRundownPreview(options: ImportMap): Promise<PreviewSpreadsheetResponse> {
const response: AxiosResponse<PreviewSpreadsheetResponse> = await axios.post(`${excelPath}/preview`, {
options,
});
return response.data;
}
apps/client/src/common/api/sheets.ts
+10 -2
View File
@@ -1,4 +1,4 @@
import axios from 'axios';
import axios, { AxiosResponse } from 'axios';
import { AuthenticationStatus, CustomFields, OntimeRundown } from 'ontime-types';
import { ImportMap } from 'ontime-utils';
@@ -9,7 +9,10 @@ const sheetsPath = `${apiEntryUrl}/sheets`;
/**
* HTTP request to verify whether we are authenticated with Google Sheet service
*/
export const verifyAuthenticationStatus = async (): Promise<{ authenticated: AuthenticationStatus }> => {
export const verifyAuthenticationStatus = async (): Promise<{
authenticated: AuthenticationStatus;
sheetId: string;
}> => {
const response = await axios.get(`${sheetsPath}/connect`);
return response.data;
};
@@ -58,6 +61,11 @@ export const previewRundown = async (
return response.data;
};
export const getWorksheetNames = async (sheetId: string): Promise<string[]> => {
const response: AxiosResponse<string[]> = await axios.post(`${sheetsPath}/${sheetId}/worksheets`);
return response.data;
};
/**
* HTTP request to upload the rundown to a google sheet
*/
apps/client/src/common/utils/uploadUtils.ts
+2 -2
View File
@@ -2,7 +2,7 @@
* Collection of rules for pre-validating a spreadsheet
* @param file
*/
export function validateSpreadsheetImport(file: File) {
export function validateExcelImport(file: File) {
if (!isExcelFile(file)) {
throw new Error('Unknown file type');
}
@@ -12,7 +12,7 @@ export function validateSpreadsheetImport(file: File) {
throw new Error('File is empty');
}
// Limit file size of an excel file to around 10MB
// Limit file size of an Excel file to around 10MB
if (file.size > 10_000_000) {
throw new Error('File size limit (10MB) exceeded');
}
apps/client/src/features/app-settings/panel/sources-panel/GSheetSetup.tsx
+6 -1
View File
@@ -3,6 +3,7 @@ import { Button, Input } from '@chakra-ui/react';
import { IoCheckmark } from '@react-icons/all-files/io5/IoCheckmark';
import { IoShieldCheckmarkOutline } from '@react-icons/all-files/io5/IoShieldCheckmarkOutline';
import { getWorksheetNames } from '../../../../common/api/sheets';
import CopyTag from '../../../../common/components/copy-tag/CopyTag';
import { openLink } from '../../../../common/utils/linkUtils';
import * as Panel from '../PanelUtils';
@@ -27,6 +28,7 @@ export default function GSheetSetup(props: GSheetSetupProps) {
const sheetId = useSheetStore((state) => state.sheetId);
const setSheetId = useSheetStore((state) => state.setSheetId);
const setWorksheets = useSheetStore((state) => state.setWorksheets);
const authenticationStatus = useSheetStore((state) => state.authenticationStatus);
const setAuthenticationStatus = useSheetStore((state) => state.setAuthenticationStatus);
@@ -53,7 +55,6 @@ export default function GSheetSetup(props: GSheetSetupProps) {
};
const handleCancelFlow = async () => {
await handleRevoke();
onCancel();
};
@@ -89,6 +90,10 @@ export default function GSheetSetup(props: GSheetSetupProps) {
if (result?.authenticated) {
setAuthenticationStatus(result.authenticated);
if (result.authenticated !== 'pending') {
if (result.authenticated == 'authenticated') {
const names = await getWorksheetNames(result.sheetId);
setWorksheets(names);
}
setLoading('');
return;
}
apps/client/src/features/app-settings/panel/sources-panel/ImportReview.tsx
+3 -2
View File
@@ -14,10 +14,11 @@ interface ImportReviewProps {
rundown: OntimeRundown;
customFields: CustomFields;
onFinished: () => void;
onCancel: () => void;
}
export default function ImportReview(props: ImportReviewProps) {
const { rundown, customFields, onFinished } = props;
const { rundown, customFields, onFinished, onCancel } = props;
const [loading, setLoading] = useState(false);
const { importRundown } = useGoogleSheet();
@@ -25,7 +26,7 @@ export default function ImportReview(props: ImportReviewProps) {
const handleCancel = () => {
resetPreview();
onFinished();
onCancel();
};
const applyImport = async () => {
apps/client/src/features/app-settings/panel/sources-panel/SourcesPanel.tsx
+60 -24
View File
@@ -4,9 +4,14 @@ import { IoCloudOutline } from '@react-icons/all-files/io5/IoCloudOutline';
import { IoDownloadOutline } from '@react-icons/all-files/io5/IoDownloadOutline';
import { ImportMap, unpackError } from 'ontime-utils';
import { importSpreadsheetPreview } from '../../../../common/api/db';
import {
getWorksheetNames as getWorksheetNamesExcel,
importRundownPreview as importRundownPreviewExcel,
upload as uploadExcel,
} from '../../../../common/api/excel';
import { getWorksheetNames } from '../../../../common/api/sheets';
import { maybeAxiosError } from '../../../../common/api/utils';
import { validateSpreadsheetImport } from '../../../../common/utils/uploadUtils';
import { validateExcelImport } from '../../../../common/utils/uploadUtils';
import * as Panel from '../PanelUtils';
import ImportMapForm from './import-map/ImportMapForm';
@@ -21,36 +26,43 @@ import style from './SourcesPanel.module.scss';
export default function SourcesPanel() {
const [importFlow, setImportFlow] = useState<'none' | 'excel' | 'gsheet' | 'finished'>('none');
const [error, setError] = useState('');
const [hasFile, setHasFile] = useState<'none' | 'loading' | 'done'>('none');
const { exportRundown, importRundownPreview, revoke, verifyAuth } = useGoogleSheet();
const { exportRundown, importRundownPreview, verifyAuth } = useGoogleSheet();
const spreadsheet = useSheetStore((state) => state.spreadsheet);
const setSpreadsheet = useSheetStore((state) => state.setSpreadsheet);
const setWorksheets = useSheetStore((state) => state.setWorksheets);
const authenticationStatus = useSheetStore((state) => state.authenticationStatus);
const setAuthenticationStatus = useSheetStore((state) => state.setAuthenticationStatus);
const rundown = useSheetStore((state) => state.rundown);
const setRundown = useSheetStore((state) => state.setRundown);
const customFields = useSheetStore((state) => state.customFields);
const setCustomFields = useSheetStore((state) => state.setCustomFields);
const setSheetId = useSheetStore((state) => state.setSheetId);
const sheetId = useSheetStore((state) => state.sheetId);
const fileInputRef = useRef<HTMLInputElement>(null);
const handleFile = (event: ChangeEvent<HTMLInputElement>) => {
const handleFile = async (event: ChangeEvent<HTMLInputElement>) => {
const fileToUpload = event.target.files?.[0];
if (!fileToUpload) {
setSpreadsheet(null);
setWorksheets(null);
setHasFile('none');
return;
}
try {
validateSpreadsheetImport(fileToUpload);
setSpreadsheet(fileToUpload);
setHasFile('loading');
validateExcelImport(fileToUpload);
await uploadExcel(fileToUpload);
const names = await getWorksheetNamesExcel();
setWorksheets(names);
setImportFlow('excel');
setHasFile('done');
} catch (error) {
const errorMessage = unpackError(error);
setError(`Error uploading file: ${errorMessage}`);
setSpreadsheet(null);
setWorksheets(null);
setHasFile('none');
}
};
@@ -58,7 +70,16 @@ export default function SourcesPanel() {
fileInputRef.current?.click();
};
const openGSheetFlow = () => {
const openGSheetFlow = async () => {
const result = await verifyAuth();
if (result) {
setAuthenticationStatus(result.authenticated);
setSheetId(result.sheetId);
if (result.authenticated === 'authenticated' && result.sheetId) {
const names = await getWorksheetNames(result.sheetId);
setWorksheets(names);
}
}
setImportFlow('gsheet');
};
@@ -68,9 +89,8 @@ export default function SourcesPanel() {
const handleSubmitImportPreview = async (importMap: ImportMap) => {
if (importFlow === 'excel') {
if (!spreadsheet) return;
try {
const previewData = await importSpreadsheetPreview(spreadsheet, importMap);
const previewData = await importRundownPreviewExcel(importMap);
setRundown(previewData.rundown);
setCustomFields(previewData.customFields);
} catch (error) {
@@ -86,12 +106,9 @@ export default function SourcesPanel() {
const cancelImportMap = async () => {
setImportFlow('none');
if (spreadsheet) {
setSpreadsheet(null);
}
setHasFile('none');
setWorksheets(null);
if (authenticationStatus === 'authenticated') {
await revoke();
const result = await verifyAuth();
if (result) {
setAuthenticationStatus(result.authenticated);
@@ -102,7 +119,8 @@ export default function SourcesPanel() {
const handleFinished = () => {
setImportFlow('finished');
setRundown(null);
setSpreadsheet(null);
setHasFile('none');
setWorksheets(null);
setCustomFields(null);
};
@@ -113,12 +131,11 @@ export default function SourcesPanel() {
const isExcelFlow = importFlow === 'excel';
const isGSheetFlow = importFlow === 'gsheet';
const hasFile = Boolean(spreadsheet);
const isAuthenticated = authenticationStatus === 'authenticated';
const showInput = importFlow === 'none';
const showSuccess = importFlow === 'finished';
const showAuth = isGSheetFlow && !isAuthenticated;
const showImportMap = (isGSheetFlow && isAuthenticated) || (isExcelFlow && hasFile);
const showImportMap = (isGSheetFlow && isAuthenticated) || (isExcelFlow && hasFile === 'done');
const showReview = rundown !== null && customFields !== null;
return (
@@ -141,13 +158,25 @@ export default function SourcesPanel() {
/>
<div className={style.uploadSection}>
<div>
<Button variant='ontime-filled' size='sm' leftIcon={<IoDownloadOutline />} onClick={handleUpload}>
<Button
variant='ontime-filled'
size='sm'
leftIcon={<IoDownloadOutline />}
onClick={handleUpload}
isLoading={hasFile === 'loading'}
>
Import from spreadsheet
</Button>
<Panel.Description>Accepts .xlsx files</Panel.Description>
</div>
<div>
<Button variant='ontime-filled' size='sm' leftIcon={<IoCloudOutline />} onClick={openGSheetFlow}>
<Button
variant='ontime-filled'
size='sm'
leftIcon={<IoCloudOutline />}
onClick={openGSheetFlow}
isDisabled={hasFile !== 'none'}
>
Synchronise with Google
</Button>
<Panel.Description>Start authentication process</Panel.Description>
@@ -172,7 +201,14 @@ export default function SourcesPanel() {
onSubmitImport={handleSubmitImportPreview}
/>
)}
{showReview && <ImportReview rundown={rundown} customFields={customFields} onFinished={handleFinished} />}
{showReview && (
<ImportReview
rundown={rundown}
customFields={customFields}
onFinished={handleFinished}
onCancel={cancelImportMap}
/>
)}
</Panel.Card>
</Panel.Section>
</>
apps/client/src/features/app-settings/panel/sources-panel/import-map/ImportMapForm.tsx
+41 -2
View File
@@ -1,12 +1,13 @@
import { useState } from 'react';
import { useFieldArray, useForm } from 'react-hook-form';
import { Button, IconButton, Input } from '@chakra-ui/react';
import { Button, IconButton, Input, Select, Tooltip } from '@chakra-ui/react';
import { IoAdd } from '@react-icons/all-files/io5/IoAdd';
import { IoTrash } from '@react-icons/all-files/io5/IoTrash';
import { ImportMap } from 'ontime-utils';
import { isAlphanumeric } from '../../../../../common/utils/regex';
import * as Panel from '../../PanelUtils';
import useGoogleSheet from '../useGoogleSheet';
import { useSheetStore } from '../useSheetStore';
import { convertToImportMap, getPersistedOptions, NamedImportMap, persistImportMap } from './importMapUtils';
@@ -23,7 +24,7 @@ interface ImportMapFormProps {
export default function ImportMapForm(props: ImportMapFormProps) {
const { isSpreadsheet, onCancel, onSubmitExport, onSubmitImport } = props;
const namedImportMap = getPersistedOptions();
const { revoke } = useGoogleSheet();
const {
control,
handleSubmit,
@@ -41,6 +42,7 @@ export default function ImportMapForm(props: ImportMapFormProps) {
});
const stepData = useSheetStore((state) => state.stepData);
const worksheetNames = useSheetStore((state) => state.worksheetNames);
const [loading, setLoading] = useState<'' | 'export' | 'import'>('');
@@ -52,6 +54,11 @@ export default function ImportMapForm(props: ImportMapFormProps) {
setLoading('');
};
const handleRevoke = async () => {
await revoke();
onCancel();
};
const handleImportPreview = async (values: NamedImportMap) => {
setLoading('import');
const importMap = convertToImportMap(values);
@@ -78,6 +85,13 @@ export default function ImportMapForm(props: ImportMapFormProps) {
<Panel.Title>
Import options
<div className={style.buttonRow}>
{!isSpreadsheet && (
<Tooltip label='Revoke the google authentication'>
<Button variant='ontime-subtle' size='sm' onClick={handleRevoke} isDisabled={isLoading}>
Revoke
</Button>
</Tooltip>
)}
<Button variant='ontime-subtle' size='sm' onClick={onCancel} isDisabled={isLoading}>
Cancel
</Button>
@@ -116,6 +130,31 @@ export default function ImportMapForm(props: ImportMapFormProps) {
if (label === 'custom') {
return null;
}
if (label === 'Worksheet') {
return (
<tr key={importName as string}>
<td>{label}</td>
<td>
<Select
variant='ontime'
id={importName as string}
size='sm'
{...register(label as keyof NamedImportMap)}
>
{worksheetNames &&
worksheetNames.map((name) => {
return (
<option key={name} value={name}>
{name}
</option>
);
})}
</Select>
</td>
<td className={style.singleActionCell} />
</tr>
);
}
return (
<tr key={importName as string}>
<td>{label}</td>
apps/client/src/features/app-settings/panel/sources-panel/useGoogleSheet.ts
+1 -1
View File
@@ -23,7 +23,7 @@ export default function useGoogleSheet() {
const setCustomFields = useSheetStore((state) => state.setCustomFields);
/** whether the current session has been authenticated */
const verifyAuth = async (): Promise<{ authenticated: AuthenticationStatus } | void> => {
const verifyAuth = async (): Promise<{ authenticated: AuthenticationStatus; sheetId: string } | void> => {
try {
return verifyAuthenticationStatus();
} catch (_error) {
apps/client/src/features/app-settings/panel/sources-panel/useSheetStore.ts
+5 -6
View File
@@ -5,13 +5,12 @@ import { create } from 'zustand';
type SheetStore = {
stepData: typeof initialStepData;
patchStepData: (patch: Partial<typeof initialStepData>) => void;
setWorksheets: (worksheetNames: string[] | null) => void;
worksheetNames: string[] | null;
spreadsheet: File | null;
setSpreadsheet: (spreadsheet: File | null) => void;
//gSheet
sheetId: string | null;
setSheetId: (sheetId: string | null) => void;
authenticationStatus: AuthenticationStatus;
setAuthenticationStatus: (status: AuthenticationStatus) => void;
@@ -39,7 +38,7 @@ const initialStepData = {
const initialState = {
stepData: initialStepData,
spreadsheet: null,
worksheetNames: null,
sheetId: null,
authenticationStatus: 'not_authenticated' as AuthenticationStatus,
rundown: null,
@@ -55,7 +54,7 @@ export const useSheetStore = create<SheetStore>((set, get) => ({
set({ stepData: { ...stepData, ...patch } });
},
setSpreadsheet: (spreadsheet: File | null) => set({ spreadsheet }),
setWorksheets: (worksheetNames: string[] | null) => set({ worksheetNames }),
setSheetId: (sheetId: string | null) => set({ sheetId }),
apps/server/src/api-data/db/db.controller.ts
-26
View File
@@ -8,7 +8,6 @@ import {
} from 'ontime-types';
import type { Request, Response } from 'express';
import fs from 'fs';
import { failEmptyObjects } from '../../utils/routerUtils.js';
import { resolveDbPath, resolveProjectsDirectory } from '../../setup/index.js';
@@ -17,7 +16,6 @@ import * as projectService from '../../services/project-service/ProjectService.j
import { ensureJsonExtension } from '../../utils/fileManagement.js';
import { generateUniqueFileName } from '../../utils/generateUniqueFilename.js';
import { appStateService } from '../../services/app-state-service/AppStateService.js';
import { handleMaybeExcel } from '../../utils/parser.js';
export async function patchPartialProjectFile(req: Request, res: Response<DatabaseModel | ErrorResponse>) {
// all fields are optional in validation
@@ -245,27 +243,3 @@ export async function getInfo(_req: Request, res: Response<GetInfo>) {
const info = await projectService.getInfo();
res.status(200).send(info);
}
/**
* uploads and parses an excel spreadsheet
* @returns parsed result
*/
export async function previewSpreadsheet(req: Request, res: Response) {
if (!req.file) {
res.status(400).send({ message: 'File not found' });
return;
}
try {
const filePath = req.file.path;
if (!fs.existsSync(filePath)) {
throw new Error('Upload failed');
}
const options = JSON.parse(req.body.options);
const { data } = handleMaybeExcel(filePath, options);
res.status(200).send(data);
} catch (error) {
res.status(500).send({ message: String(error) });
}
}
apps/server/src/api-data/db/db.middleware.ts
+1 -14
View File
@@ -1,7 +1,7 @@
import { Request } from 'express';
import multer, { FileFilterCallback } from 'multer';
import { EXCEL_MIME, JSON_MIME } from '../../utils/parser.js';
import { JSON_MIME } from '../../utils/parser.js';
import { storage } from '../../utils/upload.js';
const filterProjectFile = (_req: Request, file: Express.Multer.File, cb: FileFilterCallback) => {
@@ -12,21 +12,8 @@ const filterProjectFile = (_req: Request, file: Express.Multer.File, cb: FileFil
}
};
const filterSpreadsheet = (_req: Request, file: Express.Multer.File, cb: FileFilterCallback) => {
if (file.mimetype.includes(EXCEL_MIME)) {
cb(null, true);
} else {
cb(null, false);
}
};
// Build multer uploader for a single file
export const uploadProjectFile = multer({
storage,
fileFilter: filterProjectFile,
}).single('project');
export const uploadSpreadsheet = multer({
storage,
fileFilter: filterSpreadsheet,
}).single('spreadsheet');
apps/server/src/api-data/db/db.router.ts
+5 -9
View File
@@ -2,18 +2,17 @@ import express from 'express';
import {
createProjectFile,
projectDownload,
deleteProjectFile,
duplicateProjectFile,
getInfo,
listProjects,
patchPartialProjectFile,
previewSpreadsheet,
loadProject,
duplicateProjectFile,
renameProjectFile,
patchPartialProjectFile,
postProjectFile,
projectDownload,
renameProjectFile,
} from './db.controller.js';
import { uploadProjectFile, uploadSpreadsheet } from './db.middleware.js';
import { uploadProjectFile } from './db.middleware.js';
import {
projectSanitiser,
sanitizeProjectFilename,
@@ -39,6 +38,3 @@ router.put('/:filename/rename', validateProjectRename, sanitizeProjectFilename,
router.delete('/:filename', sanitizeProjectFilename, deleteProjectFile);
router.get('/info', getInfo);
// TODO: validate import map
router.post('/spreadsheet/preview', uploadSpreadsheet, previewSpreadsheet);
apps/server/src/api-data/excel/excel.controller.ts
+40
View File
@@ -0,0 +1,40 @@
/**
* This module encapsulates logic related to
* Google Sheets
*/
import { Request, Response } from 'express';
import { generateRundownPreview, listWorksheets, saveExcelFile } from './excel.service.js';
export async function postExcel(req: Request, res: Response) {
try {
const filePath = req.file.path;
await saveExcelFile(filePath);
res.status(200).send();
} catch (error) {
res.status(500).send({ message: String(error) });
}
}
export async function getWorksheets(req: Request, res: Response) {
try {
const names = listWorksheets();
res.status(200).send(names);
} catch (error) {
res.status(500).send({ message: String(error) });
}
}
/**
* parses an Excel spreadsheet
* @returns parsed result
*/
export async function previewExcel(req: Request, res: Response) {
try {
const { options } = req.body;
const data = generateRundownPreview(options);
res.status(200).send(data);
} catch (error) {
res.status(500).send({ message: String(error) });
}
}
apps/server/src/api-data/excel/excel.middleware.ts
+17
View File
@@ -0,0 +1,17 @@
import { Request } from 'express';
import multer, { FileFilterCallback } from 'multer';
import { EXCEL_MIME } from '../../utils/parser.js';
import { storage } from '../../utils/upload.js';
const filterExcel = (_req: Request, file: Express.Multer.File, cb: FileFilterCallback) => {
if (file.mimetype.includes(EXCEL_MIME)) {
cb(null, true);
} else {
cb(null, false);
}
};
export const uploadExcel = multer({
storage,
fileFilter: filterExcel,
}).single('excel');
apps/server/src/api-data/excel/excel.router.ts
+16
View File
@@ -0,0 +1,16 @@
/**
* This is a feature specific router for integration with Excel
*/
import express from 'express';
import { uploadExcel } from './excel.middleware.js';
import { getWorksheets, postExcel, previewExcel } from './excel.controller.js';
import { validateFileExists, validateImportMapOptions } from './excel.validation.js';
export const router = express.Router();
router.post('/upload', uploadExcel, validateFileExists, postExcel);
router.get('/worksheets', getWorksheets);
router.post('/preview', validateImportMapOptions, previewExcel);
// TODO: validate import map
apps/server/src/api-data/excel/excel.service.ts
+53
View File
@@ -0,0 +1,53 @@
/**
* This module encapsulates logic related to
* Google Sheets
*/
import { extname } from 'path';
import { existsSync } from 'fs';
import { ImportMap } from 'ontime-utils';
import xlsx from 'node-xlsx';
import { parseExcel } from '../../utils/parser.js';
import { parseCustomFields, parseRundown } from '../../utils/parserFunctions.js';
import { deleteFile } from '../../utils/parserUtils.js';
let excelData: { name: string; data: unknown[][] }[] = [];
export async function saveExcelFile(filePath: string) {
if (!existsSync(filePath)) {
throw new Error('Upload of excel file failed');
}
if (extname(filePath) != '.xlsx') {
throw new Error('Wrong file format');
}
excelData = xlsx.parse(filePath, { cellDates: true });
await deleteFile(filePath);
}
export function listWorksheets() {
return excelData.map((value) => value.name);
}
export function generateRundownPreview(options: ImportMap) {
const data = excelData.find(({ name }) => name.toLowerCase() === options.worksheet.toLowerCase())?.data;
if (!data) {
throw new Error(`Could not find data to import, maybe the worksheet name is incorrect: ${options.worksheet}`);
}
const dataFromExcel = parseExcel(data, options);
// we run the parsed data through an extra step to ensure the objects shape
const result = { rundown: [], customFields: {} };
result.rundown = parseRundown(dataFromExcel);
if (result.rundown.length < 1) {
throw new Error(`Could not find data to import in the worksheet: ${options.worksheet}`);
}
result.customFields = parseCustomFields(dataFromExcel);
//clear the data
excelData = [];
return result;
}
apps/server/src/api-data/excel/excel.validation.ts
+28
View File
@@ -0,0 +1,28 @@
import { isImportMap } from 'ontime-utils';
import { body, validationResult } from 'express-validator';
import { NextFunction, Request, Response } from 'express';
export const validateFileExists = [
(req: Request, res: Response, next: NextFunction) => {
if (!req.file) {
return res.status(422).json({ errors: 'File not found' });
}
next();
},
];
export const validateImportMapOptions = [
body('options')
.exists()
.isObject()
.custom((content) => {
return isImportMap(content);
}),
(req: Request, res: Response, next: NextFunction) => {
const errors = validationResult(req);
if (!errors.isEmpty()) return res.status(422).json({ errors: errors.array() });
next();
},
];
apps/server/src/api-data/index.ts
+2
View File
@@ -9,6 +9,7 @@ import { router as projectRouter } from './project/project.router.js';
import { router as rundownRouter } from './rundown/rundown.router.js';
import { router as settingsRouter } from './settings/settings.router.js';
import { router as sheetsRouter } from './sheets/sheets.router.js';
import { router as excelRouter } from './excel/excel.router.js';
import { router as viewSettingsRouter } from './view-settings/viewSettings.router.js';
export const appRouter = express.Router();
@@ -21,5 +22,6 @@ appRouter.use('/project', projectRouter);
appRouter.use('/rundown', rundownRouter);
appRouter.use('/settings', settingsRouter);
appRouter.use('/sheets', sheetsRouter);
appRouter.use('/excel', excelRouter);
appRouter.use('/url-presets', urlPresetsRouter);
appRouter.use('/view-settings', viewSettingsRouter);
apps/server/src/api-data/sheets/sheets.controller.ts
+11
View File
@@ -14,6 +14,7 @@ import {
hasAuth,
download,
upload,
getWorksheetOptions,
} from '../../services/sheet-service/SheetService.js';
export async function requestConnection(req: Request, res: Response) {
@@ -56,6 +57,16 @@ export async function revokeAuthentication(_req: Request, res: Response) {
}
}
export async function getWorksheetNamesFromSheet(req: Request, res: Response) {
try {
const { sheetId } = req.params;
const { worksheetOptions } = await getWorksheetOptions(sheetId);
res.status(200).send(worksheetOptions);
} catch (error) {
res.status(500).send({ message: String(error) });
}
}
export async function readFromSheet(req: Request, res: Response) {
try {
const { sheetId } = req.params;
apps/server/src/api-data/sheets/sheets.router.ts
+4 -1
View File
@@ -5,6 +5,7 @@
import express from 'express';
import {
getWorksheetNamesFromSheet,
readFromSheet,
requestConnection,
revokeAuthentication,
@@ -12,7 +13,7 @@ import {
writeToSheet,
} from './sheets.controller.js';
import { uploadClientSecret } from './sheets.middleware.js';
import { validateRequestConnection, validateSheetOptions } from './sheets.validation.js';
import { validateRequestConnection, validateSheetId, validateSheetOptions } from './sheets.validation.js';
export const router = express.Router();
@@ -21,5 +22,7 @@ router.post('/:sheetId/connect', uploadClientSecret, validateRequestConnection,
router.post('/revoke', revokeAuthentication);
router.post('/:sheetId/worksheets', validateSheetId, getWorksheetNamesFromSheet);
router.post('/:sheetId/read', validateSheetOptions, readFromSheet);
router.post('/:sheetId/write', validateSheetOptions, writeToSheet);
apps/server/src/api-data/sheets/sheets.validation.ts
+10
View File
@@ -20,6 +20,16 @@ export const validateRequestConnection = [
},
];
export const validateSheetId = [
param('sheetId').exists().isString(),
(req: Request, res: Response, next: NextFunction) => {
const errors = validationResult(req);
if (!errors.isEmpty()) return res.status(422).json({ errors: errors.array() });
next();
},
];
export const validateSheetOptions = [
param('sheetId').exists().isString(),
body('options')
apps/server/src/services/sheet-service/SheetService.ts
+4 -4
View File
@@ -12,7 +12,7 @@ import got from 'got';
import { resolveSheetsDirectory } from '../../setup/index.js';
import { ensureDirectory } from '../../utils/fileManagement.js';
import { type ClientSecret, cellRequestFromEvent, getA1Notation, validateClientSecret } from './sheetUtils.js';
import { cellRequestFromEvent, type ClientSecret, getA1Notation, validateClientSecret } from './sheetUtils.js';
import { ImportMap } from 'ontime-utils';
import { parseExcel } from '../../utils/parser.js';
import { logger } from '../../classes/Logger.js';
@@ -190,11 +190,11 @@ function verifyConnection(
}
}
export function hasAuth(): { authenticated: AuthenticationStatus } {
export function hasAuth(): { authenticated: AuthenticationStatus; sheetId: string } {
if (cleanupTimeout) {
return { authenticated: 'pending' };
return { authenticated: 'pending', sheetId: currentSheetId };
}
return { authenticated: currentAuthClient ? 'authenticated' : 'not_authenticated' };
return { authenticated: currentAuthClient ? 'authenticated' : 'not_authenticated', sheetId: currentSheetId };
}
async function verifySheet(
apps/server/src/services/sheet-service/__tests__/sheetUtils.test.ts
+3 -3
View File
@@ -108,7 +108,7 @@ describe('cellRequestFromEvent()', () => {
expect(result).toStrictEqual(millisToString(event.duration));
});
test('boolean to x', () => {
test('boolean to TRUE', () => {
const event: OntimeEvent = {
type: SupportedEvent.Event,
cue: '1',
@@ -149,8 +149,8 @@ describe('cellRequestFromEvent()', () => {
timeDanger: { row: 1, col: 41 },
};
const result = cellRequestFromEvent(event, 1, 1234, metadata);
expect(result.updateCells.rows[0].values[11].userEnteredValue.stringValue).toStrictEqual('x');
expect(result.updateCells.rows[0].values[12].userEnteredValue.stringValue).toStrictEqual('');
expect(result.updateCells.rows[0].values[11].userEnteredValue.boolValue).toStrictEqual(true);
expect(result.updateCells.rows[0].values[12].userEnteredValue.boolValue).toStrictEqual(false);
});
test('spacing in metadata', () => {
apps/server/src/services/sheet-service/sheetUtils.ts
+2 -2
View File
@@ -1,4 +1,4 @@
import { OntimeRundownEntry, isOntimeBlock, isOntimeEvent } from 'ontime-types';
import { isOntimeBlock, isOntimeEvent, OntimeRundownEntry } from 'ontime-types';
import { millisToString } from 'ontime-utils';
import { sheets_v4 } from '@googleapis/sheets';
@@ -111,7 +111,7 @@ export function cellRequestFromEvent(
});
} else if (typeof event[key] === 'boolean') {
returnRows.push({
userEnteredValue: { stringValue: event[key] ? 'x' : '' },
userEnteredValue: { boolValue: event[key] },
});
} else {
returnRows.push({});
apps/server/src/utils/__tests__/parser.test.ts
+64 -20
View File
@@ -5,20 +5,20 @@ import {
DatabaseModel,
EndAction,
OntimeEvent,
OntimeRundown,
ProjectData,
Settings,
SupportedEvent,
TimeStrategy,
TimerType,
TimeStrategy,
ViewSettings,
OntimeRundown,
} from 'ontime-types';
import { dbModel } from '../../models/dataModel.js';
import { parseExcel, parseJson, createEvent, getCustomFieldData } from '../parser.js';
import { createEvent, getCustomFieldData, parseExcel, parseJson } from '../parser.js';
import { makeString } from '../parserUtils.js';
import { parseUrlPresets, parseViewSettings } from '../parserFunctions.js';
import { parseRundown, parseUrlPresets, parseViewSettings } from '../parserFunctions.js';
describe('test json parser with valid def', () => {
const testData: Partial<DatabaseModel> = {
@@ -795,8 +795,8 @@ describe('parseExcel()', () => {
'Public',
'Skip',
'Notes',
'test0',
'test1',
't0',
'UpperCaseFromSheet',
'test2',
'test3',
'test4',
@@ -809,8 +809,8 @@ describe('parseExcel()', () => {
'cue',
],
[
'1899-12-30T07:00:00.000Z',
'1899-12-30T08:00:10.000Z',
'07:00:00',
'08:00:10',
'Guest Welcome',
'',
'',
@@ -831,8 +831,8 @@ describe('parseExcel()', () => {
101,
],
[
'1899-12-30T08:00:00.000Z',
'1899-12-30T08:30:00.000Z',
'08:00:00',
'08:30:00',
'A song from the hearth',
'load-next',
'clock',
@@ -858,9 +858,9 @@ describe('parseExcel()', () => {
// partial import map with only custom fields
const importMap = {
custom: {
user0: 'test0',
user1: 'test1',
user2: 'test2',
user0: 't0',
user1: 'UpperCaseFromSheet',
UpperCaseFromOntime: 'test2',
user3: 'test3',
user4: 'test4',
user5: 'test5',
@@ -874,8 +874,8 @@ describe('parseExcel()', () => {
// TODO: update tests once import is resolved
const expectedParsedRundown = [
{
//timeStart: 28800000,
//timeEnd: 32410000,
timeStart: 25200000,
timeEnd: 28810000,
title: 'Guest Welcome',
timerType: 'count-down',
endAction: 'none',
@@ -885,7 +885,7 @@ describe('parseExcel()', () => {
custom: {
user0: { value: 'a0' },
user1: { value: 'a1' },
user2: { value: 'a2' },
UpperCaseFromOntime: { value: 'a2' },
user3: { value: 'a3' },
user4: { value: 'a4' },
user5: { value: 'a5' },
@@ -899,8 +899,8 @@ describe('parseExcel()', () => {
cue: '101',
},
{
//timeStart: 32400000,
//timeEnd: 34200000,
timeStart: 28800000,
timeEnd: 30600000,
title: 'A song from the hearth',
timerType: 'clock',
endAction: 'load-next',
@@ -929,10 +929,10 @@ describe('parseExcel()', () => {
colour: '',
label: 'user1',
},
user2: {
UpperCaseFromOntime: {
type: 'string',
colour: '',
label: 'user2',
label: 'UpperCaseFromOntime',
},
user3: {
type: 'string',
@@ -1363,4 +1363,48 @@ describe('parseExcel()', () => {
expect(result.rundown.at(1).type).toBe(SupportedEvent.Event);
expect((result.rundown.at(1) as OntimeEvent).timerType).toBe(TimerType.CountDown);
});
it('am/pm conversion to 24h', () => {
const testData = [
['Time Start', 'Time End', 'Title', 'End Action', 'Public', 'Skip', 'Notes', 'Colour', 'cue'],
['4:30:00', '4:36:00', 'A song from the hearth', 'load-next', 'x', '', 'Rainbow chase', '#F00', 102],
['9:45:00', '10:56:00', 'Green grass', 'load-next', 'x', '', 'Rainbow chase', '#0F0', 103],
['16:30:00', '16:36:00', 'A song from the hearth', 'load-next', 'x', '', 'Rainbow chase', '#F00', 102],
['21:45:00', '22:56:00', 'Green grass', 'load-next', 'x', '', 'Rainbow chase', '#0F0', 103],
['4:30:00AM', '4:36:00AM', 'A song from the hearth', 'load-next', 'x', '', 'Rainbow chase', '#F00', 102],
['9:45:00AM', '10:56:00AM', 'Green grass', 'load-next', 'x', '', 'Rainbow chase', '#0F0', 103],
['4:30:00PM', '4:36:00PM', 'A song from the hearth', 'load-next', 'x', '', 'Rainbow chase', '#F00', 102],
['9:45:00PM', '10:56:00PM', 'Green grass', 'load-next', 'x', '', 'Rainbow chase', '#0F0', 103],
[],
];
const importMap = {
worksheet: 'event schedule',
timeStart: 'time start',
timeEnd: 'time end',
duration: 'duration',
cue: 'cue',
title: 'title',
isPublic: 'public',
skip: 'skip',
note: 'notes',
colour: 'colour',
endAction: 'end action',
timerType: 'timer type',
timeWarning: 'warning time',
timeDanger: 'danger time',
custom: {},
};
const result = parseExcel(testData, importMap);
const rundown = parseRundown(result);
const events = rundown.filter((e) => e.type === SupportedEvent.Event) as OntimeEvent[];
expect(events.at(0).timeStart).toEqual(16200000);
expect(events.at(1).timeStart).toEqual(35100000);
expect(events.at(2).timeStart).toEqual(59400000);
expect(events.at(3).timeStart).toEqual(78300000);
expect(events.at(4).timeStart).toEqual(16200000);
expect(events.at(5).timeStart).toEqual(35100000);
expect(events.at(6).timeStart).toEqual(59400000);
expect(events.at(7).timeStart).toEqual(78300000);
});
});
apps/server/src/utils/__tests__/time.test.ts
+1 -1
View File
@@ -37,7 +37,7 @@ describe('parseExcelDate', () => {
});
describe('parses a time string that passes validation', () => {
const validFields = ['10:00:00', '10:00'];
const validFields = ['10:00:00', '10:00', '10:00AM', '10:00am', '10:00PM', '10:00pm'];
validFields.forEach((field) => {
it(`handles ${field}`, () => {
const millis = parseExcelDate(field);
apps/server/src/utils/parser.ts
+25 -68
View File
@@ -1,40 +1,37 @@
import {
generateId,
isImportMap,
type ImportMap,
defaultImportMap,
validateEndAction,
validateTimerType,
type ImportOptions,
validateTimes,
generateId,
type ImportMap,
isKnownTimerType,
validateEndAction,
validateLinkStart,
validateTimerType,
validateTimes,
} from 'ontime-utils';
import {
CustomFields,
DatabaseModel,
EventCustomFields,
OntimeBlock,
OntimeEvent,
OntimeRundown,
SupportedEvent,
TimeStrategy,
CustomFields,
EventCustomFields,
TimerType,
TimeStrategy,
} from 'ontime-types';
import xlsx from 'node-xlsx';
import { event as eventDef } from '../models/eventsDefinition.js';
import { dbModel } from '../models/dataModel.js';
import { deleteFile, makeString } from './parserUtils.js';
import { makeString } from './parserUtils.js';
import {
parseUrlPresets,
parseProject,
parseOsc,
parseCustomFields,
parseHttp,
parseOsc,
parseProject,
parseRundown,
parseSettings,
parseUrlPresets,
parseViewSettings,
parseCustomFields,
} from './parserFunctions.js';
import { parseExcelDate } from './time.js';
import { coerceBoolean } from './coerceType.js';
@@ -192,10 +189,6 @@ export const parseExcel = (excelData: unknown[][], options?: Partial<ImportMap>)
}
} else if (j === titleIndex) {
event.title = makeString(column, '');
// if this is a block, we have nothing else to import
if (event.type === SupportedEvent.Block) {
continue;
}
} else if (j === timeStartIndex) {
event.timeStart = parseExcelDate(column);
} else if (j === timeEndIndex) {
@@ -237,8 +230,8 @@ export const parseExcel = (excelData: unknown[][], options?: Partial<ImportMap>)
}
// check if it is a custom field
if (columnText in customFieldImportKeys) {
handlers.custom(rowIndex, j, columnText);
if (column in customFieldImportKeys) {
handlers.custom(rowIndex, j, column);
}
// else. we don't know how to handle this column
@@ -250,11 +243,16 @@ export const parseExcel = (excelData: unknown[][], options?: Partial<ImportMap>)
// if any data was found in row, push to array
const keysFound = Object.keys(event).length + Object.keys(eventCustomFields).length;
if (keysFound > 0) {
if (timerTypeIndex === null) {
event.timerType = TimerType.CountDown;
event.type = SupportedEvent.Event;
// if it is a Block type drop all other filed
if (event.type === SupportedEvent.Block) {
rundown.push({ type: event.type, id: event.id, title: event.title } as OntimeBlock);
} else {
if (timerTypeIndex === null) {
event.timerType = TimerType.CountDown;
event.type = SupportedEvent.Event;
}
rundown.push({ ...event, custom: { ...eventCustomFields } });
}
rundown.push({ ...event, custom: { ...eventCustomFields } });
}
});
@@ -363,44 +361,3 @@ export const createEvent = (eventArgs: Partial<OntimeEvent>, cueFallback: string
const event = createPatch(baseEvent, eventArgs);
return event;
};
type ResponseOK = {
data: Partial<DatabaseModel>;
};
/**
* Validates and calls parse on an excel file
*/
export function handleMaybeExcel(file: string, options: ImportOptions) {
const res: Partial<ResponseOK> = {};
if (!file.endsWith('.xlsx')) {
throw new Error('unexpected extension for spreadsheet');
}
// we need to check that the options are applicable
if (!isImportMap(options)) {
throw new Error('Got incorrect options for spreadsheet import');
}
const excelData = xlsx
.parse(file, { cellDates: true })
.find(({ name }) => name.toLowerCase() === options.worksheet.toLowerCase());
if (!excelData?.data) {
throw new Error(`Could not find data to import, maybe the worksheet name is incorrect: ${options.worksheet}`);
}
const dataFromExcel = parseExcel(excelData.data, options);
// we run the parsed data through an extra step to ensure the objects shape
res.data = {};
res.data.rundown = parseRundown(dataFromExcel);
if (res.data.rundown.length < 1) {
throw new Error(`Could not find data to import in the worksheet: ${options.worksheet}`);
}
res.data.customFields = parseCustomFields(dataFromExcel);
deleteFile(file);
return res;
}
apps/server/src/utils/time.ts
+25 -7
View File
@@ -35,22 +35,40 @@ const parse = (valueAsString: string): number => {
return Math.abs(parsed);
};
const stripAMPM = (value: string) => {
const lowerValue = value.toLowerCase();
if (lowerValue.endsWith('am')) {
return { sansPostfix: lowerValue.substring(0, lowerValue.length - 2), pastNoon: false };
} else if (lowerValue.endsWith('pm')) {
return { sansPostfix: lowerValue.substring(0, lowerValue.length - 2), pastNoon: true };
} else {
return { sansPostfix: lowerValue, pastNoon: false };
}
};
/**
* @description Parses a time string to millis, copied from client code
* @param {string} value - time string
* @param {boolean} fillLeft - autofill left = hours / right = seconds
* @returns {number} - time string in millis
*/
export const forgivingStringToMillis = (value: string, fillLeft = true): number => {
export const forgivingStringToMillis = (value: string, fillLeft: boolean = true): number => {
let millis = 0;
// check for AM/PM indicators
const { sansPostfix, pastNoon } = stripAMPM(value);
//if past noon indicated add 12 hours
if (pastNoon) {
millis = mth * 12;
}
// split string at known separators : , .
const separatorRegex = /[\s,:.]+/;
const [first, second, third] = value.split(separatorRegex);
const [first, second, third] = sansPostfix.split(separatorRegex);
if (first != null && second != null && third != null) {
// if string has three sections, treat as [hours] [minutes] [seconds]
millis = parse(first) * mth;
millis += parse(first) * mth;
millis += parse(second) * mtm;
millis += parse(third) * mts;
} else if (first != null && second == null && third == null) {
@@ -60,23 +78,23 @@ export const forgivingStringToMillis = (value: string, fillLeft = true): number
const hours = first.substring(0, 2);
const minutes = first.substring(2, 4);
const seconds = first.substring(4);
millis = parse(hours) * mth;
millis += parse(hours) * mth;
millis += parse(minutes) * mtm;
millis += parse(seconds) * mts;
} else {
// otherwise lets treat as [minutes]
millis = parse(first) * mtm;
millis += parse(first) * mtm;
}
}
if (first != null && second != null && third == null) {
// if string has two sections
if (fillLeft) {
// treat as [hours] [minutes]
millis = parse(first) * mth;
millis += parse(first) * mth;
millis += parse(second) * mtm;
} else {
// treat as [minutes] [seconds]
millis = parse(first) * mtm;
millis += parse(first) * mtm;
millis += parse(second) * mts;
}
}
e2e/tests/features/207-spreadsheet-import.spec.ts
+37
View File
@@ -0,0 +1,37 @@
import { expect, test } from '@playwright/test';
const fileToUpload = 'e2e/tests/fixtures/test-sheet.xlsx';
test('sheet file upload', async ({ page }) => {
await page.goto('http://localhost:4001/editor');
await page.getByRole('button', { name: 'Rundown menu' }).click();
await page.getByRole('menuitem', { name: 'Delete all events' }).click();
await page.getByRole('button', { name: 'Application settings' }).click();
await page.getByRole('button', { name: 'Import spreadsheet' }).click();
// workaround to upload file on hidden input
// https://playwright.dev/docs/api/class-filechooser
const fileChooserPromise = page.waitForEvent('filechooser');
await page.getByRole('button', { name: 'Import from spreadsheet', exact: true }).click();
const fileChooser = await fileChooserPromise;
await fileChooser.setFiles(fileToUpload);
await page.locator('[id="event\\ schedule"]').selectOption('Sheet2');
await page.locator('[id="event\\ schedule"]').selectOption('test');
await page.getByRole('button', { name: 'Import preview' }).click();
await page.getByRole('button', { name: 'Apply' }).click();
await page.getByRole('button', { name: 'Return' }).click();
await page.getByLabel('close').click();
// asset test events
const firstTitle = page.getByTestId('entry-1').getByTestId('block__title');
await expect(firstTitle).toHaveValue('Attempt light check');
const secondTitle = page.getByTestId('entry-2').getByTestId('block__title');
await expect(secondTitle).toHaveValue('Preset');
const thirdTitle = page.getByTestId('entry-3').getByTestId('block__title');
await expect(thirdTitle).toHaveValue('Albania');
});
e2e/tests/fixtures/test-sheet.xlsx
Vendored
BIN
View File
Binary file not shown.
packages/utils/src/date-utils/isTimeString.test.ts
+9
View File
@@ -24,3 +24,12 @@ describe('test isTimeString() function handle different separators', () => {
});
}
});
describe('test isTimeString() function handle AM/PM', () => {
const ts = ['2:10AM', '2:10PM', '2:10'];
for (const s of ts) {
it(`it handles ${s}`, () => {
expect(isTimeString(s)).toBe(true);
});
}
});
packages/utils/src/date-utils/isTimeString.ts
+1 -11
View File
@@ -4,16 +4,6 @@
* @returns {boolean} string represents time
*/
export const isTimeString = (text: string): boolean => {
// ^ # Start of string
// (?: # Try to match...
// (?: # Try to match...
// ([01]?\d|2[0-3]): # HH:
// )? # (optionally).
// ([0-5]?\d): # MM: (required)
// )? # (entire group optional, so either HH:MM:, MM: or nothing)
// ([0-5]?\d) # SS (required)
// $ # End of string
const regex = /^(?:(?:([01]?\d|2[0-3])[:,.])?([0-5]?\d)[:,.])?([0-5]?\d)$/;
const regex = /^(?:(?:([01]?\d|2[0-3])[:,.])?([0-5]?\d)[:,.])?([0-5]?\d)?(\s)?([APap][Mm])?$/;
return regex.test(text);
};