Improve spreadsheet import flow (#839)

* feat: add dropdown to worksheet selection

* feat: move revoke to separate button

* refactor: extract excel flow to separate controller and service

* chore: show spinner when uploading sheet and disable the other option

* chore: don't show "success" when canceling the import

* chore: export boolean types properly to sheets

apps/client/src/common/api/db.ts

@@ -1,14 +1,11 @@
 import axios, { AxiosResponse } from 'axios';
 import {
-  CustomFields,
   DatabaseModel,
   GetInfo,
   MessageResponse,
-  OntimeRundown,
   ProjectData,
   ProjectFileListResponse,
 } from 'ontime-types';
-import { ImportMap } from 'ontime-utils';
 
 import { makeCSV, makeTable } from '../../features/cuesheet/cuesheetUtils';
 
@@ -155,32 +152,6 @@ export async function getInfo(): Promise<GetInfo> {
   return res.data;
 }
 
-type PreviewSpreadsheetResponse = {
-  rundown: OntimeRundown;
-  customFields: CustomFields;
-};
-
-/**
- * Make patch changes to the objects in the db
- */
-export async function importSpreadsheetPreview(file: File, options: ImportMap): Promise<PreviewSpreadsheetResponse> {
-  const formData = new FormData();
-  formData.append('spreadsheet', file);
-  formData.append('options', JSON.stringify(options));
-
-  const response: AxiosResponse<PreviewSpreadsheetResponse> = await axios.post(
-    `${dbPath}/spreadsheet/preview`,
-    formData,
-    {
-      headers: {
-        'Content-Type': 'multipart/form-data',
-      },
-    },
-  );
-
-  return response.data;
-}
-
 /**
  * Utility function gets project from db
  * @param fileName

apps/client/src/common/api/excel.ts

@@ -0,0 +1,42 @@
+import axios, { AxiosResponse } from 'axios';
+import { CustomFields, OntimeRundown } from 'ontime-types';
+import { ImportMap } from 'ontime-utils';
+
+import { apiEntryUrl } from './constants';
+
+const excelPath = `${apiEntryUrl}/excel`;
+
+type PreviewSpreadsheetResponse = {
+  rundown: OntimeRundown;
+  customFields: CustomFields;
+};
+
+/**
+ * upload Excel file to server
+ * @return string - file ID op the uploaded file
+ */
+export async function upload(file: File) {
+  const formData = new FormData();
+  formData.append('excel', file);
+  await axios.post(`${excelPath}/upload`, formData, {
+    headers: {
+      'Content-Type': 'multipart/form-data',
+    },
+  });
+}
+
+/**
+ * Get Worksheet names
+ * @return string[] - array of available worksheets
+ */
+export async function getWorksheetNames(): Promise<string[]> {
+  const response: AxiosResponse<string[]> = await axios.get(`${excelPath}/worksheets`);
+  return response.data;
+}
+
+export async function importRundownPreview(options: ImportMap): Promise<PreviewSpreadsheetResponse> {
+  const response: AxiosResponse<PreviewSpreadsheetResponse> = await axios.post(`${excelPath}/preview`, {
+    options,
+  });
+  return response.data;
+}

apps/client/src/common/api/sheets.ts

@@ -1,4 +1,4 @@
-import axios from 'axios';
+import axios, { AxiosResponse } from 'axios';
 import { AuthenticationStatus, CustomFields, OntimeRundown } from 'ontime-types';
 import { ImportMap } from 'ontime-utils';
 
@@ -9,7 +9,10 @@ const sheetsPath = `${apiEntryUrl}/sheets`;
 /**
  * HTTP request to verify whether we are authenticated with Google Sheet service
  */
-export const verifyAuthenticationStatus = async (): Promise<{ authenticated: AuthenticationStatus }> => {
+export const verifyAuthenticationStatus = async (): Promise<{
+  authenticated: AuthenticationStatus;
+  sheetId: string;
+}> => {
   const response = await axios.get(`${sheetsPath}/connect`);
   return response.data;
 };
@@ -58,6 +61,11 @@ export const previewRundown = async (
   return response.data;
 };
 
+export const getWorksheetNames = async (sheetId: string): Promise<string[]> => {
+  const response: AxiosResponse<string[]> = await axios.post(`${sheetsPath}/${sheetId}/worksheets`);
+  return response.data;
+};
+
 /**
  * HTTP request to upload the rundown to a google sheet
  */

apps/client/src/common/utils/uploadUtils.ts

@@ -2,7 +2,7 @@
  * Collection of rules for pre-validating a spreadsheet
  * @param file
  */
-export function validateSpreadsheetImport(file: File) {
+export function validateExcelImport(file: File) {
   if (!isExcelFile(file)) {
     throw new Error('Unknown file type');
   }
@@ -12,7 +12,7 @@ export function validateSpreadsheetImport(file: File) {
     throw new Error('File is empty');
   }
 
-  // Limit file size of an excel file to around 10MB
+  // Limit file size of an Excel file to around 10MB
   if (file.size > 10_000_000) {
     throw new Error('File size limit (10MB) exceeded');
   }

apps/client/src/features/app-settings/panel/sources-panel/GSheetSetup.tsx

@@ -3,6 +3,7 @@ import { Button, Input } from '@chakra-ui/react';
 import { IoCheckmark } from '@react-icons/all-files/io5/IoCheckmark';
 import { IoShieldCheckmarkOutline } from '@react-icons/all-files/io5/IoShieldCheckmarkOutline';
 
+import { getWorksheetNames } from '../../../../common/api/sheets';
 import CopyTag from '../../../../common/components/copy-tag/CopyTag';
 import { openLink } from '../../../../common/utils/linkUtils';
 import * as Panel from '../PanelUtils';
@@ -27,6 +28,7 @@ export default function GSheetSetup(props: GSheetSetupProps) {
 
   const sheetId = useSheetStore((state) => state.sheetId);
   const setSheetId = useSheetStore((state) => state.setSheetId);
+  const setWorksheets = useSheetStore((state) => state.setWorksheets);
 
   const authenticationStatus = useSheetStore((state) => state.authenticationStatus);
   const setAuthenticationStatus = useSheetStore((state) => state.setAuthenticationStatus);
@@ -53,7 +55,6 @@ export default function GSheetSetup(props: GSheetSetupProps) {
   };
 
   const handleCancelFlow = async () => {
-    await handleRevoke();
     onCancel();
   };
 
@@ -89,6 +90,10 @@ export default function GSheetSetup(props: GSheetSetupProps) {
     if (result?.authenticated) {
       setAuthenticationStatus(result.authenticated);
       if (result.authenticated !== 'pending') {
+        if (result.authenticated == 'authenticated') {
+          const names = await getWorksheetNames(result.sheetId);
+          setWorksheets(names);
+        }
         setLoading('');
         return;
       }

apps/client/src/features/app-settings/panel/sources-panel/ImportReview.tsx

@@ -14,10 +14,11 @@ interface ImportReviewProps {
   rundown: OntimeRundown;
   customFields: CustomFields;
   onFinished: () => void;
+  onCancel: () => void;
 }
 
 export default function ImportReview(props: ImportReviewProps) {
-  const { rundown, customFields, onFinished } = props;
+  const { rundown, customFields, onFinished, onCancel } = props;
 
   const [loading, setLoading] = useState(false);
   const { importRundown } = useGoogleSheet();
@@ -25,7 +26,7 @@ export default function ImportReview(props: ImportReviewProps) {
 
   const handleCancel = () => {
     resetPreview();
-    onFinished();
+    onCancel();
   };
 
   const applyImport = async () => {

apps/client/src/features/app-settings/panel/sources-panel/SourcesPanel.tsx

@@ -4,9 +4,14 @@ import { IoCloudOutline } from '@react-icons/all-files/io5/IoCloudOutline';
 import { IoDownloadOutline } from '@react-icons/all-files/io5/IoDownloadOutline';
 import { ImportMap, unpackError } from 'ontime-utils';
 
-import { importSpreadsheetPreview } from '../../../../common/api/db';
+import {
+  getWorksheetNames as getWorksheetNamesExcel,
+  importRundownPreview as importRundownPreviewExcel,
+  upload as uploadExcel,
+} from '../../../../common/api/excel';
+import { getWorksheetNames } from '../../../../common/api/sheets';
 import { maybeAxiosError } from '../../../../common/api/utils';
-import { validateSpreadsheetImport } from '../../../../common/utils/uploadUtils';
+import { validateExcelImport } from '../../../../common/utils/uploadUtils';
 import * as Panel from '../PanelUtils';
 
 import ImportMapForm from './import-map/ImportMapForm';
@@ -21,36 +26,43 @@ import style from './SourcesPanel.module.scss';
 export default function SourcesPanel() {
   const [importFlow, setImportFlow] = useState<'none' | 'excel' | 'gsheet' | 'finished'>('none');
   const [error, setError] = useState('');
+  const [hasFile, setHasFile] = useState<'none' | 'loading' | 'done'>('none');
 
-  const { exportRundown, importRundownPreview, revoke, verifyAuth } = useGoogleSheet();
+  const { exportRundown, importRundownPreview, verifyAuth } = useGoogleSheet();
 
-  const spreadsheet = useSheetStore((state) => state.spreadsheet);
-  const setSpreadsheet = useSheetStore((state) => state.setSpreadsheet);
+  const setWorksheets = useSheetStore((state) => state.setWorksheets);
   const authenticationStatus = useSheetStore((state) => state.authenticationStatus);
   const setAuthenticationStatus = useSheetStore((state) => state.setAuthenticationStatus);
   const rundown = useSheetStore((state) => state.rundown);
   const setRundown = useSheetStore((state) => state.setRundown);
   const customFields = useSheetStore((state) => state.customFields);
   const setCustomFields = useSheetStore((state) => state.setCustomFields);
+  const setSheetId = useSheetStore((state) => state.setSheetId);
   const sheetId = useSheetStore((state) => state.sheetId);
 
   const fileInputRef = useRef<HTMLInputElement>(null);
 
-  const handleFile = (event: ChangeEvent<HTMLInputElement>) => {
+  const handleFile = async (event: ChangeEvent<HTMLInputElement>) => {
     const fileToUpload = event.target.files?.[0];
 
     if (!fileToUpload) {
-      setSpreadsheet(null);
+      setWorksheets(null);
+      setHasFile('none');
       return;
     }
     try {
-      validateSpreadsheetImport(fileToUpload);
-      setSpreadsheet(fileToUpload);
+      setHasFile('loading');
+      validateExcelImport(fileToUpload);
+      await uploadExcel(fileToUpload);
+      const names = await getWorksheetNamesExcel();
+      setWorksheets(names);
       setImportFlow('excel');
+      setHasFile('done');
     } catch (error) {
       const errorMessage = unpackError(error);
       setError(`Error uploading file: ${errorMessage}`);
-      setSpreadsheet(null);
+      setWorksheets(null);
+      setHasFile('none');
     }
   };
 
@@ -58,7 +70,16 @@ export default function SourcesPanel() {
     fileInputRef.current?.click();
   };
 
-  const openGSheetFlow = () => {
+  const openGSheetFlow = async () => {
+    const result = await verifyAuth();
+    if (result) {
+      setAuthenticationStatus(result.authenticated);
+      setSheetId(result.sheetId);
+      if (result.authenticated === 'authenticated' && result.sheetId) {
+        const names = await getWorksheetNames(result.sheetId);
+        setWorksheets(names);
+      }
+    }
     setImportFlow('gsheet');
   };
 
@@ -68,9 +89,8 @@ export default function SourcesPanel() {
 
   const handleSubmitImportPreview = async (importMap: ImportMap) => {
     if (importFlow === 'excel') {
-      if (!spreadsheet) return;
       try {
-        const previewData = await importSpreadsheetPreview(spreadsheet, importMap);
+        const previewData = await importRundownPreviewExcel(importMap);
         setRundown(previewData.rundown);
         setCustomFields(previewData.customFields);
       } catch (error) {
@@ -86,12 +106,9 @@ export default function SourcesPanel() {
 
   const cancelImportMap = async () => {
     setImportFlow('none');
-    if (spreadsheet) {
-      setSpreadsheet(null);
-    }
-
+    setHasFile('none');
+    setWorksheets(null);
     if (authenticationStatus === 'authenticated') {
-      await revoke();
       const result = await verifyAuth();
       if (result) {
         setAuthenticationStatus(result.authenticated);
@@ -102,7 +119,8 @@ export default function SourcesPanel() {
   const handleFinished = () => {
     setImportFlow('finished');
     setRundown(null);
-    setSpreadsheet(null);
+    setHasFile('none');
+    setWorksheets(null);
     setCustomFields(null);
   };
 
@@ -113,12 +131,11 @@ export default function SourcesPanel() {
 
   const isExcelFlow = importFlow === 'excel';
   const isGSheetFlow = importFlow === 'gsheet';
-  const hasFile = Boolean(spreadsheet);
   const isAuthenticated = authenticationStatus === 'authenticated';
   const showInput = importFlow === 'none';
   const showSuccess = importFlow === 'finished';
   const showAuth = isGSheetFlow && !isAuthenticated;
-  const showImportMap = (isGSheetFlow && isAuthenticated) || (isExcelFlow && hasFile);
+  const showImportMap = (isGSheetFlow && isAuthenticated) || (isExcelFlow && hasFile === 'done');
   const showReview = rundown !== null && customFields !== null;
 
   return (
@@ -141,13 +158,25 @@ export default function SourcesPanel() {
               />
               <div className={style.uploadSection}>
                 <div>
-                  <Button variant='ontime-filled' size='sm' leftIcon={<IoDownloadOutline />} onClick={handleUpload}>
+                  <Button
+                    variant='ontime-filled'
+                    size='sm'
+                    leftIcon={<IoDownloadOutline />}
+                    onClick={handleUpload}
+                    isLoading={hasFile === 'loading'}
+                  >
                     Import from spreadsheet
                   </Button>
                   <Panel.Description>Accepts .xlsx files</Panel.Description>
                 </div>
                 <div>
-                  <Button variant='ontime-filled' size='sm' leftIcon={<IoCloudOutline />} onClick={openGSheetFlow}>
+                  <Button
+                    variant='ontime-filled'
+                    size='sm'
+                    leftIcon={<IoCloudOutline />}
+                    onClick={openGSheetFlow}
+                    isDisabled={hasFile !== 'none'}
+                  >
                     Synchronise with Google
                   </Button>
                   <Panel.Description>Start authentication process</Panel.Description>
@@ -172,7 +201,14 @@ export default function SourcesPanel() {
               onSubmitImport={handleSubmitImportPreview}
             />
           )}
-          {showReview && <ImportReview rundown={rundown} customFields={customFields} onFinished={handleFinished} />}
+          {showReview && (
+            <ImportReview
+              rundown={rundown}
+              customFields={customFields}
+              onFinished={handleFinished}
+              onCancel={cancelImportMap}
+            />
+          )}
         </Panel.Card>
       </Panel.Section>
     </>

apps/client/src/features/app-settings/panel/sources-panel/import-map/ImportMapForm.tsx

@@ -1,12 +1,13 @@
 import { useState } from 'react';
 import { useFieldArray, useForm } from 'react-hook-form';
-import { Button, IconButton, Input } from '@chakra-ui/react';
+import { Button, IconButton, Input, Select, Tooltip } from '@chakra-ui/react';
 import { IoAdd } from '@react-icons/all-files/io5/IoAdd';
 import { IoTrash } from '@react-icons/all-files/io5/IoTrash';
 import { ImportMap } from 'ontime-utils';
 
 import { isAlphanumeric } from '../../../../../common/utils/regex';
 import * as Panel from '../../PanelUtils';
+import useGoogleSheet from '../useGoogleSheet';
 import { useSheetStore } from '../useSheetStore';
 
 import { convertToImportMap, getPersistedOptions, NamedImportMap, persistImportMap } from './importMapUtils';
@@ -23,7 +24,7 @@ interface ImportMapFormProps {
 export default function ImportMapForm(props: ImportMapFormProps) {
   const { isSpreadsheet, onCancel, onSubmitExport, onSubmitImport } = props;
   const namedImportMap = getPersistedOptions();
-
+  const { revoke } = useGoogleSheet();
   const {
     control,
     handleSubmit,
@@ -41,6 +42,7 @@ export default function ImportMapForm(props: ImportMapFormProps) {
   });
 
   const stepData = useSheetStore((state) => state.stepData);
+  const worksheetNames = useSheetStore((state) => state.worksheetNames);
 
   const [loading, setLoading] = useState<'' | 'export' | 'import'>('');
 
@@ -52,6 +54,11 @@ export default function ImportMapForm(props: ImportMapFormProps) {
     setLoading('');
   };
 
+  const handleRevoke = async () => {
+    await revoke();
+    onCancel();
+  };
+
   const handleImportPreview = async (values: NamedImportMap) => {
     setLoading('import');
     const importMap = convertToImportMap(values);
@@ -78,6 +85,13 @@ export default function ImportMapForm(props: ImportMapFormProps) {
       <Panel.Title>
         Import options
         <div className={style.buttonRow}>
+          {!isSpreadsheet && (
+            <Tooltip label='Revoke the google authentication'>
+              <Button variant='ontime-subtle' size='sm' onClick={handleRevoke} isDisabled={isLoading}>
+                Revoke
+              </Button>
+            </Tooltip>
+          )}
           <Button variant='ontime-subtle' size='sm' onClick={onCancel} isDisabled={isLoading}>
             Cancel
           </Button>
@@ -116,6 +130,31 @@ export default function ImportMapForm(props: ImportMapFormProps) {
             if (label === 'custom') {
               return null;
             }
+            if (label === 'Worksheet') {
+              return (
+                <tr key={importName as string}>
+                  <td>{label}</td>
+                  <td>
+                    <Select
+                      variant='ontime'
+                      id={importName as string}
+                      size='sm'
+                      {...register(label as keyof NamedImportMap)}
+                    >
+                      {worksheetNames &&
+                        worksheetNames.map((name) => {
+                          return (
+                            <option key={name} value={name}>
+                              {name}
+                            </option>
+                          );
+                        })}
+                    </Select>
+                  </td>
+                  <td className={style.singleActionCell} />
+                </tr>
+              );
+            }
             return (
               <tr key={importName as string}>
                 <td>{label}</td>

apps/client/src/features/app-settings/panel/sources-panel/useGoogleSheet.ts

@@ -23,7 +23,7 @@ export default function useGoogleSheet() {
   const setCustomFields = useSheetStore((state) => state.setCustomFields);
 
   /** whether the current session has been authenticated */
-  const verifyAuth = async (): Promise<{ authenticated: AuthenticationStatus } | void> => {
+  const verifyAuth = async (): Promise<{ authenticated: AuthenticationStatus; sheetId: string } | void> => {
     try {
       return verifyAuthenticationStatus();
     } catch (_error) {

apps/client/src/features/app-settings/panel/sources-panel/useSheetStore.ts

@@ -5,13 +5,12 @@ import { create } from 'zustand';
 type SheetStore = {
   stepData: typeof initialStepData;
   patchStepData: (patch: Partial<typeof initialStepData>) => void;
+  setWorksheets: (worksheetNames: string[] | null) => void;
+  worksheetNames: string[] | null;
 
-  spreadsheet: File | null;
-  setSpreadsheet: (spreadsheet: File | null) => void;
-
+  //gSheet
   sheetId: string | null;
   setSheetId: (sheetId: string | null) => void;
-
   authenticationStatus: AuthenticationStatus;
   setAuthenticationStatus: (status: AuthenticationStatus) => void;
 
@@ -39,7 +38,7 @@ const initialStepData = {
 
 const initialState = {
   stepData: initialStepData,
-  spreadsheet: null,
+  worksheetNames: null,
   sheetId: null,
   authenticationStatus: 'not_authenticated' as AuthenticationStatus,
   rundown: null,
@@ -55,7 +54,7 @@ export const useSheetStore = create<SheetStore>((set, get) => ({
     set({ stepData: { ...stepData, ...patch } });
   },
 
-  setSpreadsheet: (spreadsheet: File | null) => set({ spreadsheet }),
+  setWorksheets: (worksheetNames: string[] | null) => set({ worksheetNames }),
 
   setSheetId: (sheetId: string | null) => set({ sheetId }),