Merge pull request #23 from hakumizuki/feat/g2g-after-transfer-complete

feat: G2G after transfer complete

packages/app/src/components/Admin/G2GDataTransfer.tsx

@@ -27,9 +27,8 @@ const G2GDataTransfer = (): JSX.Element => {
   const [selectedCollections, setSelectedCollections] = useState<Set<string>>(new Set());
   const [optionsMap, setOptionsMap] = useState<any>({});
   const [isShowExportForm, setShowExportForm] = useState(false);
-  const [isExporting, setExporting] = useState(false);
-  // TODO: データのエクスポートが完了したことが分かるようにする
-  const [isExported, setExported] = useState(false);
+  const [isTransferring, setTransferring] = useState(false);
+  const [statusMessage, setStatusMessage] = useState('');
 
   const updateSelectedCollections = (newSelectedCollections: Set<string>) => {
     setSelectedCollections(newSelectedCollections);
@@ -56,35 +55,25 @@ const G2GDataTransfer = (): JSX.Element => {
 
     setCollections(filteredCollections);
     setSelectedCollections(new Set(filteredCollections));
-    setExporting(statusData.status.isExporting);
   }, []);
 
   const setupWebsocketEventHandler = useCallback(() => {
     if (socket != null) {
-      // websocket event
-      socket.on('admin:onProgressForExport', ({ currentCount, totalCount, progressList }) => {
-        setExporting(true);
+      socket.on('admin:onStartTransferMongoData', () => {
+        setTransferring(true);
+        setStatusMessage(t('Transferring DB data ...'));
       });
 
-      // websocket event
-      socket.on('admin:onTerminateForExport', ({ addedZipFileStat }) => {
-
-        setExporting(false);
-        setExported(true);
-
-        // TODO: toastSuccess, toastError
-        toastr.success(undefined, `New Archive Data '${addedZipFileStat.fileName}' is added`, {
-          closeButton: true,
-          progressBar: true,
-          newestOnTop: false,
-          showDuration: '100',
-          hideDuration: '100',
-          timeOut: '1200',
-          extendedTimeOut: '150',
-        });
+      socket.on('admin:onStartTransferAttachments', () => {
+        setStatusMessage(t('Transferring attachment files ...'));
+      });
+
+      socket.on('admin:onFinishTransfer', () => {
+        setTransferring(false);
+        setStatusMessage(t('Successfully transferred GROWI. Now you can use new GROWI !'));
       });
     }
-  }, [socket]);
+  }, [socket, t]);
 
   const { transferKey, generateTransferKeyWithThrottle } = useGenerateTransferKeyWithThrottle();
 
@@ -98,7 +87,7 @@ const G2GDataTransfer = (): JSX.Element => {
     try {
       await customAxios.post('/_api/v3/g2g-transfer/transfer', {
         transferKey: startTransferKey,
-        collections: selectedCollections,
+        collections: Array.from(selectedCollections),
         optionsMap,
       });
     }
@@ -117,7 +106,7 @@ const G2GDataTransfer = (): JSX.Element => {
     <div data-testid="admin-export-archive-data">
       <h2 className="border-bottom">{t('admin:g2g_data_transfer.transfer_data_to_another_growi')}</h2>
 
-      <button type="button" className="btn btn-outline-secondary mt-4" disabled={isExporting} onClick={() => setShowExportForm(!isShowExportForm)}>
+      <button type="button" className="btn btn-outline-secondary mt-4" disabled={isTransferring} onClick={() => setShowExportForm(!isShowExportForm)}>
         {t('admin:g2g_data_transfer.advanced_options')}
       </button>
 
@@ -150,6 +139,14 @@ const G2GDataTransfer = (): JSX.Element => {
         </div>
       </form>
 
+
+      {statusMessage != null && <p>{statusMessage}</p>}
+      {isTransferring && (
+        <div className="text-muted text-center">
+          <i className="fa fa-2x fa-spinner fa-pulse mr-1"></i>
+        </div>
+      )}
+
       <h2 className="border-bottom mt-5">{t('admin:g2g_data_transfer.transfer_data_to_this_growi')}</h2>
 
       <div className="form-group row mt-4">

packages/app/src/server/routes/apiv3/g2g-transfer.ts

@@ -204,6 +204,12 @@ module.exports = (crowi: Crowi): Router => {
         if (collectionName === 'pages' && options.mode === 'insert') {
           throw Error('`insert` is not available as an import setting for pages collection');
         }
+        if (collectionName === 'attachmentFiles.chunks') {
+          throw Error('`attachmentFiles.chunks` must not be transferred. Please omit it from request body `collections`.');
+        }
+        if (collectionName === 'attachmentFiles.files') {
+          throw Error('`attachmentFiles.files` must not be transferred. Please omit it from request body `collections`.');
+        }
 
         const importSettings = importService.generateImportSettings(options.mode);
 
@@ -224,27 +230,20 @@ module.exports = (crowi: Crowi): Router => {
      * import
      */
     try {
-      importService.import(collections, importSettingsMap);
+      await importService.import(collections, importSettingsMap);
+      await crowi?.setUpFileUpload(true);
+      await crowi?.appService?.setupAfterInstall();
     }
     catch (err) {
       logger.error(err);
-      return;
+      return res.apiv3Err(new ErrorV3('Failed to import.', 'failed_to_import'), 500);
     }
 
-    // try {
-    //   await g2gTransferReceiverService.receive(file.stream);
-    // }
-    // catch (err) {
-    //   logger.error(err);
-    //   return res.apiv3Err(new ErrorV3('Error occurred while importing transfer data.', 'failed_to_receive'));
-    // }
-
     return res.apiv3({ message: 'Successfully started to receive transfer data.' });
   });
 
-  // TODO: verify transfer key
   // This endpoint uses multer's MemoryStorage since the received data should be persisted directly on attachment storage.
-  receiveRouter.post('/attachment', uploadsForAttachment.single('content'), /* verifyAndExtractTransferKey, */
+  receiveRouter.post('/attachment', uploadsForAttachment.single('content'), verifyAndExtractTransferKey,
     async(req: Request & { transferKey: TransferKey }, res: ApiV3Response) => {
       const { file } = req;
       const { attachmentMetadata } = req.body;
@@ -346,7 +345,7 @@ module.exports = (crowi: Crowi): Router => {
 
     // Start transfer
     try {
-      await g2gTransferPusherService.startTransfer(tk, req.user, collections, optionsMap);
+      await g2gTransferPusherService.startTransfer(tk, req.user, toGROWIInfo, collections, optionsMap);
     }
     catch (err) {
       logger.error(err);

packages/app/src/server/service/g2g-transfer.ts

@@ -24,7 +24,12 @@ export const X_GROWI_TRANSFER_KEY_HEADER_NAME = 'x-growi-transfer-key';
 export type IDataGROWIInfo = {
   version: string
   userUpperLimit: number | null // Handle null as Infinity
-  attachmentInfo: any
+  attachmentInfo: {
+    type: string,
+    bucket?: string,
+    customEndpoint?: string, // for S3
+    uploadNamespace?: string, // for GCS
+  };
 }
 
 interface Pusher {
@@ -49,7 +54,7 @@ interface Pusher {
    * @param {string[]} collections Collection name string array
    * @param {any} optionsMap Options map
    */
-  startTransfer(tk: TransferKey, user: any, collections: string[], optionsMap: any): Promise<void>
+  startTransfer(tk: TransferKey, user: any, toGROWIInfo: IDataGROWIInfo, collections: string[], optionsMap: any): Promise<void>
 }
 
 interface Receiver {
@@ -89,6 +94,7 @@ const generateAxiosRequestConfigWithTransferKey = (tk: TransferKey, additionalHe
       ...additionalHeaders,
       [X_GROWI_TRANSFER_KEY_HEADER_NAME]: key,
     },
+    maxBodyLength: Infinity,
   };
 };
 
@@ -168,7 +174,6 @@ export class G2GTransferPusherService implements Pusher {
   public async transferAttachments(tk: TransferKey): Promise<void> {
     const BATCH_SIZE = 100;
 
-    const { appUrl, key } = tk;
     const { fileUploadService } = this.crowi;
     const Attachment = this.crowi.model('Attachment');
 
@@ -192,7 +197,7 @@ export class G2GTransferPusherService implements Pusher {
         // TODO: refresh transfer key per 1 hour
         // post each attachment file data to receiver
         try {
-          this.doTransferAttachment(tk, attachment, fileStream);
+          await this.doTransferAttachment(tk, attachment, fileStream);
         }
         catch (errs) {
           logger.error(`Error occured when uploading attachment(ID=${attachment.id})`, errs);
@@ -212,8 +217,37 @@ export class G2GTransferPusherService implements Pusher {
     }
   }
 
-  public async startTransfer(tk: TransferKey, user: any, collections: string[], optionsMap: any): Promise<void> {
-    const { appUrl, key } = tk;
+  // eslint-disable-next-line max-len
+  public async startTransfer(tk: TransferKey, user: any, toGROWIInfo: IDataGROWIInfo, collections: string[], optionsMap: any, shouldEmit = true): Promise<void> {
+    const socket = this.crowi.socketIoService.getDefaultSocket();
+
+    if (shouldEmit) socket.emit('admin:onStartTransferMongoData', {});
+
+    if (toGROWIInfo.attachmentInfo.type === 'none') {
+      try {
+        const targetConfigKeys = [
+          'app:fileUploadType',
+          'app:useOnlyEnvVarForFileUploadType',
+          'aws:referenceFileWithRelayMode',
+          'aws:lifetimeSecForTemporaryUrl',
+          'gcs:apiKeyJsonPath',
+          'gcs:bucket',
+          'gcs:uploadNamespace',
+          'gcs:referenceFileWithRelayMode',
+          'gcs:useOnlyEnvVarsForSomeOptions',
+        ];
+
+        const updateConfigs = Object.fromEntries(targetConfigKeys.map((key) => {
+          return [key, this.crowi.configManager.getConfig('crowi', key)];
+        }));
+
+        await this.crowi.configManager.updateConfigsInTheSameNamespace('crowi', updateConfigs);
+      }
+      catch (err) {
+        logger.error(err);
+        throw err;
+      }
+    }
 
     let zipFileStream: ReadStream;
     try {
@@ -244,7 +278,7 @@ export class G2GTransferPusherService implements Pusher {
       logger.error(errs);
       if (!Array.isArray(errs)) {
         // TODO: socker.emit(failed_to_transfer);
-        return;
+        throw errs;
       }
 
       const err = errs[0];
@@ -252,8 +286,22 @@ export class G2GTransferPusherService implements Pusher {
 
 
       // TODO: socker.emit(failed_to_transfer);
-      return;
+      throw errs;
+    }
+
+    if (shouldEmit) socket.emit('admin:onStartTransferAttachments', {});
+
+    try {
+      await this.transferAttachments(tk);
     }
+    catch (err) {
+      logger.error(err);
+      throw err;
+    }
+
+    if (shouldEmit) socket.emit('admin:onFinishTransfer', {});
+
+    return;
   }
 
   /**

packages/app/src/server/service/import.js

@@ -212,6 +212,8 @@ class ImportService {
 
     this.currentProgressingStatus = null;
     this.emitTerminateEvent();
+
+    await this.crowi.configManager.loadConfigs();
   }
 
   /**