Merge pull request #14 from hakumizuki/feat/g2g-nextjs-copy-attachments-2

Copy attachment data

packages/app/config/logger/config.dev.js

@@ -27,6 +27,7 @@ module.exports = {
   'growi-plugin:*': 'debug',
   // 'growi:InterceptorManager': 'debug',
   'growi:service:search-delegator:elasticsearch': 'debug',
+  'growi:service:g2g-transfer': 'debug',
 
   /*
    * configure level for client

packages/app/src/server/routes/apiv3/g2g-transfer.ts

@@ -1,4 +1,5 @@
 import path from 'path';
+import { Readable } from 'stream';
 
 import express, { NextFunction, Request, Router } from 'express';
 import { body } from 'express-validator';
@@ -65,6 +66,10 @@ module.exports = (crowi: Crowi): Router => {
     },
   });
 
+  const uploadsForAttachment = multer({
+    storage: multer.memoryStorage(),
+  });
+
   const isInstalled = crowi.configManager?.getConfig('crowi', 'app:installed');
 
   const accessTokenParser = require('../../middlewares/access-token-parser')(crowi);
@@ -255,6 +260,29 @@ module.exports = (crowi: Crowi): Router => {
     return res.apiv3({ message: 'Successfully started to receive transfer data.' });
   });
 
+  // TODO: verify transfer key
+  // This endpoint uses multer's MemoryStorage since the received data should be persisted directly on attachment storage.
+  receiveRouter.post('/attachment', uploadsForAttachment.single('content'), /* verifyAndExtractTransferKey, */
+    async(req: Request & { transferKey: TransferKey }, res: ApiV3Response) => {
+      const { file } = req;
+      const { attachmentMetadata } = req.body;
+
+      let attachmentMap;
+      try {
+        attachmentMap = JSON.parse(attachmentMetadata);
+      }
+      catch (err) {
+        logger.error(err);
+        return res.apiv3Err(new ErrorV3('Failed to parse body.', 'parse_failed'), 500);
+      }
+
+      // convert Buffer to stream
+      // see: https://stackoverflow.com/a/62143160
+      await g2gTransferReceiverService.receiveAttachment(Readable.from(file.buffer), attachmentMap);
+
+      return res.apiv3({ message: 'Successfully imported attached file.' });
+    });
+
   receiveRouter.get('/growi-info', verifyAndExtractTransferKey, async(req: Request & { transferKey: TransferKey }, res: ApiV3Response) => {
     let growiInfo: IDataGROWIInfo;
     try {

packages/app/src/server/service/g2g-transfer.ts

@@ -111,7 +111,57 @@ export class G2GTransferPusherService implements Pusher {
     return false;
   }
 
-  public async transferAttachments(): Promise<void> { return }
+  public async transferAttachments(tk: TransferKey): Promise<void> {
+    const { appUrl, key } = tk;
+    const { fileUploadService } = this.crowi;
+    const Attachment = this.crowi.model('Attachment');
+
+    // TODO: batch get
+    const attachments = await Attachment.find();
+    for await (const attachment of attachments) {
+      logger.debug(`processing attachment: ${attachment}`);
+      let fileStream;
+      try {
+        // get read stream of each attachment
+        fileStream = await fileUploadService.findDeliveryFile(attachment);
+      }
+      catch (err) {
+        logger.warn(`Error occured when getting Attachment(ID=${attachment.id}), skipping: `, err);
+        continue;
+      }
+      // TODO: get attachmentLists from destination GROWI to avoid transferring files that the dest GROWI has
+      // TODO: refresh transfer key per 1 hour
+      // post each attachment file data to receiver
+      try {
+        // Use FormData to immitate browser's form data object
+        const form = new FormData();
+
+        form.append('content', fileStream, attachment.fileName);
+        form.append('attachmentMetadata', JSON.stringify(attachment));
+        await rawAxios.post('/_api/v3/g2g-transfer/attachment', form, {
+          baseURL: appUrl.origin,
+          headers: {
+            ...form.getHeaders(), // This generates a unique boundary for multi part form data
+            [X_GROWI_TRANSFER_KEY_HEADER_NAME]: key,
+          },
+        });
+      }
+      catch (errs) {
+        logger.error(`Error occured when uploading attachment(ID=${attachment.id})`, errs);
+        if (!Array.isArray(errs)) {
+          // TODO: socker.emit(failed_to_transfer);
+          return;
+        }
+
+        const err = errs[0];
+        logger.error(err);
+
+
+        // TODO: socker.emit(failed_to_transfer);
+        return;
+      }
+    }
+  }
 
   public async startTransfer(tk: TransferKey, user: any, collections: string[], optionsMap: any): Promise<void> {
     const { appUrl, key } = tk;
@@ -192,16 +242,18 @@ export class G2GTransferReceiverService implements Receiver {
   }
 
   public async answerGROWIInfo(): Promise<IDataGROWIInfo> {
-    const configManager = this.crowi.configManager;
+    // TODO: add attachment file limit, storage total limit
+    const { configManager } = this.crowi;
     const userUpperLimit = configManager.getConfig('crowi', 'security:userUpperLimit');
     const version = this.crowi.version;
     const attachmentInfo = {
       type: configManager.getConfig('crowi', 'app:fileUploadType'),
       bucket: undefined,
-      customEndpoint: undefined,
+      customEndpoint: undefined, // for S3
+      uploadNamespace: undefined, // for GCS
     };
 
-    // put storage location info to check identificat
+    // put storage location info to check storage identification
     switch (attachmentInfo.type) {
       case 'aws':
         attachmentInfo.bucket = configManager.getConfig('crowi', 'aws:s3Bucket');
@@ -209,6 +261,7 @@ export class G2GTransferReceiverService implements Receiver {
         break;
       case 'gcs':
         attachmentInfo.bucket = configManager.getConfig('crowi', 'gcs:bucket');
+        attachmentInfo.uploadNamespace = configManager.getConfig('crowi', 'gcs:uploadNamespace');
         break;
       default:
     }
@@ -249,6 +302,17 @@ export class G2GTransferReceiverService implements Receiver {
     return;
   }
 
+  /**
+   *
+   * @param content Pushed attachment data from source GROWI
+   * @param attachmentMap Map-ped Attachment instance
+   * @returns
+   */
+  public async receiveAttachment(content: Readable, attachmentMap): Promise<void> {
+    const { fileUploadService } = this.crowi;
+    return fileUploadService.uploadFile(content, attachmentMap);
+  }
+
   /**
    * Sync DB, etc.
    * @returns {Promise<void>}