Merge pull request #6881 from mizozobu/feat/g2g-retry-uploading-attachments

feat: skip dupe attachment transfer

packages/app/src/server/models/vo/g2g-transfer-error.ts

@@ -3,6 +3,7 @@ import ExtensibleCustomError from 'extensible-custom-error';
 export const G2GTransferErrorCode = {
   INVALID_TRANSFER_KEY_STRING: 'INVALID_TRANSFER_KEY_STRING',
   FAILED_TO_RETREIVE_GROWI_INFO: 'FAILED_TO_RETREIVE_GROWI_INFO',
+  FAILED_TO_RETREIVE_ATTACHMENTS: 'FAILED_TO_RETREIVE_ATTACHMENTS',
 } as const;
 
 export type G2GTransferErrorCode = typeof G2GTransferErrorCode[keyof typeof G2GTransferErrorCode];

packages/app/src/server/routes/apiv3/g2g-transfer.ts

@@ -4,6 +4,7 @@ import path from 'path';
 import { ErrorV3 } from '@growi/core';
 import express, { NextFunction, Request, Router } from 'express';
 import { body } from 'express-validator';
+import { type Document } from 'mongoose';
 import multer from 'multer';
 
 import { SupportedAction } from '~/interfaces/activity';
@@ -146,6 +147,18 @@ module.exports = (crowi: Crowi): Router => {
   const receiveRouter = express.Router();
   const pushRouter = express.Router();
 
+  // eslint-disable-next-line max-len
+  receiveRouter.get('/attachments', verifyAndExtractTransferKey, async(req: Request & { transferKey: TransferKey, operatorUserId: string }, res: ApiV3Response) => {
+    const transform = (doc: Document) => JSON.stringify(doc._id.toString());
+    const readStream = crowi.exportService.createExportCollectionStream(
+      'attachments',
+      undefined,
+      { projection: { _id: 1 } },
+      transform,
+    );
+    return readStream.pipe(res);
+  });
+
   // Auto import
   // eslint-disable-next-line max-len
   receiveRouter.post('/', uploads.single('transferDataZipFile'), verifyAndExtractTransferKey, async(req: Request & { transferKey: TransferKey, operatorUserId: string }, res: ApiV3Response) => {
@@ -385,9 +398,12 @@ module.exports = (crowi: Crowi): Router => {
       return res.apiv3Err(new ErrorV3(transferability.reason, 'growi_incompatible_to_transfer'));
     }
 
+    // get attachments from new growi
+    const attachmentIdsFromNewGrowi = await g2gTransferPusherService.getAttachments(tk);
+
     // Start transfer
     try {
-      await g2gTransferPusherService.startTransfer(tk, req.user, toGROWIInfo, collections, optionsMap);
+      await g2gTransferPusherService.startTransfer(tk, req.user, toGROWIInfo, collections, optionsMap, attachmentIdsFromNewGrowi);
     }
     catch (err) {
       logger.error(err);

packages/app/src/server/service/export.js

@@ -164,6 +164,26 @@ class ExportService {
     return transformStream;
   }
 
+  /**
+   * dump a mongodb collection into json
+   *
+   * @memberOf ExportService
+   * @param {string} collectionName collection name
+   * @param {Filter<TSchema>} filter find filter
+   * @param {FindOptions} options find options
+   * @param {CursorStreamOptions.transform} transform a transformation method applied to each document emitted by the stream
+   * @return {NodeJS.ReadStream} readstream for the collection
+   */
+  createExportCollectionStream(collectionName, filter, options, transform = JSON.stringify) {
+    const collection = mongoose.connection.collection(collectionName);
+    const nativeCursor = collection.find(filter, options);
+    const readStream = nativeCursor.stream({ transform });
+    const transformStream = this.generateTransformStream();
+
+    return readStream
+      .pipe(transformStream);
+  }
+
   /**
    * dump a mongodb collection into json
    *

packages/app/src/server/service/g2g-transfer.ts

@@ -47,6 +47,12 @@ export type IDataGROWIInfo = {
   };
 }
 
+/**
+ * Attachment data already exsisting in the new GROWI
+ */
+// TODO: use Attachemnt model type
+export type Attachment = any;
+
 /**
  * Return type for {@link Pusher.getTransferability}
  */
@@ -67,14 +73,21 @@ interface Pusher {
    * Transfer all Attachment data to destination GROWI
    * @param {TransferKey} tk Transfer key
    */
-  transferAttachments(tk: TransferKey): Promise<void>
+  transferAttachments(tk: TransferKey, attachmentIdsFromNewGrowi: string[]): Promise<void>
   /**
    * Start transfer data between GROWIs
    * @param {TransferKey} tk TransferKey object
    * @param {string[]} collections Collection name string array
    * @param {any} optionsMap Options map
    */
-  startTransfer(tk: TransferKey, user: any, toGROWIInfo: IDataGROWIInfo, collections: string[], optionsMap: any): Promise<void>
+  startTransfer(
+    tk: TransferKey,
+    user: any,
+    toGROWIInfo: IDataGROWIInfo,
+    collections: string[],
+    optionsMap: any,
+    attachmentIdsFromNewGrowi: string[]
+  ): Promise<void>
 }
 
 interface Receiver {
@@ -264,15 +277,25 @@ export class G2GTransferPusherService implements Pusher {
     return { canTransfer: true };
   }
 
-  public async transferAttachments(tk: TransferKey): Promise<void> {
+  public async getAttachments(tk: TransferKey): Promise<string[]> {
+    try {
+      const { data } = await axios.get<string[]>('/_api/v3/g2g-transfer/attachments', generateAxiosRequestConfigWithTransferKey(tk));
+      return data;
+    }
+    catch (err) {
+      logger.error(err);
+      throw new G2GTransferError('Failed to retreive attachments', G2GTransferErrorCode.FAILED_TO_RETREIVE_ATTACHMENTS);
+    }
+  }
+
+  public async transferAttachments(tk: TransferKey, attachmentIdsFromNewGrowi: string[]): Promise<void> {
     const BATCH_SIZE = 100;
 
-    const socket = this.crowi.socketIoService.getAdminSocket();
     const { fileUploadService } = this.crowi;
     const Attachment = this.crowi.model('Attachment');
 
     // batch get
-    const attachmentsCursor = await Attachment.find().cursor();
+    const attachmentsCursor = await Attachment.find({ _id: { $nin: attachmentIdsFromNewGrowi } }).cursor();
     const batchStream = createBatchStream(BATCH_SIZE);
 
     for await (const attachmentBatch of attachmentsCursor.pipe(batchStream)) {
@@ -301,7 +324,7 @@ export class G2GTransferPusherService implements Pusher {
   }
 
   // eslint-disable-next-line max-len
-  public async startTransfer(tk: TransferKey, user: any, toGROWIInfo: IDataGROWIInfo, collections: string[], optionsMap: any, shouldEmit = true): Promise<void> {
+  public async startTransfer(tk: TransferKey, user: any, toGROWIInfo: IDataGROWIInfo, collections: string[], optionsMap: any, attachmentIdsFromNewGrowi: string[], shouldEmit = true): Promise<void> {
     const socket = this.crowi.socketIoService.getAdminSocket();
 
     if (shouldEmit) socket.emit('admin:onStartTransferMongoData', {});
@@ -348,7 +371,7 @@ export class G2GTransferPusherService implements Pusher {
     if (shouldEmit) socket.emit('admin:onStartTransferAttachments', {});
 
     try {
-      await this.transferAttachments(tk);
+      await this.transferAttachments(tk, attachmentIdsFromNewGrowi);
     }
     catch (err) {
       logger.error(err);