refactor: for g2g, list actual files instead of attachment collection

packages/app/src/server/models/vo/g2g-transfer-error.ts

@@ -2,8 +2,8 @@ import ExtensibleCustomError from 'extensible-custom-error';
 
 export const G2GTransferErrorCode = {
   INVALID_TRANSFER_KEY_STRING: 'INVALID_TRANSFER_KEY_STRING',
-  FAILED_TO_RETREIVE_GROWI_INFO: 'FAILED_TO_RETREIVE_GROWI_INFO',
-  FAILED_TO_RETREIVE_ATTACHMENTS: 'FAILED_TO_RETREIVE_ATTACHMENTS',
+  FAILED_TO_RETRIEVE_GROWI_INFO: 'FAILED_TO_RETRIEVE_GROWI_INFO',
+  FAILED_TO_RETRIEVE_FILE_METADATA: 'FAILED_TO_RETRIEVE_FILE_METADATA',
 } as const;
 
 export type G2GTransferErrorCode = typeof G2GTransferErrorCode[keyof typeof G2GTransferErrorCode];

packages/app/src/server/routes/apiv3/g2g-transfer.ts

@@ -4,7 +4,6 @@ import path from 'path';
 import { ErrorV3 } from '@growi/core';
 import express, { NextFunction, Request, Router } from 'express';
 import { body } from 'express-validator';
-import { type Document } from 'mongoose';
 import multer from 'multer';
 
 import { SupportedAction } from '~/interfaces/activity';
@@ -149,14 +148,8 @@ module.exports = (crowi: Crowi): Router => {
 
   // eslint-disable-next-line max-len
   receiveRouter.get('/attachments', verifyAndExtractTransferKey, async(req: Request & { transferKey: TransferKey, operatorUserId: string }, res: ApiV3Response) => {
-    const transform = (doc: Document) => JSON.stringify(doc._id.toString());
-    const readStream = crowi.exportService.createExportCollectionStream(
-      'attachments',
-      undefined,
-      { projection: { _id: 1 } },
-      transform,
-    );
-    return readStream.pipe(res);
+    const files = await crowi.fileUploadService.listFiles();
+    return res.apiv3(files);
   });
 
   // Auto import
@@ -398,12 +391,9 @@ module.exports = (crowi: Crowi): Router => {
       return res.apiv3Err(new ErrorV3(transferability.reason, 'growi_incompatible_to_transfer'));
     }
 
-    // get attachments from new growi
-    const attachmentIdsFromNewGrowi = await g2gTransferPusherService.getAttachments(tk);
-
     // Start transfer
     try {
-      await g2gTransferPusherService.startTransfer(tk, req.user, toGROWIInfo, collections, optionsMap, attachmentIdsFromNewGrowi);
+      await g2gTransferPusherService.startTransfer(tk, req.user, toGROWIInfo, collections, optionsMap);
     }
     catch (err) {
       logger.error(err);

packages/app/src/server/service/export.js

@@ -164,26 +164,6 @@ class ExportService {
     return transformStream;
   }
 
-  /**
-   * dump a mongodb collection into json
-   *
-   * @memberOf ExportService
-   * @param {string} collectionName collection name
-   * @param {Filter<TSchema>} filter find filter
-   * @param {FindOptions} options find options
-   * @param {CursorStreamOptions.transform} transform a transformation method applied to each document emitted by the stream
-   * @return {NodeJS.ReadStream} readstream for the collection
-   */
-  createExportCollectionStream(collectionName, filter, options, transform = JSON.stringify) {
-    const collection = mongoose.connection.collection(collectionName);
-    const nativeCursor = collection.find(filter, options);
-    const readStream = nativeCursor.stream({ transform });
-    const transformStream = this.generateTransformStream();
-
-    return readStream
-      .pipe(transformStream);
-  }
-
   /**
    * dump a mongodb collection into json
    *

packages/app/src/server/service/file-uploader/aws.ts

@@ -230,5 +230,15 @@ module.exports = (crowi) => {
     return lib.doCheckLimit(uploadFileSize, maxFileSize, totalLimit);
   };
 
+  /**
+   * List files in storage
+   * TODO: implement
+   */
+  lib.listFiles = async() => {
+    return [
+      { filePath: '', fileSize: '' },
+    ];
+  };
+
   return lib;
 };

packages/app/src/server/service/file-uploader/gcs.js

@@ -184,5 +184,15 @@ module.exports = function(crowi) {
     return lib.doCheckLimit(uploadFileSize, maxFileSize, gcsTotalLimit);
   };
 
+  /**
+   * List files in storage
+   * TODO: implement
+   */
+  lib.listFiles = async() => {
+    return [
+      { filePath: '', fileSize: '' },
+    ];
+  };
+
   return lib;
 };

packages/app/src/server/service/file-uploader/gridfs.js

@@ -125,5 +125,15 @@ module.exports = function(crowi) {
     return AttachmentFile.read({ _id: attachmentFile._id });
   };
 
+  /**
+   * List files in storage
+   * TODO: implement
+   */
+  lib.listFiles = async() => {
+    return [
+      { filePath: '', fileSize: '' },
+    ];
+  };
+
   return lib;
 };

packages/app/src/server/service/file-uploader/local.js

@@ -4,6 +4,7 @@ const logger = loggerFactory('growi:service:fileUploaderLocal');
 
 const fs = require('fs');
 const path = require('path');
+
 const mkdir = require('mkdirp');
 const streamToPromise = require('stream-to-promise');
 const urljoin = require('url-join');
@@ -126,5 +127,15 @@ module.exports = function(crowi) {
     return res.end();
   };
 
+  /**
+   * List files in storage
+   * TODO: implement
+   */
+  lib.listFiles = async() => {
+    return [
+      { filePath: '', fileSize: '' },
+    ];
+  };
+
   return lib;
 };

packages/app/src/server/service/g2g-transfer.ts

@@ -49,10 +49,12 @@ export type IDataGROWIInfo = {
 }
 
 /**
- * Attachment data already exsisting in the new GROWI
+ * File metadata in storage
  */
-// TODO: use Attachemnt model type
-export type Attachment = any;
+interface FileMeta {
+  filePath: string;
+  fileSize: number;
+}
 
 /**
  * Return type for {@link Pusher.getTransferability}
@@ -70,11 +72,16 @@ interface Pusher {
    * @param {IDataGROWIInfo} fromGROWIInfo
    */
   getTransferability(fromGROWIInfo: IDataGROWIInfo): Promise<IGetTransferabilityReturn>
+  /**
+   * List files in the storage
+   * @param {TransferKey} tk Transfer key
+   */
+  listFilesInStorage(tk: TransferKey): Promise<FileMeta[]>
   /**
    * Transfer all Attachment data to destination GROWI
    * @param {TransferKey} tk Transfer key
    */
-  transferAttachments(tk: TransferKey, attachmentIdsFromNewGrowi: string[]): Promise<void>
+  transferAttachments(tk: TransferKey): Promise<void>
   /**
    * Start transfer data between GROWIs
    * @param {TransferKey} tk TransferKey object
@@ -87,7 +94,6 @@ interface Pusher {
     toGROWIInfo: IDataGROWIInfo,
     collections: string[],
     optionsMap: any,
-    attachmentIdsFromNewGrowi: string[]
   ): Promise<void>
 }
 
@@ -225,7 +231,7 @@ export class G2GTransferPusherService implements Pusher {
     }
     catch (err) {
       logger.error(err);
-      throw new G2GTransferError('Failed to retreive growi info.', G2GTransferErrorCode.FAILED_TO_RETREIVE_GROWI_INFO);
+      throw new G2GTransferError('Failed to retrieve growi info.', G2GTransferErrorCode.FAILED_TO_RETRIEVE_GROWI_INFO);
     }
 
     return toGROWIInfo;
@@ -278,25 +284,23 @@ export class G2GTransferPusherService implements Pusher {
     return { canTransfer: true };
   }
 
-  public async getAttachments(tk: TransferKey): Promise<string[]> {
+  public async listFilesInStorage(tk: TransferKey): Promise<FileMeta[]> {
     try {
-      const { data } = await axios.get<string[]>('/_api/v3/g2g-transfer/attachments', generateAxiosRequestConfigWithTransferKey(tk));
+      const { data } = await axios.get<FileMeta[]>('/_api/v3/g2g-transfer/attachments', generateAxiosRequestConfigWithTransferKey(tk));
       return data;
     }
     catch (err) {
       logger.error(err);
-      throw new G2GTransferError('Failed to retreive attachments', G2GTransferErrorCode.FAILED_TO_RETREIVE_ATTACHMENTS);
+      throw new G2GTransferError('Failed to retrieve file metadata', G2GTransferErrorCode.FAILED_TO_RETRIEVE_FILE_METADATA);
     }
   }
 
-  public async transferAttachments(tk: TransferKey, attachmentIdsFromNewGrowi: string[]): Promise<void> {
+  public async transferAttachments(tk: TransferKey): Promise<void> {
     const BATCH_SIZE = 100;
-
     const { fileUploadService } = this.crowi;
     const Attachment = this.crowi.model('Attachment');
-
-    // batch get
-    const attachmentsCursor = await Attachment.find({ _id: { $nin: attachmentIdsFromNewGrowi } }).cursor();
+    const filesFromNewGrowi = await this.listFilesInStorage(tk);
+    const attachmentsCursor = await Attachment.find({ _id: { $nin: filesFromNewGrowi } }).cursor();
     const batchStream = createBatchStream(BATCH_SIZE);
 
     for await (const attachmentBatch of attachmentsCursor.pipe(batchStream)) {
@@ -325,7 +329,7 @@ export class G2GTransferPusherService implements Pusher {
   }
 
   // eslint-disable-next-line max-len
-  public async startTransfer(tk: TransferKey, user: any, toGROWIInfo: IDataGROWIInfo, collections: string[], optionsMap: any, attachmentIdsFromNewGrowi: string[], shouldEmit = true): Promise<void> {
+  public async startTransfer(tk: TransferKey, user: any, toGROWIInfo: IDataGROWIInfo, collections: string[], optionsMap: any, shouldEmit = true): Promise<void> {
     const socket = this.crowi.socketIoService.getAdminSocket();
 
     if (shouldEmit) {
@@ -390,7 +394,7 @@ export class G2GTransferPusherService implements Pusher {
     }
 
     try {
-      await this.transferAttachments(tk, attachmentIdsFromNewGrowi);
+      await this.transferAttachments(tk);
     }
     catch (err) {
       logger.error(err);