Merge pull request #6779 from mizozobu/feat/g2g-check-storage-limit

feat: check storage limit before g2g

packages/app/src/components/Admin/G2GDataTransfer.tsx

@@ -4,10 +4,8 @@ import { useTranslation } from 'react-i18next';
 
 import { useGenerateTransferKeyWithThrottle } from '~/client/services/g2g-transfer';
 import { toastError } from '~/client/util/apiNotification';
-import { apiv3Get } from '~/client/util/apiv3-client';
+import { apiv3Get, apiv3Post } from '~/client/util/apiv3-client';
 import { useAdminSocket } from '~/stores/socket-io';
-import customAxios from '~/utils/axios';
-
 
 import CustomCopyToClipBoard from '../Common/CustomCopyToClipBoard';
 
@@ -94,14 +92,14 @@ const G2GDataTransfer = (): JSX.Element => {
     e.preventDefault();
 
     try {
-      await customAxios.post('/_api/v3/g2g-transfer/transfer', {
+      await apiv3Post('/g2g-transfer/transfer', {
         transferKey: startTransferKey,
         collections: Array.from(selectedCollections),
         optionsMap,
       });
     }
     catch (errs) {
-      toastError('Failed to transfer');
+      toastError(errs);
     }
   }, [startTransferKey, selectedCollections, optionsMap]);
 

packages/app/src/server/models/user.js

@@ -447,7 +447,7 @@ module.exports = function(crowi) {
 
     const userUpperLimit = configManager.getConfig('crowi', 'security:userUpperLimit');
 
-    const activeUsers = await this.countListByStatus(STATUS_ACTIVE);
+    const activeUsers = await this.countActiveUsers();
     if (userUpperLimit <= activeUsers) {
       return true;
     }
@@ -455,6 +455,10 @@ module.exports = function(crowi) {
     return false;
   };
 
+  userSchema.statics.countActiveUsers = async function() {
+    return this.countListByStatus(STATUS_ACTIVE);
+  };
+
   userSchema.statics.countListByStatus = async function(status) {
     const User = this;
     const conditions = { status };

packages/app/src/server/routes/apiv3/g2g-transfer.ts

@@ -379,10 +379,10 @@ module.exports = (crowi: Crowi): Router => {
     }
 
     // Check if can transfer
-    const canTransfer = await g2gTransferPusherService.canTransfer(toGROWIInfo);
-    if (!canTransfer) {
+    const transferability = await g2gTransferPusherService.getTransferability(toGROWIInfo);
+    if (!transferability.canTransfer) {
       logger.debug('Could not transfer.');
-      return res.apiv3Err(new ErrorV3('GROWI is incompatible to transfer data.', 'growi_incompatible_to_transfer'));
+      return res.apiv3Err(new ErrorV3(transferability.reason, 'growi_incompatible_to_transfer'));
     }
 
     // Start transfer

packages/app/src/server/service/config-manager.ts

@@ -4,9 +4,10 @@ import loggerFactory from '~/utils/logger';
 
 import ConfigModel from '../models/config';
 import S2sMessage from '../models/vo/s2s-message';
+
+import ConfigLoader, { ConfigObject } from './config-loader';
 import { S2sMessagingService } from './s2s-messaging/base';
 import { S2sMessageHandlable } from './s2s-messaging/handlable';
-import ConfigLoader, { ConfigObject } from './config-loader';
 
 const logger = loggerFactory('growi:service:ConfigManager');
 
@@ -375,4 +376,17 @@ export default class ConfigManager implements S2sMessageHandlable {
     return this.loadConfigs();
   }
 
+  /**
+   * Returns file upload total limit in bytes.
+   * {@link https://github.com/weseek/growi/blob/798e44f14ad01544c1d75ba83d4dfb321a94aa0b/src/server/service/file-uploader/gridfs.js#L86-L88 Reference to previous implementation.}
+   * @returns file upload total limit in bytes
+   */
+  getFileUploadTotalLimit(): number {
+    const fileUploadTotalLimit = this.getConfig('crowi', 'app:fileUploadType') === 'mongodb'
+      // Use app:fileUploadTotalLimit if gridfs:totalLimit is null (default for gridfs:totalLimit is null)
+      ? this.getConfig('crowi', 'gridfs:totalLimit') ?? this.getConfig('crowi', 'app:fileUploadTotalLimit')
+      : this.getConfig('crowi', 'app:fileUploadTotalLimit');
+    return fileUploadTotalLimit;
+  }
+
 }

packages/app/src/server/service/file-uploader/gridfs.js

@@ -1,9 +1,10 @@
 import loggerFactory from '~/utils/logger';
 
 const logger = loggerFactory('growi:service:fileUploaderGridfs');
-const mongoose = require('mongoose');
 const util = require('util');
 
+const mongoose = require('mongoose');
+
 module.exports = function(crowi) {
   const Uploader = require('./uploader');
   const lib = new Uploader(crowi);
@@ -85,11 +86,8 @@ module.exports = function(crowi) {
    */
   lib.checkLimit = async(uploadFileSize) => {
     const maxFileSize = crowi.configManager.getConfig('crowi', 'app:maxFileSize');
-
-    // Use app:fileUploadTotalLimit if gridfs:totalLimit is null (default for gridfs:totalLimitd is null)
-    const gridfsTotalLimit = crowi.configManager.getConfig('crowi', 'gridfs:totalLimit')
-      || crowi.configManager.getConfig('crowi', 'app:fileUploadTotalLimit');
-    return lib.doCheckLimit(uploadFileSize, maxFileSize, gridfsTotalLimit);
+    const totalLimit = crowi.configManager.getFileUploadTotalLimit();
+    return lib.doCheckLimit(uploadFileSize, maxFileSize, totalLimit);
   };
 
   lib.uploadFile = async function(fileStream, attachment) {

packages/app/src/server/service/file-uploader/uploader.js

@@ -33,6 +33,23 @@ class Uploader {
     throw new Error('Implemnt this');
   }
 
+  /**
+   * Get total file size
+   * @returns Total file size
+   */
+  async getTotalFileSize() {
+    const Attachment = this.crowi.model('Attachment');
+
+    // Get attachment total file size
+    const res = await Attachment.aggregate().group({
+      _id: null,
+      total: { $sum: '$fileSize' },
+    });
+
+    // res is [] if not using
+    return res.length === 0 ? 0 : res[0].total;
+  }
+
   /**
    * Check files size limits for all uploaders
    *
@@ -46,21 +63,13 @@ class Uploader {
     if (uploadFileSize > maxFileSize) {
       return { isUploadable: false, errorMessage: 'File size exceeds the size limit per file' };
     }
-    const Attachment = this.crowi.model('Attachment');
-    // Get attachment total file size
-    const res = await Attachment.aggregate().group({
-      _id: null,
-      total: { $sum: '$fileSize' },
-    });
-    // Return res is [] if not using
-    const usingFilesSize = res.length === 0 ? 0 : res[0].total;
 
+    const usingFilesSize = await this.getTotalFileSize();
     if (usingFilesSize + uploadFileSize > totalLimit) {
       return { isUploadable: false, errorMessage: 'Uploading files reaches limit' };
     }
 
     return { isUploadable: true };
-
   }
 
   /**

packages/app/src/server/service/g2g-transfer.ts

@@ -37,6 +37,8 @@ export const uploadConfigKeys = [
 export type IDataGROWIInfo = {
   version: string
   userUpperLimit: number | null // Handle null as Infinity
+  fileUploadDisabled: boolean;
+  fileUploadTotalLimit: number | null // Handle null as Infinity
   attachmentInfo: {
     type: string,
     bucket?: string,
@@ -45,6 +47,11 @@ export type IDataGROWIInfo = {
   };
 }
 
+/**
+ * Return type for {@link Pusher.getTransferability}
+ */
+type IGetTransferabilityReturn = { canTransfer: true; } | { canTransfer: false; reason: string; };
+
 interface Pusher {
   /**
    * Send to-growi a request to get growi info
@@ -55,7 +62,7 @@ interface Pusher {
    * Check if transfering is proceedable
    * @param {IDataGROWIInfo} fromGROWIInfo
    */
-  canTransfer(fromGROWIInfo: IDataGROWIInfo): Promise<boolean>
+  getTransferability(fromGROWIInfo: IDataGROWIInfo): Promise<IGetTransferabilityReturn>
   /**
    * Transfer all Attachment data to destination GROWI
    * @param {TransferKey} tk Transfer key
@@ -117,7 +124,7 @@ const generateAxiosRequestConfigWithTransferKey = (tk: TransferKey, additionalHe
  * @param crowi Crowi instance
  * @returns Whether the storage is writable
  */
-const getWritePermission = async(crowi: any): Promise<boolean> => {
+const hasWritePermission = async(crowi: any): Promise<boolean> => {
   const { fileUploadService } = crowi;
 
   let writable = true;
@@ -148,11 +155,13 @@ const getWritePermission = async(crowi: any): Promise<boolean> => {
  * @returns
  */
 const generateGROWIInfo = async(crowi: any): Promise<IDataGROWIInfo> => {
-  // TODO: add attachment file limit, storage total limit
+  // TODO: add attachment file limit
   const { configManager } = crowi;
   const userUpperLimit = configManager.getConfig('crowi', 'security:userUpperLimit');
+  const fileUploadDisabled = configManager.getConfig('crowi', 'app:fileUploadDisabled');
+  const fileUploadTotalLimit = configManager.getFileUploadTotalLimit();
   const version = crowi.version;
-  const writable = await getWritePermission(crowi);
+  const writable = await hasWritePermission(crowi);
 
   const attachmentInfo = {
     type: configManager.getConfig('crowi', 'app:fileUploadType'),
@@ -175,7 +184,13 @@ const generateGROWIInfo = async(crowi: any): Promise<IDataGROWIInfo> => {
     default:
   }
 
-  return { userUpperLimit, version, attachmentInfo };
+  return {
+    userUpperLimit,
+    fileUploadDisabled,
+    fileUploadTotalLimit,
+    version,
+    attachmentInfo,
+  };
 };
 
 export class G2GTransferPusherService implements Pusher {
@@ -202,21 +217,51 @@ export class G2GTransferPusherService implements Pusher {
     return toGROWIInfo;
   }
 
-  public async canTransfer(toGROWIInfo: IDataGROWIInfo): Promise<boolean> {
-    // TODO: check FILE_UPLOAD_TOTAL_LIMIT, FILE_UPLOAD_DISABLED
-    const configManager = this.crowi.configManager;
-    const userUpperLimit = configManager.getConfig('crowi', 'security:userUpperLimit');
-    const version = this.crowi.version;
+  /**
+   * Returns whether g2g transfer is possible and reason for failure
+   * @param toGROWIInfo to-growi info
+   * @returns Whether g2g transfer is possible and reason for failure
+   */
+  public async getTransferability(toGROWIInfo: IDataGROWIInfo): Promise<IGetTransferabilityReturn> {
+    const { fileUploadService } = this.crowi;
 
+    const version = this.crowi.version;
     if (version !== toGROWIInfo.version) {
-      return false;
+      return {
+        canTransfer: false,
+        // TODO: i18n for reason
+        reason: `Growi versions mismatch. This Growi: ${version} / new Growi: ${toGROWIInfo.version}.`,
+      };
+    }
+
+    const activeUserCount = await this.crowi.model('User').countActiveUsers();
+    if ((toGROWIInfo.userUpperLimit ?? Infinity) < activeUserCount) {
+      return {
+        canTransfer: false,
+        // TODO: i18n for reason
+        reason: `The number of active users (${activeUserCount} users) exceeds the limit of new Growi (to up ${toGROWIInfo.userUpperLimit} users).`,
+      };
+    }
+
+    if (toGROWIInfo.fileUploadDisabled) {
+      return {
+        canTransfer: false,
+        // TODO: i18n for reason
+        reason: 'File upload is disabled in new Growi.',
+      };
     }
 
-    if ((userUpperLimit ?? Infinity) < (toGROWIInfo.userUpperLimit ?? 0)) {
-      return false;
+    const totalFileSize = await fileUploadService.getTotalFileSize();
+    if ((toGROWIInfo.fileUploadTotalLimit ?? Infinity) < totalFileSize) {
+      return {
+        canTransfer: false,
+        // TODO: i18n for reason
+        // eslint-disable-next-line max-len
+        reason: `Total file size exceeds file upload limit of new Growi. Requires ${totalFileSize.toLocaleString()} bytes, but got ${(toGROWIInfo.fileUploadTotalLimit ?? Infinity).toLocaleString()} bytes.`,
+      };
     }
 
-    return true;
+    return { canTransfer: true };
   }
 
   public async transferAttachments(tk: TransferKey): Promise<void> {