Merge pull request #725 from weseek/feat/limit-amount-of-gridfs-use

Feat/limit amount of gridfs use

README.md

@@ -168,6 +168,7 @@ Environment Variables
       * `mongodb` : MongoDB GridFS (Setting-less)
       * `local` : Server's Local file system (Setting-less)
       * `none` : Disable file uploading
+    * MONGODB_GRIDFS_LIMIT: Limit amount of uploaded file with GridFS: `Infinity`
 * **Option to integrate with external systems**
     * HACKMD_URI: URI to connect to [HackMD(CodiMD)](https://hackmd.io/) server.
         * **This server must load the GROWI agent. [Here's how to prepare it](https://docs.growi.org/management-cookbook/integrate-with-hackmd).**

config/env.dev.js

@@ -1,6 +1,7 @@
 module.exports = {
   NODE_ENV: 'development',
   FILE_UPLOAD: 'mongodb',
+  MONGODB_GRIDFS_LIMIT: Infinity,
   // MATHJAX: 1,
   ELASTICSEARCH_URI: 'http://localhost:9200/growi',
   HACKMD_URI: 'http://localhost:3010',

src/client/js/components/PageEditor.js

@@ -9,6 +9,8 @@ import { EditorOptions, PreviewOptions } from './PageEditor/OptionsSelector';
 import Editor from './PageEditor/Editor';
 import Preview from './PageEditor/Preview';
 import scrollSyncHelper from './PageEditor/ScrollSyncHelper';
+import * as toastr from 'toastr';
+
 
 export default class PageEditor extends React.Component {
 
@@ -42,6 +44,7 @@ export default class PageEditor extends React.Component {
     this.onPreviewScroll = this.onPreviewScroll.bind(this);
     this.saveDraft = this.saveDraft.bind(this);
     this.clearDraft = this.clearDraft.bind(this);
+    this.apiErrorHandler = this.apiErrorHandler.bind(this);
 
     // for scrolling
     this.lastScrolledDateWithCursor = null;
@@ -114,41 +117,54 @@ export default class PageEditor extends React.Component {
    * the upload event handler
    * @param {any} files
    */
-  onUpload(file) {
-    const endpoint = '/attachments.add';
-
-    // create a FromData instance
-    const formData = new FormData();
-    formData.append('_csrf', this.props.crowi.csrfToken);
-    formData.append('file', file);
-    formData.append('path', this.props.pagePath);
-    formData.append('page_id', this.state.pageId || 0);
-
-    // post
-    this.props.crowi.apiPost(endpoint, formData)
-      .then((res) => {
-        const url = res.url;
-        const attachment = res.attachment;
-        const fileName = attachment.originalName;
-
-        let insertText = `[${fileName}](${url})`;
-        // when image
-        if (attachment.fileFormat.startsWith('image/')) {
-          // modify to "![fileName](url)" syntax
-          insertText = '!' + insertText;
-        }
-        this.refs.editor.insertText(insertText);
-
-        // when if created newly
-        if (res.pageCreated) {
-          // do nothing
-        }
-      })
-      .catch(this.apiErrorHandler)
-      // finally
-      .then(() => {
-        this.refs.editor.terminateUploadingState();
-      });
+  async onUpload(file) {
+    try {
+      let res  = await this.props.crowi.apiGet('/attachments.limit', {_csrf: this.props.crowi.csrfToken, fileSize: file.size});
+      if (!res.isUploadable) {
+        toastr.error(undefined, 'MongoDB for uploading files reaches limit', {
+          closeButton: true,
+          progressBar: true,
+          newestOnTop: false,
+          showDuration: '100',
+          hideDuration: '100',
+          timeOut: '5000',
+        });
+        throw new Error('MongoDB for uploading files reaches limit');
+      }
+      const endpoint = '/attachments.add';
+
+      // create a FromData instance
+      const formData = new FormData();
+      formData.append('_csrf', this.props.crowi.csrfToken);
+      formData.append('file', file);
+      formData.append('path', this.props.pagePath);
+      formData.append('page_id', this.state.pageId || 0);
+
+      // post
+      res = await this.props.crowi.apiPost(endpoint, formData)
+      const url = res.url;
+      const attachment = res.attachment;
+      const fileName = attachment.originalName;
+
+      let insertText = `[${fileName}](${url})`;
+      // when image
+      if (attachment.fileFormat.startsWith('image/')) {
+        // modify to "![fileName](url)" syntax
+        insertText = '!' + insertText;
+      }
+      this.refs.editor.insertText(insertText);
+
+      // when if created newly
+      if (res.pageCreated) {
+        // do nothing
+      }
+    }
+    catch (e) {
+      this.apiErrorHandler(e);
+    }
+    finally {
+      this.refs.editor.terminateUploadingState();
+    }
   }
 
   /**
@@ -292,6 +308,17 @@ export default class PageEditor extends React.Component {
 
   }
 
+  apiErrorHandler(error) {
+    toastr.error(error.message, 'Error occured', {
+      closeButton: true,
+      progressBar: true,
+      newestOnTop: false,
+      showDuration: '100',
+      hideDuration: '100',
+      timeOut: '3000',
+    });
+  }
+
   render() {
     const emojiStrategy = this.props.crowi.getEmojiStrategy();
 

src/server/routes/attachment.js

@@ -106,6 +106,16 @@ module.exports = function(crowi, app) {
     });
   };
 
+  /**
+   * @api {get} /attachments.limit get available capacity of uploaded file with GridFS
+   * @apiName AddAttachments
+   * @apiGroup Attachment
+   */
+  api.limit = async function(req, res) {
+    const isUploadable = await fileUploader.checkCapacity(req.query.fileSize);
+    return res.json(ApiResponse.success({isUploadable: isUploadable}));
+  };
+
   /**
    * @api {post} /attachments.add Add attachment to the page
    * @apiName AddAttachments
@@ -114,7 +124,7 @@ module.exports = function(crowi, app) {
    * @apiParam {String} page_id
    * @apiParam {File} file
    */
-  api.add = function(req, res) {
+  api.add = async function(req, res) {
     var id = req.body.page_id || 0,
       path = decodeURIComponent(req.body.path) || null,
       pageCreated = false,
@@ -123,11 +133,14 @@ module.exports = function(crowi, app) {
     debug('id and path are: ', id, path);
 
     var tmpFile = req.file || null;
+    const isUploadable = await fileUploader.checkCapacity(tmpFile.size);
+    if (!isUploadable) {
+      return res.json(ApiResponse.error('MongoDB for uploading files reaches limit'));
+    }
     debug('Uploaded tmpFile: ', tmpFile);
     if (!tmpFile) {
       return res.json(ApiResponse.error('File error.'));
     }
-
     new Promise(function(resolve, reject) {
       if (id == 0) {
         if (path === null) {

src/server/routes/index.js

@@ -212,6 +212,7 @@ module.exports = function(crowi, app) {
   app.get( '/_api/attachments.list'   , accessTokenParser , loginRequired(crowi, app, false) , attachment.api.list);
   app.post('/_api/attachments.add'    , uploads.single('file'), accessTokenParser, loginRequired(crowi, app) ,csrf, attachment.api.add);
   app.post('/_api/attachments.remove' , accessTokenParser , loginRequired(crowi, app) , csrf, attachment.api.remove);
+  app.get( '/_api/attachments.limit' , accessTokenParser , loginRequired(crowi, app) , csrf, attachment.api.limit);
 
   app.get( '/_api/revisions.get'      , accessTokenParser , loginRequired(crowi, app, false) , revision.api.get);
   app.get( '/_api/revisions.ids'      , accessTokenParser , loginRequired(crowi, app, false) , revision.api.ids);

src/server/service/file-uploader/aws.js

@@ -161,6 +161,13 @@ module.exports = function(crowi) {
     return false;
   };
 
+  /**
+   * chech storage for fileUpload reaches MONGODB_GRIDFS_LIMIT (for gridfs)
+   */
+  lib.checkCapacity = async(uploadFileSize) => {
+    return true;
+  };
+
   return lib;
 };
 

src/server/service/file-uploader/gridfs.js

@@ -18,6 +18,7 @@ module.exports = function(crowi) {
 
   // obtain a model
   const AttachmentFile = gridfs.model;
+  const Chunks = mongoose.model('Chunks', gridfs.schema, 'attachmentFiles.chunks');
 
   // delete a file
   lib.deleteFile = async function(fileId, filePath) {
@@ -44,6 +45,34 @@ module.exports = function(crowi) {
     }
   };
 
+  /**
+   * get size of data uploaded files using (Promise wrapper)
+   */
+  const getCollectionSize = () => {
+    return new Promise((resolve, reject) => {
+      Chunks.collection.stats((err, data) => {
+        if (err) {
+          reject(err);
+        }
+        resolve(data.size);
+      });
+    });
+  };
+
+  /**
+   * chech storage for fileUpload reaches MONGODB_GRIDFS_LIMIT (for gridfs)
+   */
+  lib.checkCapacity = async(uploadFileSize) => {
+    const usingFilesSize = await getCollectionSize();
+    if (process.env.MONGODB_GRIDFS_LIMIT != false) {
+      return true;
+    }
+    else if (+process.env.MONGODB_GRIDFS_LIMIT > usingFilesSize + +uploadFileSize) {
+      return true;
+    }
+    return false;
+  };
+
   lib.uploadFile = async function(filePath, contentType, fileStream, options) {
     debug('File uploading: ' + filePath);
     await writeFile(filePath, contentType, fileStream);

src/server/service/file-uploader/local.js

@@ -55,6 +55,13 @@ module.exports = function(crowi) {
     return Promise.resolve(lib.generateUrl(filePath));
   };
 
+  /**
+   * chech storage for fileUpload reaches MONGODB_GRIDFS_LIMIT (for gridfs)
+   */
+  lib.checkCapacity = async(uploadFileSize) => {
+    return true;
+  };
+
   return lib;
 };