Merge pull request #695 from weseek/feat/use-MongoDB-GridFS-for-file-storage

Feat/use mongo db grid fs for file storage

src/server/models/attachment.js

@@ -21,7 +21,7 @@ module.exports = function(crowi) {
     originalName: { type: String },
     fileFormat: { type: String, required: true },
     fileSize: { type: Number, default: 0 },
-    createdAt: { type: Date, default: Date.now }
+    createdAt: { type: Date, default: Date.now },
   }, {
     toJSON: {
       virtuals: true
@@ -52,7 +52,6 @@ module.exports = function(crowi) {
         }
         return resolve(data);
       });
-
     });
   };
 

src/server/routes/attachment.js

@@ -28,7 +28,7 @@ module.exports = function(crowi, app) {
             if (fileName.match(/^\/uploads/)) {
               return res.download(path.join(crowi.publicDir, fileName), data.originalName);
             }
-            // aws
+            // aws or gridfs
             else {
               const options = {
                 headers: {
@@ -47,6 +47,30 @@ module.exports = function(crowi, app) {
       });
   };
 
+  /**
+   * @api {get} /attachments.get get attachments from mongoDB
+   * @apiName get
+   * @apiGroup Attachment
+   *
+   * @apiParam {String} pageId, fileName
+   */
+  api.get = async function(req, res) {
+    if (process.env.FILE_UPLOAD != 'gridfs') {
+      return res.status(400);
+    }
+    const pageId = req.params.pageId;
+    const fileName = req.params.fileName;
+    const filePath = `attachment/${pageId}/${fileName}`;
+    try {
+      const fileData = await fileUploader.getFileData(filePath);
+      res.set('Content-Type', fileData.contentType);
+      return res.send(ApiResponse.success(fileData.data));
+    }
+    catch (e) {
+      return res.json(ApiResponse.error('attachment not found'));
+    }
+  };
+
   /**
    * @api {get} /attachments.list Get attachments of the page
    * @apiName ListAttachments

src/server/routes/index.js

@@ -176,6 +176,7 @@ module.exports = function(crowi, app) {
   app.get( '/:id([0-9a-z]{24})'       , loginRequired(crowi, app, false) , page.api.redirector);
   app.get( '/_r/:id([0-9a-z]{24})'    , loginRequired(crowi, app, false) , page.api.redirector); // alias
   app.get( '/download/:id([0-9a-z]{24})' , loginRequired(crowi, app, false) , attachment.api.download);
+  app.get( '/attachment/:pageId/:fileName'  , loginRequired(crowi, app, false), attachment.api.get);
 
   app.get( '/_search'                 , loginRequired(crowi, app, false) , search.searchPage);
   app.get( '/_api/search'             , accessTokenParser , loginRequired(crowi, app, false) , search.api.search);

src/server/service/file-uploader/gridfs.js

@@ -3,15 +3,16 @@
 module.exports = function(crowi) {
   'use strict';
 
-  var debug = require('debug')('growi:service:fileUploaderLocal')
+  var debug = require('debug')('growi:service:fileUploadergridfs')
   var mongoose = require('mongoose');
   var path = require('path');
+  var fs = require('fs');
   var lib = {};
   var AttachmentFile = {};
 
   // instantiate mongoose-gridfs
   var gridfs = require('mongoose-gridfs')({
-    collection: 'attachments',
+    collection: 'attachmentFiles',
     model: 'AttachmentFile',
     mongooseConnection: mongoose.connection
   });
@@ -19,30 +20,133 @@ module.exports = function(crowi) {
   // obtain a model
   AttachmentFile = gridfs.model;
 
-  // // delete a file
-  // lib.deleteFile = async function(fileId, filePath) {
-  //   debug('File deletion: ' + fileId);
-  //   await AttachmentFile.unlinkById(fileId, function(error, unlinkedAttachment) {
-  //     if (error) {
-  //       throw new Error(error);
-  //     }
-  //   });
-  // };
-
-  // create or save a file
+  // delete a file
+  lib.deleteFile = async function(fileId, filePath) {
+    debug('File deletion: ' + fileId);
+    const file = await getFile(filePath);
+    const id = file.id;
+    AttachmentFile.unlinkById(id, function(error, unlinkedAttachment) {
+      if (error) {
+        throw new Error(error);
+      }
+    });
+    clearCache(fileId);
+  };
+
+  const clearCache = (fileId) => {
+    const cacheFile = createCacheFileName(fileId);
+    const stats = fs.statSync(crowi.cacheDir);
+    if (stats.isFile(`attachment-${fileId}`)) {
+      fs.unlink(cacheFile, (err) => {
+        if (err) {
+          throw new Error('fail to delete cache file', err);
+        }
+      });
+    }
+  };
+
   lib.uploadFile = async function(filePath, contentType, fileStream, options) {
     debug('File uploading: ' + filePath);
-    await AttachmentFile.write({filename: filePath, contentType: contentType}, fileStream,
+    await writeFile(filePath, contentType, fileStream);
+  };
+
+  /**
+   * write file to MongoDB with GridFS (Promise wrapper)
+   */
+  const writeFile = (filePath, contentType, fileStream) => {
+    return new Promise((resolve, reject) => {
+      AttachmentFile.write({
+        filename: filePath,
+        contentType: contentType
+      }, fileStream,
       function(error, createdFile) {
         if (error) {
-          throw new Error('Failed to upload ' + createdFile + 'to gridFS', error);
+          reject(error);
+        }
+        resolve();
+      });
+    });
+  };
+
+  lib.getFileData = async function(filePath) {
+    const file = await getFile(filePath);
+    const id = file.id;
+    const contentType = file.contentType;
+    const data = await readFileData(id);
+    return {
+      data,
+      contentType
+    };
+  };
+
+  /**
+   * get file from MongoDB (Promise wrapper)
+   */
+  const getFile = (filePath) => {
+    return new Promise((resolve, reject) => {
+      AttachmentFile.findOne({
+        filename: filePath
+      }, function(err, file) {
+        if (err) {
+          reject(err);
         }
-        return createdFile._id;
+        resolve(file);
       });
+    });
+  };
+
+  /**
+   * read File in MongoDB (Promise wrapper)
+   */
+  const readFileData = (id) => {
+    return new Promise((resolve, reject) => {
+      let buf;
+      const stream = AttachmentFile.readById(id);
+      stream.on('error', function(error) {
+        reject(error);
+      });
+      stream.on('data', function(data) {
+        if (buf) {
+          buf = Buffer.concat([buf, data]);
+        }
+        else {
+          buf = data;
+        }
+      });
+      stream.on('close', function() {
+        debug('GridFS readstream closed');
+        resolve(buf);
+      });
+    });
+  };
+
+  lib.findDeliveryFile = async function(fileId, filePath) {
+    const cacheFile = createCacheFileName(fileId);
+    debug('Load attachement file into local cache file', cacheFile);
+    const fileStream = fs.createWriteStream(cacheFile);
+    const file = await getFile(filePath);
+    const id = file.id;
+    const buf = await readFileData(id);
+    await writeCacheFile(fileStream, buf);
+    return cacheFile;
+  };
+
+  const createCacheFileName = (fileId) => {
+    return path.join(crowi.cacheDir, `attachment-${fileId}`);
+  };
+
+  /**
+   * write cache file (Promise wrapper)
+   */
+  const writeCacheFile = (fileStream, data) => {
+    return new Promise((resolve, reject) => {
+      fileStream.write(data);
+      resolve();
+    });
   };
 
   lib.generateUrl = function(filePath) {
-    return path.posix.join('/uploads', filePath);
+    return `/${filePath}`;
   };
 
   return lib;