WIP: refactor attachment

ensure that findDeliveryFile methods return readable stream

src/server/models/attachment.js

@@ -3,6 +3,8 @@ module.exports = function(crowi) {
   const mongoose = require('mongoose');
   const ObjectId = mongoose.Schema.Types.ObjectId;
 
+  const urljoin = require('url-join');
+
   const fileUploader = require('../service/file-uploader')(crowi);
 
   let attachmentSchema;
@@ -33,6 +35,17 @@ module.exports = function(crowi) {
     return `/files/${this._id}`;
   });
 
+  attachmentSchema.virtual('filePathOnStorage').get(function() {
+    if (this.filePath != null) {
+      return this.filePath;
+    }
+
+    const pageId = this.page._id || this.page;
+    const filePath = urljoin('/attachment', pageId.toString(), this.fileName);
+
+    return filePath;
+  });
+
   attachmentSchema.statics.findById = function(id) {
     var Attachment = this;
 

src/server/routes/attachment.js

@@ -3,10 +3,6 @@ const logger = require('@alias/logger')('growi:routes:attachment');
 
 const path = require('path');
 const fs = require('fs');
-const util = require('util');
-const https = require('https');
-const httpsGet = util.promisify(https.get);
-const urljoin = require('url-join');
 
 const ApiResponse = require('../util/apiResponse');
 
@@ -19,13 +15,6 @@ module.exports = function(crowi, app) {
   const actions = {};
   const api = {};
 
-  async function findDeliveryFile(attachment) {
-    const pageId = attachment.page._id || attachment.page;
-    const filePath = urljoin('/attachment', pageId.toString(), attachment.fileName);
-
-    return fileUploader.findDeliveryFile(attachment._id, filePath);
-  }
-
   actions.api = api;
 
   api.download = function(req, res) {
@@ -78,23 +67,17 @@ module.exports = function(crowi, app) {
 
     // TODO consider page restrection
 
-    res.set('Content-Type', attachment.fileFormat);
-
+    let fileStream;
     try {
-      const file = await findDeliveryFile(attachment);
-
-      // redirect if string
-      if (typeof file === 'string') {
-        const httpsGetResponse = await httpsGet(file);
-        return res.pipe(httpsGetResponse);
-      }
-
-      return res.send(ApiResponse.success(file.data));
+      fileStream = await fileUploader.findDeliveryFile(attachment);
     }
     catch (e) {
       // TODO handle errors
       return res.json(ApiResponse.error(e.message));
     }
+
+    res.set('Content-Type', attachment.fileFormat);
+    return fileStream.pipe(res);
   };
 
   /**

src/server/service/file-uploader/aws.js

@@ -1,5 +1,7 @@
 const debug = require('debug')('growi:service:fileUploaderAws');
+const logger = require('@alias/logger')('growi:service:fileUploaderAws');
 
+const axios = require('axios');
 const urljoin = require('url-join');
 const aws = require('aws-sdk');
 
@@ -81,14 +83,28 @@ module.exports = function(crowi) {
   };
 
   /**
-   * return local file path string
+   * Find data substance
+   *
+   * @param {Attachment} attachment
+   * @return {stream.Readable} readable stream
    */
-  lib.findDeliveryFile = async function(attachmentId, filePath) {
+  lib.findDeliveryFile = async function(attachment) {
+    // construct url
     const awsConfig = getAwsConfig();
     const baseUrl = `https://${awsConfig.bucket}.s3.amazonaws.com`;
-    const url = urljoin(baseUrl, filePath);
+    const url = urljoin(baseUrl, attachment.filePathOnStorage);
 
-    return url;
+    let response;
+    try {
+      response = await axios.get(url, { responseType: 'stream' });
+    }
+    catch (err) {
+      logger.error(err);
+      throw new Error(`Coudn't get file from AWS for the Attachment (${attachment._id.toString()})`);
+    }
+
+    // return stream.Readable
+    return response.data;
   };
 
   /**

src/server/service/file-uploader/gridfs.js

@@ -1,12 +1,9 @@
-// crowi-fileupload-gridFS
+const debug = require('debug')('growi:service:fileUploaderGridfs');
+const mongoose = require('mongoose');
 
 module.exports = function(crowi) {
   'use strict';
 
-  const debug = require('debug')('growi:service:fileUploaderGridfs');
-  const mongoose = require('mongoose');
-  const path = require('path');
-  const fs = require('fs');
   const lib = {};
 
   // instantiate mongoose-gridfs
@@ -30,19 +27,7 @@ module.exports = function(crowi) {
         throw new Error(error);
       }
     });
-    clearCache(fileId);
-  };
-
-  const clearCache = (fileId) => {
-    const cacheFile = createCacheFileName(fileId);
-    const stats = fs.statSync(crowi.cacheDir);
-    if (stats.isFile(`attachment-${fileId}`)) {
-      fs.unlink(cacheFile, (err) => {
-        if (err) {
-          throw new Error('fail to delete cache file', err);
-        }
-      });
-    }
+    // clearCache(fileId);
   };
 
   /**
@@ -95,85 +80,21 @@ module.exports = function(crowi) {
     });
   };
 
-  lib.getFileData = async function(filePath) {
-    const file = await getFile(filePath);
-    const id = file.id;
-    const contentType = file.contentType;
-    const data = await readFileData(id);
-    return {
-      data,
-      contentType
-    };
-  };
-
-  /**
-   * get file from MongoDB (Promise wrapper)
-   */
-  const getFile = (filePath) => {
-    return new Promise((resolve, reject) => {
-      AttachmentFile.findOne({
-        filename: filePath
-      }, function(err, file) {
-        if (err) {
-          reject(err);
-        }
-        resolve(file);
-      });
-    });
-  };
-
   /**
-   * read File in MongoDB (Promise wrapper)
+   * Find data substance
+   *
+   * @param {Attachment} attachment
+   * @return {stream.Readable} readable stream
    */
-  const readFileData = (id) => {
-    return new Promise((resolve, reject) => {
-      let buf;
-      const stream = AttachmentFile.readById(id);
-      stream.on('error', function(error) {
-        reject(error);
-      });
-      stream.on('data', function(data) {
-        if (buf) {
-          buf = Buffer.concat([buf, data]);
-        }
-        else {
-          buf = data;
-        }
-      });
-      stream.on('close', function() {
-        debug('GridFS readstream closed');
-        resolve(buf);
-      });
-    });
-  };
-
-  lib.findDeliveryFile = async function(fileId, filePath) {
-    const cacheFile = createCacheFileName(fileId);
-    debug('Load attachement file into local cache file', cacheFile);
-    const fileStream = fs.createWriteStream(cacheFile);
-    const file = await getFile(filePath);
-    const id = file.id;
-    const buf = await readFileData(id);
-    await writeCacheFile(fileStream, buf);
-    return cacheFile;
-  };
+  lib.findDeliveryFile = async function(attachment) {
+    const attachmentFile = await AttachmentFile.findOne({ fileName: attachment.fileName });
 
-  const createCacheFileName = (fileId) => {
-    return path.join(crowi.cacheDir, `attachment-${fileId}`);
-  };
-
-  /**
-   * write cache file (Promise wrapper)
-   */
-  const writeCacheFile = (fileStream, data) => {
-    return new Promise((resolve, reject) => {
-      fileStream.write(data);
-      resolve();
-    });
-  };
+    if (attachmentFile == null) {
+      throw new Error(`Any AttachmentFile that relate to the Attachment (${attachment._id.toString()}) does not exist in GridFS`);
+    }
 
-  lib.generateUrl = function(filePath) {
-    return `/${filePath}`;
+    // return stream.Readable
+    return AttachmentFile.readById(attachmentFile.id);
   };
 
   return lib;

src/server/service/file-uploader/index.js

@@ -2,7 +2,9 @@ const envToModuleMappings = {
   aws:     'aws',
   local:   'local',
   none:    'none',
+  mongo:   'gridfs',
   mongodb: 'gridfs',
+  gridfs:  'gridfs',
 };
 
 class FileUploaderFactory {

src/server/service/file-uploader/local.js

@@ -1,14 +1,14 @@
-// crowi-fileupload-local
+const debug = require('debug')('growi:service:fileUploaderLocal');
+
+const fs = require('fs');
+const path = require('path');
+const mkdir = require('mkdirp');
 
 module.exports = function(crowi) {
   'use strict';
 
-  var debug = require('debug')('growi:service:fileUploaderLocal')
-    , fs = require('fs')
-    , path = require('path')
-    , mkdir = require('mkdirp')
-    , lib = {}
-    , basePath = path.posix.join(crowi.publicDir, 'uploads'); // TODO: to configurable
+  const lib = {};
+  const basePath = path.posix.join(crowi.publicDir, 'uploads'); // TODO: to configurable
 
   lib.deleteFile = function(fileId, filePath) {
     debug('File deletion: ' + filePath);
@@ -48,10 +48,25 @@ module.exports = function(crowi) {
   };
 
   /**
-   * return local file path string
+   * Find data substance
+   *
+   * @param {Attachment} attachment
+   * @return {stream.Readable} readable stream
    */
-  lib.findDeliveryFile = async function(attachmentId, filePath) {
-    return path.posix.join('/uploads', filePath);
+  lib.findDeliveryFile = async function(attachment) {
+    const uploadDir = path.posix.join(crowi.publicDir, 'uploads');
+    const filePath = path.posix.join(uploadDir, attachment.filePathOnStorage);
+
+    // check file exists
+    try {
+      fs.statSync(filePath);
+    }
+    catch (err) {
+      throw new Error(`Any AttachmentFile that relate to the Attachment (${attachment._id.toString()}) does not exist in local fs`);
+    }
+
+    // return stream.Readable
+    return fs.createReadStream(filePath);
   };
 
   /**