Merge pull request #1138 from tats-u/minio

Support S3-compatible object storage services like MinIO

resource/locales/en-US/translation.json

@@ -427,6 +427,8 @@
     "change_setting": "Caution:if you change this setting not completed, you will not be able to access files you have uploaded so far.",
     "region": "Region",
     "bucket name": "Bucket name",
+    "custom endpoint": "Custom endpoint",
+    "custom_endpoint_change": "Input the URL of the endpoint of an object storage service like MinIO that has a S3-compatible API.  Amazon S3 is used if empty.",
     "Plugin settings": "Plugin settings",
     "Enable plugin loading": "Enable plugin loading",
     "Load plugins": "Load plugins",

resource/locales/ja/translation.json

@@ -425,6 +425,8 @@
     "change_setting": "この設定を途中で変更すると、これまでにアップロードしたファイル等へのアクセスができなくなりますのでご注意下さい。",
     "region": "リージョン",
     "bucket name": "バケット名",
+    "custom endpoint": "カスタムエンドポイント",
+    "custom_endpoint_change": "MinIOなど、S3互換APIを持つ他のオブジェクトストレージサービスを使用する場合のみ、そのエンドポイントのURLを入力してください。空欄の場合は、Amazon S3を使用します。",
     "Plugin settings": "プラグイン設定",
     "Enable plugin loading": "プラグインの読み込みを有効にします。",
     "Load plugins": "プラグインを読み込む",

src/server/form/admin/aws.js

@@ -4,6 +4,7 @@ const field = form.field;
 
 module.exports = form(
   field('settingForm[aws:region]', 'リージョン').trim().is(/^[a-z]+-[a-z]+-\d+$/, 'リージョンには、AWSリージョン名を入力してください。 例: ap-northeast-1'),
+  field('settingForm[aws:customEndpoint]', 'カスタムエンドポイント').trim().is(/^(https?:\/\/[^/]+|)$/, 'カスタムエンドポイントは、http(s)://で始まるURLを指定してください。また、末尾の/は不要です。'),
   field('settingForm[aws:bucket]', 'バケット名').trim(),
   field('settingForm[aws:accessKeyId]', 'Access Key Id').trim().is(/^[\da-zA-Z]+$/),
   field('settingForm[aws:secretAccessKey]', 'Secret Access Key').trim(),

src/server/models/config.js

@@ -75,6 +75,7 @@ module.exports = function(crowi) {
       'aws:region'          : 'ap-northeast-1',
       'aws:accessKeyId'     : undefined,
       'aws:secretAccessKey' : undefined,
+      'aws:customEndpoint'  : undefined,
 
       'mail:from'         : undefined,
       'mail:smtpHost'     : undefined,

src/server/service/file-uploader/aws.js

@@ -1,6 +1,5 @@
 const logger = require('@alias/logger')('growi:service:fileUploaderAws');
 
-const axios = require('axios');
 const urljoin = require('url-join');
 const aws = require('aws-sdk');
 
@@ -15,6 +14,7 @@ module.exports = function(crowi) {
       secretAccessKey: configManager.getConfig('crowi', 'aws:secretAccessKey'),
       region: configManager.getConfig('crowi', 'aws:region'),
       bucket: configManager.getConfig('crowi', 'aws:bucket'),
+      customEndpoint: configManager.getConfig('crowi', 'aws:customEndpoint'),
     };
   }
 
@@ -29,9 +29,11 @@ module.exports = function(crowi) {
       accessKeyId: awsConfig.accessKeyId,
       secretAccessKey: awsConfig.secretAccessKey,
       region: awsConfig.region,
+      s3ForcePathStyle: awsConfig.customEndpoint ? true : undefined,
     });
 
-    return new aws.S3();
+    // undefined & null & '' => default endpoint (genuine S3)
+    return new aws.S3({ endpoint: awsConfig.customEndpoint || undefined });
   }
 
   function getFilePathOnStorage(attachment) {
@@ -89,14 +91,17 @@ module.exports = function(crowi) {
    * @return {stream.Readable} readable stream
    */
   lib.findDeliveryFile = async function(attachment) {
-    // construct url
+    const s3 = S3Factory(this.getIsUploadable());
     const awsConfig = getAwsConfig();
-    const baseUrl = `https://${awsConfig.bucket}.s3.amazonaws.com`;
-    const url = urljoin(baseUrl, getFilePathOnStorage(attachment));
+    const filePath = getFilePathOnStorage(attachment);
 
-    let response;
+    let stream;
     try {
-      response = await axios.get(url, { responseType: 'stream' });
+      const params = {
+        Bucket: awsConfig.bucket,
+        Key: filePath,
+      };
+      stream = s3.getObject(params).createReadStream();
     }
     catch (err) {
       logger.error(err);
@@ -104,7 +109,7 @@ module.exports = function(crowi) {
     }
 
     // return stream.Readable
-    return response.data;
+    return stream;
   };
 
   /**

src/server/service/file-uploader/uploader.js

@@ -13,7 +13,9 @@ class Uploader {
     if (method === 'aws' && (
       !this.configManager.getConfig('crowi', 'aws:accessKeyId')
         || !this.configManager.getConfig('crowi', 'aws:secretAccessKey')
-        || !this.configManager.getConfig('crowi', 'aws:region')
+        || (
+          !this.configManager.getConfig('crowi', 'aws:region')
+            && !this.configManager.getConfig('crowi', 'aws:customEndpoint'))
         || !this.configManager.getConfig('crowi', 'aws:bucket'))) {
       return false;
     }

src/server/util/middlewares.js

@@ -274,7 +274,7 @@ module.exports = (crowi, app) => {
 
   middlewares.awsEnabled = function() {
     return function(req, res, next) {
-      if (configManager.getConfig('crowi', 'aws:region') !== ''
+      if ((configManager.getConfig('crowi', 'aws:region') !== '' || this.configManager.getConfig('crowi', 'aws:customEndpoint') !== '')
           && configManager.getConfig('crowi', 'aws:bucket') !== ''
           && configManager.getConfig('crowi', 'aws:accessKeyId') !== ''
           && configManager.getConfig('crowi', 'aws:secretAccessKey') !== '') {

src/server/views/admin/app.html

@@ -252,6 +252,19 @@
           </div>
         </div>
 
+        <div class="form-group">
+          <label for="settingForm[aws:customEndpoint]" class="col-xs-3 control-label">{{ t('app_setting.custom endpoint') }}</label>
+          <div class="col-xs-6">
+            <input class="form-control"
+                   id="settingForm[aws:customEndpoint]"
+                   type="text"
+                   name="settingForm[aws:customEndpoint]"
+                   placeholder="例: http://localhost:9000"
+                   value="{{ getConfig('crowi', 'aws:customEndpoint') | default('') }}">
+                   <p class="help-block">{{ t("app_setting.custom_endpoint_change") }}</p>
+          </div>
+        </div>
+
         <div class="form-group">
           <label for="settingForm[aws:bucket]" class="col-xs-3 control-label">{{ t('app_setting.bucket name') }}</label>
           <div class="col-xs-6">