Merge pull request #7077 from mizozobu/imprv/get-writable

imprv: mv logics

packages/app/src/server/routes/apiv3/g2g-transfer.ts

@@ -7,7 +7,6 @@ import { body } from 'express-validator';
 import multer from 'multer';
 
 import GrowiArchiveImportOption from '~/models/admin/growi-archive-import-option';
-import TransferKeyModel from '~/server/models/transfer-key';
 import { isG2GTransferError } from '~/server/models/vo/g2g-transfer-error';
 import { IDataGROWIInfo, uploadConfigKeys, X_GROWI_TRANSFER_KEY_HEADER_NAME } from '~/server/service/g2g-transfer';
 import loggerFactory from '~/utils/logger';
@@ -110,23 +109,13 @@ module.exports = (crowi: Crowi): Router => {
 
   // Local middleware to check if key is valid or not
   const validateTransferKey = async(req: Request, res: ApiV3Response, next: NextFunction) => {
-    const key = req.headers[X_GROWI_TRANSFER_KEY_HEADER_NAME];
+    const transferKey = req.headers[X_GROWI_TRANSFER_KEY_HEADER_NAME] as string;
 
-    if (typeof key !== 'string') {
-      return res.apiv3Err(new ErrorV3('Invalid transfer key or not set.', 'invalid_transfer_key'), 400);
-    }
-
-    let transferKey;
     try {
-      transferKey = await (TransferKeyModel as any).findOneActiveTransferKey(key); // TODO: Improve TS of models
+      await g2gTransferReceiverService.validateTransferKey(transferKey);
     }
     catch (err) {
-      logger.error(err);
-      return res.apiv3Err(new ErrorV3('Error occurred while trying to fing a transfer key.', 'failed_to_find_transfer_key'), 500);
-    }
-
-    if (transferKey == null) {
-      return res.apiv3Err(new ErrorV3('Transfer key has expired or not found.', 'transfer_key_expired_or_not_found'), 404);
+      return res.apiv3Err(new ErrorV3('Invalid transfer key', 'invalid_transfer_key'), 403);
     }
 
     next();
@@ -350,12 +339,12 @@ module.exports = (crowi: Crowi): Router => {
   // TODO: Use socket to send progress info to the client
   // eslint-disable-next-line max-len
   pushRouter.post('/transfer', accessTokenParser, loginRequiredStrictly, adminRequired, validator.transfer, apiV3FormValidator, async(req: AuthorizedRequest, res: ApiV3Response) => {
-    const { transferKey: transferKeyString, collections, optionsMap } = req.body;
+    const { transferKey, collections, optionsMap } = req.body;
 
     // Parse transfer key
     let tk: TransferKey;
     try {
-      tk = TransferKey.parse(transferKeyString);
+      tk = TransferKey.parse(transferKey);
     }
     catch (err) {
       logger.error(err);

packages/app/src/server/service/attachment.js

@@ -35,7 +35,7 @@ class AttachmentService {
     let attachment;
     try {
       attachment = Attachment.createWithoutSave(pageId, user, fileStream, file.originalname, file.mimetype, file.size, attachmentType);
-      await fileUploadService.uploadFile(fileStream, attachment);
+      await fileUploadService.uploadAttachment(fileStream, attachment);
       await attachment.save();
     }
     catch (err) {

packages/app/src/server/service/file-uploader/aws.ts

@@ -86,7 +86,7 @@ module.exports = (crowi) => {
     return true;
   };
 
-  lib.isValidUploadSettings = () => {
+  lib.isValidUploadSettings = function() {
     return configManager.getConfig('crowi', 'aws:s3AccessKeyId') != null
       && configManager.getConfig('crowi', 'aws:s3SecretAccessKey') != null
       && (
@@ -96,11 +96,11 @@ module.exports = (crowi) => {
       && configManager.getConfig('crowi', 'aws:s3Bucket') != null;
   };
 
-  lib.canRespond = () => {
+  lib.canRespond = function() {
     return !configManager.getConfig('crowi', 'aws:referenceFileWithRelayMode');
   };
 
-  lib.respond = async(res, attachment) => {
+  lib.respond = async function(res, attachment) {
     if (!lib.getIsUploadable()) {
       throw new Error('AWS is not configured.');
     }
@@ -134,12 +134,12 @@ module.exports = (crowi) => {
 
   };
 
-  lib.deleteFile = async(attachment) => {
+  lib.deleteFile = async function(attachment) {
     const filePath = getFilePathOnStorage(attachment);
     return lib.deleteFileByFilePath(filePath);
   };
 
-  lib.deleteFiles = async(attachments) => {
+  lib.deleteFiles = async function(attachments) {
     if (!lib.getIsUploadable()) {
       throw new Error('AWS is not configured.');
     }
@@ -157,7 +157,7 @@ module.exports = (crowi) => {
     return s3.send(new DeleteObjectsCommand(totalParams));
   };
 
-  lib.deleteFileByFilePath = async(filePath) => {
+  lib.deleteFileByFilePath = async function(filePath) {
     if (!lib.getIsUploadable()) {
       throw new Error('AWS is not configured.');
     }
@@ -179,7 +179,7 @@ module.exports = (crowi) => {
     return s3.send(new DeleteObjectCommand(params));
   };
 
-  lib.uploadFile = async(fileStream, attachment) => {
+  lib.uploadAttachment = async function(fileStream, attachment) {
     if (!lib.getIsUploadable()) {
       throw new Error('AWS is not configured.');
     }
@@ -201,7 +201,22 @@ module.exports = (crowi) => {
     return s3.send(new PutObjectCommand(params));
   };
 
-  lib.findDeliveryFile = async(attachment) => {
+  lib.saveFile = async function({ filePath, contentType, data }) {
+    const s3 = S3Factory();
+    const awsConfig = getAwsConfig();
+
+    const params = {
+      Bucket: awsConfig.bucket,
+      ContentType: contentType,
+      Key: filePath,
+      Body: data,
+      ACL: 'public-read',
+    };
+
+    return s3.send(new PutObjectCommand(params));
+  };
+
+  lib.findDeliveryFile = async function(attachment) {
     if (!lib.getIsReadable()) {
       throw new Error('AWS is not configured.');
     }
@@ -234,16 +249,16 @@ module.exports = (crowi) => {
     return stream;
   };
 
-  lib.checkLimit = async(uploadFileSize) => {
-    const maxFileSize = crowi.configManager.getConfig('crowi', 'app:maxFileSize');
-    const totalLimit = crowi.configManager.getConfig('crowi', 'app:fileUploadTotalLimit');
+  lib.checkLimit = async function(uploadFileSize) {
+    const maxFileSize = configManager.getConfig('crowi', 'app:maxFileSize');
+    const totalLimit = configManager.getConfig('crowi', 'app:fileUploadTotalLimit');
     return lib.doCheckLimit(uploadFileSize, maxFileSize, totalLimit);
   };
 
   /**
    * List files in storage
    */
-  lib.listFiles = async() => {
+  lib.listFiles = async function() {
     if (!lib.getIsReadable()) {
       throw new Error('AWS is not configured.');
     }

packages/app/src/server/service/file-uploader/gcs.js

@@ -50,16 +50,16 @@ module.exports = function(crowi) {
   }
 
   lib.isValidUploadSettings = function() {
-    return this.configManager.getConfig('crowi', 'gcs:apiKeyJsonPath') != null
-      && this.configManager.getConfig('crowi', 'gcs:bucket') != null;
+    return configManager.getConfig('crowi', 'gcs:apiKeyJsonPath') != null
+      && configManager.getConfig('crowi', 'gcs:bucket') != null;
   };
 
   lib.canRespond = function() {
-    return !this.configManager.getConfig('crowi', 'gcs:referenceFileWithRelayMode');
+    return !configManager.getConfig('crowi', 'gcs:referenceFileWithRelayMode');
   };
 
   lib.respond = async function(res, attachment) {
-    if (!this.getIsUploadable()) {
+    if (!lib.getIsUploadable()) {
       throw new Error('GCS is not configured.');
     }
     const temporaryUrl = attachment.getValidTemporaryUrl();
@@ -71,7 +71,7 @@ module.exports = function(crowi) {
     const myBucket = gcs.bucket(getGcsBucket());
     const filePath = getFilePathOnStorage(attachment);
     const file = myBucket.file(filePath);
-    const lifetimeSecForTemporaryUrl = this.configManager.getConfig('crowi', 'gcs:lifetimeSecForTemporaryUrl');
+    const lifetimeSecForTemporaryUrl = configManager.getConfig('crowi', 'gcs:lifetimeSecForTemporaryUrl');
 
     // issue signed url (default: expires 120 seconds)
     // https://cloud.google.com/storage/docs/access-control/signed-urls
@@ -104,7 +104,7 @@ module.exports = function(crowi) {
   };
 
   lib.deleteFilesByFilePaths = function(filePaths) {
-    if (!this.getIsUploadable()) {
+    if (!lib.getIsUploadable()) {
       throw new Error('GCS is not configured.');
     }
 
@@ -120,8 +120,8 @@ module.exports = function(crowi) {
     });
   };
 
-  lib.uploadFile = function(fileStream, attachment) {
-    if (!this.getIsUploadable()) {
+  lib.uploadAttachment = function(fileStream, attachment) {
+    if (!lib.getIsUploadable()) {
       throw new Error('GCS is not configured.');
     }
 
@@ -137,6 +137,13 @@ module.exports = function(crowi) {
     return myBucket.upload(fileStream.path, options);
   };
 
+  lib.saveFile = async function({ filePath, contentType, data }) {
+    const gcs = getGcsInstance();
+    const myBucket = gcs.bucket(getGcsBucket());
+
+    return myBucket.file(filePath).save(data, { resumable: false });
+  };
+
   /**
    * Find data substance
    *
@@ -144,7 +151,7 @@ module.exports = function(crowi) {
    * @return {stream.Readable} readable stream
    */
   lib.findDeliveryFile = async function(attachment) {
-    if (!this.getIsReadable()) {
+    if (!lib.getIsReadable()) {
       throw new Error('GCS is not configured.');
     }
 
@@ -178,17 +185,17 @@ module.exports = function(crowi) {
    * In detail, the followings are checked.
    * - per-file size limit (specified by MAX_FILE_SIZE)
    */
-  lib.checkLimit = async(uploadFileSize) => {
-    const maxFileSize = crowi.configManager.getConfig('crowi', 'app:maxFileSize');
-    const gcsTotalLimit = crowi.configManager.getConfig('crowi', 'app:fileUploadTotalLimit');
+  lib.checkLimit = async function(uploadFileSize) {
+    const maxFileSize = configManager.getConfig('crowi', 'app:maxFileSize');
+    const gcsTotalLimit = configManager.getConfig('crowi', 'app:fileUploadTotalLimit');
     return lib.doCheckLimit(uploadFileSize, maxFileSize, gcsTotalLimit);
   };
 
   /**
    * List files in storage
    */
-  lib.listFiles = async() => {
-    if (!this.getIsReadable()) {
+  lib.listFiles = async function() {
+    if (!lib.getIsReadable()) {
       throw new Error('GCS is not configured.');
     }
 

packages/app/src/server/service/file-uploader/gridfs.js

@@ -1,3 +1,5 @@
+import { Readable } from 'stream';
+
 import loggerFactory from '~/utils/logger';
 
 const logger = loggerFactory('growi:service:fileUploaderGridfs');
@@ -7,6 +9,7 @@ const mongoose = require('mongoose');
 
 module.exports = function(crowi) {
   const Uploader = require('./uploader');
+  const { configManager } = crowi;
   const lib = new Uploader(crowi);
   const COLLECTION_NAME = 'attachmentFiles';
   const CHUNK_COLLECTION_NAME = `${COLLECTION_NAME}.chunks`;
@@ -84,13 +87,13 @@ module.exports = function(crowi) {
    * - per-file size limit (specified by MAX_FILE_SIZE)
    * - mongodb(gridfs) size limit (specified by MONGO_GRIDFS_TOTAL_LIMIT)
    */
-  lib.checkLimit = async(uploadFileSize) => {
-    const maxFileSize = crowi.configManager.getConfig('crowi', 'app:maxFileSize');
-    const totalLimit = crowi.configManager.getFileUploadTotalLimit();
+  lib.checkLimit = async function(uploadFileSize) {
+    const maxFileSize = configManager.getConfig('crowi', 'app:maxFileSize');
+    const totalLimit = configManager.getFileUploadTotalLimit();
     return lib.doCheckLimit(uploadFileSize, maxFileSize, totalLimit);
   };
 
-  lib.uploadFile = async function(fileStream, attachment) {
+  lib.uploadAttachment = async function(fileStream, attachment) {
     logger.debug(`File uploading: fileName=${attachment.fileName}`);
 
     return AttachmentFile.promisifiedWrite(
@@ -102,6 +105,20 @@ module.exports = function(crowi) {
     );
   };
 
+  lib.saveFile = async function({ filePath, contentType, data }) {
+    const readable = new Readable();
+    readable.push(data);
+    readable.push(null); // EOF
+
+    return AttachmentFile.promisifiedWrite(
+      {
+        filename: filePath,
+        contentType,
+      },
+      readable,
+    );
+  };
+
   /**
    * Find data substance
    *
@@ -128,7 +145,7 @@ module.exports = function(crowi) {
   /**
    * List files in storage
    */
-  lib.listFiles = async() => {
+  lib.listFiles = async function() {
     const attachmentFiles = await AttachmentFile.find();
     return attachmentFiles.map(({ filename: name, length: size }) => ({
       name, size,

packages/app/src/server/service/file-uploader/local.js

@@ -1,3 +1,5 @@
+import { Readable } from 'stream';
+
 import loggerFactory from '~/utils/logger';
 
 const logger = loggerFactory('growi:service:fileUploaderLocal');
@@ -12,6 +14,7 @@ const urljoin = require('url-join');
 
 module.exports = function(crowi) {
   const Uploader = require('./uploader');
+  const { configManager } = crowi;
   const lib = new Uploader(crowi);
   const basePath = path.posix.join(crowi.publicDir, 'uploads');
 
@@ -51,7 +54,7 @@ module.exports = function(crowi) {
 
   lib.deleteFiles = async function(attachments) {
     attachments.map((attachment) => {
-      return this.deleteFile(attachment);
+      return lib.deleteFile(attachment);
     });
   };
 
@@ -68,7 +71,7 @@ module.exports = function(crowi) {
     return fs.unlinkSync(filePath);
   };
 
-  lib.uploadFile = async function(fileStream, attachment) {
+  lib.uploadAttachment = async function(fileStream, attachment) {
     logger.debug(`File uploading: fileName=${attachment.fileName}`);
 
     const filePath = getFilePathOnStorage(attachment);
@@ -81,6 +84,20 @@ module.exports = function(crowi) {
     return streamToPromise(stream);
   };
 
+  lib.saveFile = async function({ filePath, contentType, data }) {
+    const absFilePath = path.posix.join(basePath, filePath);
+    const dirpath = path.posix.dirname(absFilePath);
+
+    // mkdir -p
+    mkdir.sync(dirpath);
+
+    const fileStream = new Readable();
+    fileStream.push(data);
+    fileStream.push(null); // EOF
+    const stream = fileStream.pipe(fs.createWriteStream(absFilePath));
+    return streamToPromise(stream);
+  };
+
   /**
    * Find data substance
    *
@@ -108,18 +125,18 @@ module.exports = function(crowi) {
    * In detail, the followings are checked.
    * - per-file size limit (specified by MAX_FILE_SIZE)
    */
-  lib.checkLimit = async(uploadFileSize) => {
-    const maxFileSize = crowi.configManager.getConfig('crowi', 'app:maxFileSize');
-    const totalLimit = crowi.configManager.getConfig('crowi', 'app:fileUploadTotalLimit');
+  lib.checkLimit = async function(uploadFileSize) {
+    const maxFileSize = configManager.getConfig('crowi', 'app:maxFileSize');
+    const totalLimit = configManager.getConfig('crowi', 'app:fileUploadTotalLimit');
     return lib.doCheckLimit(uploadFileSize, maxFileSize, totalLimit);
   };
 
   /**
    * Checks if Uploader can respond to the HTTP request.
    */
-  lib.canRespond = () => {
+  lib.canRespond = function() {
     // Check whether to use internal redirect of nginx or Apache.
-    return lib.configManager.getConfig('crowi', 'fileUpload:local:useInternalRedirect');
+    return configManager.getConfig('crowi', 'fileUpload:local:useInternalRedirect');
   };
 
   /**
@@ -127,11 +144,11 @@ module.exports = function(crowi) {
    * @param {Response} res
    * @param {Response} attachment
    */
-  lib.respond = (res, attachment) => {
+  lib.respond = function(res, attachment) {
     // Responce using internal redirect of nginx or Apache.
     const storagePath = getFilePathOnStorage(attachment);
     const relativePath = path.relative(crowi.publicDir, storagePath);
-    const internalPathRoot = lib.configManager.getConfig('crowi', 'fileUpload:local:internalRedirectPath');
+    const internalPathRoot = configManager.getConfig('crowi', 'fileUpload:local:internalRedirectPath');
     const internalPath = urljoin(internalPathRoot, relativePath);
     res.set('X-Accel-Redirect', internalPath);
     res.set('X-Sendfile', storagePath);
@@ -141,7 +158,7 @@ module.exports = function(crowi) {
   /**
    * List files in storage
    */
-  lib.listFiles = async() => {
+  lib.listFiles = async function() {
     // `mkdir -p` to avoid ENOENT error
     await mkdir(basePath);
     const filePaths = await readdirRecursively(basePath);

packages/app/src/server/service/file-uploader/uploader.js

@@ -1,3 +1,9 @@
+import { randomUUID } from 'crypto';
+
+import loggerFactory from '~/utils/logger';
+
+const logger = loggerFactory('growi:service:fileUploader');
+
 // file uploader virtual class
 // 各アップローダーで共通のメソッドはここで定義する
 
@@ -12,6 +18,30 @@ class Uploader {
     return !this.configManager.getConfig('crowi', 'app:fileUploadDisabled') && this.isValidUploadSettings();
   }
 
+  /**
+   * Returns whether write opration to the storage is permitted
+   * @returns Whether write opration to the storage is permitted
+   */
+  async isWritable() {
+    const filePath = `${randomUUID()}.growi`;
+    const data = 'This file was created during g2g transfer to check write permission. You can safely remove this file.';
+
+    try {
+      await this.saveFile({
+        filePath,
+        contentType: 'text/plain',
+        data,
+      });
+      // TODO: delete tmp file in background
+
+      return true;
+    }
+    catch (err) {
+      logger.error(err);
+      return false;
+    }
+  }
+
   // File reading is possible even if uploading is disabled
   getIsReadable() {
     return this.isValidUploadSettings();

packages/app/src/server/service/g2g-transfer.ts

@@ -1,4 +1,3 @@
-import { randomUUID } from 'crypto';
 import { createReadStream, ReadStream } from 'fs';
 import { basename } from 'path';
 import { Readable } from 'stream';
@@ -140,81 +139,6 @@ const generateAxiosRequestConfigWithTransferKey = (tk: TransferKey, additionalHe
   };
 };
 
-
-/**
- * Check whether the storage is writable
- * @param crowi Crowi instance
- * @returns Whether the storage is writable
- */
-const hasWritePermission = async(crowi: any): Promise<boolean> => {
-  const { fileUploadService } = crowi;
-
-  let writable = true;
-  const fileStream = new Readable();
-  fileStream.push('This file was created during g2g transfer to check write permission. You can safely remove this file.');
-  fileStream.push(null); // EOF
-  const attachment = {
-    fileName: `${randomUUID()}.growi`,
-    filePath: '',
-    fileFormat: 'text/plain',
-  };
-
-  try {
-    await fileUploadService.uploadFile(fileStream, attachment);
-    // TODO: remove tmp file
-  }
-  catch (err) {
-    writable = false;
-    logger.error(err);
-  }
-
-  return writable;
-};
-
-/**
- * generate GROWIInfo
- * @param crowi Crowi instance
- * @returns
- */
-const generateGROWIInfo = async(crowi: any): Promise<IDataGROWIInfo> => {
-  // TODO: add attachment file limit
-  const { configManager } = crowi;
-  const userUpperLimit = configManager.getConfig('crowi', 'security:userUpperLimit');
-  const fileUploadDisabled = configManager.getConfig('crowi', 'app:fileUploadDisabled');
-  const fileUploadTotalLimit = configManager.getFileUploadTotalLimit();
-  const version = crowi.version;
-  const writable = await hasWritePermission(crowi);
-
-  const attachmentInfo = {
-    type: configManager.getConfig('crowi', 'app:fileUploadType'),
-    bucket: undefined,
-    customEndpoint: undefined, // for S3
-    uploadNamespace: undefined, // for GCS
-    writable,
-  };
-
-  // put storage location info to check storage identification
-  switch (attachmentInfo.type) {
-    case 'aws':
-      attachmentInfo.bucket = configManager.getConfig('crowi', 'aws:s3Bucket');
-      attachmentInfo.customEndpoint = configManager.getConfig('crowi', 'aws:s3CustomEndpoint');
-      break;
-    case 'gcs':
-      attachmentInfo.bucket = configManager.getConfig('crowi', 'gcs:bucket');
-      attachmentInfo.uploadNamespace = configManager.getConfig('crowi', 'gcs:uploadNamespace');
-      break;
-    default:
-  }
-
-  return {
-    userUpperLimit,
-    fileUploadDisabled,
-    fileUploadTotalLimit,
-    version,
-    attachmentInfo,
-  };
-};
-
 export class G2GTransferPusherService implements Pusher {
 
   crowi: any;
@@ -491,15 +415,62 @@ export class G2GTransferReceiverService implements Receiver {
     this.crowi = crowi;
   }
 
-  public async validateTransferKey(transferKeyString: string): Promise<void> {
-    // Parse to tk
-    // Find active tkd
+  public async validateTransferKey(key: string): Promise<void> {
+    const transferKey = await (TransferKeyModel as any).findOne({ key });
 
-    return;
+    if (transferKey == null) {
+      throw new Error(`Transfer key "${key}" was expired or not found`);
+    }
+
+    try {
+      TransferKey.parse(transferKey.keyString);
+    }
+    catch (err) {
+      logger.error(err);
+      throw new Error(`Transfer key "${key}" is invalid`);
+    }
   }
 
+  /**
+   * generate GROWIInfo
+   * @returns
+   */
   public async answerGROWIInfo(): Promise<IDataGROWIInfo> {
-    return generateGROWIInfo(this.crowi);
+    // TODO: add attachment file limit
+    const { version, configManager, fileUploadService } = this.crowi;
+    const userUpperLimit = configManager.getConfig('crowi', 'security:userUpperLimit');
+    const fileUploadDisabled = configManager.getConfig('crowi', 'app:fileUploadDisabled');
+    const fileUploadTotalLimit = configManager.getFileUploadTotalLimit();
+    const isWritable = await fileUploadService.isWritable();
+
+    const attachmentInfo = {
+      type: configManager.getConfig('crowi', 'app:fileUploadType'),
+      bucket: undefined,
+      customEndpoint: undefined, // for S3
+      uploadNamespace: undefined, // for GCS
+      writable: isWritable,
+    };
+
+    // put storage location info to check storage identification
+    switch (attachmentInfo.type) {
+      case 'aws':
+        attachmentInfo.bucket = configManager.getConfig('crowi', 'aws:s3Bucket');
+        attachmentInfo.customEndpoint = configManager.getConfig('crowi', 'aws:s3CustomEndpoint');
+        break;
+      case 'gcs':
+        attachmentInfo.bucket = configManager.getConfig('crowi', 'gcs:bucket');
+        attachmentInfo.uploadNamespace = configManager.getConfig('crowi', 'gcs:uploadNamespace');
+        break;
+      default:
+    }
+
+    return {
+      userUpperLimit,
+      fileUploadDisabled,
+      fileUploadTotalLimit,
+      version,
+      attachmentInfo,
+    };
   }
 
   public async createTransferKey(appSiteUrlOrigin: string): Promise<string> {
@@ -535,7 +506,7 @@ export class G2GTransferReceiverService implements Receiver {
   public async receiveAttachment(content: Readable, attachmentMap): Promise<void> {
     // TODO: test with S3, local
     const { fileUploadService } = this.crowi;
-    return fileUploadService.uploadFile(content, attachmentMap);
+    return fileUploadService.uploadAttachment(content, attachmentMap);
   }
 
   /**