Merge pull request #1212 from weseek/feat/export-n-import-revision-6

Feat/export n import revision 6

src/server/routes/apiv3/export.js

@@ -2,6 +2,7 @@ const loggerFactory = require('@alias/logger');
 
 const logger = loggerFactory('growi:routes:apiv3:export'); // eslint-disable-line no-unused-vars
 const path = require('path');
+const fs = require('fs');
 
 const express = require('express');
 
@@ -22,14 +23,19 @@ module.exports = (crowi) => {
    *  /export/status:
    *    get:
    *      tags: [Export]
-   *      description: get properties of zip files for export
-   *      produces:
-   *        - application/json
+   *      description: get properties of stored zip files for export
    *      responses:
    *        200:
-   *          description: export cache status
+   *          description: the zip file statuses
    *          content:
    *            application/json:
+   *              schema:
+   *                properties:
+   *                  zipFileStats:
+   *                    type: array
+   *                    items:
+   *                      type: object
+   *                      description: the property of each file
    */
   router.get('/status', async(req, res) => {
     const zipFileStats = await exportService.getStatus();
@@ -44,21 +50,24 @@ module.exports = (crowi) => {
    *  /export:
    *    post:
    *      tags: [Export]
-   *      description: generate a zipped json for multiple collections
-   *      produces:
-   *        - application/json
+   *      description: generate zipped jsons for collections
    *      responses:
    *        200:
    *          description: a zip file is generated
    *          content:
    *            application/json:
+   *              schema:
+   *                properties:
+   *                  zipFileStat:
+   *                    type: object
+   *                    description: the property of the zip file
    */
   router.post('/', async(req, res) => {
     // TODO: add express validator
     try {
       const { collections } = req.body;
       // get model for collection
-      const models = collections.map(collectionName => exportService.getModelFromCollectionName(collectionName));
+      const models = collections.map(collectionName => growiBridgeService.getModelFromCollectionName(collectionName));
 
       const [metaJson, jsonFiles] = await Promise.all([
         exportService.createMetaJson(),
@@ -90,23 +99,24 @@ module.exports = (crowi) => {
   /**
    * @swagger
    *
-   *  /export:
+   *  /export/{fileName}:
    *    delete:
    *      tags: [Export]
-   *      description: unlink all json and zip files for exports
-   *      produces:
-   *        - application/json
+   *      description: delete the file
    *      parameters:
    *        - name: fileName
    *          in: path
-   *          description: file name of zip file
+   *          description: the file name of zip file
+   *          required: true
    *          schema:
    *            type: string
    *      responses:
    *        200:
-   *          description: the json and zip file are deleted
+   *          description: the file is deleted
    *          content:
    *            application/json:
+   *              schema:
+   *                type: object
    */
   router.delete('/:fileName', async(req, res) => {
     // TODO: add express validator
@@ -114,7 +124,7 @@ module.exports = (crowi) => {
 
     try {
       const zipFile = exportService.getFile(fileName);
-      exportService.deleteZipFile(zipFile);
+      fs.unlinkSync(zipFile);
 
       // TODO: use res.apiv3
       return res.status(200).send({ ok: true });

src/server/routes/apiv3/import.js

@@ -2,6 +2,7 @@ const loggerFactory = require('@alias/logger');
 
 const logger = loggerFactory('growi:routes:apiv3:import'); // eslint-disable-line no-unused-vars
 const path = require('path');
+const fs = require('fs');
 const multer = require('multer');
 const { ObjectId } = require('mongoose').Types;
 
@@ -85,13 +86,13 @@ module.exports = (crowi) => {
    *    post:
    *      tags: [Import]
    *      description: import a collection from a zipped json
-   *      produces:
-   *        - application/json
    *      responses:
    *        200:
-   *          description: data is successfully imported
+   *          description: the data is successfully imported
    *          content:
    *            application/json:
+   *              schema:
+   *                type: object
    */
   router.post('/', async(req, res) => {
     // TODO: add express validator
@@ -112,7 +113,7 @@ module.exports = (crowi) => {
       importService.validate(meta);
 
       await Promise.all(filteredFileStats.map(async({ fileName, collectionName, size }) => {
-        const Model = importService.getModelFromCollectionName(collectionName);
+        const Model = growiBridgeService.getModelFromCollectionName(collectionName);
         const jsonFile = importService.getFile(fileName);
 
         let overwriteParams;
@@ -141,27 +142,24 @@ module.exports = (crowi) => {
    *    post:
    *      tags: [Import]
    *      description: upload a zip file
-   *      produces:
-   *        - application/json
    *      responses:
    *        200:
-   *          description: file is uploaded
+   *          description: the file is uploaded
    *          content:
    *            application/json:
    *              schema:
    *                properties:
-   *                  properties:
-   *                    meta:
+   *                  meta:
+   *                    type: object
+   *                    description: the meta data of the uploaded file
+   *                  fileName:
+   *                    type: string
+   *                    description: the base name of the uploaded file
+   *                  fileStats:
+   *                    type: array
+   *                    items:
    *                      type: object
-   *                      description: meta data of the uploaded file
-   *                    fileName:
-   *                      type: string
-   *                      description: base name of the uploaded file
-   *                    fileStats:
-   *                      type: array
-   *                      items:
-   *                        type: object
-   *                        description: property of each extracted file
+   *                      description: the property of each extracted file
    */
   router.post('/upload', uploads.single('file'), async(req, res) => {
     const { file } = req;
@@ -189,30 +187,31 @@ module.exports = (crowi) => {
   /**
    * @swagger
    *
-   *  /import/upload:
+   *  /import/{fileName}:
    *    post:
    *      tags: [Import]
    *      description: delete a zip file
-   *      produces:
-   *        - application/json
    *      parameters:
    *        - name: fileName
    *          in: path
-   *          description: file name of zip file
+   *          description: the file name of zip file
+   *          required: true
    *          schema:
    *            type: string
    *      responses:
    *        200:
-   *          description: file is deleted
+   *          description: the file is deleted
    *          content:
    *            application/json:
+   *              schema:
+   *                type: object
    */
   router.delete('/:fileName', async(req, res) => {
     const { fileName } = req.params;
 
     try {
       const zipFile = importService.getFile(fileName);
-      importService.deleteZipFile(zipFile);
+      fs.unlinkSync(zipFile);
 
       // TODO: use res.apiv3
       return res.send({

src/server/routes/apiv3/mongo.js

@@ -20,11 +20,9 @@ module.exports = (crowi) => {
    *    get:
    *      tags: [Mongo]
    *      description: get mongodb collections names
-   *      produces:
-   *        - application/json
    *      responses:
    *        200:
-   *          description: list of collections in mongoDB
+   *          description: a list of collections in mongoDB
    *          content:
    *            application/json:
    *              schema:

src/server/service/export.js

@@ -11,16 +11,11 @@ class ExportService {
     this.crowi = crowi;
     this.appService = crowi.appService;
     this.growiBridgeService = crowi.growiBridgeService;
+    this.getFile = this.growiBridgeService.getFile.bind(this);
     this.baseDir = path.join(crowi.tmpDir, 'downloads');
-    this.metaFileName = 'meta.json';
-    this.encoding = 'utf-8';
     this.per = 100;
     this.zlibLevel = 9; // 0(min) - 9(max)
 
-    // { pages: Page, users: User, ... }
-    this.collectionMap = {};
-    this.initCollectionMap(crowi.models);
-
     // this.files = {
     //   configs: path.join(this.baseDir, 'configs.json'),
     //   pages: path.join(this.baseDir, 'pages.json'),
@@ -34,18 +29,6 @@ class ExportService {
     });
   }
 
-  /**
-   * initialize collection map
-   *
-   * @memberOf ExportService
-   * @param {object} models from models/index.js
-   */
-  initCollectionMap(models) {
-    for (const model of Object.values(models)) {
-      this.collectionMap[model.collection.collectionName] = model;
-    }
-  }
-
   /**
    * parse all zip files in downloads dir
    *
@@ -69,8 +52,8 @@ class ExportService {
    * @return {string} path to meta.json
    */
   async createMetaJson() {
-    const metaJson = path.join(this.baseDir, this.metaFileName);
-    const writeStream = fs.createWriteStream(metaJson, { encoding: this.encoding });
+    const metaJson = path.join(this.baseDir, this.growiBridgeService.getMetaFileName());
+    const writeStream = fs.createWriteStream(metaJson, { encoding: this.growiBridgeService.getEncoding() });
 
     const metaData = {
       version: this.crowi.version,
@@ -98,7 +81,7 @@ class ExportService {
    */
   async export(file, readStream, total) {
     let n = 0;
-    const ws = fs.createWriteStream(file, { encoding: this.encoding });
+    const ws = fs.createWriteStream(file, { encoding: this.growiBridgeService.getEncoding() });
 
     // open an array
     ws.write('[');
@@ -223,49 +206,6 @@ class ExportService {
     return zipFile;
   }
 
-  /**
-   * get the absolute path to a file
-   *
-   * @memberOf ExportService
-   * @param {string} fileName base name of file
-   * @return {string} absolute path to the file
-   */
-  getFile(fileName) {
-    const jsonFile = path.join(this.baseDir, fileName);
-
-    // throws err if the file does not exist
-    fs.accessSync(jsonFile);
-
-    return jsonFile;
-  }
-
-  /**
-   * get a model from collection name
-   *
-   * @memberOf ExportService
-   * @param {string} collectionName collection name
-   * @return {object} instance of mongoose model
-   */
-  getModelFromCollectionName(collectionName) {
-    const Model = this.collectionMap[collectionName];
-
-    if (Model == null) {
-      throw new Error(`cannot find a model for collection name "${collectionName}"`);
-    }
-
-    return Model;
-  }
-
-  /**
-   * remove zip file from downloads dir
-   *
-   * @param {string} zipFile absolute path to zip file
-   * @memberOf ExportService
-   */
-  deleteZipFile(zipFile) {
-    fs.unlinkSync(zipFile);
-  }
-
 }
 
 module.exports = ExportService;

src/server/service/growi-bridge.js

@@ -1,4 +1,4 @@
-const logger = require('@alias/logger')('growi:services:ImportService'); // eslint-disable-line no-unused-vars
+const logger = require('@alias/logger')('growi:services:GrowiBridgeService'); // eslint-disable-line no-unused-vars
 const fs = require('fs');
 const path = require('path');
 const streamToPromise = require('stream-to-promise');
@@ -11,13 +11,88 @@ const unzipper = require('unzipper');
 class GrowiBridgeService {
 
   constructor(crowi) {
+    this.encoding = 'utf-8';
     this.metaFileName = 'meta.json';
+
+    // { pages: Page, users: User, ... }
+    this.collectionMap = {};
+    this.initCollectionMap(crowi.models);
+  }
+
+  /**
+   * initialize collection map
+   *
+   * @memberOf GrowiBridgeService
+   * @param {object} models from models/index.js
+   */
+  initCollectionMap(models) {
+    for (const model of Object.values(models)) {
+      this.collectionMap[model.collection.collectionName] = model;
+    }
+  }
+
+  /**
+   * getter for encoding
+   *
+   * @memberOf GrowiBridgeService
+   * @return {string} encoding
+   */
+  getEncoding() {
+    return this.encoding;
+  }
+
+  /**
+   * getter for metaFileName
+   *
+   * @memberOf GrowiBridgeService
+   * @return {string} base name of meta file
+   */
+  getMetaFileName() {
+    return this.metaFileName;
+  }
+
+  /**
+   * get a model from collection name
+   *
+   * @memberOf GrowiBridgeService
+   * @param {string} collectionName collection name
+   * @return {object} instance of mongoose model
+   */
+  getModelFromCollectionName(collectionName) {
+    const Model = this.collectionMap[collectionName];
+
+    if (Model == null) {
+      throw new Error(`cannot find a model for collection name "${collectionName}"`);
+    }
+
+    return Model;
+  }
+
+  /**
+   * get the absolute path to a file
+   * this method must must be bound to the caller (this.baseDir is undefined in this service)
+   *
+   * @memberOf GrowiBridgeService
+   * @param {string} fileName base name of file
+   * @return {string} absolute path to the file
+   */
+  getFile(fileName) {
+    if (this.baseDir == null) {
+      throw new Error('baseDir is not defined');
+    }
+
+    const jsonFile = path.join(this.baseDir, fileName);
+
+    // throws err if the file does not exist
+    fs.accessSync(jsonFile);
+
+    return jsonFile;
   }
 
   /**
    * parse a zip file
    *
-   * @memberOf ImportService
+   * @memberOf GrowiBridgeService
    * @param {string} zipFile path to zip file
    * @return {object} meta{object} and files{Array.<object>}
    */
@@ -31,7 +106,7 @@ class GrowiBridgeService {
       const fileName = entry.path;
       const size = entry.vars.uncompressedSize; // There is also compressedSize;
 
-      if (fileName === this.metaFileName) {
+      if (fileName === this.getMetaFileName()) {
         meta = JSON.parse((await entry.buffer()).toString());
       }
       else {

src/server/service/import.js

@@ -10,33 +10,17 @@ class ImportService {
 
   constructor(crowi) {
     this.crowi = crowi;
+    this.growiBridgeService = crowi.growiBridgeService;
+    this.getFile = this.growiBridgeService.getFile.bind(this);
     this.baseDir = path.join(crowi.tmpDir, 'imports');
-    this.metaFileName = 'meta.json';
-    this.encoding = 'utf-8';
     this.per = 100;
     this.keepOriginal = this.keepOriginal.bind(this);
 
-    // { pages: Page, users: User, ... }
-    this.collectionMap = {};
-    this.initCollectionMap(crowi.models);
-
     // { pages: { _id: ..., path: ..., ...}, users: { _id: ..., username: ..., }, ... }
     this.convertMap = {};
     this.initConvertMap(crowi.models);
   }
 
-  /**
-   * initialize collection map
-   *
-   * @memberOf ImportService
-   * @param {object} models from models/index.js
-   */
-  initCollectionMap(models) {
-    for (const model of Object.values(models)) {
-      this.collectionMap[model.collection.collectionName] = model;
-    }
-  }
-
   /**
    * initialize convert map. set keepOriginal as default
    *
@@ -93,7 +77,7 @@ class ImportService {
     let failedIds = [];
     let unorderedBulkOp = Model.collection.initializeUnorderedBulkOp();
 
-    const readStream = fs.createReadStream(jsonFile, { encoding: this.encoding });
+    const readStream = fs.createReadStream(jsonFile, { encoding: this.growiBridgeService.getEncoding() });
     const jsonStream = readStream.pipe(JSONStream.parse('*'));
 
     jsonStream.on('data', async(document) => {
@@ -137,7 +121,7 @@ class ImportService {
     await streamToPromise(readStream);
 
     // clean up tmp directory
-    this.deleteZipFile(jsonFile);
+    fs.unlinkSync(jsonFile);
   }
 
   /**
@@ -155,13 +139,13 @@ class ImportService {
     unzipStream.on('entry', (entry) => {
       const fileName = entry.path;
 
-      if (fileName === this.metaFileName) {
+      if (fileName === this.growiBridgeService.getMetaFileName()) {
         // skip meta.json
         entry.autodrain();
       }
       else {
         const jsonFile = path.join(this.baseDir, fileName);
-        const writeStream = fs.createWriteStream(jsonFile, { encoding: this.encoding });
+        const writeStream = fs.createWriteStream(jsonFile, { encoding: this.growiBridgeService.getEncoding() });
         entry.pipe(writeStream);
         files.push(jsonFile);
       }
@@ -251,49 +235,6 @@ class ImportService {
     return document;
   }
 
-  /**
-   * get a model from collection name
-   *
-   * @memberOf ImportService
-   * @param {string} collectionName collection name
-   * @return {object} instance of mongoose model
-   */
-  getModelFromCollectionName(collectionName) {
-    const Model = this.collectionMap[collectionName];
-
-    if (Model == null) {
-      throw new Error(`cannot find a model for collection name "${collectionName}"`);
-    }
-
-    return Model;
-  }
-
-  /**
-   * get the absolute path to a file
-   *
-   * @memberOf ImportService
-   * @param {string} fileName base name of file
-   * @return {string} absolute path to the file
-   */
-  getFile(fileName) {
-    const jsonFile = path.join(this.baseDir, fileName);
-
-    // throws err if the file does not exist
-    fs.accessSync(jsonFile);
-
-    return jsonFile;
-  }
-
-  /**
-   * remove zip file from imports dir
-   *
-   * @memberOf ImportService
-   * @param {string} zipFile absolute path to zip file
-   */
-  deleteZipFile(zipFile) {
-    fs.unlinkSync(zipFile);
-  }
-
   /**
    * validate using meta.json
    * to pass validation, all the criteria must be met