Merge branch 'feat/export-n-import-revision' into feat/export-n-import-revision-2

src/client/js/components/Admin/Export/ExportAsZip.jsx

@@ -1,6 +1,7 @@
 import React, { Fragment } from 'react';
 import PropTypes from 'prop-types';
 import { withTranslation } from 'react-i18next';
+import { format } from 'date-fns';
 
 import { createSubscribedElement } from '../../UnstatedUtils';
 import AppContainer from '../../../services/AppContainer';
@@ -13,9 +14,12 @@ class ExportPage extends React.Component {
 
     this.state = {
       files: {},
+      collections: new Set(),
     };
 
-    this.createZipFile = this.createZipFile.bind(this);
+    this.toggleCheckbox = this.toggleCheckbox.bind(this);
+    this.exportSingle = this.exportSingle.bind(this);
+    this.exportMultiple = this.exportMultiple.bind(this);
     this.deleteZipFile = this.deleteZipFile.bind(this);
   }
 
@@ -25,9 +29,40 @@ class ExportPage extends React.Component {
     this.setState({ files: res.files });
   }
 
-  async createZipFile() {
+  toggleCheckbox(e) {
+    const { target } = e;
+    const { name, checked } = target;
+
+    this.setState((prevState) => {
+      const collections = new Set(prevState.collections);
+      if (checked) {
+        collections.add(name);
+      }
+      else {
+        collections.delete(name);
+      }
+
+      return { collections };
+    });
+  }
+
+  async exportSingle(collection) {
+    // TODO use appContainer.apiv3.post
+    const res = await this.props.appContainer.apiPost(`/v3/export/${collection}`, {});
+    // TODO toastSuccess, toastError
+    this.setState((prevState) => {
+      return {
+        files: {
+          ...prevState.files,
+          [res.collection]: res.file,
+        },
+      };
+    });
+  }
+
+  async exportMultiple() {
     // TODO use appContainer.apiv3.post
-    const res = await this.props.appContainer.apiPost('/v3/export/pages', {});
+    const res = await this.props.appContainer.apiPost('/v3/export', { collections: Array.from(this.state.collections) });
     // TODO toastSuccess, toastError
     this.setState((prevState) => {
       return {
@@ -51,29 +86,36 @@ class ExportPage extends React.Component {
       <Fragment>
         <h2>Export Data as Zip</h2>
         <form className="my-5">
-          {Object.keys(this.state.files).map((file) => {
-            const disabled = file !== 'pages';
+          {Object.keys(this.state.files).map((collectionName) => {
+            const disabled = !(collectionName === 'pages' || collectionName === 'revisions');
+            const stat = this.state.files[collectionName] || {};
             return (
-              <div className="form-check" key={file}>
+              <div className="checkbox checkbox-info" key={collectionName}>
                 <input
-                  type="radio"
-                  id={file}
-                  name="collection"
+                  type="checkbox"
+                  id={collectionName}
+                  name={collectionName}
                   className="form-check-input"
-                  value={file}
+                  value={collectionName}
                   disabled={disabled}
-                  checked={!disabled}
-                  onChange={() => {}}
+                  checked={this.state.collections.has(collectionName)}
+                  onChange={this.toggleCheckbox}
                 />
-                <label className={`form-check-label ml-3 ${disabled ? 'text-muted' : ''}`} htmlFor={file}>
-                  {file} ({this.state.files[file] || 'not found'})
+                <label className={`form-check-label ml-3 ${disabled ? 'text-muted' : ''}`} htmlFor={collectionName}>
+                  {collectionName} ({stat.name || 'not found'}) ({stat.mtime ? format(new Date(stat.mtime), 'yyyy/MM/dd HH:mm:ss') : ''})
                 </label>
+                <button type="button" className="btn btn-sm btn-primary" onClick={() => this.exportSingle(collectionName)} disabled={disabled}>
+                  Create zip file
+                </button>
+                <a href={`/_api/v3/export/${collectionName}`}>
+                  <button type="button" className="btn btn-sm btn-primary ml-2">Download</button>
+                </a>
               </div>
             );
           })}
         </form>
-        <button type="button" className="btn btn-sm btn-default" onClick={this.createZipFile}>Generate</button>
-        <a href="/_api/v3/export/pages">
+        <button type="button" className="btn btn-sm btn-default" onClick={this.exportMultiple}>Generate</button>
+        <a href="/_api/v3/export/download">
           <button type="button" className="btn btn-sm btn-primary ml-2">Download</button>
         </a>
         {/* <button type="button" className="btn btn-sm btn-danger ml-2" onClick={this.deleteZipFile}>Clear</button> */}

src/server/routes/apiv3/export.js

@@ -15,7 +15,6 @@ const router = express.Router();
 
 module.exports = (crowi) => {
   const { exportService } = crowi;
-  const { Page } = crowi.models;
 
   /**
    * @swagger
@@ -26,6 +25,12 @@ module.exports = (crowi) => {
    *      description: get mongodb collections names and zip files for them
    *      produces:
    *        - application/json
+   *      parameters:
+   *        - name: collectionName
+   *          in: path
+   *          description: collection name
+   *          schema:
+   *            type: string
    *      responses:
    *        200:
    *          description: export cache info
@@ -42,10 +47,10 @@ module.exports = (crowi) => {
   /**
    * @swagger
    *
-   *  /export/pages:
+   *  /export/download:
    *    get:
    *      tags: [Export]
-   *      description: download a zipped json for page collection
+   *      description: download a zipped json for multiple collections
    *      produces:
    *        - application/json
    *      responses:
@@ -54,16 +59,95 @@ module.exports = (crowi) => {
    *          content:
    *            application/zip:
    */
-  router.get('/pages', async(req, res) => {
-    // TODO: rename path to "/:collection" and add express validator
+  router.get('/download', async(req, res) => {
+    // TODO: add express validator
     try {
-      const file = exportService.getZipFile(Page);
+      return res.download(exportService.zipFile);
+    }
+    catch (err) {
+      // TODO: use ApiV3Error
+      logger.error(err);
+      return res.status(500).send({ status: 'ERROR' });
+    }
+  });
+
+  /**
+   * @swagger
+   *
+   *  /export/:collectionName:
+   *    get:
+   *      tags: [Export]
+   *      description: download a zipped json for a single collection
+   *      produces:
+   *        - application/json
+   *      parameters:
+   *        - name: collectionName
+   *          in: path
+   *          description: collection name
+   *          schema:
+   *            type: string
+   *      responses:
+   *        200:
+   *          description: a zip file
+   *          content:
+   *            application/zip:
+   */
+  router.get('/:collectionName', async(req, res) => {
+    // TODO: add express validator
+    try {
+      const { collectionName } = req.params;
+      // get model for collection
+      const Model = exportService.getModelFromCollectionName(collectionName);
+      // get zip file path
+      const zipFile = exportService.getZipFile(Model);
+
+      return res.download(zipFile);
+    }
+    catch (err) {
+      // TODO: use ApiV3Error
+      logger.error(err);
+      return res.status(500).send({ status: 'ERROR' });
+    }
+  });
 
-      if (file == null) {
-        throw new Error('the target file does not exist');
-      }
+  /**
+   * @swagger
+   *
+   *  /export/:collectionName:
+   *    post:
+   *      tags: [Export]
+   *      description: generate a zipped json for a single collection
+   *      produces:
+   *        - application/json
+   *      parameters:
+   *        - name: collectionName
+   *          in: path
+   *          description: collection name
+   *          schema:
+   *            type: string
+   *      responses:
+   *        200:
+   *          description: a zip file is generated
+   *          content:
+   *            application/json:
+   */
+  router.post('/:collectionName', async(req, res) => {
+    // TODO: add express validator
+    try {
+      const { collectionName } = req.params;
+      // get model for collection
+      const Model = exportService.getModelFromCollectionName(collectionName);
+      // export into json
+      const jsonFile = await exportService.exportCollectionToJson(Model);
+      // zip json
+      const zipFile = await exportService.zipSingleFile(jsonFile);
 
-      return res.download(file);
+      // TODO: use res.apiv3
+      return res.status(200).json({
+        ok: true,
+        collection: [Model.collection.collectionName],
+        file: path.basename(zipFile),
+      });
     }
     catch (err) {
       // TODO: use ApiV3Error
@@ -75,10 +159,10 @@ module.exports = (crowi) => {
   /**
    * @swagger
    *
-   *  /export/pages:
+   *  /export:
    *    post:
    *      tags: [Export]
-   *      description: generate a zipped json for page collection
+   *      description: generate a zipped json for multiple collections
    *      produces:
    *        - application/json
    *      responses:
@@ -87,15 +171,23 @@ module.exports = (crowi) => {
    *          content:
    *            application/json:
    */
-  router.post('/pages', async(req, res) => {
-    // TODO: rename path to "/:collection" and add express validator
+  router.post('/', async(req, res) => {
+    // TODO: add express validator
     try {
-      const file = await exportService.exportCollection(Page);
+      const { collections } = req.body;
+      // get model for collection
+      const models = collections.map(collectionName => exportService.getModelFromCollectionName(collectionName));
+      // export into json
+      const jsonFiles = await exportService.exportMultipleCollectionsToJsons(models);
+      // zip json
+      const configs = jsonFiles.map((jsonFile) => { return { from: jsonFile, as: path.basename(jsonFile) } });
+      const zipFile = await exportService.zipMultipleFiles(configs);
+
       // TODO: use res.apiv3
       return res.status(200).json({
         ok: true,
-        collection: [Page.collection.collectionName],
-        file: path.basename(file),
+        // collection: [Model.collection.collectionName],
+        file: path.basename(zipFile),
       });
     }
     catch (err) {
@@ -108,21 +200,30 @@ module.exports = (crowi) => {
   /**
    * @swagger
    *
-   *  /export/pages:
+   *  /export/:collectionName:
    *    delete:
    *      tags: [Export]
-   *      description: unlink a json and zip file for page collection
+   *      description: unlink a json and zip file for a single collection
    *      produces:
    *        - application/json
+   *      parameters:
+   *        - name: collectionName
+   *          in: path
+   *          description: collection name
+   *          schema:
+   *            type: string
    *      responses:
    *        200:
    *          description: the json and zip file are removed
    *          content:
    *            application/json:
    */
-  // router.delete('/pages', async(req, res) => {
-  //   // TODO: rename path to "/:collection" and add express validator
+  // router.delete('/:collectionName', async(req, res) => {
+  //   // TODO: add express validator
   //   try {
+  //     const { collectionName } = req.params;
+  //     // get model for collection
+  //     const Model = exportService.getModelFromCollectionName(collectionName);
   //     // remove .json and .zip for collection
   //     // TODO: use res.apiv3
   //     return res.status(200).send({ status: 'DONE' });

src/server/service/export.js

@@ -7,28 +7,44 @@ const archiver = require('archiver');
 class ExportService {
 
   constructor(crowi) {
+    this.appService = crowi.appService;
     this.baseDir = path.join(crowi.tmpDir, 'downloads');
-    this.extension = 'json';
     this.encoding = 'utf-8';
     this.per = 100;
     this.zlibLevel = 9; // 0(min) - 9(max)
 
-    this.files = {};
-    // populate this.files
+    // path to zip file for exporting multiple collection
+    this.zipFile = path.join(this.baseDir, 'GROWI.zip');
+
+    // { pages: Page, users: User, ... }
+    this.collectionMap = {};
+    this.initCollectionMap(crowi.models);
+
     // this.files = {
     //   configs: path.join(this.baseDir, 'configs.json'),
     //   pages: path.join(this.baseDir, 'pages.json'),
     //   pagetagrelations: path.join(this.baseDir, 'pagetagrelations.json'),
     //   ...
     // };
-    // TODO: handle 3 globalnotificationsettings collection properly
-    // see Object.values(crowi.models).forEach((m) => { return console.log(m.collection.collectionName) });
+    this.files = {};
     Object.values(crowi.models).forEach((m) => {
       const name = m.collection.collectionName;
-      this.files[name] = path.join(this.baseDir, `${name}.${this.extension}`);
+      this.files[name] = path.join(this.baseDir, `${name}.json`);
     });
   }
 
+  /**
+   * initialize collection map
+   *
+   * @memberOf ExportService
+   * @param {object} models from models/index.js
+   */
+  initCollectionMap(models) {
+    for (const model of Object.values(models)) {
+      this.collectionMap[model.collection.collectionName] = model;
+    }
+  }
+
   /**
    * dump a collection into json
    *
@@ -49,6 +65,12 @@ class ExportService {
       status[path.basename(file, '.zip')] = file;
     });
 
+    files.forEach((file) => {
+      const stats = fs.statSync(path.join(this.baseDir, file));
+      stats.name = file;
+      status[path.basename(file, '.zip')] = stats;
+    });
+
     return status;
   }
 
@@ -59,6 +81,7 @@ class ExportService {
    * @param {string} file path to json file to be written
    * @param {readStream} readStream  read stream
    * @param {number} [total] number of target items (optional)
+   * @return {string} path to the exported json file
    */
   async export(file, readStream, total) {
     let n = 0;
@@ -81,6 +104,8 @@ class ExportService {
     });
 
     await streamToPromise(readStream);
+
+    return file;
   }
 
   /**
@@ -90,19 +115,20 @@ class ExportService {
    * @param {object} Model instance of mongoose model
    * @return {string} path to zip file
    */
-  async exportCollection(Model) {
-    const modelName = Model.collection.collectionName;
-    const file = this.files[modelName];
+  async exportCollectionToJson(Model) {
+    const { collectionName } = Model.collection;
+    const targetFile = this.files[collectionName];
     const total = await Model.countDocuments();
     const readStream = Model.find().cursor();
 
-    await this.export(file, readStream, total);
-
-    const { file: zipFile, size } = await this.zipSingleFile(file);
+    const file = await this.export(targetFile, readStream, total);
 
-    logger.info(`exported ${modelName} collection into ${zipFile} (${size} bytes)`);
+    return file;
+  }
 
-    return zipFile;
+  async exportMultipleCollectionsToJsons(models) {
+    const jsonFiles = await Promise.all(models.map(Model => this.exportCollectionToJson(Model)));
+    return jsonFiles;
   }
 
   /**
@@ -134,7 +160,7 @@ class ExportService {
    * @param {string} from path to input file
    * @param {string} [to=`${path.join(path.dirname(from), `${path.basename(from, path.extname(from))}.zip`)}`] path to output file
    * @param {string} [as=path.basename(from)] file name after unzipped
-   * @return {object} file path and file size
+   * @return {string} path to zip file
    * @see https://www.archiverjs.com/#quick-start
    */
   async zipSingleFile(from, to = this.replaceExtension(from, 'zip'), as = path.basename(from)) {
@@ -165,10 +191,52 @@ class ExportService {
 
     await streamToPromise(archive);
 
-    return {
-      file: to,
-      size: archive.pointer(),
-    };
+    logger.debug(`zipped ${from} into ${to} (${archive.pointer()} bytes)`);
+
+    return to;
+  }
+
+  /**
+   * zip a file
+   *
+   * @memberOf ExportService
+   * @param {array} configs array of object { from: "path to source file", as: "file name appears after unzipped" }
+   * @return {string} path to zip file
+   */
+  async zipMultipleFiles(configs) {
+    const archive = archiver('zip', {
+      zlib: { level: this.zlibLevel },
+    });
+    const output = fs.createWriteStream(this.zipFile);
+
+    // good practice to catch warnings (ie stat failures and other non-blocking errors)
+    archive.on('warning', (err) => {
+      if (err.code === 'ENOENT') logger.error(err);
+      else throw err;
+    });
+
+    // good practice to catch this error explicitly
+    archive.on('error', (err) => { throw err });
+
+    for (const { from, as } of configs) {
+      const input = fs.createReadStream(from);
+
+      // append a file from stream
+      archive.append(input, { name: as });
+    }
+
+    // pipe archive data to the file
+    archive.pipe(output);
+
+    // finalize the archive (ie we are done appending files but streams have to finish yet)
+    // 'close', 'end' or 'finish' may be fired right after calling this method so register to them beforehand
+    archive.finalize();
+
+    await streamToPromise(archive);
+
+    logger.debug(`zipped growi data into ${this.zipFile} (${archive.pointer()} bytes)`);
+
+    return this.zipFile;
   }
 
   /**
@@ -194,12 +262,29 @@ class ExportService {
     const json = this.files[Model.collection.collectionName];
     const zip = this.replaceExtension(json, 'zip');
     if (!fs.existsSync(zip)) {
-      return null;
+      throw new Error(`${zip} does not exist`);
     }
 
     return zip;
   }
 
+  /**
+   * get a model from collection name
+   *
+   * @memberOf ExportService
+   * @param {object} collectionName collection name
+   * @return {object} instance of mongoose model
+   */
+  getModelFromCollectionName(collectionName) {
+    const Model = this.collectionMap[collectionName];
+
+    if (Model == null) {
+      throw new Error(`cannot find a model for collection name "${collectionName}"`);
+    }
+
+    return Model;
+  }
+
 }
 
 module.exports = ExportService;