Merge pull request #1198 from weseek/feat/export-n-import-revision

Feat/export n import revision

src/client/js/components/Admin/Export/ExportAsZip.jsx

@@ -1,6 +1,7 @@
 import React, { Fragment } from 'react';
 import PropTypes from 'prop-types';
 import { withTranslation } from 'react-i18next';
+import { format } from 'date-fns';
 
 import { createSubscribedElement } from '../../UnstatedUtils';
 import AppContainer from '../../../services/AppContainer';
@@ -13,21 +14,40 @@ class ExportPage extends React.Component {
 
     this.state = {
       files: {},
+      collections: new Set(),
     };
 
-    this.createZipFile = this.createZipFile.bind(this);
+    this.toggleCheckbox = this.toggleCheckbox.bind(this);
+    this.exportMultiple = this.exportMultiple.bind(this);
     this.deleteZipFile = this.deleteZipFile.bind(this);
   }
 
   async componentDidMount() {
-    const res = await this.props.appContainer.apiGet('/v3/export', {});
+    const res = await this.props.appContainer.apiGet('/v3/export/status', {});
 
     this.setState({ files: res.files });
   }
 
-  async createZipFile() {
+  toggleCheckbox(e) {
+    const { target } = e;
+    const { name, checked } = target;
+
+    this.setState((prevState) => {
+      const collections = new Set(prevState.collections);
+      if (checked) {
+        collections.add(name);
+      }
+      else {
+        collections.delete(name);
+      }
+
+      return { collections };
+    });
+  }
+
+  async exportMultiple() {
     // TODO use appContainer.apiv3.post
-    const res = await this.props.appContainer.apiPost('/v3/export/pages', {});
+    const res = await this.props.appContainer.apiPost('/v3/export', { collections: Array.from(this.state.collections) });
     // TODO toastSuccess, toastError
     this.setState((prevState) => {
       return {
@@ -51,29 +71,30 @@ class ExportPage extends React.Component {
       <Fragment>
         <h2>Export Data as Zip</h2>
         <form className="my-5">
-          {Object.keys(this.state.files).map((file) => {
-            const disabled = file !== 'pages';
+          {Object.keys(this.state.files).map((collectionName) => {
+            const disabled = !(collectionName === 'pages' || collectionName === 'revisions');
+            const stat = this.state.files[collectionName] || {};
             return (
-              <div className="form-check" key={file}>
+              <div className="checkbox checkbox-info" key={collectionName}>
                 <input
-                  type="radio"
-                  id={file}
-                  name="collection"
+                  type="checkbox"
+                  id={collectionName}
+                  name={collectionName}
                   className="form-check-input"
-                  value={file}
+                  value={collectionName}
                   disabled={disabled}
-                  checked={!disabled}
-                  onChange={() => {}}
+                  checked={this.state.collections.has(collectionName)}
+                  onChange={this.toggleCheckbox}
                 />
-                <label className={`form-check-label ml-3 ${disabled ? 'text-muted' : ''}`} htmlFor={file}>
-                  {file} ({this.state.files[file] || 'not found'})
+                <label className={`form-check-label ml-3 ${disabled ? 'text-muted' : ''}`} htmlFor={collectionName}>
+                  {collectionName} ({stat.name || 'not found'}) ({stat.mtime ? format(new Date(stat.mtime), 'yyyy/MM/dd HH:mm:ss') : ''})
                 </label>
               </div>
             );
           })}
         </form>
-        <button type="button" className="btn btn-sm btn-default" onClick={this.createZipFile}>Generate</button>
-        <a href="/_api/v3/export/pages">
+        <button type="button" className="btn btn-sm btn-default" onClick={this.exportMultiple}>Generate</button>
+        <a href="/_api/v3/export">
           <button type="button" className="btn btn-sm btn-primary ml-2">Download</button>
         </a>
         {/* <button type="button" className="btn btn-sm btn-danger ml-2" onClick={this.deleteZipFile}>Clear</button> */}

src/lib/util/toArrayIfNot.js

@@ -0,0 +1,15 @@
+// converts non-array item to array
+
+const toArrayIfNot = (item) => {
+  if (item == null) {
+    return [];
+  }
+
+  if (Array.isArray(item)) {
+    return item;
+  }
+
+  return [item];
+};
+
+module.exports = toArrayIfNot;

src/server/routes/apiv3/export.js

@@ -15,12 +15,11 @@ const router = express.Router();
 
 module.exports = (crowi) => {
   const { exportService } = crowi;
-  const { Page } = crowi.models;
 
   /**
    * @swagger
    *
-   *  /export:
+   *  /export/status:
    *    get:
    *      tags: [Export]
    *      description: get mongodb collections names and zip files for them
@@ -28,11 +27,11 @@ module.exports = (crowi) => {
    *        - application/json
    *      responses:
    *        200:
-   *          description: export cache info
+   *          description: export cache status
    *          content:
    *            application/json:
    */
-  router.get('/', async(req, res) => {
+  router.get('/status', async(req, res) => {
     const files = exportService.getStatus();
 
     // TODO: use res.apiv3
@@ -42,10 +41,10 @@ module.exports = (crowi) => {
   /**
    * @swagger
    *
-   *  /export/pages:
+   *  /export/download:
    *    get:
    *      tags: [Export]
-   *      description: download a zipped json for page collection
+   *      description: download a zipped json for multiple collections
    *      produces:
    *        - application/json
    *      responses:
@@ -54,16 +53,10 @@ module.exports = (crowi) => {
    *          content:
    *            application/zip:
    */
-  router.get('/pages', async(req, res) => {
-    // TODO: rename path to "/:collection" and add express validator
+  router.get('/', async(req, res) => {
+    // TODO: add express validator
     try {
-      const file = exportService.getZipFile(Page);
-
-      if (file == null) {
-        throw new Error('the target file does not exist');
-      }
-
-      return res.download(file);
+      return res.download(exportService.zipFile);
     }
     catch (err) {
       // TODO: use ApiV3Error
@@ -75,10 +68,10 @@ module.exports = (crowi) => {
   /**
    * @swagger
    *
-   *  /export/pages:
+   *  /export:
    *    post:
    *      tags: [Export]
-   *      description: generate a zipped json for page collection
+   *      description: generate a zipped json for multiple collections
    *      produces:
    *        - application/json
    *      responses:
@@ -87,15 +80,23 @@ module.exports = (crowi) => {
    *          content:
    *            application/json:
    */
-  router.post('/pages', async(req, res) => {
-    // TODO: rename path to "/:collection" and add express validator
+  router.post('/', async(req, res) => {
+    // TODO: add express validator
     try {
-      const file = await exportService.exportCollection(Page);
+      const { collections } = req.body;
+      // get model for collection
+      const models = collections.map(collectionName => exportService.getModelFromCollectionName(collectionName));
+      // export into json
+      const jsonFiles = await exportService.exportMultipleCollectionsToJsons(models);
+      // zip json
+      const configs = jsonFiles.map((jsonFile) => { return { from: jsonFile, as: path.basename(jsonFile) } });
+      const zipFile = await exportService.zipFiles(configs);
+
       // TODO: use res.apiv3
       return res.status(200).json({
         ok: true,
-        collection: [Page.collection.collectionName],
-        file: path.basename(file),
+        // collection: [Model.collection.collectionName],
+        file: path.basename(zipFile),
       });
     }
     catch (err) {
@@ -108,20 +109,20 @@ module.exports = (crowi) => {
   /**
    * @swagger
    *
-   *  /export/pages:
+   *  /export:
    *    delete:
    *      tags: [Export]
-   *      description: unlink a json and zip file for page collection
+   *      description: unlink all json and zip files for exports
    *      produces:
    *        - application/json
    *      responses:
    *        200:
-   *          description: the json and zip file are removed
+   *          description: the json and zip file are deleted
    *          content:
    *            application/json:
    */
-  // router.delete('/pages', async(req, res) => {
-  //   // TODO: rename path to "/:collection" and add express validator
+  // router.delete('/', async(req, res) => {
+  //   // TODO: add express validator
   //   try {
   //     // remove .json and .zip for collection
   //     // TODO: use res.apiv3

src/server/service/export.js

@@ -3,32 +3,49 @@ const fs = require('fs');
 const path = require('path');
 const streamToPromise = require('stream-to-promise');
 const archiver = require('archiver');
+const toArrayIfNot = require('../../lib/util/toArrayIfNot');
 
 class ExportService {
 
   constructor(crowi) {
+    this.appService = crowi.appService;
     this.baseDir = path.join(crowi.tmpDir, 'downloads');
-    this.extension = 'json';
     this.encoding = 'utf-8';
     this.per = 100;
     this.zlibLevel = 9; // 0(min) - 9(max)
 
-    this.files = {};
-    // populate this.files
+    // path to zip file for exporting multiple collection
+    this.zipFile = path.join(this.baseDir, 'GROWI.zip');
+
+    // { pages: Page, users: User, ... }
+    this.collectionMap = {};
+    this.initCollectionMap(crowi.models);
+
     // this.files = {
     //   configs: path.join(this.baseDir, 'configs.json'),
     //   pages: path.join(this.baseDir, 'pages.json'),
     //   pagetagrelations: path.join(this.baseDir, 'pagetagrelations.json'),
     //   ...
     // };
-    // TODO: handle 3 globalnotificationsettings collection properly
-    // see Object.values(crowi.models).forEach((m) => { return console.log(m.collection.collectionName) });
+    this.files = {};
     Object.values(crowi.models).forEach((m) => {
       const name = m.collection.collectionName;
-      this.files[name] = path.join(this.baseDir, `${name}.${this.extension}`);
+      this.files[name] = path.join(this.baseDir, `${name}.json`);
     });
   }
 
+  /**
+   * initialize collection map
+   *
+   * @memberOf ExportService
+   * @param {object} models from models/index.js
+   */
+  initCollectionMap(models) {
+    for (const model of Object.values(models)) {
+      this.collectionMap[model.collection.collectionName] = model;
+    }
+  }
+
   /**
    * dump a collection into json
    *
@@ -49,6 +66,12 @@ class ExportService {
       status[path.basename(file, '.zip')] = file;
     });
 
+    files.forEach((file) => {
+      const stats = fs.statSync(path.join(this.baseDir, file));
+      stats.name = file;
+      status[path.basename(file, '.zip')] = stats;
+    });
+
     return status;
   }
 
@@ -59,6 +82,7 @@ class ExportService {
    * @param {string} file path to json file to be written
    * @param {readStream} readStream  read stream
    * @param {number} [total] number of target items (optional)
+   * @return {string} path to the exported json file
    */
   async export(file, readStream, total) {
     let n = 0;
@@ -81,6 +105,8 @@ class ExportService {
     });
 
     await streamToPromise(readStream);
+
+    return file;
   }
 
   /**
@@ -90,19 +116,20 @@ class ExportService {
    * @param {object} Model instance of mongoose model
    * @return {string} path to zip file
    */
-  async exportCollection(Model) {
-    const modelName = Model.collection.collectionName;
-    const file = this.files[modelName];
+  async exportCollectionToJson(Model) {
+    const { collectionName } = Model.collection;
+    const targetFile = this.files[collectionName];
     const total = await Model.countDocuments();
     const readStream = Model.find().cursor();
 
-    await this.export(file, readStream, total);
-
-    const { file: zipFile, size } = await this.zipSingleFile(file);
+    const file = await this.export(targetFile, readStream, total);
 
-    logger.info(`exported ${modelName} collection into ${zipFile} (${size} bytes)`);
+    return file;
+  }
 
-    return zipFile;
+  async exportMultipleCollectionsToJsons(models) {
+    const jsonFiles = await Promise.all(models.map(Model => this.exportCollectionToJson(Model)));
+    return jsonFiles;
   }
 
   /**
@@ -128,21 +155,18 @@ class ExportService {
   }
 
   /**
-   * zip a file
+   * zip files into one zip file
    *
    * @memberOf ExportService
-   * @param {string} from path to input file
-   * @param {string} [to=`${path.join(path.dirname(from), `${path.basename(from, path.extname(from))}.zip`)}`] path to output file
-   * @param {string} [as=path.basename(from)] file name after unzipped
-   * @return {object} file path and file size
+   * @param {object|array<object>} configs array of object { from: "path to source file", as: "file name after unzipped" }
+   * @return {string} path to zip file
    * @see https://www.archiverjs.com/#quick-start
    */
-  async zipSingleFile(from, to = this.replaceExtension(from, 'zip'), as = path.basename(from)) {
+  async zipFiles(_configs) {
+    const configs = toArrayIfNot(_configs);
     const archive = archiver('zip', {
       zlib: { level: this.zlibLevel },
     });
-    const input = fs.createReadStream(from);
-    const output = fs.createWriteStream(to);
 
     // good practice to catch warnings (ie stat failures and other non-blocking errors)
     archive.on('warning', (err) => {
@@ -153,8 +177,14 @@ class ExportService {
     // good practice to catch this error explicitly
     archive.on('error', (err) => { throw err });
 
-    // append a file from stream
-    archive.append(input, { name: as });
+    for (const { from, as } of configs) {
+      const input = fs.createReadStream(from);
+
+      // append a file from stream
+      archive.append(input, { name: as });
+    }
+
+    const output = fs.createWriteStream(this.zipFile);
 
     // pipe archive data to the file
     archive.pipe(output);
@@ -165,10 +195,9 @@ class ExportService {
 
     await streamToPromise(archive);
 
-    return {
-      file: to,
-      size: archive.pointer(),
-    };
+    logger.debug(`zipped growi data into ${this.zipFile} (${archive.pointer()} bytes)`);
+
+    return this.zipFile;
   }
 
   /**
@@ -194,12 +223,29 @@ class ExportService {
     const json = this.files[Model.collection.collectionName];
     const zip = this.replaceExtension(json, 'zip');
     if (!fs.existsSync(zip)) {
-      return null;
+      throw new Error(`${zip} does not exist`);
     }
 
     return zip;
   }
 
+  /**
+   * get a model from collection name
+   *
+   * @memberOf ExportService
+   * @param {object} collectionName collection name
+   * @return {object} instance of mongoose model
+   */
+  getModelFromCollectionName(collectionName) {
+    const Model = this.collectionMap[collectionName];
+
+    if (Model == null) {
+      throw new Error(`cannot find a model for collection name "${collectionName}"`);
+    }
+
+    return Model;
+  }
+
 }
 
 module.exports = ExportService;