Merge pull request #1204 from weseek/feat/export-n-import-revision-3

Feat/export n import revision 3

src/client/js/components/Admin/Export/ExportAsZip.jsx

@@ -7,25 +7,24 @@ import { createSubscribedElement } from '../../UnstatedUtils';
 import AppContainer from '../../../services/AppContainer';
 // import { toastSuccess, toastError } from '../../../util/apiNotification';
 
-class ExportPage extends React.Component {
+class ExportAsZip extends React.Component {
 
   constructor(props) {
     super(props);
 
     this.state = {
-      files: {},
+      zipFileStats: [],
       collections: new Set(),
     };
 
     this.toggleCheckbox = this.toggleCheckbox.bind(this);
-    this.exportMultiple = this.exportMultiple.bind(this);
+    this.export = this.export.bind(this);
     this.deleteZipFile = this.deleteZipFile.bind(this);
   }
 
   async componentDidMount() {
-    const res = await this.props.appContainer.apiGet('/v3/export/status', {});
-
-    this.setState({ files: res.files });
+    const { zipFileStats } = await this.props.appContainer.apiGet('/v3/export/status', {});
+    this.setState({ zipFileStats });
   }
 
   toggleCheckbox(e) {
@@ -45,35 +44,41 @@ class ExportPage extends React.Component {
     });
   }
 
-  async exportMultiple() {
+  async export() {
     // TODO use appContainer.apiv3.post
-    const res = await this.props.appContainer.apiPost('/v3/export', { collections: Array.from(this.state.collections) });
+    const { zipFileStat } = await this.props.appContainer.apiPost('/v3/export', { collections: Array.from(this.state.collections) });
     // TODO toastSuccess, toastError
     this.setState((prevState) => {
       return {
-        files: {
-          ...prevState.files,
-          [res.collection]: res.file,
-        },
+        zipFileStats: [
+          ...prevState.zipFileStats,
+          zipFileStat,
+        ],
       };
     });
   }
 
-  async deleteZipFile() {
+  async deleteZipFile(zipFile) {
     // TODO use appContainer.apiv3.delete
+    await this.props.appContainer.apiRequest('delete', `/v3/export/${zipFile}`, {});
+
+    this.setState((prevState) => {
+      return {
+        zipFileStats: prevState.zipFileStats.filter(stat => stat.fileName !== zipFile),
+      };
+    });
     // TODO toastSuccess, toastError
   }
 
   render() {
     // const { t } = this.props;
+    const collections = ['pages', 'revisions'];
 
     return (
       <Fragment>
         <h2>Export Data as Zip</h2>
         <form className="my-5">
-          {Object.keys(this.state.files).map((collectionName) => {
-            const disabled = !(collectionName === 'pages' || collectionName === 'revisions');
-            const stat = this.state.files[collectionName] || {};
+          {collections.map((collectionName) => {
             return (
               <div className="checkbox checkbox-info" key={collectionName}>
                 <input
@@ -82,40 +87,63 @@ class ExportPage extends React.Component {
                   name={collectionName}
                   className="form-check-input"
                   value={collectionName}
-                  disabled={disabled}
                   checked={this.state.collections.has(collectionName)}
                   onChange={this.toggleCheckbox}
                 />
-                <label className={`form-check-label ml-3 ${disabled ? 'text-muted' : ''}`} htmlFor={collectionName}>
-                  {collectionName} ({stat.name || 'not found'}) ({stat.mtime ? format(new Date(stat.mtime), 'yyyy/MM/dd HH:mm:ss') : ''})
+                <label className="form-check-label ml-3" htmlFor={collectionName}>
+                  {collectionName}
                 </label>
               </div>
             );
           })}
         </form>
-        <button type="button" className="btn btn-sm btn-default" onClick={this.exportMultiple}>Generate</button>
-        <a href="/_api/v3/export">
-          <button type="button" className="btn btn-sm btn-primary ml-2">Download</button>
-        </a>
-        {/* <button type="button" className="btn btn-sm btn-danger ml-2" onClick={this.deleteZipFile}>Clear</button> */}
+        <button type="button" className="btn btn-sm btn-default" onClick={this.export}>Generate</button>
+
+        <table className="table table-bordered">
+          <thead>
+            <tr>
+              <th>File</th>
+              <th>Growi Version</th>
+              <th>Collections</th>
+              <th>Exported At</th>
+              <th></th>
+            </tr>
+          </thead>
+          <tbody>
+            {this.state.zipFileStats.map(({ meta, fileName, fileStats }) => {
+              return (
+                <tr key={meta}>
+                  <th>{fileName}</th>
+                  <td>{meta.version}</td>
+                  <td>{fileStats.map(fileStat => fileStat.collectionName).join(', ')}</td>
+                  <td>{meta.exportedAt ? format(new Date(meta.exportedAt), 'yyyy/MM/dd HH:mm:ss') : ''}</td>
+                  <td>
+                    <a href="/_api/v3/export">
+                      <button type="button" className="btn btn-sm btn-primary ml-2">Download</button>
+                    </a>
+                    <button type="button" className="btn btn-sm btn-danger ml-2" onClick={() => this.deleteZipFile(fileName)}>Delete</button>
+                  </td>
+                </tr>
+              );
+            })}
+          </tbody>
+        </table>
       </Fragment>
     );
   }
 
 }
 
-ExportPage.propTypes = {
+ExportAsZip.propTypes = {
   t: PropTypes.func.isRequired, // i18next
   appContainer: PropTypes.instanceOf(AppContainer).isRequired,
-
-  isAclEnabled: PropTypes.bool,
 };
 
 /**
  * Wrapper component for using unstated
  */
-const ExportPageWrapper = (props) => {
-  return createSubscribedElement(ExportPage, props, [AppContainer]);
+const ExportAsZipWrapper = (props) => {
+  return createSubscribedElement(ExportAsZip, props, [AppContainer]);
 };
 
-export default withTranslation()(ExportPageWrapper);
+export default withTranslation()(ExportAsZipWrapper);

src/client/js/components/Admin/Import/GrowiImportForm.jsx

@@ -63,8 +63,8 @@ class GrowiImportForm extends React.Component {
     formData.append('file', this.inputRef.current.files[0]);
 
     // TODO use appContainer.apiv3.post
-    const { file, data } = await this.props.appContainer.apiPost('/v3/import/upload', formData);
-    this.setState({ meta: data.meta, zipFileName: file, fileStats: data.fileStats });
+    const { data } = await this.props.appContainer.apiPost('/v3/import/upload', formData);
+    this.setState({ meta: data.meta, zipFileName: data.fileName, fileStats: data.fileStats });
     // TODO toastSuccess, toastError
   }
 
@@ -133,6 +133,7 @@ class GrowiImportForm extends React.Component {
             <table className="table table-bordered table-mapping">
               <thead>
                 <tr>
+                  <th></th>
                   <th>File</th>
                   <th>Collection</th>
                 </tr>

src/server/crowi/index.js

@@ -46,6 +46,7 @@ function Crowi(rootdir) {
   this.appService = null;
   this.fileUploadService = null;
   this.restQiitaAPIService = null;
+  this.growiBridgeService = null;
   this.exportService = null;
   this.importService = null;
   this.cdnResourcesService = new CdnResourcesService();
@@ -87,10 +88,12 @@ Crowi.prototype.init = async function() {
   // customizeService depends on AppService and XssService
   // passportService depends on appService
   // slack depends on setUpSlacklNotification
+  // export and import depends on setUpGrowiBridge
   await Promise.all([
     this.setUpApp(),
     this.setUpXss(),
     this.setUpSlacklNotification(),
+    this.setUpGrowiBridge(),
   ]);
 
   await Promise.all([
@@ -126,6 +129,7 @@ Crowi.prototype.initForTest = async function() {
     this.setUpApp(),
     // this.setUpXss(),
     // this.setUpSlacklNotification(),
+    // this.setUpGrowiBridge(),
   ]);
 
   await Promise.all([
@@ -539,6 +543,13 @@ Crowi.prototype.setupUserGroup = async function() {
   }
 };
 
+Crowi.prototype.setUpGrowiBridge = async function() {
+  const GrowiBridgeService = require('../service/growi-bridge');
+  if (this.growiBridgeService == null) {
+    this.growiBridgeService = new GrowiBridgeService(this);
+  }
+};
+
 Crowi.prototype.setupExport = async function() {
   const ExportService = require('../service/export');
   if (this.exportService == null) {

src/server/routes/apiv3/export.js

@@ -14,7 +14,7 @@ const router = express.Router();
  */
 
 module.exports = (crowi) => {
-  const { exportService } = crowi;
+  const { growiBridgeService, exportService } = crowi;
 
   /**
    * @swagger
@@ -32,10 +32,10 @@ module.exports = (crowi) => {
    *            application/json:
    */
   router.get('/status', async(req, res) => {
-    const files = exportService.getStatus();
+    const zipFileStats = await exportService.getStatus();
 
     // TODO: use res.apiv3
-    return res.json({ ok: true, files });
+    return res.json({ ok: true, zipFileStats });
   });
 
   /**
@@ -86,17 +86,25 @@ module.exports = (crowi) => {
       const { collections } = req.body;
       // get model for collection
       const models = collections.map(collectionName => exportService.getModelFromCollectionName(collectionName));
-      // export into json
-      const jsonFiles = await exportService.exportMultipleCollectionsToJsons(models);
+
+      const [metaJson, jsonFiles] = await Promise.all([
+        exportService.createMetaJson(),
+        exportService.exportMultipleCollectionsToJsons(models),
+      ]);
+
       // zip json
       const configs = jsonFiles.map((jsonFile) => { return { from: jsonFile, as: path.basename(jsonFile) } });
+      // add meta.json in zip
+      configs.push({ from: metaJson, as: path.basename(metaJson) });
+      // exec zip
       const zipFile = await exportService.zipFiles(configs);
+      // get stats for the zip file
+      const zipFileStat = await growiBridgeService.parseZipFile(zipFile);
 
       // TODO: use res.apiv3
       return res.status(200).json({
         ok: true,
-        // collection: [Model.collection.collectionName],
-        file: path.basename(zipFile),
+        zipFileStat,
       });
     }
     catch (err) {
@@ -115,25 +123,35 @@ module.exports = (crowi) => {
    *      description: unlink all json and zip files for exports
    *      produces:
    *        - application/json
+   *      parameters:
+   *        - name: fileName
+   *          in: path
+   *          description: file name of zip file
+   *          schema:
+   *            type: string
    *      responses:
    *        200:
    *          description: the json and zip file are deleted
    *          content:
    *            application/json:
    */
-  // router.delete('/', async(req, res) => {
-  //   // TODO: add express validator
-  //   try {
-  //     // remove .json and .zip for collection
-  //     // TODO: use res.apiv3
-  //     return res.status(200).send({ status: 'DONE' });
-  //   }
-  //   catch (err) {
-  //     // TODO: use ApiV3Error
-  //     logger.error(err);
-  //     return res.status(500).send({ status: 'ERROR' });
-  //   }
-  // });
+  router.delete('/:fileName', async(req, res) => {
+    // TODO: add express validator
+    const { fileName } = req.params;
+
+    try {
+      const zipFile = exportService.getFile(fileName);
+      exportService.deleteZipFile(zipFile);
+
+      // TODO: use res.apiv3
+      return res.status(200).send({ ok: true });
+    }
+    catch (err) {
+      // TODO: use ApiV3Error
+      logger.error(err);
+      return res.status(500).send({ ok: false });
+    }
+  });
 
   return router;
 };

src/server/routes/apiv3/import.js

@@ -16,7 +16,7 @@ const router = express.Router();
  */
 
 module.exports = (crowi) => {
-  const { importService } = crowi;
+  const { growiBridgeService, importService } = crowi;
   const uploads = multer({
     storage: multer.diskStorage({
       destination: (req, file, cb) => {
@@ -102,14 +102,15 @@ module.exports = (crowi) => {
     // unzip
     await importService.unzip(zipFile);
     // eslint-disable-next-line no-unused-vars
-    const { meta, fileStats } = await importService.parseZipFile(zipFile);
+    const { meta, fileStats } = await growiBridgeService.parseZipFile(zipFile);
 
     // filter fileStats
     const filteredFileStats = fileStats.filter(({ fileName, collectionName, size }) => { return collections.includes(collectionName) });
 
-    // TODO: validate using meta data
-
     try {
+      // validate with meta.json
+      importService.validate(meta);
+
       await Promise.all(filteredFileStats.map(async({ fileName, collectionName, size }) => {
         const Model = importService.getModelFromCollectionName(collectionName);
         const jsonFile = importService.getFile(fileName);
@@ -138,12 +139,14 @@ module.exports = (crowi) => {
     const zipFile = importService.getFile(file.filename);
 
     try {
-      const data = await importService.parseZipFile(zipFile);
+      const data = await growiBridgeService.parseZipFile(zipFile);
+
+      // validate with meta.json
+      importService.validate(data.meta);
 
       // TODO: use res.apiv3
       return res.send({
         ok: true,
-        file: file.filename,
         data,
       });
     }

src/server/service/export.js

@@ -8,7 +8,9 @@ const toArrayIfNot = require('../../lib/util/toArrayIfNot');
 class ExportService {
 
   constructor(crowi) {
+    this.crowi = crowi;
     this.appService = crowi.appService;
+    this.growiBridgeService = crowi.growiBridgeService;
     this.baseDir = path.join(crowi.tmpDir, 'downloads');
     this.zipFileName = 'GROWI.zip';
     this.metaFileName = 'meta.json';
@@ -46,32 +48,44 @@ class ExportService {
   }
 
   /**
-   * dump a collection into json
+   * parse all zip files in downloads dir
    *
    * @memberOf ExportService
-   * @return {object} cache info for exported zip files
+   * @return {Array.<object>} info for zip files
    */
-  getStatus() {
-    const status = {};
-    const collections = Object.keys(this.files);
-    collections.forEach((file) => {
-      status[path.basename(file, '.zip')] = null;
-    });
+  async getStatus() {
+    const zipFiles = fs.readdirSync(this.baseDir).filter((file) => { return path.extname(file) === '.zip' });
+    const zipFileStats = await Promise.all(zipFiles.map((file) => {
+      const zipFile = this.getFile(file);
+      return this.growiBridgeService.parseZipFile(zipFile);
+    }));
+
+    return zipFileStats;
+  }
 
-    // extract ${collectionName}.zip
-    const files = fs.readdirSync(this.baseDir).filter((file) => { return path.extname(file) === '.zip' && collections.includes(path.basename(file, '.zip')) });
+  /**
+   * create meta.json
+   *
+   * @memberOf ExportService
+   * @return {string} path to meta.json
+   */
+  async createMetaJson() {
+    const metaJson = path.join(this.baseDir, this.metaFileName);
+    const writeStream = fs.createWriteStream(metaJson, { encoding: this.encoding });
 
-    files.forEach((file) => {
-      status[path.basename(file, '.zip')] = file;
-    });
+    const metaData = {
+      version: this.crowi.version,
+      url: this.appService.getSiteUrl(),
+      passwordSeed: this.crowi.env.PASSWORD_SEED,
+      exportedAt: new Date(),
+    };
 
-    files.forEach((file) => {
-      const stats = fs.statSync(path.join(this.baseDir, file));
-      stats.name = file;
-      status[path.basename(file, '.zip')] = stats;
-    });
+    writeStream.write(JSON.stringify(metaData));
+    writeStream.close();
 
-    return status;
+    await streamToPromise(writeStream);
+
+    return metaJson;
   }
 
   /**
@@ -126,8 +140,16 @@ class ExportService {
     return file;
   }
 
+  /**
+   * export multiple collections
+   *
+   * @memberOf ExportService
+   * @param {Array.<object>} models array of instances of mongoose model
+   * @return {Array.<string>} paths to json files created
+   */
   async exportMultipleCollectionsToJsons(models) {
     const jsonFiles = await Promise.all(models.map(Model => this.exportCollectionToJson(Model)));
+
     return jsonFiles;
   }
 
@@ -157,8 +179,8 @@ class ExportService {
    * zip files into one zip file
    *
    * @memberOf ExportService
-   * @param {object|array<object>} configs array of object { from: "path to source file", as: "file name after unzipped" }
-   * @return {string} path to zip file
+   * @param {object|array<object>} configs object or array of object { from: "path to source file", as: "file name after unzipped" }
+   * @return {string} absolute path to the zip file
    * @see https://www.archiverjs.com/#quick-start
    */
   async zipFiles(_configs) {
@@ -200,10 +222,26 @@ class ExportService {
     return zipFile;
   }
 
+  /**
+   * get the absolute path to a file
+   *
+   * @memberOf ExportService
+   * @param {string} fileName base name of file
+   * @return {string} absolute path to the file
+   */
+  getFile(fileName) {
+    const jsonFile = path.join(this.baseDir, fileName);
+
+    // throws err if the file does not exist
+    fs.accessSync(jsonFile);
+
+    return jsonFile;
+  }
+
   /**
    * get the absolute path to the zip file
    *
-   * @memberOf ImportService
+   * @memberOf ExportService
    * @return {string} absolute path to the zip file
    */
   getZipFile() {
@@ -219,7 +257,7 @@ class ExportService {
    * get a model from collection name
    *
    * @memberOf ExportService
-   * @param {object} collectionName collection name
+   * @param {string} collectionName collection name
    * @return {object} instance of mongoose model
    */
   getModelFromCollectionName(collectionName) {
@@ -232,6 +270,16 @@ class ExportService {
     return Model;
   }
 
+  /**
+   * remove zip file from downloads dir
+   *
+   * @param {string} zipFile absolute path to zip file
+   * @memberOf ExportService
+   */
+  deleteZipFile(zipFile) {
+    fs.unlinkSync(zipFile);
+  }
+
 }
 
 module.exports = ExportService;

src/server/service/growi-bridge.js

@@ -0,0 +1,59 @@
+const logger = require('@alias/logger')('growi:services:ImportService'); // eslint-disable-line no-unused-vars
+const fs = require('fs');
+const path = require('path');
+const streamToPromise = require('stream-to-promise');
+const unzipper = require('unzipper');
+
+/**
+ * the service class for bridging GROWIs (export and import)
+ * common properties and methods between export service and import service are defined in this service
+ */
+class GrowiBridgeService {
+
+  constructor(crowi) {
+    this.metaFileName = 'meta.json';
+  }
+
+  /**
+   * parse a zip file
+   *
+   * @memberOf ImportService
+   * @param {string} zipFile path to zip file
+   * @return {object} meta{object} and files{Array.<object>}
+   */
+  async parseZipFile(zipFile) {
+    const readStream = fs.createReadStream(zipFile);
+    const unzipStream = readStream.pipe(unzipper.Parse());
+    const fileStats = [];
+    let meta = {};
+
+    unzipStream.on('entry', async(entry) => {
+      const fileName = entry.path;
+      const size = entry.vars.uncompressedSize; // There is also compressedSize;
+
+      if (fileName === this.metaFileName) {
+        meta = JSON.parse((await entry.buffer()).toString());
+      }
+      else {
+        fileStats.push({
+          fileName,
+          collectionName: path.basename(fileName, '.json'),
+          size,
+        });
+      }
+
+      entry.autodrain();
+    });
+
+    await streamToPromise(unzipStream);
+
+    return {
+      meta,
+      fileName: path.basename(zipFile),
+      fileStats,
+    };
+  }
+
+}
+
+module.exports = GrowiBridgeService;

src/server/service/import.js

@@ -9,6 +9,7 @@ const { ObjectId } = require('mongoose').Types;
 class ImportService {
 
   constructor(crowi) {
+    this.crowi = crowi;
     this.baseDir = path.join(crowi.tmpDir, 'imports');
     this.metaFileName = 'meta.json';
     this.encoding = 'utf-8';
@@ -80,6 +81,7 @@ class ImportService {
    *
    * @memberOf ImportService
    * @param {object} Model instance of mongoose model
+   * @param {string} jsonFile absolute path to the jsonFile being imported
    * @param {object} overwriteParams overwrite each document with unrelated value. e.g. { creator: req.user }
    */
   async import(Model, jsonFile, overwriteParams = {}) {
@@ -138,50 +140,11 @@ class ImportService {
     fs.unlinkSync(jsonFile);
   }
 
-  /**
-   * parse a zip file
-   *
-   * @memberOf ImportService
-   * @param {string} zipFile path to zip file
-   * @return {object} meta{object} and files{array<object>}
-   */
-  async parseZipFile(zipFile) {
-    const readStream = fs.createReadStream(zipFile);
-    const unzipStream = readStream.pipe(unzipper.Parse());
-    const fileStats = [];
-
-    unzipStream.on('entry', (entry) => {
-      const fileName = entry.path;
-      const size = entry.vars.uncompressedSize; // There is also compressedSize;
-
-      if (fileName === this.metaFileName) {
-        // TODO: parse meta.json
-        entry.autodrain();
-      }
-      else {
-        fileStats.push({
-          fileName,
-          collectionName: path.basename(fileName, '.json'),
-          size,
-        });
-      }
-
-      entry.autodrain();
-    });
-
-    await streamToPromise(unzipStream);
-
-    return {
-      meta: {},
-      fileStats,
-    };
-  }
-
   /**
    * extract a zip file
    *
    * @memberOf ImportService
-   * @param {string} zipFile path to zip file
+   * @param {string} zipFile absolute path to zip file
    * @return {Array.<string>} array of absolute paths to extracted files
    */
   async unzip(zipFile) {
@@ -193,7 +156,7 @@ class ImportService {
       const fileName = entry.path;
 
       if (fileName === this.metaFileName) {
-        // TODO: parse meta.json
+        // skip meta.json
         entry.autodrain();
       }
       else {
@@ -214,7 +177,7 @@ class ImportService {
    *
    * @memberOf ImportService
    * @param {object} unorderedBulkOp result of Model.collection.initializeUnorderedBulkOp()
-   * @return {{nInserted: number, failed: string[]}} number of docuemnts inserted and failed
+   * @return {{nInserted: number, failed: Array.<string>}} number of docuemnts inserted and failed
    */
   async execUnorderedBulkOpSafely(unorderedBulkOp) {
     // keep the number of documents inserted and failed for logger
@@ -321,6 +284,24 @@ class ImportService {
     return jsonFile;
   }
 
+  /**
+   * validate using meta.json
+   * to pass validation, all the criteria must be met
+   *   - ${version of this growi} === ${version of growi that exported data}
+   *
+   * @memberOf ImportService
+   * @param {object} meta meta data from meta.json
+   */
+  validate(meta) {
+    if (meta.version !== this.crowi.version) {
+      throw new Error('the version of this growi and the growi that exported the data are not met');
+    }
+
+    // TODO: check if all migrations are completed
+    // - export: throw err if there are pending migrations
+    // - import: throw err if there are pending migrations
+  }
+
 }
 
 module.exports = ImportService;