Merge branch 'feat/export-n-import-revision-2' into feat/export-n-import-revision-3

src/client/js/components/Admin/Import/GrowiImportForm.jsx

@@ -13,7 +13,9 @@ class GrowiImportForm extends React.Component {
 
     this.initialState = {
       meta: {},
-      files: [],
+      zipFileName: '',
+      collections: new Set(),
+      fileStats: [],
       schema: {
         pages: {},
         revisions: {},
@@ -25,6 +27,7 @@ class GrowiImportForm extends React.Component {
     this.inputRef = React.createRef();
 
     this.changeFileName = this.changeFileName.bind(this);
+    this.toggleCheckbox = this.toggleCheckbox.bind(this);
     this.uploadZipFile = this.uploadZipFile.bind(this);
     this.import = this.import.bind(this);
     this.validateForm = this.validateForm.bind(this);
@@ -36,6 +39,22 @@ class GrowiImportForm extends React.Component {
     this.setState({ name: e.target.files[0].name });
   }
 
+  toggleCheckbox(e) {
+    const { target } = e;
+    const { name, checked } = target;
+
+    this.setState((prevState) => {
+      const collections = new Set(prevState.collections);
+      if (checked) {
+        collections.add(name);
+      }
+      else {
+        collections.delete(name);
+      }
+      return { collections };
+    });
+  }
+
   async uploadZipFile(e) {
     e.preventDefault();
 
@@ -44,8 +63,8 @@ class GrowiImportForm extends React.Component {
     formData.append('file', this.inputRef.current.files[0]);
 
     // TODO use appContainer.apiv3.post
-    const { data } = await this.props.appContainer.apiPost('/v3/import/upload', formData);
-    this.setState({ meta: data.meta, files: data.files });
+    const { file, data } = await this.props.appContainer.apiPost('/v3/import/upload', formData);
+    this.setState({ meta: data.meta, zipFileName: file, fileStats: data.fileStats });
     // TODO toastSuccess, toastError
   }
 
@@ -54,13 +73,9 @@ class GrowiImportForm extends React.Component {
 
     // TODO use appContainer.apiv3.post
     await this.props.appContainer.apiPost('/v3/import', {
-      meta: this.state.meta,
-      options: this.state.files.map((option) => {
-        return {
-          ...option,
-          schema: this.state.schema[option.collectionName],
-        };
-      }),
+      fileName: this.state.zipFileName,
+      collections: Array.from(this.state.collections),
+      schema: this.state.schema,
     });
     // TODO toastSuccess, toastError
     this.setState(this.initialState);
@@ -110,9 +125,10 @@ class GrowiImportForm extends React.Component {
         </form>
 
         {/* TODO: move to another component 1 */}
-        {this.state.files.length > 0 && (
+        {this.state.fileStats.length > 0 && (
           <Fragment>
             {/* TODO: move to another component 2 */}
+            <div>{this.state.zipFileName}</div>
             <div>{JSON.stringify(this.state.meta)}</div>
             <table className="table table-bordered table-mapping">
               <thead>
@@ -122,11 +138,23 @@ class GrowiImportForm extends React.Component {
                 </tr>
               </thead>
               <tbody>
-                {this.state.files.map((file) => {
+                {this.state.fileStats.map((file) => {
+                  const { fileName, collectionName } = file;
                   return (
-                    <tr key={file.fileName}>
-                      <td>{file.fileName}</td>
-                      <td>{file.collectionName}</td>
+                    <tr key={fileName}>
+                      <td>
+                        <input
+                          type="checkbox"
+                          id={collectionName}
+                          name={collectionName}
+                          className="form-check-input"
+                          value={collectionName}
+                          checked={this.state.collections.has(collectionName)}
+                          onChange={this.toggleCheckbox}
+                        />
+                      </td>
+                      <td>{fileName}</td>
+                      <td>{collectionName}</td>
                     </tr>
                   );
                 })}

src/server/routes/apiv3/export.js

@@ -56,7 +56,7 @@ module.exports = (crowi) => {
   router.get('/', async(req, res) => {
     // TODO: add express validator
     try {
-      return res.download(exportService.getZipFile(true));
+      return res.download(exportService.getZipFile());
     }
     catch (err) {
       // TODO: use ApiV3Error

src/server/routes/apiv3/import.js

@@ -3,7 +3,6 @@ const loggerFactory = require('@alias/logger');
 const logger = loggerFactory('growi:routes:apiv3:import'); // eslint-disable-line no-unused-vars
 const path = require('path');
 const multer = require('multer');
-const autoReap = require('multer-autoreap');
 const { ObjectId } = require('mongoose').Types;
 
 const express = require('express');
@@ -19,7 +18,15 @@ const router = express.Router();
 module.exports = (crowi) => {
   const { importService } = crowi;
   const uploads = multer({
-    dest: importService.baseDir,
+    storage: multer.diskStorage({
+      destination: (req, file, cb) => {
+        cb(null, importService.baseDir);
+      },
+      filename(req, file, cb) {
+        // to prevent hashing the file name. files with same name will be overwritten.
+        cb(null, file.originalname);
+      },
+    }),
     fileFilter: (req, file, cb) => {
       if (path.extname(file.originalname) === '.zip') {
         return cb(null, true);
@@ -88,20 +95,29 @@ module.exports = (crowi) => {
    */
   router.post('/', async(req, res) => {
     // TODO: add express validator
+
+    const { fileName, collections, schema } = req.body;
+    const zipFile = importService.getFile(fileName);
+
+    // unzip
+    await importService.unzip(zipFile);
     // eslint-disable-next-line no-unused-vars
-    const { meta, options } = req.body;
+    const { meta, fileStats } = await importService.parseZipFile(zipFile);
+
+    // filter fileStats
+    const filteredFileStats = fileStats.filter(({ fileName, collectionName, size }) => { return collections.includes(collectionName) });
 
     // TODO: validate using meta data
 
     try {
-      await Promise.all(options.map(async({ collectionName, fileName, schema }) => {
+      await Promise.all(filteredFileStats.map(async({ fileName, collectionName, size }) => {
         const Model = importService.getModelFromCollectionName(collectionName);
-        const jsonFile = importService.getJsonFile(fileName, true);
+        const jsonFile = importService.getFile(fileName);
 
         let overwriteParams;
         if (overwriteParamsFn[collectionName] != null) {
           // await in case overwriteParamsFn[collection] is a Promise
-          overwriteParams = await overwriteParamsFn(Model, schema, req);
+          overwriteParams = await overwriteParamsFn(Model, schema[collectionName], req);
         }
 
         await importService.import(Model, jsonFile, overwriteParams);
@@ -117,15 +133,19 @@ module.exports = (crowi) => {
     }
   });
 
-  router.post('/upload', uploads.single('file'), autoReap, async(req, res) => {
+  router.post('/upload', uploads.single('file'), async(req, res) => {
     const { file } = req;
-    const zipFile = path.join(file.destination, file.filename);
+    const zipFile = importService.getFile(file.filename);
 
     try {
-      const data = await importService.unzip(zipFile);
+      const data = await importService.parseZipFile(zipFile);
 
       // TODO: use res.apiv3
-      return res.send({ ok: true, data });
+      return res.send({
+        ok: true,
+        file: file.filename,
+        data,
+      });
     }
     catch (err) {
       // TODO: use ApiV3Error

src/server/service/export.js

@@ -197,7 +197,7 @@ class ExportService {
    */
   async zipFiles(_configs) {
     const configs = toArrayIfNot(_configs);
-    const zipFile = this.getZipFile();
+    const zipFile = path.join(this.baseDir, this.zipFileName);
     const archive = archiver('zip', {
       zlib: { level: this.zlibLevel },
     });
@@ -238,27 +238,13 @@ class ExportService {
    * get the absolute path to the zip file
    *
    * @memberOf ImportService
-   * @param {boolean} [validate=false] boolean to check if the file exists
    * @return {string} absolute path to the zip file
    */
-  getZipFile(validate = false) {
+  getZipFile() {
     const zipFile = path.join(this.baseDir, this.zipFileName);
 
-    if (validate) {
-      try {
-        fs.accessSync(zipFile);
-      }
-      catch (err) {
-        if (err.code === 'ENOENT') {
-          logger.error(`${zipFile} does not exist`, err);
-        }
-        else {
-          logger.error(err);
-        }
-
-        throw err;
-      }
-    }
+    // throws err if the file does not exist
+    fs.accessSync(zipFile);
 
     return zipFile;
   }

src/server/service/import.js

@@ -139,16 +139,16 @@ class ImportService {
   }
 
   /**
-   * extract a zip file
+   * parse a zip file
    *
    * @memberOf ImportService
    * @param {string} zipFile path to zip file
    * @return {object} meta{object} and files{Array.<object>}
    */
-  async unzip(zipFile) {
+  async parseZipFile(zipFile) {
     const readStream = fs.createReadStream(zipFile);
     const unzipStream = readStream.pipe(unzipper.Parse());
-    const files = [];
+    const fileStats = [];
 
     unzipStream.on('entry', (entry) => {
       const fileName = entry.path;
@@ -159,25 +159,56 @@ class ImportService {
         entry.autodrain();
       }
       else {
-        const writeStream = fs.createWriteStream(this.getJsonFile(fileName), { encoding: this.encoding });
-        entry.pipe(writeStream);
-
-        files.push({
+        fileStats.push({
           fileName,
           collectionName: path.basename(fileName, '.json'),
           size,
         });
       }
+
+      entry.autodrain();
     });
 
     await streamToPromise(unzipStream);
 
     return {
       meta: {},
-      files,
+      fileStats,
     };
   }
 
+  /**
+   * extract a zip file
+   *
+   * @memberOf ImportService
+   * @param {string} zipFile path to zip file
+   * @return {Array.<string>} array of absolute paths to extracted files
+   */
+  async unzip(zipFile) {
+    const readStream = fs.createReadStream(zipFile);
+    const unzipStream = readStream.pipe(unzipper.Parse());
+    const files = [];
+
+    unzipStream.on('entry', (entry) => {
+      const fileName = entry.path;
+
+      if (fileName === this.metaFileName) {
+        // TODO: parse meta.json
+        entry.autodrain();
+      }
+      else {
+        const jsonFile = path.join(this.baseDir, fileName);
+        const writeStream = fs.createWriteStream(jsonFile, { encoding: this.encoding });
+        entry.pipe(writeStream);
+        files.push(jsonFile);
+      }
+    });
+
+    await streamToPromise(unzipStream);
+
+    return files;
+  }
+
   /**
    * execute unorderedBulkOp and ignore errors
    *
@@ -279,27 +310,13 @@ class ImportService {
    *
    * @memberOf ImportService
    * @param {string} fileName base name of file
-   * @param {boolean} [validate=false] boolean to check if the file exists
    * @return {string} absolute path to the file
    */
-  getJsonFile(fileName, validate = false) {
+  getFile(fileName) {
     const jsonFile = path.join(this.baseDir, fileName);
 
-    if (validate) {
-      try {
-        fs.accessSync(jsonFile);
-      }
-      catch (err) {
-        if (err.code === 'ENOENT') {
-          logger.error(`${jsonFile} does not exist`, err);
-        }
-        else {
-          logger.error(err);
-        }
-
-        throw err;
-      }
-    }
+    // throws err if the file does not exist
+    fs.accessSync(jsonFile);
 
     return jsonFile;
   }