upload

src/client/js/components/Admin/Import/GrowiImportForm.jsx

@@ -13,6 +13,7 @@ class GrowiImportForm extends React.Component {
 
     this.initialState = {
       meta: {},
+      zipFileName: '',
       files: [],
       schema: {
         pages: {},
@@ -44,8 +45,8 @@ class GrowiImportForm extends React.Component {
     formData.append('file', this.inputRef.current.files[0]);
 
     // TODO use appContainer.apiv3.post
-    const { data } = await this.props.appContainer.apiPost('/v3/import/upload', formData);
-    this.setState({ meta: data.meta, files: data.files });
+    const { file, data } = await this.props.appContainer.apiPost('/v3/import/upload', formData);
+    this.setState({ meta: data.meta, zipFileName: file, files: data.files });
     // TODO toastSuccess, toastError
   }
 
@@ -54,13 +55,8 @@ class GrowiImportForm extends React.Component {
 
     // TODO use appContainer.apiv3.post
     await this.props.appContainer.apiPost('/v3/import', {
-      meta: this.state.meta,
-      options: this.state.files.map((option) => {
-        return {
-          ...option,
-          schema: this.state.schema[option.collectionName],
-        };
-      }),
+      zipFile: this.state.zipFileName,
+      schema: this.state.schema,
     });
     // TODO toastSuccess, toastError
     this.setState(this.initialState);
@@ -113,6 +109,7 @@ class GrowiImportForm extends React.Component {
         {this.state.files.length > 0 && (
           <Fragment>
             {/* TODO: move to another component 2 */}
+            <div>{this.state.zipFileName}</div>
             <div>{JSON.stringify(this.state.meta)}</div>
             <table className="table table-bordered table-mapping">
               <thead>

src/server/routes/apiv3/import.js

@@ -3,7 +3,6 @@ const loggerFactory = require('@alias/logger');
 const logger = loggerFactory('growi:routes:apiv3:import'); // eslint-disable-line no-unused-vars
 const path = require('path');
 const multer = require('multer');
-const autoReap = require('multer-autoreap');
 const { ObjectId } = require('mongoose').Types;
 
 const express = require('express');
@@ -19,7 +18,15 @@ const router = express.Router();
 module.exports = (crowi) => {
   const { importService } = crowi;
   const uploads = multer({
-    dest: importService.baseDir,
+    storage: multer.diskStorage({
+      destination: (req, file, cb) => {
+        cb(null, importService.baseDir);
+      },
+      filename(req, file, cb) {
+        // to prevent hashing the file name. files with same name will be overwritten.
+        cb(null, file.originalname);
+      },
+    }),
     fileFilter: (req, file, cb) => {
       if (path.extname(file.originalname) === '.zip') {
         return cb(null, true);
@@ -88,23 +95,27 @@ module.exports = (crowi) => {
    */
   router.post('/', async(req, res) => {
     // TODO: add express validator
+
+    const { zipFile, schema } = req.body;
+
+    // unzip
+    await importService.unzip(zipFile);
     // eslint-disable-next-line no-unused-vars
-    const { meta, options } = req.body;
+    const { meta, files } = await importService.parseZipFile(zipFile);
 
     // TODO: validate using meta data
 
     try {
-      await Promise.all(options.map(async({ collectionName, fileName, schema }) => {
+      await Promise.all(files.map(async({ fileName, collectionName, size }) => {
         const Model = importService.getModelFromCollectionName(collectionName);
-        const jsonFile = importService.getJsonFile(fileName, true);
 
         let overwriteParams;
         if (overwriteParamsFn[collectionName] != null) {
           // await in case overwriteParamsFn[collection] is a Promise
-          overwriteParams = await overwriteParamsFn(Model, schema, req);
+          overwriteParams = await overwriteParamsFn(Model, schema[collectionName], req);
         }
 
-        await importService.import(Model, jsonFile, overwriteParams);
+        await importService.import(Model, fileName, overwriteParams);
       }));
 
       // TODO: use res.apiv3
@@ -117,15 +128,19 @@ module.exports = (crowi) => {
     }
   });
 
-  router.post('/upload', uploads.single('file'), autoReap, async(req, res) => {
+  router.post('/upload', uploads.single('file'), async(req, res) => {
     const { file } = req;
     const zipFile = path.join(file.destination, file.filename);
 
     try {
-      const data = await importService.unzip(zipFile);
+      const data = await importService.parseZipFile(zipFile);
 
       // TODO: use res.apiv3
-      return res.send({ ok: true, data });
+      return res.send({
+        ok: true,
+        file: file.filename,
+        data,
+      });
     }
     catch (err) {
       // TODO: use ApiV3Error

src/server/service/import.js

@@ -139,13 +139,13 @@ class ImportService {
   }
 
   /**
-   * extract a zip file
+   * parse a zip file
    *
    * @memberOf ImportService
    * @param {string} zipFile path to zip file
    * @return {object} meta{object} and files{array<object>}
    */
-  async unzip(zipFile) {
+  async parseZipFile(zipFile) {
     const readStream = fs.createReadStream(zipFile);
     const unzipStream = readStream.pipe(unzipper.Parse());
     const files = [];
@@ -159,15 +159,14 @@ class ImportService {
         entry.autodrain();
       }
       else {
-        const writeStream = fs.createWriteStream(this.getJsonFile(fileName), { encoding: this.encoding });
-        entry.pipe(writeStream);
-
         files.push({
           fileName,
           collectionName: path.basename(fileName, '.json'),
           size,
         });
       }
+
+      entry.autodrain();
     });
 
     await streamToPromise(unzipStream);
@@ -178,6 +177,38 @@ class ImportService {
     };
   }
 
+  /**
+   * extract a zip file
+   *
+   * @memberOf ImportService
+   * @param {string} zipFile path to zip file
+   * @return {Array.<string>} array of absolute paths to extracted files
+   */
+  async unzip(zipFile) {
+    const readStream = fs.createReadStream(zipFile);
+    const unzipStream = readStream.pipe(unzipper.Parse());
+    const files = [];
+
+    unzipStream.on('entry', (entry) => {
+      const fileName = entry.path;
+
+      if (fileName === this.metaFileName) {
+        // TODO: parse meta.json
+        entry.autodrain();
+      }
+      else {
+        const jsonFile = path.join(this.baseDir, fileName);
+        const writeStream = fs.createWriteStream(jsonFile, { encoding: this.encoding });
+        entry.pipe(writeStream);
+        files.push(jsonFile);
+      }
+    });
+
+    await streamToPromise(unzipStream);
+
+    return files;
+  }
+
   /**
    * execute unorderedBulkOp and ignore errors
    *