Merge pull request #1213 from weseek/feat/export-n-import-revision-7

Feat/export n import revision 7

src/client/js/components/Admin/Export/ExportPage.jsx

@@ -1,6 +1,7 @@
 import React, { Fragment } from 'react';
 import PropTypes from 'prop-types';
 import { withTranslation } from 'react-i18next';
+import * as toastr from 'toastr';
 
 import ExportZipFormModal from './ExportZipFormModal';
 import ZipFileTable from './ZipFileTable';
@@ -26,14 +27,16 @@ class ExportPage extends React.Component {
   }
 
   async componentDidMount() {
-    // TODO: use apiv3.get
+    // TODO:: use apiv3.get
+    // eslint-disable-next-line no-unused-vars
     const [{ collections }, { zipFileStats }] = await Promise.all([
       this.props.appContainer.apiGet('/v3/mongo/collections', {}),
       this.props.appContainer.apiGet('/v3/export/status', {}),
     ]);
-    // TODO toastSuccess, toastError
+    // TODO: toastSuccess, toastError
 
-    this.setState({ collections, zipFileStats });
+    this.setState({ collections: ['pages', 'revisions'], zipFileStats }); // FIXME: delete this line and uncomment the line below
+    // this.setState({ collections, zipFileStats });
   }
 
   onZipFileStatAdd(newStat) {
@@ -45,13 +48,37 @@ class ExportPage extends React.Component {
   }
 
   async onZipFileStatRemove(fileName) {
-    await this.props.appContainer.apiRequest('delete', `/v3/export/${fileName}`, {});
-
-    this.setState((prevState) => {
-      return {
-        zipFileStats: prevState.zipFileStats.filter(stat => stat.fileName !== fileName),
-      };
-    });
+    try {
+      await this.props.appContainer.apiRequest('delete', `/v3/export/${fileName}`, {});
+
+      this.setState((prevState) => {
+        return {
+          zipFileStats: prevState.zipFileStats.filter(stat => stat.fileName !== fileName),
+        };
+      });
+
+      // TODO: toastSuccess, toastError
+      toastr.success(undefined, `Deleted ${fileName}`, {
+        closeButton: true,
+        progressBar: true,
+        newestOnTop: false,
+        showDuration: '100',
+        hideDuration: '100',
+        timeOut: '1200',
+        extendedTimeOut: '150',
+      });
+    }
+    catch (err) {
+      // TODO: toastSuccess, toastError
+      toastr.error(err, 'Error', {
+        closeButton: true,
+        progressBar: true,
+        newestOnTop: false,
+        showDuration: '100',
+        hideDuration: '100',
+        timeOut: '3000',
+      });
+    }
   }
 
   openExportModal() {

src/client/js/components/Admin/Export/ExportZipFormModal.jsx

@@ -2,6 +2,7 @@ import React from 'react';
 import PropTypes from 'prop-types';
 import { withTranslation } from 'react-i18next';
 import Modal from 'react-bootstrap/es/Modal';
+import * as toastr from 'toastr';
 
 import { createSubscribedElement } from '../../UnstatedUtils';
 import AppContainer from '../../../services/AppContainer';
@@ -51,11 +52,35 @@ class ExportZipFormModal extends React.Component {
   async export(e) {
     e.preventDefault();
 
-    // TODO use appContainer.apiv3.post
-    const { zipFileStat } = await this.props.appContainer.apiPost('/v3/export', { collections: Array.from(this.state.collections) });
-    // TODO toastSuccess, toastError
-    this.props.onZipFileStatAdd(zipFileStat);
-    this.props.onClose();
+    try {
+      // TODO: use appContainer.apiv3.post
+      const { zipFileStat } = await this.props.appContainer.apiPost('/v3/export', { collections: Array.from(this.state.collections) });
+      // TODO: toastSuccess, toastError
+      this.props.onZipFileStatAdd(zipFileStat);
+      this.props.onClose();
+
+      // TODO: toastSuccess, toastError
+      toastr.success(undefined, `Generated ${zipFileStat.fileName}`, {
+        closeButton: true,
+        progressBar: true,
+        newestOnTop: false,
+        showDuration: '100',
+        hideDuration: '100',
+        timeOut: '1200',
+        extendedTimeOut: '150',
+      });
+    }
+    catch (err) {
+      // TODO: toastSuccess, toastError
+      toastr.error(err, 'Error', {
+        closeButton: true,
+        progressBar: true,
+        newestOnTop: false,
+        showDuration: '100',
+        hideDuration: '100',
+        timeOut: '3000',
+      });
+    }
   }
 
   validateForm() {

src/client/js/components/Admin/Import/GrowiZipImportForm.jsx

@@ -1,6 +1,7 @@
 import React, { Fragment } from 'react';
 import PropTypes from 'prop-types';
 import { withTranslation } from 'react-i18next';
+import * as toastr from 'toastr';
 
 import { createSubscribedElement } from '../../UnstatedUtils';
 import AppContainer from '../../../services/AppContainer';
@@ -46,14 +47,50 @@ class GrowiImportForm extends React.Component {
   async import(e) {
     e.preventDefault();
 
-    // TODO use appContainer.apiv3.post
-    await this.props.appContainer.apiPost('/v3/import', {
-      fileName: this.props.fileName,
-      collections: Array.from(this.state.collections),
-      schema: this.state.schema,
-    });
-    // TODO toastSuccess, toastError
-    this.setState(this.initialState);
+    try {
+      // TODO: use appContainer.apiv3.post
+      const { results } = await this.props.appContainer.apiPost('/v3/import', {
+        fileName: this.props.fileName,
+        collections: Array.from(this.state.collections),
+        schema: this.state.schema,
+      });
+
+      this.setState(this.initialState);
+      this.props.onPostImport();
+
+      // TODO: toastSuccess, toastError
+      toastr.success(undefined, 'Imported documents', {
+        closeButton: true,
+        progressBar: true,
+        newestOnTop: false,
+        showDuration: '100',
+        hideDuration: '100',
+        timeOut: '1200',
+        extendedTimeOut: '150',
+      });
+
+      for (const { collectionName, failedIds } of results) {
+        if (failedIds.length > 0) {
+          toastr.error(`failed to insert ${failedIds.join(', ')}`, collectionName, {
+            closeButton: true,
+            progressBar: true,
+            newestOnTop: false,
+            timeOut: '30000',
+          });
+        }
+      }
+    }
+    catch (err) {
+      // TODO: toastSuccess, toastError
+      toastr.error(err, 'Error', {
+        closeButton: true,
+        progressBar: true,
+        newestOnTop: false,
+        showDuration: '100',
+        hideDuration: '100',
+        timeOut: '3000',
+      });
+    }
   }
 
   validateForm() {
@@ -131,6 +168,7 @@ GrowiImportForm.propTypes = {
   fileName: PropTypes.string,
   fileStats: PropTypes.arrayOf(PropTypes.object).isRequired,
   onDiscard: PropTypes.func.isRequired,
+  onPostImport: PropTypes.func.isRequired,
 };
 
 /**

src/client/js/components/Admin/Import/GrowiZipImportSection.jsx

@@ -1,6 +1,7 @@
 import React, { Fragment } from 'react';
 import PropTypes from 'prop-types';
 import { withTranslation } from 'react-i18next';
+import * as toastr from 'toastr';
 
 import GrowiZipUploadForm from './GrowiZipUploadForm';
 import GrowiZipImportForm from './GrowiZipImportForm';
@@ -22,6 +23,7 @@ class GrowiZipImportSection extends React.Component {
 
     this.handleUpload = this.handleUpload.bind(this);
     this.discardData = this.discardData.bind(this);
+    this.resetState = this.resetState.bind(this);
   }
 
   handleUpload({ meta, fileName, fileStats }) {
@@ -32,7 +34,36 @@ class GrowiZipImportSection extends React.Component {
   }
 
   async discardData() {
-    await this.props.appContainer.apiRequest('delete', `/v3/import/${this.state.fileName}`, {});
+    try {
+      const { fileName } = this.state;
+      await this.props.appContainer.apiRequest('delete', `/v3/import/${this.state.fileName}`, {});
+      this.resetState();
+
+      // TODO: toastSuccess, toastError
+      toastr.success(undefined, `Deleted ${fileName}`, {
+        closeButton: true,
+        progressBar: true,
+        newestOnTop: false,
+        showDuration: '100',
+        hideDuration: '100',
+        timeOut: '1200',
+        extendedTimeOut: '150',
+      });
+    }
+    catch (err) {
+      // TODO: toastSuccess, toastError
+      toastr.error(err, 'Error', {
+        closeButton: true,
+        progressBar: true,
+        newestOnTop: false,
+        showDuration: '100',
+        hideDuration: '100',
+        timeOut: '3000',
+      });
+    }
+  }
+
+  resetState() {
     this.setState(this.initialState);
   }
 
@@ -54,6 +85,7 @@ class GrowiZipImportSection extends React.Component {
               fileName={this.state.fileName}
               fileStats={this.state.fileStats}
               onDiscard={this.discardData}
+              onPostImport={this.resetState}
             />
           </Fragment>
         ) : (

src/client/js/components/Admin/Import/GrowiZipUploadForm.jsx

@@ -31,10 +31,10 @@ class GrowiZipUploadForm extends React.Component {
     formData.append('_csrf', this.props.appContainer.csrfToken);
     formData.append('file', this.inputRef.current.files[0]);
 
-    // TODO use appContainer.apiv3.post
+    // TODO: use appContainer.apiv3.post
     const { data } = await this.props.appContainer.apiPost('/v3/import/upload', formData);
     this.props.onUpload(data);
-    // TODO toastSuccess, toastError
+    // TODO: toastSuccess, toastError
   }
 
   validateForm() {

src/server/routes/apiv3/import.js

@@ -92,7 +92,12 @@ module.exports = (crowi) => {
    *          content:
    *            application/json:
    *              schema:
-   *                type: object
+   *                properties:
+   *                  results:
+   *                    type: array
+   *                    items:
+   *                      type: object
+   *                      description: collectionName, insertedIds, failedIds
    */
   router.post('/', async(req, res) => {
     // TODO: add express validator
@@ -105,6 +110,9 @@ module.exports = (crowi) => {
     // eslint-disable-next-line no-unused-vars
     const { meta, fileStats } = await growiBridgeService.parseZipFile(zipFile);
 
+    // delete zip file after unzipping and parsing it
+    fs.unlinkSync(zipFile);
+
     // filter fileStats
     const filteredFileStats = fileStats.filter(({ fileName, collectionName, size }) => { return collections.includes(collectionName) });
 
@@ -112,7 +120,7 @@ module.exports = (crowi) => {
       // validate with meta.json
       importService.validate(meta);
 
-      await Promise.all(filteredFileStats.map(async({ fileName, collectionName, size }) => {
+      const results = await Promise.all(filteredFileStats.map(async({ fileName, collectionName, size }) => {
         const Model = growiBridgeService.getModelFromCollectionName(collectionName);
         const jsonFile = importService.getFile(fileName);
 
@@ -122,11 +130,17 @@ module.exports = (crowi) => {
           overwriteParams = await overwriteParamsFn(Model, schema[collectionName], req);
         }
 
-        await importService.import(Model, jsonFile, overwriteParams);
+        const { insertedIds, failedIds } = await importService.import(Model, jsonFile, overwriteParams);
+
+        return {
+          collectionName,
+          insertedIds,
+          failedIds,
+        };
       }));
 
       // TODO: use res.apiv3
-      return res.send({ ok: true });
+      return res.send({ ok: true, results });
     }
     catch (err) {
       // TODO: use ApiV3Error

src/server/service/export.js

@@ -15,18 +15,6 @@ class ExportService {
     this.baseDir = path.join(crowi.tmpDir, 'downloads');
     this.per = 100;
     this.zlibLevel = 9; // 0(min) - 9(max)
-
-    // this.files = {
-    //   configs: path.join(this.baseDir, 'configs.json'),
-    //   pages: path.join(this.baseDir, 'pages.json'),
-    //   pagetagrelations: path.join(this.baseDir, 'pagetagrelations.json'),
-    //   ...
-    // };
-    this.files = {};
-    Object.values(crowi.models).forEach((m) => {
-      const name = m.collection.collectionName;
-      this.files[name] = path.join(this.baseDir, `${name}.json`);
-    });
   }
 
   /**
@@ -76,32 +64,32 @@ class ExportService {
    * @memberOf ExportService
    * @param {string} file path to json file to be written
    * @param {readStream} readStream  read stream
-   * @param {number} [total] number of target items (optional)
+   * @param {number} total number of target items (optional)
+   * @param {function} [getLogText] (n, total) => { ... }
    * @return {string} path to the exported json file
    */
-  async export(file, readStream, total) {
+  async export(writeStream, readStream, total, getLogText) {
     let n = 0;
-    const ws = fs.createWriteStream(file, { encoding: this.growiBridgeService.getEncoding() });
 
     // open an array
-    ws.write('[');
+    writeStream.write('[');
 
     readStream.on('data', (chunk) => {
-      if (n !== 0) ws.write(',');
-      ws.write(JSON.stringify(chunk));
+      if (n !== 0) writeStream.write(',');
+      writeStream.write(JSON.stringify(chunk));
       n++;
-      this.logProgress(n, total);
+      this.logProgress(n, total, getLogText);
     });
 
     readStream.on('end', () => {
       // close the array
-      ws.write(']');
-      ws.close();
+      writeStream.write(']');
+      writeStream.close();
     });
 
     await streamToPromise(readStream);
 
-    return file;
+    return writeStream.path;
   }
 
   /**
@@ -113,13 +101,15 @@ class ExportService {
    */
   async exportCollectionToJson(Model) {
     const { collectionName } = Model.collection;
-    const targetFile = this.files[collectionName];
-    const total = await Model.countDocuments();
+    const jsonFileToWrite = path.join(this.baseDir, `${collectionName}.json`);
+    const writeStream = fs.createWriteStream(jsonFileToWrite, { encoding: this.growiBridgeService.getEncoding() });
     const readStream = Model.find().cursor();
+    const total = await Model.countDocuments();
+    const getLogText = (n, total) => `${collectionName}: ${n}/${total} written`;
 
-    const file = await this.export(targetFile, readStream, total);
+    const jsonFileWritten = await this.export(writeStream, readStream, total, getLogText);
 
-    return file;
+    return jsonFileWritten;
   }
 
   /**
@@ -140,16 +130,11 @@ class ExportService {
    *
    * @memberOf ExportService
    * @param {number} n number of items exported
-   * @param {number} [total] number of target items (optional)
+   * @param {number} total number of target items (optional)
+   * @param {function} [getLogText] (n, total) => { ... }
    */
-  logProgress(n, total) {
-    let output;
-    if (total) {
-      output = `${n}/${total} written`;
-    }
-    else {
-      output = `${n} items written`;
-    }
+  logProgress(n, total, getLogText) {
+    const output = getLogText ? getLogText(n, total) : `${n}/${total} items written`;
 
     // output every this.per items
     if (n % this.per === 0) logger.debug(output);
@@ -201,7 +186,12 @@ class ExportService {
 
     await streamToPromise(archive);
 
-    logger.debug(`zipped growi data into ${zipFile} (${archive.pointer()} bytes)`);
+    logger.info(`zipped growi data into ${zipFile} (${archive.pointer()} bytes)`);
+
+    // delete json files
+    for (const { from } of configs) {
+      fs.unlinkSync(from);
+    }
 
     return zipFile;
   }

src/server/service/import.js

@@ -67,61 +67,66 @@ class ImportService {
    * @param {object} Model instance of mongoose model
    * @param {string} jsonFile absolute path to the jsonFile being imported
    * @param {object} overwriteParams overwrite each document with unrelated value. e.g. { creator: req.user }
+   * @return {insertedIds: Array.<string>, failedIds: Array.<string>}
    */
   async import(Model, jsonFile, overwriteParams = {}) {
-    const { collectionName } = Model.collection;
-
-    let counter = 0;
-    let nInsertedTotal = 0;
-
-    let failedIds = [];
-    let unorderedBulkOp = Model.collection.initializeUnorderedBulkOp();
-
-    const readStream = fs.createReadStream(jsonFile, { encoding: this.growiBridgeService.getEncoding() });
-    const jsonStream = readStream.pipe(JSONStream.parse('*'));
-
-    jsonStream.on('data', async(document) => {
-      // documents are not persisted until unorderedBulkOp.execute()
-      unorderedBulkOp.insert(this.convertDocuments(Model, document, overwriteParams));
-
-      counter++;
-
-      if (counter % this.per === 0) {
-        // puase jsonStream to prevent more items to be added to unorderedBulkOp
-        jsonStream.pause();
-
-        const { nInserted, failed } = await this.execUnorderedBulkOpSafely(unorderedBulkOp);
-        nInsertedTotal += nInserted;
-        failedIds = [...failedIds, ...failed];
-
-        // reset initializeUnorderedBulkOp
-        unorderedBulkOp = Model.collection.initializeUnorderedBulkOp();
-
-        // resume jsonStream
-        jsonStream.resume();
-      }
+    // streamToPromise(jsonStream) throws an error, use new Promise instead
+    return new Promise((resolve, reject) => {
+      const { collectionName } = Model.collection;
+
+      let counter = 0;
+      let insertedIds = [];
+      let failedIds = [];
+      let unorderedBulkOp = Model.collection.initializeUnorderedBulkOp();
+
+      const readStream = fs.createReadStream(jsonFile, { encoding: this.growiBridgeService.getEncoding() });
+      const jsonStream = readStream.pipe(JSONStream.parse('*'));
+
+      jsonStream.on('data', async(document) => {
+        // documents are not persisted until unorderedBulkOp.execute()
+        unorderedBulkOp.insert(this.convertDocuments(Model, document, overwriteParams));
+
+        counter++;
+
+        if (counter % this.per === 0) {
+          // puase jsonStream to prevent more items to be added to unorderedBulkOp
+          jsonStream.pause();
+
+          const { insertedIds: _insertedIds, failedIds: _failedIds } = await this.execUnorderedBulkOpSafely(unorderedBulkOp);
+          insertedIds = [...insertedIds, ..._insertedIds];
+          failedIds = [...failedIds, ..._failedIds];
+
+          // reset initializeUnorderedBulkOp
+          unorderedBulkOp = Model.collection.initializeUnorderedBulkOp();
+
+          // resume jsonStream
+          jsonStream.resume();
+        }
+      });
+
+      jsonStream.on('end', async(data) => {
+        // insert the rest. avoid errors when unorderedBulkOp has no items
+        if (unorderedBulkOp.s.currentBatch !== null) {
+          const { insertedIds: _insertedIds, failedIds: _failedIds } = await this.execUnorderedBulkOpSafely(unorderedBulkOp);
+          insertedIds = [...insertedIds, ..._insertedIds];
+          failedIds = [...failedIds, ..._failedIds];
+        }
+
+        logger.info(`Done. Inserted ${insertedIds.length} ${collectionName}.`);
+
+        if (failedIds.length > 0) {
+          logger.error(`Failed to insert ${failedIds.length} ${collectionName}: ${failedIds.join(', ')}.`);
+        }
+
+        // clean up tmp directory
+        fs.unlinkSync(jsonFile);
+
+        return resolve({
+          insertedIds,
+          failedIds,
+        });
+      });
     });
-
-    jsonStream.on('end', async(data) => {
-      // insert the rest. avoid errors when unorderedBulkOp has no items
-      if (unorderedBulkOp.s.currentBatch !== null) {
-        const { nInserted, failed } = await this.execUnorderedBulkOpSafely(unorderedBulkOp);
-        nInsertedTotal += nInserted;
-        failedIds = [...failedIds, ...failed];
-      }
-
-      logger.info(`Done. Inserted ${nInsertedTotal} ${collectionName}.`);
-
-      if (failedIds.length > 0) {
-        logger.error(`Failed to insert ${failedIds.length} ${collectionName}: ${failedIds.join(', ')}.`);
-      }
-    });
-
-    // streamToPromise(jsonStream) throws error, so await readStream instead
-    await streamToPromise(readStream);
-
-    // clean up tmp directory
-    fs.unlinkSync(jsonFile);
   }
 
   /**
@@ -165,28 +170,34 @@ class ImportService {
    */
   async execUnorderedBulkOpSafely(unorderedBulkOp) {
     // keep the number of documents inserted and failed for logger
-    let nInserted = 0;
-    const failed = [];
+    let insertedIds = [];
+    let failedIds = [];
 
     // try catch to skip errors
     try {
       const log = await unorderedBulkOp.execute();
-      nInserted = log.result.nInserted;
+      const _insertedIds = log.result.insertedIds.map(op => op._id);
+      insertedIds = [...insertedIds, ..._insertedIds];
     }
     catch (err) {
+      const collectionName = unorderedBulkOp.s.namespace;
+
       for (const error of err.result.result.writeErrors) {
-        logger.error(error.errmsg);
-        failed.push(error.err.op._id);
+        logger.error(`${collectionName}: ${error.errmsg}`);
       }
 
-      nInserted = err.result.result.nInserted;
+      const _failedIds = err.result.result.writeErrors.map(err => err.err.op._id);
+      const _insertedIds = err.result.result.insertedIds.filter(op => !_failedIds.includes(op._id)).map(op => op._id);
+
+      failedIds = [...failedIds, ..._failedIds];
+      insertedIds = [...insertedIds, ..._insertedIds];
     }
 
-    logger.debug(`Importing ${unorderedBulkOp.s.collection.s.name}. Inserted: ${nInserted}. Failed: ${failed.length}.`);
+    logger.debug(`Importing ${unorderedBulkOp.s.collection.s.name}. Inserted: ${insertedIds.length}. Failed: ${failedIds.length}.`);
 
     return {
-      nInserted,
-      failed,
+      insertedIds,
+      failedIds,
     };
   }