download each single collection

src/client/js/components/Admin/Export/ExportAsZip.jsx

@@ -86,27 +86,30 @@ class ExportPage extends React.Component {
       <Fragment>
         <h2>Export Data as Zip</h2>
         <form className="my-5">
-          {Object.keys(this.state.files).map((file) => {
-            const disabled = !(file === 'pages' || file === 'revisions');
-            const stat = this.state.files[file] || {};
+          {Object.keys(this.state.files).map((collectionName) => {
+            const disabled = !(collectionName === 'pages' || collectionName === 'revisions');
+            const stat = this.state.files[collectionName] || {};
             return (
-              <div className="checkbox checkbox-info" key={file}>
+              <div className="checkbox checkbox-info" key={collectionName}>
                 <input
                   type="checkbox"
-                  id={file}
-                  name={file}
+                  id={collectionName}
+                  name={collectionName}
                   className="form-check-input"
-                  value={file}
+                  value={collectionName}
                   disabled={disabled}
-                  checked={this.state.collections.has(file)}
+                  checked={this.state.collections.has(collectionName)}
                   onChange={this.toggleCheckbox}
                 />
-                <label className={`form-check-label ml-3 ${disabled ? 'text-muted' : ''}`} htmlFor={file}>
-                  {file} ({stat.name || 'not found'}) ({stat.mtime ? format(new Date(stat.mtime), 'yyyy/MM/dd HH:mm:ss') : ''})
+                <label className={`form-check-label ml-3 ${disabled ? 'text-muted' : ''}`} htmlFor={collectionName}>
+                  {collectionName} ({stat.name || 'not found'}) ({stat.mtime ? format(new Date(stat.mtime), 'yyyy/MM/dd HH:mm:ss') : ''})
                 </label>
-                <button type="button" className="btn btn-sm btn-primary" onClick={() => this.exportSingle(file)} disabled={disabled}>
+                <button type="button" className="btn btn-sm btn-primary" onClick={() => this.exportSingle(collectionName)} disabled={disabled}>
                   Create zip file
                 </button>
+                <a href={`/_api/v3/export/${collectionName}`}>
+                  <button type="button" className="btn btn-sm btn-primary ml-2">Download</button>
+                </a>
               </div>
             );
           })}

src/server/routes/apiv3/export.js

@@ -15,7 +15,6 @@ const router = express.Router();
 
 module.exports = (crowi) => {
   const { exportService } = crowi;
-  const { Page } = crowi.models;
 
   /**
    * @swagger
@@ -42,7 +41,7 @@ module.exports = (crowi) => {
   /**
    * @swagger
    *
-   *  /export/pages:
+   *  /export/:collection:
    *    get:
    *      tags: [Export]
    *      description: download a zipped json for page collection
@@ -54,16 +53,16 @@ module.exports = (crowi) => {
    *          content:
    *            application/zip:
    */
-  router.get('/pages', async(req, res) => {
-    // TODO: rename path to "/:collection" and add express validator
+  router.get('/:collection', async(req, res) => {
+    // TODO: add express validator
     try {
-      const file = exportService.getZipFile(Page);
-
-      if (file == null) {
-        throw new Error('the target file does not exist');
-      }
+      const { collection: collectionName } = req.params;
+      // get model for collection
+      const Model = exportService.getModelFromCollectionName(collectionName);
+      // get zip file path
+      const zipFile = exportService.getZipFile(Model);
 
-      return res.download(file);
+      return res.download(zipFile);
     }
     catch (err) {
       // TODO: use ApiV3Error
@@ -90,15 +89,19 @@ module.exports = (crowi) => {
   router.post('/:collection', async(req, res) => {
     // TODO: add express validator
     try {
-      const { collection } = req.params;
-      const Model = exportService.getModelFromCollectionName(collection);
+      const { collection: collectionName } = req.params;
+      // get model for collection
+      const Model = exportService.getModelFromCollectionName(collectionName);
+      // export into json
+      const jsonFile = await exportService.exportCollectionToJson(Model);
+      // zip json
+      const zipFile = await exportService.zipSingleFile(jsonFile);
 
-      const file = await exportService.exportCollection(Model);
       // TODO: use res.apiv3
       return res.status(200).json({
         ok: true,
         collection: [Model.collection.collectionName],
-        file: path.basename(file),
+        file: path.basename(zipFile),
       });
     }
     catch (err) {
@@ -111,7 +114,48 @@ module.exports = (crowi) => {
   /**
    * @swagger
    *
-   *  /export/pages:
+   *  /export:
+   *    post:
+   *      tags: [Export]
+   *      description: generate a zipped json for multiple collections
+   *      produces:
+   *        - application/json
+   *      responses:
+   *        200:
+   *          description: a zip file is generated
+   *          content:
+   *            application/json:
+   */
+  router.post('/', async(req, res) => {
+    // TODO: add express validator
+    try {
+      const { collections } = req.body;
+      // get model for collection
+      const models = collections.map(collectionName => exportService.getModelFromCollectionName(collectionName));
+      // export into json
+      const jsonFiles = await exportService.exportMultipleCollectionsToJsons(models);
+      // zip json
+      const configs = jsonFiles.map((jsonFile) => { return { from: jsonFile, as: path.basename(jsonFile) } });
+      const zipFile = await exportService.zipMultipleFiles(configs);
+
+      // TODO: use res.apiv3
+      return res.status(200).json({
+        ok: true,
+        // collection: [Model.collection.collectionName],
+        file: path.basename(zipFile),
+      });
+    }
+    catch (err) {
+      // TODO: use ApiV3Error
+      logger.error(err);
+      return res.status(500).send({ status: 'ERROR' });
+    }
+  });
+
+  /**
+   * @swagger
+   *
+   *  /export/:collection:
    *    delete:
    *      tags: [Export]
    *      description: unlink a json and zip file for page collection
@@ -123,9 +167,12 @@ module.exports = (crowi) => {
    *          content:
    *            application/json:
    */
-  // router.delete('/pages', async(req, res) => {
-  //   // TODO: rename path to "/:collection" and add express validator
+  // router.delete('/:collection', async(req, res) => {
+  //   // TODO: add express validator
   //   try {
+  //     const { collection: collectionName } = req.params;
+  //     // get model for collection
+  //     const Model = exportService.getModelFromCollectionName(collectionName);
   //     // remove .json and .zip for collection
   //     // TODO: use res.apiv3
   //     return res.status(200).send({ status: 'DONE' });

src/server/service/export.js

@@ -7,6 +7,7 @@ const archiver = require('archiver');
 class ExportService {
 
   constructor(crowi) {
+    this.appService = crowi.appService;
     this.baseDir = path.join(crowi.tmpDir, 'downloads');
     this.encoding = 'utf-8';
     this.per = 100;
@@ -77,6 +78,7 @@ class ExportService {
    * @param {string} file path to json file to be written
    * @param {readStream} readStream  read stream
    * @param {number} [total] number of target items (optional)
+   * @return {string} path to the exported json file
    */
   async export(file, readStream, total) {
     let n = 0;
@@ -99,6 +101,8 @@ class ExportService {
     });
 
     await streamToPromise(readStream);
+
+    return file;
   }
 
   /**
@@ -108,19 +112,20 @@ class ExportService {
    * @param {object} Model instance of mongoose model
    * @return {string} path to zip file
    */
-  async exportCollection(Model) {
-    const modelName = Model.collection.collectionName;
-    const file = this.files[modelName];
+  async exportCollectionToJson(Model) {
+    const { collectionName } = Model.collection;
+    const targetFile = this.files[collectionName];
     const total = await Model.countDocuments();
     const readStream = Model.find().cursor();
 
-    await this.export(file, readStream, total);
-
-    const { file: zipFile, size } = await this.zipSingleFile(file);
+    const file = await this.export(targetFile, readStream, total);
 
-    logger.info(`exported ${modelName} collection into ${zipFile} (${size} bytes)`);
+    return file;
+  }
 
-    return zipFile;
+  async exportMultipleCollectionsToJsons(models) {
+    const jsonFiles = await Promise.all(models.map(Model => this.exportCollectionToJson(Model)));
+    return jsonFiles;
   }
 
   /**
@@ -183,10 +188,45 @@ class ExportService {
 
     await streamToPromise(archive);
 
-    return {
-      file: to,
-      size: archive.pointer(),
-    };
+    logger.debug(`zipped ${from} into ${to} (${archive.pointer()} bytes)`);
+
+    return to;
+  }
+
+  async zipMultipleFiles(configs, to = path.join(this.baseDir, `${this.appService.getAppTitle()}-${(new Date()).getTime()}.zip`)) {
+    const archive = archiver('zip', {
+      zlib: { level: this.zlibLevel },
+    });
+    const output = fs.createWriteStream(to);
+
+    // good practice to catch warnings (ie stat failures and other non-blocking errors)
+    archive.on('warning', (err) => {
+      if (err.code === 'ENOENT') logger.error(err);
+      else throw err;
+    });
+
+    // good practice to catch this error explicitly
+    archive.on('error', (err) => { throw err });
+
+    for (const { from, as } of configs) {
+      const input = fs.createReadStream(from);
+
+      // append a file from stream
+      archive.append(input, { name: as });
+    }
+
+    // pipe archive data to the file
+    archive.pipe(output);
+
+    // finalize the archive (ie we are done appending files but streams have to finish yet)
+    // 'close', 'end' or 'finish' may be fired right after calling this method so register to them beforehand
+    archive.finalize();
+
+    await streamToPromise(archive);
+
+    logger.debug(`zipped growi data into ${to} (${archive.pointer()} bytes)`);
+
+    return to;
   }
 
   /**
@@ -212,7 +252,7 @@ class ExportService {
     const json = this.files[Model.collection.collectionName];
     const zip = this.replaceExtension(json, 'zip');
     if (!fs.existsSync(zip)) {
-      return null;
+      throw new Error(`${zip} does not exist`);
     }
 
     return zip;