Merge pull request #1173 from weseek/feat/zip-download-1

Feat/zip download 1

src/server/crowi/index.js

@@ -46,6 +46,7 @@ function Crowi(rootdir) {
   this.appService = null;
   this.fileUploadService = null;
   this.restQiitaAPIService = null;
+  this.exportService = null;
   this.cdnResourcesService = new CdnResourcesService();
   this.interceptorManager = new InterceptorManager();
   this.xss = new Xss();
@@ -103,6 +104,7 @@ Crowi.prototype.init = async function() {
     this.setUpCustomize(),
     this.setUpRestQiitaAPI(),
     this.setupUserGroup(),
+    this.setupExport(),
   ]);
 
   // globalNotification depends on slack and mailer
@@ -536,4 +538,11 @@ Crowi.prototype.setupUserGroup = async function() {
   }
 };
 
+Crowi.prototype.setupExport = async function() {
+  const ExportService = require('../service/export');
+  if (this.exportService == null) {
+    this.exportService = new ExportService(this);
+  }
+};
+
 module.exports = Crowi;

src/server/routes/apiv3/export.js

@@ -0,0 +1,49 @@
+const loggerFactory = require('@alias/logger');
+
+const logger = loggerFactory('growi:routes:apiv3:export'); // eslint-disable-line no-unused-vars
+
+const express = require('express');
+
+const router = express.Router();
+
+/**
+ * @swagger
+ *  tags:
+ *    name: Export
+ */
+
+module.exports = (crowi) => {
+  const { exportService } = crowi;
+  const { Page } = crowi.models;
+
+  /**
+   * @swagger
+   *
+   *  /export/pages:
+   *    get:
+   *      tags: [Page]
+   *      description: generate a zipped json for page collection
+   *      produces:
+   *        - application/json
+   *      responses:
+   *        200:
+   *          description: a zip file is generated
+   *          content:
+   *            application/json:
+   */
+  router.get('/pages', async(req, res) => {
+    // TODO: rename path to "/:collection" and add express validator
+    try {
+      await exportService.exportCollection(Page);
+      // TODO:use res.apiv3
+      return res.status(200).send({ status: 'DONE' });
+    }
+    catch (err) {
+      // TODO: use ApiV3Error
+      logger.error(err);
+      return res.status(500).send({ status: 'ERROR' });
+    }
+  });
+
+  return router;
+};

src/server/routes/apiv3/index.js

@@ -8,5 +8,8 @@ const router = express.Router();
 
 module.exports = (crowi) => {
   router.use('/healthcheck', require('./healthcheck')(crowi));
+
+  router.use('/export', require('./export')(crowi));
+
   return router;
 };

src/server/service/export.js

@@ -0,0 +1,102 @@
+const logger = require('@alias/logger')('growi:services:ExportService'); // eslint-disable-line no-unused-vars
+const fs = require('fs');
+const path = require('path');
+const streamToPromise = require('stream-to-promise');
+
+class ExportService {
+
+  constructor(crowi) {
+    this.baseDir = path.join(crowi.tmpDir, 'downloads');
+    this.extension = 'json';
+    this.encoding = 'utf-8';
+    this.per = 100;
+
+    this.files = {};
+    // populate this.files
+    // this.files = {
+    //   configs: path.join(this.baseDir, 'configs.json'),
+    //   pages: path.join(this.baseDir, 'pages.json'),
+    //   pagetagrelations: path.join(this.baseDir, 'pagetagrelations.json'),
+    //   ...
+    // };
+    // TODO: handle 3 globalnotificationsettings collection properly
+    // see Object.values(crowi.models).forEach((m) => { return console.log(m.collection.collectionName) });
+    Object.values(crowi.models).forEach((m) => {
+      const name = m.collection.collectionName;
+      this.files[name] = path.join(this.baseDir, `${name}.${this.extension}`);
+    });
+  }
+
+  /**
+   * dump a collection into json
+   *
+   * @memberOf ExportService
+   * @param {string} file path to json file to be written
+   * @param {readStream} readStream  read stream
+   * @param {number} [total] number of target items (optional)
+   */
+  async export(file, readStream, total) {
+    const ws = fs.createWriteStream(file, { encoding: this.encoding });
+    let n = 0;
+
+    // open an array
+    ws.write('[');
+
+    await streamToPromise(
+      readStream
+        .on('data', (chunk) => {
+          if (n !== 0) ws.write(',');
+          ws.write(JSON.stringify(chunk));
+          n++;
+          this.logProgress(n, total);
+        })
+        .on('end', () => {
+        // close the array
+          ws.write(']');
+          ws.close();
+        }),
+    );
+  }
+
+  /**
+   * log export progress
+   *
+   * @memberOf ExportService
+   * @param {number} n number of items exported
+   * @param {number} [total] number of target items (optional)
+   */
+  logProgress(n, total) {
+    let output;
+    if (total) {
+      output = `${n}/${total} written`;
+    }
+    else {
+      output = `${n} items written`;
+    }
+
+    // output every this.per items and last item
+    if (n % this.per === 0 || n === total) {
+      logger.debug(output);
+    }
+  }
+
+  /**
+   * dump a mongodb collection into json
+   *
+   * @memberOf ExportService
+   * @param {object} Model instance of mongoose model
+   */
+  async exportCollection(Model) {
+    const modelName = Model.collection.collectionName;
+    const file = this.files[modelName];
+    const total = await Model.countDocuments();
+    const readStream = Model.find().cursor();
+
+    await this.export(file, readStream, total);
+
+    logger.debug(`exported ${modelName} collection into ${file}`);
+  }
+
+}
+
+module.exports = ExportService;