export page collection

src/server/crowi/index.js

@@ -46,6 +46,7 @@ function Crowi(rootdir) {
   this.appService = null;
   this.fileUploadService = null;
   this.restQiitaAPIService = null;
+  this.exportService = null;
   this.cdnResourcesService = new CdnResourcesService();
   this.interceptorManager = new InterceptorManager();
   this.xss = new Xss();
@@ -103,6 +104,7 @@ Crowi.prototype.init = async function() {
     this.setUpCustomize(),
     this.setUpRestQiitaAPI(),
     this.setupUserGroup(),
+    this.setupExport(),
   ]);
 
   // globalNotification depends on slack and mailer
@@ -536,4 +538,11 @@ Crowi.prototype.setupUserGroup = async function() {
   }
 };
 
+Crowi.prototype.setupExport = async function() {
+  const ExportService = require('../service/export');
+  if (this.exportService == null) {
+    this.exportService = new ExportService(this);
+  }
+};
+
 module.exports = Crowi;

src/server/routes/apiv3/index.js

@@ -8,5 +8,8 @@ const router = express.Router();
 
 module.exports = (crowi) => {
   router.use('/healthcheck', require('./healthcheck')(crowi));
+
+  router.use('/pages', require('./page')(crowi));
+
   return router;
 };

src/server/routes/apiv3/page.js

@@ -0,0 +1,46 @@
+const loggerFactory = require('@alias/logger');
+
+const logger = loggerFactory('growi:routes:apiv3:page'); // eslint-disable-line no-unused-vars
+
+const express = require('express');
+
+const router = express.Router();
+
+/**
+ * @swagger
+ *  tags:
+ *    name: Page
+ */
+
+module.exports = (crowi) => {
+  const { exportService } = crowi;
+
+  /**
+   * @swagger
+   *
+   *  /page/dump:
+   *    get:
+   *      tags: [Page]
+   *      description: generate a zipped json for page collection
+   *      produces:
+   *        - application/json
+   *      responses:
+   *        200:
+   *          description: a zip file is generated
+   *          content:
+   *            application/json:
+   */
+  router.get('/dump', async(req, res) => {
+    try {
+      await exportService.exportPageCollection();
+      return res.status(200).send({ status: 'DONE' });
+    }
+    catch (err) {
+      // TODO:user ApiV3Error
+      logger.error(err);
+      return res.status(500).send({ status: 'ERROR' });
+    }
+  });
+
+  return router;
+};

src/server/service/export.js

@@ -0,0 +1,114 @@
+const logger = require('@alias/logger')('growi:services:ExportService'); // eslint-disable-line no-unused-vars
+
+const fs = require('fs');
+const path = require('path');
+const mongoose = require('mongoose');
+
+const Page = mongoose.model('Page');
+
+class ExportService {
+
+  constructor(crowi) {
+    this.baseDir = crowi.tmpDir;
+    this.limit = 100;
+
+    this.files = {
+      pages: path.join(this.baseDir, 'pages.json'),
+    };
+  }
+
+  /**
+   * conver an array into writable string
+   * e.g. [1, 2, 3,] => "1,2,3"
+   *
+   * @memberOf ExportService
+   * @param {array} array
+   */
+  stringify(array) {
+    // validate arguments
+    if (!Array.isArray(array)) {
+      throw new Error('Invalid input. Must be an array.');
+    }
+
+    const stringified = JSON.stringify(array);
+
+    // remove the leading "[" and trailing "]"
+    return stringified.substring(1, stringified.length - 1);
+  }
+
+  /**
+   * dump a collection into json
+   *
+   * @memberOf ExportService
+   * @param {string} file path to json file to write
+   * @param {func} getTotalFn function to get the total count of documents in a collection
+   * @param {func} paginatedQueryFn function to query db with pagination
+   */
+  async export(file, getTotalFn, paginatedQueryFn) {
+    // validate arguments
+    if (typeof getTotalFn !== 'function' || typeof paginatedQueryFn !== 'function') {
+      throw new Error('getTotalFn and paginatedQueryFn must be a function)');
+    }
+
+    // convert to Promise if not
+    const total = await Promise.resolve(getTotalFn());
+    let pageNum = 0;
+    let n = 0;
+
+    const ws = fs.createWriteStream(file, { encoding: 'utf-8' });
+
+    // open an array
+    ws.write('[');
+
+    while (n < total) {
+      // convert to Promise if not
+      // eslint-disable-next-line no-await-in-loop
+      const pages = await Promise.resolve(paginatedQueryFn(this.limit, pageNum));
+
+      // write comma for second chunk on
+      if (n !== 0) {
+        ws.write(',');
+      }
+
+      // wirte chunk to the file
+      ws.write(this.stringify(pages));
+
+      // increment number of items written and page number
+      n += pages.length;
+      pageNum++;
+
+      logger.debug(`page ${pageNum} ... ${n}/${total} written`);
+    }
+
+    // close the array
+    ws.write(']');
+
+    ws.close();
+  }
+
+  /**
+   * dump page collection
+   *
+   * @memberOf ExportService
+   */
+  async exportPageCollection() {
+    const file = this.files.pages;
+
+    const getTotalFn = () => {
+      return Page.countDocuments();
+    };
+
+    const paginatedQueryFn = (limit, pageNum) => {
+      return Page
+        .find()
+        .skip(limit * pageNum)
+        .limit(limit);
+    };
+
+    await this.export(file, getTotalFn, paginatedQueryFn);
+    logger.debug(`exported page collection into ${file}`);
+  }
+
+}
+
+module.exports = ExportService;