weseek__growi/src/server/service/export.js

const logger = require('@alias/logger')('growi:services:ExportService'); // eslint-disable-line no-unused-vars
const fs = require('fs');
const path = require('path');
const streamToPromise = require('stream-to-promise');
const archiver = require('archiver');
const toArrayIfNot = require('../../lib/util/toArrayIfNot');

class ExportService {

  constructor(crowi) {
    this.crowi = crowi;
    this.appService = crowi.appService;
    this.baseDir = path.join(crowi.tmpDir, 'downloads');
    this.zipFileName = 'GROWI.zip';
    this.metaFileName = 'meta.json';
    this.encoding = 'utf-8';
    this.per = 100;
    this.zlibLevel = 9; // 0(min) - 9(max)

    // { pages: Page, users: User, ... }
    this.collectionMap = {};
    this.initCollectionMap(crowi.models);

    // this.files = {
    //   configs: path.join(this.baseDir, 'configs.json'),
    //   pages: path.join(this.baseDir, 'pages.json'),
    //   pagetagrelations: path.join(this.baseDir, 'pagetagrelations.json'),
    //   ...
    // };
    this.files = {};
    Object.values(crowi.models).forEach((m) => {
      const name = m.collection.collectionName;
      this.files[name] = path.join(this.baseDir, `${name}.json`);
    });
  }

  /**
   * initialize collection map
   *
   * @memberOf ExportService
   * @param {object} models from models/index.js
   */
  initCollectionMap(models) {
    for (const model of Object.values(models)) {
      this.collectionMap[model.collection.collectionName] = model;
    }
  }

  /**
   * dump a collection into json
   *
   * @memberOf ExportService
   * @return {object} cache info for exported zip files
   */
  getStatus() {
    const status = {};
    const collections = Object.keys(this.files);
    collections.forEach((file) => {
      status[path.basename(file, '.zip')] = null;
    });

    // extract ${collectionName}.zip
    const files = fs.readdirSync(this.baseDir).filter((file) => { return path.extname(file) === '.zip' && collections.includes(path.basename(file, '.zip')) });

    files.forEach((file) => {
      status[path.basename(file, '.zip')] = file;
    });

    files.forEach((file) => {
      const stats = fs.statSync(path.join(this.baseDir, file));
      stats.name = file;
      status[path.basename(file, '.zip')] = stats;
    });

    return status;
  }

  /**
   * create meta.json
   *
   * @memberOf ExportService
   * @return {string} path to meta.json
   */
  async createMetaJson() {
    const metaJson = this.getMetaJson();
    const writeStream = fs.createWriteStream(metaJson, { encoding: this.encoding });

    const metaData = {
      version: this.crowi.version,
      url: this.appService.getSiteUrl(),
      passwordSeed: this.crowi.env.PASSWORD_SEED,
      exportedAt: new Date(),
    };

    writeStream.write(JSON.stringify(metaData));
    writeStream.close();

    await streamToPromise(writeStream);

    return metaJson;
  }

  /**
   * dump a collection into json
   *
   * @memberOf ExportService
   * @param {string} file path to json file to be written
   * @param {readStream} readStream  read stream
   * @param {number} [total] number of target items (optional)
   * @return {string} path to the exported json file
   */
  async export(file, readStream, total) {
    let n = 0;
    const ws = fs.createWriteStream(file, { encoding: this.encoding });

    // open an array
    ws.write('[');

    readStream.on('data', (chunk) => {
      if (n !== 0) ws.write(',');
      ws.write(JSON.stringify(chunk));
      n++;
      this.logProgress(n, total);
    });

    readStream.on('end', () => {
      // close the array
      ws.write(']');
      ws.close();
    });

    await streamToPromise(readStream);

    return file;
  }

  /**
   * dump a mongodb collection into json
   *
   * @memberOf ExportService
   * @param {object} Model instance of mongoose model
   * @return {string} path to zip file
   */
  async exportCollectionToJson(Model) {
    const { collectionName } = Model.collection;
    const targetFile = this.files[collectionName];
    const total = await Model.countDocuments();
    const readStream = Model.find().cursor();

    const file = await this.export(targetFile, readStream, total);

    return file;
  }

  /**
   * export multiple collections
   *
   * @memberOf ExportService
   * @param {number} models number of items exported
   * @return {Array.<string>} paths to json files created
   */
  async exportMultipleCollectionsToJsons(models) {
    const jsonFiles = await Promise.all(models.map(Model => this.exportCollectionToJson(Model)));

    return jsonFiles;
  }

  /**
   * log export progress
   *
   * @memberOf ExportService
   * @param {number} n number of items exported
   * @param {number} [total] number of target items (optional)
   */
  logProgress(n, total) {
    let output;
    if (total) {
      output = `${n}/${total} written`;
    }
    else {
      output = `${n} items written`;
    }

    // output every this.per items
    if (n % this.per === 0) logger.debug(output);
    // output last item
    else if (n === total) logger.info(output);
  }

  /**
   * zip files into one zip file
   *
   * @memberOf ExportService
   * @param {object|array<object>} configs array of object { from: "path to source file", as: "file name after unzipped" }
   * @return {string} path to zip file
   * @see https://www.archiverjs.com/#quick-start
   */
  async zipFiles(_configs) {
    const configs = toArrayIfNot(_configs);
    const zipFile = this.getZipFile();
    const archive = archiver('zip', {
      zlib: { level: this.zlibLevel },
    });

    // good practice to catch warnings (ie stat failures and other non-blocking errors)
    archive.on('warning', (err) => {
      if (err.code === 'ENOENT') logger.error(err);
      else throw err;
    });

    // good practice to catch this error explicitly
    archive.on('error', (err) => { throw err });

    for (const { from, as } of configs) {
      const input = fs.createReadStream(from);

      // append a file from stream
      archive.append(input, { name: as });
    }

    const output = fs.createWriteStream(zipFile);

    // pipe archive data to the file
    archive.pipe(output);

    // finalize the archive (ie we are done appending files but streams have to finish yet)
    // 'close', 'end' or 'finish' may be fired right after calling this method so register to them beforehand
    archive.finalize();

    await streamToPromise(archive);

    logger.debug(`zipped growi data into ${zipFile} (${archive.pointer()} bytes)`);

    return zipFile;
  }

  /**
   * get the absolute path to the zip file
   *
   * @memberOf ImportService
   * @param {boolean} [validate=false] boolean to check if the file exists
   * @return {string} absolute path to the zip file
   */
  getZipFile(validate = false) {
    const zipFile = path.join(this.baseDir, this.zipFileName);

    if (validate) {
      try {
        fs.accessSync(zipFile);
      }
      catch (err) {
        if (err.code === 'ENOENT') {
          logger.error(`${zipFile} does not exist`, err);
        }
        else {
          logger.error(err);
        }

        throw err;
      }
    }

    return zipFile;
  }

  /**
   * get the absolute path to the zip file
   *
   * @memberOf ImportService
   * @param {boolean} [validate=false] boolean to check if the file exists
   * @return {string} absolute path to meta.json
   */
  getMetaJson(validate = false) {
    const jsonFile = path.join(this.baseDir, this.metaFileName);

    if (validate) {
      try {
        fs.accessSync(jsonFile);
      }
      catch (err) {
        if (err.code === 'ENOENT') {
          logger.error(`${jsonFile} does not exist`, err);
        }
        else {
          logger.error(err);
        }

        throw err;
      }
    }

    return jsonFile;
  }

  /**
   * get a model from collection name
   *
   * @memberOf ExportService
   * @param {object} collectionName collection name
   * @return {object} instance of mongoose model
   */
  getModelFromCollectionName(collectionName) {
    const Model = this.collectionMap[collectionName];

    if (Model == null) {
      throw new Error(`cannot find a model for collection name "${collectionName}"`);
    }

    return Model;
  }

}

module.exports = ExportService;