|
@@ -1,28 +1,22 @@
|
|
|
-import fs from 'fs';
|
|
|
|
|
-import path from 'path';
|
|
|
|
|
-import { Readable, Transform } from 'stream';
|
|
|
|
|
-
|
|
|
|
|
-import archiver from 'archiver';
|
|
|
|
|
-
|
|
|
|
|
import { toArrayIfNot } from '~/utils/array-utils';
|
|
import { toArrayIfNot } from '~/utils/array-utils';
|
|
|
import { getGrowiVersion } from '~/utils/growi-version';
|
|
import { getGrowiVersion } from '~/utils/growi-version';
|
|
|
import loggerFactory from '~/utils/logger';
|
|
import loggerFactory from '~/utils/logger';
|
|
|
|
|
|
|
|
-import type CollectionProgress from '../models/vo/collection-progress';
|
|
|
|
|
-import CollectionProgressingStatus from '../models/vo/collection-progressing-status';
|
|
|
|
|
-
|
|
|
|
|
-import type AppService from './app';
|
|
|
|
|
-import type GrowiBridgeService from './growi-bridge';
|
|
|
|
|
-import type { ZipFileStat } from './interfaces/export';
|
|
|
|
|
import { configManager } from './config-manager';
|
|
import { configManager } from './config-manager';
|
|
|
import { growiInfoService } from './growi-info';
|
|
import { growiInfoService } from './growi-info';
|
|
|
|
|
|
|
|
|
|
+const logger = loggerFactory('growi:services:ExportService'); // eslint-disable-line no-unused-vars
|
|
|
|
|
|
|
|
-const logger = loggerFactory('growi:services:ExportService');
|
|
|
|
|
|
|
+const fs = require('fs');
|
|
|
|
|
+const path = require('path');
|
|
|
|
|
+const { Transform } = require('stream');
|
|
|
const { pipeline, finished } = require('stream/promises');
|
|
const { pipeline, finished } = require('stream/promises');
|
|
|
|
|
|
|
|
|
|
+const archiver = require('archiver');
|
|
|
const mongoose = require('mongoose');
|
|
const mongoose = require('mongoose');
|
|
|
|
|
|
|
|
|
|
+const CollectionProgressingStatus = require('../models/vo/collection-progressing-status');
|
|
|
|
|
+
|
|
|
class ExportProgressingStatus extends CollectionProgressingStatus {
|
|
class ExportProgressingStatus extends CollectionProgressingStatus {
|
|
|
|
|
|
|
|
async init() {
|
|
async init() {
|
|
@@ -40,27 +34,17 @@ class ExportProgressingStatus extends CollectionProgressingStatus {
|
|
|
|
|
|
|
|
class ExportService {
|
|
class ExportService {
|
|
|
|
|
|
|
|
- crowi: any;
|
|
|
|
|
-
|
|
|
|
|
- appService: AppService;
|
|
|
|
|
-
|
|
|
|
|
- growiBridgeService: GrowiBridgeService;
|
|
|
|
|
-
|
|
|
|
|
- per = 100;
|
|
|
|
|
-
|
|
|
|
|
- zlibLevel = 9; // 0(min) - 9(max)
|
|
|
|
|
-
|
|
|
|
|
- currentProgressingStatus: ExportProgressingStatus | null;
|
|
|
|
|
-
|
|
|
|
|
- baseDir: string;
|
|
|
|
|
-
|
|
|
|
|
- adminEvent: any;
|
|
|
|
|
|
|
+ /** @type {import('~/server/crowi').default} Crowi instance */
|
|
|
|
|
+ crowi;
|
|
|
|
|
|
|
|
|
|
+ /** @param {import('~/server/crowi').default} crowi Crowi instance */
|
|
|
constructor(crowi) {
|
|
constructor(crowi) {
|
|
|
this.crowi = crowi;
|
|
this.crowi = crowi;
|
|
|
this.appService = crowi.appService;
|
|
this.appService = crowi.appService;
|
|
|
this.growiBridgeService = crowi.growiBridgeService;
|
|
this.growiBridgeService = crowi.growiBridgeService;
|
|
|
this.baseDir = path.join(crowi.tmpDir, 'downloads');
|
|
this.baseDir = path.join(crowi.tmpDir, 'downloads');
|
|
|
|
|
+ this.per = 100;
|
|
|
|
|
+ this.zlibLevel = 9; // 0(min) - 9(max)
|
|
|
|
|
|
|
|
this.adminEvent = crowi.event('admin');
|
|
this.adminEvent = crowi.event('admin');
|
|
|
|
|
|
|
@@ -86,7 +70,7 @@ class ExportService {
|
|
|
const zipFiles = fs.readdirSync(this.baseDir).filter(file => path.extname(file) === '.zip');
|
|
const zipFiles = fs.readdirSync(this.baseDir).filter(file => path.extname(file) === '.zip');
|
|
|
|
|
|
|
|
// process serially so as not to waste memory
|
|
// process serially so as not to waste memory
|
|
|
- const zipFileStats: Array<ZipFileStat | null> = [];
|
|
|
|
|
|
|
+ const zipFileStats = [];
|
|
|
const parseZipFilePromises = zipFiles.map((file) => {
|
|
const parseZipFilePromises = zipFiles.map((file) => {
|
|
|
const zipFile = this.getFile(file);
|
|
const zipFile = this.getFile(file);
|
|
|
return this.growiBridgeService.parseZipFile(zipFile);
|
|
return this.growiBridgeService.parseZipFile(zipFile);
|
|
@@ -103,7 +87,7 @@ class ExportService {
|
|
|
return {
|
|
return {
|
|
|
zipFileStats: filtered,
|
|
zipFileStats: filtered,
|
|
|
isExporting,
|
|
isExporting,
|
|
|
- progressList: isExporting ? this.currentProgressingStatus?.progressList : null,
|
|
|
|
|
|
|
+ progressList: isExporting ? this.currentProgressingStatus.progressList : null,
|
|
|
};
|
|
};
|
|
|
}
|
|
}
|
|
|
|
|
|
|
@@ -113,7 +97,7 @@ class ExportService {
|
|
|
* @memberOf ExportService
|
|
* @memberOf ExportService
|
|
|
* @return {string} path to meta.json
|
|
* @return {string} path to meta.json
|
|
|
*/
|
|
*/
|
|
|
- async createMetaJson(): Promise<string> {
|
|
|
|
|
|
|
+ async createMetaJson() {
|
|
|
const metaJson = path.join(this.baseDir, this.growiBridgeService.getMetaFileName());
|
|
const metaJson = path.join(this.baseDir, this.growiBridgeService.getMetaFileName());
|
|
|
const writeStream = fs.createWriteStream(metaJson, { encoding: this.growiBridgeService.getEncoding() });
|
|
const writeStream = fs.createWriteStream(metaJson, { encoding: this.growiBridgeService.getEncoding() });
|
|
|
const passwordSeed = this.crowi.env.PASSWORD_SEED || null;
|
|
const passwordSeed = this.crowi.env.PASSWORD_SEED || null;
|
|
@@ -139,7 +123,7 @@ class ExportService {
|
|
|
* @param {ExportProgress} exportProgress
|
|
* @param {ExportProgress} exportProgress
|
|
|
* @return {Transform}
|
|
* @return {Transform}
|
|
|
*/
|
|
*/
|
|
|
- generateLogStream(exportProgress: CollectionProgress | undefined): Transform {
|
|
|
|
|
|
|
+ generateLogStream(exportProgress) {
|
|
|
const logProgress = this.logProgress.bind(this);
|
|
const logProgress = this.logProgress.bind(this);
|
|
|
|
|
|
|
|
let count = 0;
|
|
let count = 0;
|
|
@@ -160,9 +144,9 @@ class ExportService {
|
|
|
* insert beginning/ending brackets and comma separator for Json Array
|
|
* insert beginning/ending brackets and comma separator for Json Array
|
|
|
*
|
|
*
|
|
|
* @memberOf ExportService
|
|
* @memberOf ExportService
|
|
|
- * @return {Transform}
|
|
|
|
|
|
|
+ * @return {TransformStream}
|
|
|
*/
|
|
*/
|
|
|
- generateTransformStream(): Transform {
|
|
|
|
|
|
|
+ generateTransformStream() {
|
|
|
let isFirst = true;
|
|
let isFirst = true;
|
|
|
|
|
|
|
|
const transformStream = new Transform({
|
|
const transformStream = new Transform({
|
|
@@ -201,7 +185,7 @@ class ExportService {
|
|
|
* @param {string} collectionName collection name
|
|
* @param {string} collectionName collection name
|
|
|
* @return {string} path to zip file
|
|
* @return {string} path to zip file
|
|
|
*/
|
|
*/
|
|
|
- async exportCollectionToJson(collectionName: string): Promise<string> {
|
|
|
|
|
|
|
+ async exportCollectionToJson(collectionName) {
|
|
|
const collection = mongoose.connection.collection(collectionName);
|
|
const collection = mongoose.connection.collection(collectionName);
|
|
|
|
|
|
|
|
const nativeCursor = collection.find();
|
|
const nativeCursor = collection.find();
|
|
@@ -211,7 +195,7 @@ class ExportService {
|
|
|
const transformStream = this.generateTransformStream();
|
|
const transformStream = this.generateTransformStream();
|
|
|
|
|
|
|
|
// log configuration
|
|
// log configuration
|
|
|
- const exportProgress = this.currentProgressingStatus?.progressMap[collectionName];
|
|
|
|
|
|
|
+ const exportProgress = this.currentProgressingStatus.progressMap[collectionName];
|
|
|
const logStream = this.generateLogStream(exportProgress);
|
|
const logStream = this.generateLogStream(exportProgress);
|
|
|
|
|
|
|
|
// create WritableStream
|
|
// create WritableStream
|
|
@@ -220,7 +204,7 @@ class ExportService {
|
|
|
|
|
|
|
|
await pipeline(readStream, logStream, transformStream, writeStream);
|
|
await pipeline(readStream, logStream, transformStream, writeStream);
|
|
|
|
|
|
|
|
- return writeStream.path.toString();
|
|
|
|
|
|
|
+ return writeStream.path;
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
/**
|
|
@@ -228,13 +212,13 @@ class ExportService {
|
|
|
*
|
|
*
|
|
|
* @memberOf ExportService
|
|
* @memberOf ExportService
|
|
|
* @param {Array.<string>} collections array of collection name
|
|
* @param {Array.<string>} collections array of collection name
|
|
|
- * @return {Array.<ZipFileStat>} info of zip file created
|
|
|
|
|
|
|
+ * @return {Array.<string>} paths to json files created
|
|
|
*/
|
|
*/
|
|
|
- async exportCollectionsToZippedJson(collections: string[]): Promise<ZipFileStat | null> {
|
|
|
|
|
|
|
+ async exportCollectionsToZippedJson(collections) {
|
|
|
const metaJson = await this.createMetaJson();
|
|
const metaJson = await this.createMetaJson();
|
|
|
|
|
|
|
|
// process serially so as not to waste memory
|
|
// process serially so as not to waste memory
|
|
|
- const jsonFiles: string[] = [];
|
|
|
|
|
|
|
+ const jsonFiles = [];
|
|
|
const jsonFilesPromises = collections.map(collectionName => this.exportCollectionToJson(collectionName));
|
|
const jsonFilesPromises = collections.map(collectionName => this.exportCollectionToJson(collectionName));
|
|
|
for await (const jsonFile of jsonFilesPromises) {
|
|
for await (const jsonFile of jsonFilesPromises) {
|
|
|
jsonFiles.push(jsonFile);
|
|
jsonFiles.push(jsonFile);
|
|
@@ -261,7 +245,7 @@ class ExportService {
|
|
|
// TODO: remove broken zip file
|
|
// TODO: remove broken zip file
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
- async export(collections: string[]): Promise<ZipFileStat | null> {
|
|
|
|
|
|
|
+ async export(collections) {
|
|
|
if (this.currentProgressingStatus != null) {
|
|
if (this.currentProgressingStatus != null) {
|
|
|
throw new Error('There is an exporting process running.');
|
|
throw new Error('There is an exporting process running.');
|
|
|
}
|
|
}
|
|
@@ -269,7 +253,7 @@ class ExportService {
|
|
|
this.currentProgressingStatus = new ExportProgressingStatus(collections);
|
|
this.currentProgressingStatus = new ExportProgressingStatus(collections);
|
|
|
await this.currentProgressingStatus.init();
|
|
await this.currentProgressingStatus.init();
|
|
|
|
|
|
|
|
- let zipFileStat: ZipFileStat | null;
|
|
|
|
|
|
|
+ let zipFileStat;
|
|
|
try {
|
|
try {
|
|
|
zipFileStat = await this.exportCollectionsToZippedJson(collections);
|
|
zipFileStat = await this.exportCollectionsToZippedJson(collections);
|
|
|
}
|
|
}
|
|
@@ -288,9 +272,7 @@ class ExportService {
|
|
|
* @param {CollectionProgress} collectionProgress
|
|
* @param {CollectionProgress} collectionProgress
|
|
|
* @param {number} currentCount number of items exported
|
|
* @param {number} currentCount number of items exported
|
|
|
*/
|
|
*/
|
|
|
- logProgress(collectionProgress: CollectionProgress | undefined, currentCount: number): void {
|
|
|
|
|
- if (collectionProgress == null) return;
|
|
|
|
|
-
|
|
|
|
|
|
|
+ logProgress(collectionProgress, currentCount) {
|
|
|
const output = `${collectionProgress.collectionName}: ${currentCount}/${collectionProgress.totalCount} written`;
|
|
const output = `${collectionProgress.collectionName}: ${currentCount}/${collectionProgress.totalCount} written`;
|
|
|
|
|
|
|
|
// update exportProgress.currentCount
|
|
// update exportProgress.currentCount
|
|
@@ -311,11 +293,12 @@ class ExportService {
|
|
|
/**
|
|
/**
|
|
|
* emit progress event
|
|
* emit progress event
|
|
|
*/
|
|
*/
|
|
|
- emitProgressEvent(): void {
|
|
|
|
|
|
|
+ emitProgressEvent() {
|
|
|
|
|
+ const { currentCount, totalCount, progressList } = this.currentProgressingStatus;
|
|
|
const data = {
|
|
const data = {
|
|
|
- currentCount: this.currentProgressingStatus?.currentCount,
|
|
|
|
|
- totalCount: this.currentProgressingStatus?.totalCount,
|
|
|
|
|
- progressList: this.currentProgressingStatus?.progressList,
|
|
|
|
|
|
|
+ currentCount,
|
|
|
|
|
+ totalCount,
|
|
|
|
|
+ progressList,
|
|
|
};
|
|
};
|
|
|
|
|
|
|
|
// send event (in progress in global)
|
|
// send event (in progress in global)
|
|
@@ -325,7 +308,7 @@ class ExportService {
|
|
|
/**
|
|
/**
|
|
|
* emit start zipping event
|
|
* emit start zipping event
|
|
|
*/
|
|
*/
|
|
|
- emitStartZippingEvent(): void {
|
|
|
|
|
|
|
+ emitStartZippingEvent() {
|
|
|
this.adminEvent.emit('onStartZippingForExport', {});
|
|
this.adminEvent.emit('onStartZippingForExport', {});
|
|
|
}
|
|
}
|
|
|
|
|
|
|
@@ -333,7 +316,7 @@ class ExportService {
|
|
|
* emit terminate event
|
|
* emit terminate event
|
|
|
* @param {object} zipFileStat added zip file status data
|
|
* @param {object} zipFileStat added zip file status data
|
|
|
*/
|
|
*/
|
|
|
- emitTerminateEvent(zipFileStat: ZipFileStat | null): void {
|
|
|
|
|
|
|
+ emitTerminateEvent(zipFileStat) {
|
|
|
this.adminEvent.emit('onTerminateForExport', { addedZipFileStat: zipFileStat });
|
|
this.adminEvent.emit('onTerminateForExport', { addedZipFileStat: zipFileStat });
|
|
|
}
|
|
}
|
|
|
|
|
|
|
@@ -345,7 +328,7 @@ class ExportService {
|
|
|
* @return {string} absolute path to the zip file
|
|
* @return {string} absolute path to the zip file
|
|
|
* @see https://www.archiverjs.com/#quick-start
|
|
* @see https://www.archiverjs.com/#quick-start
|
|
|
*/
|
|
*/
|
|
|
- async zipFiles(_configs: {from: string, as: string}[]): Promise<string> {
|
|
|
|
|
|
|
+ async zipFiles(_configs) {
|
|
|
const configs = toArrayIfNot(_configs);
|
|
const configs = toArrayIfNot(_configs);
|
|
|
const appTitle = this.appService.getAppTitle();
|
|
const appTitle = this.appService.getAppTitle();
|
|
|
const timeStamp = (new Date()).getTime();
|
|
const timeStamp = (new Date()).getTime();
|
|
@@ -389,9 +372,10 @@ class ExportService {
|
|
|
return zipFile;
|
|
return zipFile;
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
- getReadStreamFromRevision(revision, format): Readable {
|
|
|
|
|
|
|
+ getReadStreamFromRevision(revision, format) {
|
|
|
const data = revision.body;
|
|
const data = revision.body;
|
|
|
|
|
|
|
|
|
|
+ const Readable = require('stream').Readable;
|
|
|
const readable = new Readable();
|
|
const readable = new Readable();
|
|
|
readable._read = () => {};
|
|
readable._read = () => {};
|
|
|
readable.push(data);
|
|
readable.push(data);
|
|
@@ -402,8 +386,4 @@ class ExportService {
|
|
|
|
|
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
-// eslint-disable-next-line import/no-mutable-exports
|
|
|
|
|
-export let exportService: ExportService | undefined; // singleton instance
|
|
|
|
|
-export default function instanciate(crowi: any): void {
|
|
|
|
|
- exportService = new ExportService(crowi);
|
|
|
|
|
-}
|
|
|
|
|
|
|
+module.exports = ExportService;
|