Merge pull request #8390 from weseek/imprv/138839-138840-typescriptize-export-service-related-files

typescriptize export service related files

apps/app/src/server/crowi/index.js

@@ -25,9 +25,11 @@ import { aclService as aclServiceSingletonInstance } from '../service/acl';
 import AppService from '../service/app';
 import AttachmentService from '../service/attachment';
 import { configManager as configManagerSingletonInstance } from '../service/config-manager';
-import { instanciate as instanciateExternalAccountService } from '../service/external-account';
+import instanciateExportService from '../service/export';
+import instanciateExternalAccountService from '../service/external-account';
 import { FileUploader, getUploader } from '../service/file-uploader'; // eslint-disable-line no-unused-vars
 import { G2GTransferPusherService, G2GTransferReceiverService } from '../service/g2g-transfer';
+import GrowiBridgeService from '../service/growi-bridge';
 import { InstallerService } from '../service/installer';
 import PageService from '../service/page';
 import PageGrantService from '../service/page-grant';
@@ -80,7 +82,6 @@ class Crowi {
     this.fileUploadService = null;
     this.restQiitaAPIService = null;
     this.growiBridgeService = null;
-    this.exportService = null;
     this.importService = null;
     this.pluginService = null;
     this.searchService = null;
@@ -683,17 +684,13 @@ Crowi.prototype.setupUserGroupService = async function() {
 };
 
 Crowi.prototype.setUpGrowiBridge = async function() {
-  const GrowiBridgeService = require('../service/growi-bridge');
   if (this.growiBridgeService == null) {
     this.growiBridgeService = new GrowiBridgeService(this);
   }
 };
 
 Crowi.prototype.setupExport = async function() {
-  const ExportService = require('../service/export');
-  if (this.exportService == null) {
-    this.exportService = new ExportService(this);
-  }
+  instanciateExportService(this);
 };
 
 Crowi.prototype.setupImport = async function() {

apps/app/src/server/models/vo/collection-progress.js

@@ -1,13 +0,0 @@
-class CollectionProgress {
-
-  constructor(collectionName, totalCount) {
-    this.collectionName = collectionName;
-    this.currentCount = 0;
-    this.insertedCount = 0;
-    this.modifiedCount = 0;
-    this.totalCount = totalCount;
-  }
-
-}
-
-module.exports = CollectionProgress;

apps/app/src/server/models/vo/collection-progress.ts

@@ -0,0 +1,19 @@
+class CollectionProgress {
+
+  collectionName: string;
+
+  currentCount = 0;
+
+  insertedCount = 0;
+
+  modifiedCount = 0;
+
+  totalCount = 0;
+
+  constructor(collectionName: string) {
+    this.collectionName = collectionName;
+  }
+
+}
+
+export default CollectionProgress;

apps/app/src/server/models/vo/collection-progressing-status.js → apps/app/src/server/models/vo/collection-progressing-status.ts

@@ -1,13 +1,18 @@
-const CollectionProgress = require('./collection-progress');
+import CollectionProgress from './collection-progress';
 
 class CollectionProgressingStatus {
 
-  constructor(collections) {
-    this.totalCount = 0;
+  totalCount = 0;
+
+  progressList: CollectionProgress[];
+
+  progressMap: Record<string, CollectionProgress>;
+
+  constructor(collections: string[]) {
     this.progressMap = {};
 
     this.progressList = collections.map((collectionName) => {
-      return new CollectionProgress(collectionName, 0);
+      return new CollectionProgress(collectionName);
     });
 
     // collection name to instance mapping
@@ -16,14 +21,14 @@ class CollectionProgressingStatus {
     });
   }
 
-  recalculateTotalCount() {
+  recalculateTotalCount(): void {
     this.progressList.forEach((p) => {
       this.progressMap[p.collectionName] = p;
       this.totalCount += p.totalCount;
     });
   }
 
-  get currentCount() {
+  get currentCount(): number {
     return this.progressList.reduce(
       (acc, crr) => acc + crr.currentCount,
       0,
@@ -32,4 +37,4 @@ class CollectionProgressingStatus {
 
 }
 
-module.exports = CollectionProgressingStatus;
+export default CollectionProgressingStatus;

apps/app/src/server/routes/admin.js

@@ -1,15 +1,13 @@
 import { SupportedAction } from '~/interfaces/activity';
 import loggerFactory from '~/utils/logger';
 
+import { configManager } from '../service/config-manager';
+import { exportService } from '../service/export';
+
 const logger = loggerFactory('growi:routes:admin');
 
 /* eslint-disable no-use-before-define */
 module.exports = function(crowi, app) {
-  const {
-    configManager,
-    exportService,
-  } = crowi;
-
   const ApiResponse = require('../util/apiResponse');
   const importer = require('../util/importer')(crowi);
 

apps/app/src/server/routes/apiv3/export.js

@@ -1,4 +1,5 @@
 import { SupportedAction } from '~/interfaces/activity';
+import { exportService } from '~/server/service/export';
 import loggerFactory from '~/utils/logger';
 
 import { generateAddActivityMiddleware } from '../../middlewares/add-activity';
@@ -49,7 +50,7 @@ module.exports = (crowi) => {
   const adminRequired = require('../../middlewares/admin-required')(crowi);
   const addActivity = generateAddActivityMiddleware(crowi);
 
-  const { exportService, socketIoService } = crowi;
+  const { socketIoService } = crowi;
 
   const activityEvent = crowi.event('activity');
   const adminEvent = crowi.event('admin');

apps/app/src/server/routes/apiv3/g2g-transfer.ts

@@ -7,6 +7,7 @@ import { body } from 'express-validator';
 import multer from 'multer';
 
 import { isG2GTransferError } from '~/server/models/vo/g2g-transfer-error';
+import { exportService } from '~/server/service/export';
 import { IDataGROWIInfo, X_GROWI_TRANSFER_KEY_HEADER_NAME } from '~/server/service/g2g-transfer';
 import loggerFactory from '~/utils/logger';
 import { TransferKey } from '~/utils/vo/transfer-key';
@@ -36,7 +37,7 @@ const validator = {
  */
 module.exports = (crowi: Crowi): Router => {
   const {
-    g2gTransferPusherService, g2gTransferReceiverService, exportService, importService,
+    g2gTransferPusherService, g2gTransferReceiverService, importService,
     growiBridgeService, configManager,
   } = crowi;
   if (g2gTransferPusherService == null || g2gTransferReceiverService == null || exportService == null || importService == null
@@ -165,9 +166,9 @@ module.exports = (crowi: Crowi): Router => {
       const zipFile = importService.getFile(file.filename);
       await importService.unzip(zipFile);
 
-      const { meta: parsedMeta, innerFileStats: _innerFileStats } = await growiBridgeService.parseZipFile(zipFile);
-      innerFileStats = _innerFileStats;
-      meta = parsedMeta;
+      const zipFileStat = await growiBridgeService.parseZipFile(zipFile);
+      innerFileStats = zipFileStat?.innerFileStats;
+      meta = zipFileStat?.meta;
     }
     catch (err) {
       logger.error(err);

apps/app/src/server/routes/apiv3/page.js

@@ -14,6 +14,7 @@ import { apiV3FormValidator } from '~/server/middlewares/apiv3-form-validator';
 import { excludeReadOnlyUser } from '~/server/middlewares/exclude-read-only-user';
 import Subscription from '~/server/models/subscription';
 import UserGroup from '~/server/models/user-group';
+import { exportService } from '~/server/service/export';
 import { preNotifyService } from '~/server/service/pre-notify';
 import { divideByType } from '~/server/util/granted-group';
 import loggerFactory from '~/utils/logger';
@@ -176,7 +177,7 @@ module.exports = (crowi) => {
 
   const globalNotificationService = crowi.getGlobalNotificationService();
   const { Page, GlobalNotificationSetting } = crowi.models;
-  const { pageService, exportService } = crowi;
+  const { pageService } = crowi;
 
   const activityEvent = crowi.event('activity');
 

apps/app/src/server/service/export.js → apps/app/src/server/service/export.ts

@@ -1,20 +1,25 @@
+import fs from 'fs';
+import path from 'path';
+import { Readable, Transform } from 'stream';
+
 import { toArrayIfNot } from '~/utils/array-utils';
 import loggerFactory from '~/utils/logger';
 
+import CollectionProgress from '../models/vo/collection-progress';
+import CollectionProgressingStatus from '../models/vo/collection-progressing-status';
+
+import AppService from './app';
 import ConfigLoader from './config-loader';
+import GrowiBridgeService from './growi-bridge';
+import { ZipFileStat } from './interfaces/export';
 
-const logger = loggerFactory('growi:services:ExportService'); // eslint-disable-line no-unused-vars
 
-const fs = require('fs');
-const path = require('path');
-const { Transform } = require('stream');
+const logger = loggerFactory('growi:services:ExportService'); // eslint-disable-line no-unused-vars
 
 const archiver = require('archiver');
 const mongoose = require('mongoose');
 const streamToPromise = require('stream-to-promise');
 
-const CollectionProgressingStatus = require('../models/vo/collection-progressing-status');
-
 class ExportProgressingStatus extends CollectionProgressingStatus {
 
   async init() {
@@ -32,14 +37,30 @@ class ExportProgressingStatus extends CollectionProgressingStatus {
 
 class ExportService {
 
+  crowi: any;
+
+  appService: AppService;
+
+  growiBridgeService: GrowiBridgeService;
+
+  getFile: (filename: string) => string;
+
+  per = 100;
+
+  zlibLevel = 9; // 0(min) - 9(max)
+
+  currentProgressingStatus: ExportProgressingStatus | null;
+
+  baseDir: string;
+
+  adminEvent: any;
+
   constructor(crowi) {
     this.crowi = crowi;
     this.appService = crowi.appService;
     this.growiBridgeService = crowi.growiBridgeService;
     this.getFile = this.growiBridgeService.getFile.bind(this);
     this.baseDir = path.join(crowi.tmpDir, 'downloads');
-    this.per = 100;
-    this.zlibLevel = 9; // 0(min) - 9(max)
 
     this.adminEvent = crowi.event('admin');
 
@@ -56,7 +77,7 @@ class ExportService {
     const zipFiles = fs.readdirSync(this.baseDir).filter(file => path.extname(file) === '.zip');
 
     // process serially so as not to waste memory
-    const zipFileStats = [];
+    const zipFileStats: Array<ZipFileStat | null> = [];
     const parseZipFilePromises = zipFiles.map((file) => {
       const zipFile = this.getFile(file);
       return this.growiBridgeService.parseZipFile(zipFile);
@@ -73,7 +94,7 @@ class ExportService {
     return {
       zipFileStats: filtered,
       isExporting,
-      progressList: isExporting ? this.currentProgressingStatus.progressList : null,
+      progressList: isExporting ? this.currentProgressingStatus?.progressList : null,
     };
   }
 
@@ -83,7 +104,7 @@ class ExportService {
    * @memberOf ExportService
    * @return {string} path to meta.json
    */
-  async createMetaJson() {
+  async createMetaJson(): Promise<string> {
     const metaJson = path.join(this.baseDir, this.growiBridgeService.getMetaFileName());
     const writeStream = fs.createWriteStream(metaJson, { encoding: this.growiBridgeService.getEncoding() });
     const passwordSeed = this.crowi.env.PASSWORD_SEED || null;
@@ -109,7 +130,7 @@ class ExportService {
    * @param {ExportProgress} exportProgress
    * @return {Transform}
    */
-  generateLogStream(exportProgress) {
+  generateLogStream(exportProgress: CollectionProgress | undefined): Transform {
     const logProgress = this.logProgress.bind(this);
 
     let count = 0;
@@ -130,9 +151,9 @@ class ExportService {
    * insert beginning/ending brackets and comma separator for Json Array
    *
    * @memberOf ExportService
-   * @return {TransformStream}
+   * @return {Transform}
    */
-  generateTransformStream() {
+  generateTransformStream(): Transform {
     let isFirst = true;
 
     const transformStream = new Transform({
@@ -171,7 +192,7 @@ class ExportService {
    * @param {string} collectionName collection name
    * @return {string} path to zip file
    */
-  async exportCollectionToJson(collectionName) {
+  async exportCollectionToJson(collectionName: string): Promise<string> {
     const collection = mongoose.connection.collection(collectionName);
 
     const nativeCursor = collection.find();
@@ -181,7 +202,7 @@ class ExportService {
     const transformStream = this.generateTransformStream();
 
     // log configuration
-    const exportProgress = this.currentProgressingStatus.progressMap[collectionName];
+    const exportProgress = this.currentProgressingStatus?.progressMap[collectionName];
     const logStream = this.generateLogStream(exportProgress);
 
     // create WritableStream
@@ -195,7 +216,7 @@ class ExportService {
 
     await streamToPromise(writeStream);
 
-    return writeStream.path;
+    return writeStream.path.toString();
   }
 
   /**
@@ -203,13 +224,13 @@ class ExportService {
    *
    * @memberOf ExportService
    * @param {Array.<string>} collections array of collection name
-   * @return {Array.<string>} paths to json files created
+   * @return {Array.<ZipFileStat>} info of zip file created
    */
-  async exportCollectionsToZippedJson(collections) {
+  async exportCollectionsToZippedJson(collections: string[]): Promise<ZipFileStat | null> {
     const metaJson = await this.createMetaJson();
 
     // process serially so as not to waste memory
-    const jsonFiles = [];
+    const jsonFiles: string[] = [];
     const jsonFilesPromises = collections.map(collectionName => this.exportCollectionToJson(collectionName));
     for await (const jsonFile of jsonFilesPromises) {
       jsonFiles.push(jsonFile);
@@ -236,7 +257,7 @@ class ExportService {
     // TODO: remove broken zip file
   }
 
-  async export(collections) {
+  async export(collections: string[]): Promise<ZipFileStat | null> {
     if (this.currentProgressingStatus != null) {
       throw new Error('There is an exporting process running.');
     }
@@ -244,7 +265,7 @@ class ExportService {
     this.currentProgressingStatus = new ExportProgressingStatus(collections);
     await this.currentProgressingStatus.init();
 
-    let zipFileStat;
+    let zipFileStat: ZipFileStat | null;
     try {
       zipFileStat = await this.exportCollectionsToZippedJson(collections);
     }
@@ -263,7 +284,9 @@ class ExportService {
    * @param {CollectionProgress} collectionProgress
    * @param {number} currentCount number of items exported
    */
-  logProgress(collectionProgress, currentCount) {
+  logProgress(collectionProgress: CollectionProgress | undefined, currentCount: number): void {
+    if (collectionProgress == null) return;
+
     const output = `${collectionProgress.collectionName}: ${currentCount}/${collectionProgress.totalCount} written`;
 
     // update exportProgress.currentCount
@@ -284,12 +307,11 @@ class ExportService {
   /**
    * emit progress event
    */
-  emitProgressEvent() {
-    const { currentCount, totalCount, progressList } = this.currentProgressingStatus;
+  emitProgressEvent(): void {
     const data = {
-      currentCount,
-      totalCount,
-      progressList,
+      currentCount: this.currentProgressingStatus?.currentCount,
+      totalCount: this.currentProgressingStatus?.totalCount,
+      progressList: this.currentProgressingStatus?.progressList,
     };
 
     // send event (in progress in global)
@@ -299,7 +321,7 @@ class ExportService {
   /**
    * emit start zipping event
    */
-  emitStartZippingEvent() {
+  emitStartZippingEvent(): void {
     this.adminEvent.emit('onStartZippingForExport', {});
   }
 
@@ -307,7 +329,7 @@ class ExportService {
    * emit terminate event
    * @param {object} zipFileStat added zip file status data
    */
-  emitTerminateEvent(zipFileStat) {
+  emitTerminateEvent(zipFileStat: ZipFileStat | null): void {
     this.adminEvent.emit('onTerminateForExport', { addedZipFileStat: zipFileStat });
   }
 
@@ -319,7 +341,7 @@ class ExportService {
    * @return {string} absolute path to the zip file
    * @see https://www.archiverjs.com/#quick-start
    */
-  async zipFiles(_configs) {
+  async zipFiles(_configs: {from: string, as: string}[]): Promise<string> {
     const configs = toArrayIfNot(_configs);
     const appTitle = this.appService.getAppTitle();
     const timeStamp = (new Date()).getTime();
@@ -365,10 +387,9 @@ class ExportService {
     return zipFile;
   }
 
-  getReadStreamFromRevision(revision, format) {
+  getReadStreamFromRevision(revision, format): Readable {
     const data = revision.body;
 
-    const Readable = require('stream').Readable;
     const readable = new Readable();
     readable._read = () => {};
     readable.push(data);
@@ -379,4 +400,8 @@ class ExportService {
 
 }
 
-module.exports = ExportService;
+// eslint-disable-next-line import/no-mutable-exports
+export let exportService: ExportService | undefined; // singleton instance
+export default function instanciate(crowi: any): void {
+  exportService = new ExportService(crowi);
+}

apps/app/src/server/service/external-account.ts

@@ -67,6 +67,6 @@ class ExternalAccountService {
 
 // eslint-disable-next-line import/no-mutable-exports
 export let externalAccountService: ExternalAccountService | undefined; // singleton instance
-export function instanciate(passportService: PassportService): void {
+export default function instanciate(passportService: PassportService): void {
   externalAccountService = new ExternalAccountService(passportService);
 }

apps/app/src/server/service/g2g-transfer.ts

@@ -22,6 +22,7 @@ import { Attachment } from '../models';
 import { G2GTransferError, G2GTransferErrorCode } from '../models/vo/g2g-transfer-error';
 
 import { configManager } from './config-manager';
+import { exportService } from './export';
 
 const logger = loggerFactory('growi:service:g2g-transfer');
 
@@ -432,8 +433,10 @@ export class G2GTransferPusherService implements Pusher {
 
     let zipFileStream: ReadStream;
     try {
-      const zipFileStat = await this.crowi.exportService.export(collections);
-      const zipFilePath = zipFileStat.zipFilePath;
+      const zipFileStat = await exportService?.export(collections);
+      const zipFilePath = zipFileStat?.zipFilePath;
+
+      if (zipFilePath == null) throw new Error('Failed to generate zip file');
 
       zipFileStream = createReadStream(zipFilePath);
     }

apps/app/src/server/service/growi-bridge.js → apps/app/src/server/service/growi-bridge.ts

@@ -1,9 +1,13 @@
+import fs from 'fs';
+import path from 'path';
+
+import unzipper from 'unzipper';
+
 import loggerFactory from '~/utils/logger';
 
-const fs = require('fs');
-const path = require('path');
+import { ZipFileStat } from './interfaces/export';
+
 const streamToPromise = require('stream-to-promise');
-const unzipper = require('unzipper');
 
 const logger = loggerFactory('growi:services:GrowiBridgeService'); // eslint-disable-line no-unused-vars
 
@@ -13,19 +17,25 @@ const logger = loggerFactory('growi:services:GrowiBridgeService'); // eslint-dis
  */
 class GrowiBridgeService {
 
+  crowi: any;
+
+  encoding: BufferEncoding = 'utf-8';
+
+  metaFileName = 'meta.json';
+
+  baseDir: string | undefined;
+
   constructor(crowi) {
     this.crowi = crowi;
-    this.encoding = 'utf-8';
-    this.metaFileName = 'meta.json';
   }
 
   /**
    * getter for encoding
    *
    * @memberOf GrowiBridgeService
-   * @return {string} encoding
+   * @return {BufferEncoding} encoding
    */
-  getEncoding() {
+  getEncoding(): BufferEncoding {
     return this.encoding;
   }
 
@@ -35,7 +45,7 @@ class GrowiBridgeService {
    * @memberOf GrowiBridgeService
    * @return {string} base name of meta file
    */
-  getMetaFileName() {
+  getMetaFileName(): string {
     return this.metaFileName;
   }
 
@@ -46,8 +56,8 @@ class GrowiBridgeService {
    * @param {string} collectionName collection name
    * @return {object} instance of mongoose model
    */
-  getModelFromCollectionName(collectionName) {
-    const Model = Object.values(this.crowi.models).find((m) => {
+  getModelFromCollectionName(collectionName: string) {
+    const Model = Object.values(this.crowi.models).find((m: any) => {
       return m.collection != null && m.collection.name === collectionName;
     });
 
@@ -62,7 +72,7 @@ class GrowiBridgeService {
    * @param {string} fileName base name of file
    * @return {string} absolute path to the file
    */
-  getFile(fileName) {
+  getFile(fileName: string): string {
     if (this.baseDir == null) {
       throw new Error('baseDir is not defined');
     }
@@ -82,9 +92,9 @@ class GrowiBridgeService {
    * @param {string} zipFile path to zip file
    * @return {object} meta{object} and files{Array.<object>}
    */
-  async parseZipFile(zipFile) {
+  async parseZipFile(zipFile: string): Promise<ZipFileStat | null> {
     const fileStat = fs.statSync(zipFile);
-    const innerFileStats = [];
+    const innerFileStats: {fileName: string, collectionName: string, size: number}[] = [];
     let meta = {};
 
     const readStream = fs.createReadStream(zipFile);
@@ -128,4 +138,4 @@ class GrowiBridgeService {
 
 }
 
-module.exports = GrowiBridgeService;
+export default GrowiBridgeService;

apps/app/src/server/service/import.js

@@ -2,6 +2,8 @@ import gc from 'expose-gc/function';
 
 import loggerFactory from '~/utils/logger';
 
+import CollectionProgressingStatus from '../models/vo/collection-progressing-status';
+
 const fs = require('fs');
 const path = require('path');
 const { Writable, Transform } = require('stream');
@@ -13,7 +15,6 @@ const mongoose = require('mongoose');
 const streamToPromise = require('stream-to-promise');
 const unzipper = require('unzipper');
 
-const CollectionProgressingStatus = require('../models/vo/collection-progressing-status');
 const { createBatchStream } = require('../util/batch-stream');
 
 const { ObjectId } = mongoose.Types;

apps/app/src/server/service/interfaces/export.ts

@@ -0,0 +1,13 @@
+import { Stats } from 'fs';
+
+export type ZipFileStat = {
+  meta: object;
+  fileName: string;
+  zipFilePath: string;
+  fileStat: Stats;
+  innerFileStats: {
+      fileName: string;
+      collectionName: string;
+      size: number;
+  }[];
+}

apps/app/test/integration/service/external-user-group-sync.test.ts

@@ -9,7 +9,7 @@ import ExternalUserGroupRelation from '../../../src/features/external-user-group
 import ExternalUserGroupSyncService from '../../../src/features/external-user-group/server/service/external-user-group-sync';
 import ExternalAccount from '../../../src/server/models/external-account';
 import { configManager } from '../../../src/server/service/config-manager';
-import { instanciate } from '../../../src/server/service/external-account';
+import instanciateExternalAccountService from '../../../src/server/service/external-account';
 import PassportService from '../../../src/server/service/passport';
 import { getInstance } from '../setup-crowi';
 
@@ -184,7 +184,7 @@ describe('ExternalUserGroupSyncService.syncExternalUserGroups', () => {
     crowi = await getInstance();
     await configManager.updateConfigsInTheSameNamespace('crowi', { 'app:isV5Compatible': true });
     const passportService = new PassportService(crowi);
-    instanciate(passportService);
+    instanciateExternalAccountService(passportService);
   });
 
   beforeEach(async() => {