|
|
@@ -0,0 +1,676 @@
|
|
|
+import { createReadStream, ReadStream } from 'fs';
|
|
|
+import { basename } from 'path';
|
|
|
+import { Readable } from 'stream';
|
|
|
+
|
|
|
+// eslint-disable-next-line no-restricted-imports
|
|
|
+import rawAxios, { type AxiosRequestConfig } from 'axios';
|
|
|
+import FormData from 'form-data';
|
|
|
+import { Types as MongooseTypes } from 'mongoose';
|
|
|
+
|
|
|
+import { G2G_PROGRESS_STATUS } from '~/interfaces/g2g-transfer';
|
|
|
+import GrowiArchiveImportOption from '~/models/admin/growi-archive-import-option';
|
|
|
+import TransferKeyModel from '~/server/models/transfer-key';
|
|
|
+import { generateOverwriteParams } from '~/server/routes/apiv3/import';
|
|
|
+import { type ImportSettings } from '~/server/service/import';
|
|
|
+import { createBatchStream } from '~/server/util/batch-stream';
|
|
|
+import axios from '~/utils/axios';
|
|
|
+import loggerFactory from '~/utils/logger';
|
|
|
+import { TransferKey } from '~/utils/vo/transfer-key';
|
|
|
+
|
|
|
+import { G2GTransferError, G2GTransferErrorCode } from '../models/vo/g2g-transfer-error';
|
|
|
+
|
|
|
+const logger = loggerFactory('growi:service:g2g-transfer');
|
|
|
+
|
|
|
+/**
|
|
|
+ * Header name for transfer key
|
|
|
+ */
|
|
|
+export const X_GROWI_TRANSFER_KEY_HEADER_NAME = 'x-growi-transfer-key';
|
|
|
+
|
|
|
+/**
|
|
|
+ * Keys for file upload related config
|
|
|
+ */
|
|
|
+const UPLOAD_CONFIG_KEYS = [
|
|
|
+ 'app:fileUploadType',
|
|
|
+ 'app:useOnlyEnvVarForFileUploadType',
|
|
|
+ 'aws:referenceFileWithRelayMode',
|
|
|
+ 'aws:lifetimeSecForTemporaryUrl',
|
|
|
+ 'gcs:apiKeyJsonPath',
|
|
|
+ 'gcs:bucket',
|
|
|
+ 'gcs:uploadNamespace',
|
|
|
+ 'gcs:referenceFileWithRelayMode',
|
|
|
+ 'gcs:useOnlyEnvVarsForSomeOptions',
|
|
|
+] as const;
|
|
|
+
|
|
|
+/**
|
|
|
+ * File upload related configs
|
|
|
+ */
|
|
|
+type FileUploadConfigs = { [key in typeof UPLOAD_CONFIG_KEYS[number] ]: any; }
|
|
|
+
|
|
|
+/**
|
|
|
+ * Data used for comparing to/from GROWI information
|
|
|
+ */
|
|
|
+export type IDataGROWIInfo = {
|
|
|
+ /** GROWI version */
|
|
|
+ version: string
|
|
|
+ /** Max user count */
|
|
|
+ userUpperLimit: number | null // Handle null as Infinity
|
|
|
+ /** Whether file upload is disabled */
|
|
|
+ fileUploadDisabled: boolean;
|
|
|
+ /** Total file size allowed */
|
|
|
+ fileUploadTotalLimit: number | null // Handle null as Infinity
|
|
|
+ /** Attachment infromation */
|
|
|
+ attachmentInfo: {
|
|
|
+ /** File storage type */
|
|
|
+ type: string;
|
|
|
+ /** Whether the storage is writable */
|
|
|
+ writable: boolean;
|
|
|
+ /** Bucket name (S3 and GCS only) */
|
|
|
+ bucket?: string;
|
|
|
+ /** S3 custom endpoint */
|
|
|
+ customEndpoint?: string;
|
|
|
+ /** GCS namespace */
|
|
|
+ uploadNamespace?: string;
|
|
|
+ };
|
|
|
+}
|
|
|
+
|
|
|
+/**
|
|
|
+ * File metadata in storage
|
|
|
+ * TODO: mv this to "./file-uploader/uploader"
|
|
|
+ */
|
|
|
+interface FileMeta {
|
|
|
+ /** File name */
|
|
|
+ name: string;
|
|
|
+ /** File size in bytes */
|
|
|
+ size: number;
|
|
|
+}
|
|
|
+
|
|
|
+/**
|
|
|
+ * Return type for {@link Pusher.getTransferability}
|
|
|
+ */
|
|
|
+type Transferability = { canTransfer: true; } | { canTransfer: false; reason: string; };
|
|
|
+
|
|
|
+/**
|
|
|
+ * G2g transfer pusher
|
|
|
+ */
|
|
|
+interface Pusher {
|
|
|
+ /**
|
|
|
+ * Merge axios config with transfer key
|
|
|
+ * @param {TransferKey} tk Transfer key
|
|
|
+ * @param {AxiosRequestConfig} config Axios config
|
|
|
+ */
|
|
|
+ generateAxiosConfig(tk: TransferKey, config: AxiosRequestConfig): AxiosRequestConfig
|
|
|
+ /**
|
|
|
+ * Send to-growi a request to get GROWI info
|
|
|
+ * @param {TransferKey} tk Transfer key
|
|
|
+ */
|
|
|
+ askGROWIInfo(tk: TransferKey): Promise<IDataGROWIInfo>
|
|
|
+ /**
|
|
|
+ * Check if transfering is proceedable
|
|
|
+ * @param {IDataGROWIInfo} destGROWIInfo GROWI info from dest GROWI
|
|
|
+ */
|
|
|
+ getTransferability(destGROWIInfo: IDataGROWIInfo): Promise<Transferability>
|
|
|
+ /**
|
|
|
+ * List files in the storage
|
|
|
+ * @param {TransferKey} tk Transfer key
|
|
|
+ */
|
|
|
+ listFilesInStorage(tk: TransferKey): Promise<FileMeta[]>
|
|
|
+ /**
|
|
|
+ * Transfer all Attachment data to dest GROWI
|
|
|
+ * @param {TransferKey} tk Transfer key
|
|
|
+ */
|
|
|
+ transferAttachments(tk: TransferKey): Promise<void>
|
|
|
+ /**
|
|
|
+ * Start transfer data between GROWIs
|
|
|
+ * @param {TransferKey} tk TransferKey object
|
|
|
+ * @param {any} user User operating g2g transfer
|
|
|
+ * @param {IDataGROWIInfo} destGROWIInfo GROWI info of dest GROWI
|
|
|
+ * @param {string[]} collections Collection name string array
|
|
|
+ * @param {any} optionsMap Options map
|
|
|
+ */
|
|
|
+ startTransfer(
|
|
|
+ tk: TransferKey,
|
|
|
+ user: any,
|
|
|
+ collections: string[],
|
|
|
+ optionsMap: any,
|
|
|
+ destGROWIInfo: IDataGROWIInfo,
|
|
|
+ ): Promise<void>
|
|
|
+}
|
|
|
+
|
|
|
+/**
|
|
|
+ * G2g transfer receiver
|
|
|
+ */
|
|
|
+interface Receiver {
|
|
|
+ /**
|
|
|
+ * Check if key is not expired
|
|
|
+ * @throws {import('../models/vo/g2g-transfer-error').G2GTransferError}
|
|
|
+ * @param {string} key Transfer key
|
|
|
+ */
|
|
|
+ validateTransferKey(key: string): Promise<void>
|
|
|
+ /**
|
|
|
+ * Generate GROWIInfo
|
|
|
+ * @throws {import('../models/vo/g2g-transfer-error').G2GTransferError}
|
|
|
+ */
|
|
|
+ answerGROWIInfo(): Promise<IDataGROWIInfo>
|
|
|
+ /**
|
|
|
+ * DO NOT USE TransferKeyModel.create() directly, instead, use this method to create a TransferKey document.
|
|
|
+ * This method receives appSiteUrlOrigin to create a TransferKey document and returns generated transfer key string.
|
|
|
+ * UUID is the same value as the created document's _id.
|
|
|
+ * @param {string} appSiteUrlOrigin GROWI app site URL origin
|
|
|
+ * @returns {string} Transfer key string (e.g. http://localhost:3000__grw_internal_tranferkey__<uuid>)
|
|
|
+ */
|
|
|
+ createTransferKey(appSiteUrlOrigin: string): Promise<string>
|
|
|
+ /**
|
|
|
+ * Returns a map of collection name and ImportSettings
|
|
|
+ * @param {any[]} innerFileStats
|
|
|
+ * @param {{ [key: string]: GrowiArchiveImportOption; }} optionsMap Map of collection name and GrowiArchiveImportOption
|
|
|
+ * @param {string} operatorUserId User ID
|
|
|
+ * @returns {{ [key: string]: ImportSettings; }} Map of collection name and ImportSettings
|
|
|
+ */
|
|
|
+ getImportSettingMap(
|
|
|
+ innerFileStats: any[],
|
|
|
+ optionsMap: { [key: string]: GrowiArchiveImportOption; },
|
|
|
+ operatorUserId: string,
|
|
|
+ ): { [key: string]: ImportSettings; }
|
|
|
+ /**
|
|
|
+ * Import collections
|
|
|
+ * @param {string} collections Array of collection name
|
|
|
+ * @param {{ [key: string]: ImportSettings; }} importSettingsMap Map of collection name and ImportSettings
|
|
|
+ * @param {FileUploadConfigs} sourceGROWIUploadConfigs File upload configs from src GROWI
|
|
|
+ */
|
|
|
+ importCollections(
|
|
|
+ collections: string[],
|
|
|
+ importSettingsMap: { [key: string]: ImportSettings; },
|
|
|
+ sourceGROWIUploadConfigs: FileUploadConfigs,
|
|
|
+ ): Promise<void>
|
|
|
+ /**
|
|
|
+ * Returns file upload configs
|
|
|
+ */
|
|
|
+ getFileUploadConfigs(): Promise<FileUploadConfigs>
|
|
|
+ /**
|
|
|
+ * Update file upload configs
|
|
|
+ * @param fileUploadConfigs File upload configs
|
|
|
+ */
|
|
|
+ updateFileUploadConfigs(fileUploadConfigs: FileUploadConfigs): Promise<void>
|
|
|
+ /**
|
|
|
+ * Upload attachment file
|
|
|
+ * @param {Readable} content Pushed attachment data from source GROWI
|
|
|
+ * @param {any} attachmentMap Map-ped Attachment instance
|
|
|
+ */
|
|
|
+ receiveAttachment(content: Readable, attachmentMap: any): Promise<void>
|
|
|
+}
|
|
|
+
|
|
|
+/**
|
|
|
+ * G2g transfer pusher
|
|
|
+ */
|
|
|
+export class G2GTransferPusherService implements Pusher {
|
|
|
+
|
|
|
+ crowi: any;
|
|
|
+
|
|
|
+ // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types
|
|
|
+ constructor(crowi: any) {
|
|
|
+ this.crowi = crowi;
|
|
|
+ }
|
|
|
+
|
|
|
+ public generateAxiosConfig(tk: TransferKey, baseConfig: AxiosRequestConfig = {}): AxiosRequestConfig {
|
|
|
+ const { appSiteUrlOrigin, key } = tk;
|
|
|
+
|
|
|
+ return {
|
|
|
+ ...baseConfig,
|
|
|
+ baseURL: appSiteUrlOrigin,
|
|
|
+ headers: {
|
|
|
+ ...baseConfig.headers,
|
|
|
+ [X_GROWI_TRANSFER_KEY_HEADER_NAME]: key,
|
|
|
+ },
|
|
|
+ maxBodyLength: Infinity,
|
|
|
+ };
|
|
|
+ }
|
|
|
+
|
|
|
+ public async askGROWIInfo(tk: TransferKey): Promise<IDataGROWIInfo> {
|
|
|
+ try {
|
|
|
+ const { data: { growiInfo } } = await axios.get('/_api/v3/g2g-transfer/growi-info', this.generateAxiosConfig(tk));
|
|
|
+ return growiInfo;
|
|
|
+ }
|
|
|
+ catch (err) {
|
|
|
+ logger.error(err);
|
|
|
+ throw new G2GTransferError('Failed to retrieve GROWI info.', G2GTransferErrorCode.FAILED_TO_RETRIEVE_GROWI_INFO);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ public async getTransferability(destGROWIInfo: IDataGROWIInfo): Promise<Transferability> {
|
|
|
+ const { fileUploadService, configManager } = this.crowi;
|
|
|
+
|
|
|
+ const version = this.crowi.version;
|
|
|
+ if (version !== destGROWIInfo.version) {
|
|
|
+ return {
|
|
|
+ canTransfer: false,
|
|
|
+ // TODO: i18n for reason
|
|
|
+ reason: `GROWI versions mismatch. src GROWI: ${version} / dest GROWI: ${destGROWIInfo.version}.`,
|
|
|
+ };
|
|
|
+ }
|
|
|
+
|
|
|
+ const activeUserCount = await this.crowi.model('User').countActiveUsers();
|
|
|
+ if ((destGROWIInfo.userUpperLimit ?? Infinity) < activeUserCount) {
|
|
|
+ return {
|
|
|
+ canTransfer: false,
|
|
|
+ // TODO: i18n for reason
|
|
|
+ // eslint-disable-next-line max-len
|
|
|
+ reason: `The number of active users (${activeUserCount} users) exceeds the limit of the destination GROWI (up to ${destGROWIInfo.userUpperLimit} users).`,
|
|
|
+ };
|
|
|
+ }
|
|
|
+
|
|
|
+ if (destGROWIInfo.fileUploadDisabled) {
|
|
|
+ return {
|
|
|
+ canTransfer: false,
|
|
|
+ // TODO: i18n for reason
|
|
|
+ reason: 'The file upload setting is disabled in the destination GROWI.',
|
|
|
+ };
|
|
|
+ }
|
|
|
+
|
|
|
+ if (configManager.getConfig('crowi', 'app:fileUploadType') === 'none') {
|
|
|
+ return {
|
|
|
+ canTransfer: false,
|
|
|
+ // TODO: i18n for reason
|
|
|
+ reason: 'File upload is not configured for src GROWI.',
|
|
|
+ };
|
|
|
+ }
|
|
|
+
|
|
|
+ if (destGROWIInfo.attachmentInfo.type === 'none') {
|
|
|
+ return {
|
|
|
+ canTransfer: false,
|
|
|
+ // TODO: i18n for reason
|
|
|
+ reason: 'File upload is not configured for dest GROWI.',
|
|
|
+ };
|
|
|
+ }
|
|
|
+
|
|
|
+ if (!destGROWIInfo.attachmentInfo.writable) {
|
|
|
+ return {
|
|
|
+ canTransfer: false,
|
|
|
+ // TODO: i18n for reason
|
|
|
+ reason: 'The storage of the destination GROWI is not writable.',
|
|
|
+ };
|
|
|
+ }
|
|
|
+
|
|
|
+ const totalFileSize = await fileUploadService.getTotalFileSize();
|
|
|
+ if ((destGROWIInfo.fileUploadTotalLimit ?? Infinity) < totalFileSize) {
|
|
|
+ return {
|
|
|
+ canTransfer: false,
|
|
|
+ // TODO: i18n for reason
|
|
|
+ // eslint-disable-next-line max-len
|
|
|
+ reason: `The total file size of attachments exceeds the file upload limit of the destination GROWI. Requires ${totalFileSize.toLocaleString()} bytes, but got ${(destGROWIInfo.fileUploadTotalLimit as number).toLocaleString()} bytes.`,
|
|
|
+ };
|
|
|
+ }
|
|
|
+
|
|
|
+ return { canTransfer: true };
|
|
|
+ }
|
|
|
+
|
|
|
+ public async listFilesInStorage(tk: TransferKey): Promise<FileMeta[]> {
|
|
|
+ try {
|
|
|
+ const { data: { files } } = await axios.get<{ files: FileMeta[] }>('/_api/v3/g2g-transfer/files', this.generateAxiosConfig(tk));
|
|
|
+ return files;
|
|
|
+ }
|
|
|
+ catch (err) {
|
|
|
+ logger.error(err);
|
|
|
+ throw new G2GTransferError('Failed to retrieve file metadata', G2GTransferErrorCode.FAILED_TO_RETRIEVE_FILE_METADATA);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ public async transferAttachments(tk: TransferKey): Promise<void> {
|
|
|
+ const BATCH_SIZE = 100;
|
|
|
+ const { fileUploadService, socketIoService } = this.crowi;
|
|
|
+ const socket = socketIoService.getAdminSocket();
|
|
|
+ const Attachment = this.crowi.model('Attachment');
|
|
|
+ const filesFromSrcGROWI = await this.listFilesInStorage(tk);
|
|
|
+
|
|
|
+ /**
|
|
|
+ * Given these documents,
|
|
|
+ *
|
|
|
+ * | fileName | fileSize |
|
|
|
+ * | -- | -- |
|
|
|
+ * | a.png | 1024 |
|
|
|
+ * | b.png | 2048 |
|
|
|
+ * | c.png | 1024 |
|
|
|
+ * | d.png | 2048 |
|
|
|
+ *
|
|
|
+ * this filter
|
|
|
+ *
|
|
|
+ * ```jsonc
|
|
|
+ * {
|
|
|
+ * $and: [
|
|
|
+ * // a file transferred
|
|
|
+ * {
|
|
|
+ * $or: [
|
|
|
+ * { fileName: { $ne: "a.png" } },
|
|
|
+ * { fileSize: { $ne: 1024 } }
|
|
|
+ * ]
|
|
|
+ * },
|
|
|
+ * // a file failed to transfer
|
|
|
+ * {
|
|
|
+ * $or: [
|
|
|
+ * { fileName: { $ne: "b.png" } },
|
|
|
+ * { fileSize: { $ne: 0 } }
|
|
|
+ * ]
|
|
|
+ * }
|
|
|
+ * ]
|
|
|
+ * }
|
|
|
+ * ```
|
|
|
+ *
|
|
|
+ * results in
|
|
|
+ *
|
|
|
+ * | fileName | fileSize |
|
|
|
+ * | -- | -- |
|
|
|
+ * | b.png | 2048 |
|
|
|
+ * | c.png | 1024 |
|
|
|
+ * | d.png | 2048 |
|
|
|
+ */
|
|
|
+ const filter = filesFromSrcGROWI.length > 0 ? {
|
|
|
+ $and: filesFromSrcGROWI.map(({ name, size }) => ({
|
|
|
+ $or: [
|
|
|
+ { fileName: { $ne: basename(name) } },
|
|
|
+ { fileSize: { $ne: size } },
|
|
|
+ ],
|
|
|
+ })),
|
|
|
+ } : {};
|
|
|
+ const attachmentsCursor = await Attachment.find(filter).cursor();
|
|
|
+ const batchStream = createBatchStream(BATCH_SIZE);
|
|
|
+
|
|
|
+ for await (const attachmentBatch of attachmentsCursor.pipe(batchStream)) {
|
|
|
+ for await (const attachment of attachmentBatch) {
|
|
|
+ logger.debug(`processing attachment: ${attachment}`);
|
|
|
+ let fileStream;
|
|
|
+ try {
|
|
|
+ // get read stream of each attachment
|
|
|
+ fileStream = await fileUploadService.findDeliveryFile(attachment);
|
|
|
+ }
|
|
|
+ catch (err) {
|
|
|
+ logger.warn(`Error occured when getting Attachment(ID=${attachment.id}), skipping: `, err);
|
|
|
+ socket.emit('admin:g2gError', {
|
|
|
+ message: `Error occured when uploading Attachment(ID=${attachment.id})`,
|
|
|
+ key: `Error occured when uploading Attachment(ID=${attachment.id})`,
|
|
|
+ // TODO: emit error with params
|
|
|
+ // key: 'admin:g2g:error_upload_attachment',
|
|
|
+ });
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ // post each attachment file data to receiver
|
|
|
+ try {
|
|
|
+ await this.doTransferAttachment(tk, attachment, fileStream);
|
|
|
+ }
|
|
|
+ catch (err) {
|
|
|
+ logger.error(`Error occured when uploading attachment(ID=${attachment.id})`, err);
|
|
|
+ socket.emit('admin:g2gError', {
|
|
|
+ message: `Error occured when uploading Attachment(ID=${attachment.id})`,
|
|
|
+ key: `Error occured when uploading Attachment(ID=${attachment.id})`,
|
|
|
+ // TODO: emit error with params
|
|
|
+ // key: 'admin:g2g:error_upload_attachment',
|
|
|
+ });
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ // eslint-disable-next-line max-len
|
|
|
+ public async startTransfer(tk: TransferKey, user: any, collections: string[], optionsMap: any, destGROWIInfo: IDataGROWIInfo): Promise<void> {
|
|
|
+ const socket = this.crowi.socketIoService.getAdminSocket();
|
|
|
+
|
|
|
+ socket.emit('admin:g2gProgress', {
|
|
|
+ mongo: G2G_PROGRESS_STATUS.IN_PROGRESS,
|
|
|
+ attachments: G2G_PROGRESS_STATUS.PENDING,
|
|
|
+ });
|
|
|
+
|
|
|
+ const targetConfigKeys = UPLOAD_CONFIG_KEYS;
|
|
|
+
|
|
|
+ const uploadConfigs = Object.fromEntries(targetConfigKeys.map((key) => {
|
|
|
+ return [key, this.crowi.configManager.getConfig('crowi', key)];
|
|
|
+ }));
|
|
|
+
|
|
|
+ let zipFileStream: ReadStream;
|
|
|
+ try {
|
|
|
+ const zipFileStat = await this.crowi.exportService.export(collections);
|
|
|
+ const zipFilePath = zipFileStat.zipFilePath;
|
|
|
+
|
|
|
+ zipFileStream = createReadStream(zipFilePath);
|
|
|
+ }
|
|
|
+ catch (err) {
|
|
|
+ logger.error(err);
|
|
|
+ socket.emit('admin:g2gProgress', {
|
|
|
+ mongo: G2G_PROGRESS_STATUS.ERROR,
|
|
|
+ attachments: G2G_PROGRESS_STATUS.PENDING,
|
|
|
+ });
|
|
|
+ socket.emit('admin:g2gError', { message: 'Failed to generate GROWI archive file', key: 'admin:g2g:error_generate_growi_archive' });
|
|
|
+ throw err;
|
|
|
+ }
|
|
|
+
|
|
|
+ // Send a zip file to other GROWI via axios
|
|
|
+ try {
|
|
|
+ // Use FormData to immitate browser's form data object
|
|
|
+ const form = new FormData();
|
|
|
+
|
|
|
+ const appTitle = this.crowi.appService.getAppTitle();
|
|
|
+ form.append('transferDataZipFile', zipFileStream, `${appTitle}-${Date.now}.growi.zip`);
|
|
|
+ form.append('collections', JSON.stringify(collections));
|
|
|
+ form.append('optionsMap', JSON.stringify(optionsMap));
|
|
|
+ form.append('operatorUserId', user._id.toString());
|
|
|
+ form.append('uploadConfigs', JSON.stringify(uploadConfigs));
|
|
|
+ await rawAxios.post('/_api/v3/g2g-transfer/', form, this.generateAxiosConfig(tk, { headers: form.getHeaders() }));
|
|
|
+ }
|
|
|
+ catch (err) {
|
|
|
+ logger.error(err);
|
|
|
+ socket.emit('admin:g2gProgress', {
|
|
|
+ mongo: G2G_PROGRESS_STATUS.ERROR,
|
|
|
+ attachments: G2G_PROGRESS_STATUS.PENDING,
|
|
|
+ });
|
|
|
+ socket.emit('admin:g2gError', { message: 'Failed to send GROWI archive file to the destination GROWI', key: 'admin:g2g:error_send_growi_archive' });
|
|
|
+ throw err;
|
|
|
+ }
|
|
|
+
|
|
|
+ socket.emit('admin:g2gProgress', {
|
|
|
+ mongo: G2G_PROGRESS_STATUS.COMPLETED,
|
|
|
+ attachments: G2G_PROGRESS_STATUS.IN_PROGRESS,
|
|
|
+ });
|
|
|
+
|
|
|
+ try {
|
|
|
+ await this.transferAttachments(tk);
|
|
|
+ }
|
|
|
+ catch (err) {
|
|
|
+ logger.error(err);
|
|
|
+ socket.emit('admin:g2gProgress', {
|
|
|
+ mongo: G2G_PROGRESS_STATUS.COMPLETED,
|
|
|
+ attachments: G2G_PROGRESS_STATUS.ERROR,
|
|
|
+ });
|
|
|
+ socket.emit('admin:g2gError', { message: 'Failed to transfer attachments', key: 'admin:g2g:error_upload_attachment' });
|
|
|
+ throw err;
|
|
|
+ }
|
|
|
+
|
|
|
+ socket.emit('admin:g2gProgress', {
|
|
|
+ mongo: G2G_PROGRESS_STATUS.COMPLETED,
|
|
|
+ attachments: G2G_PROGRESS_STATUS.COMPLETED,
|
|
|
+ });
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * Transfer attachment to dest GROWI
|
|
|
+ * @param {TransferKey} tk Transfer key
|
|
|
+ * @param {any} attachment Attachment model instance
|
|
|
+ * @param {Readable} fileStream Attachment data(loaded from storage)
|
|
|
+ */
|
|
|
+ private async doTransferAttachment(tk: TransferKey, attachment: any, fileStream: Readable): Promise<void> {
|
|
|
+ // Use FormData to immitate browser's form data object
|
|
|
+ const form = new FormData();
|
|
|
+
|
|
|
+ form.append('content', fileStream, attachment.fileName);
|
|
|
+ form.append('attachmentMetadata', JSON.stringify(attachment));
|
|
|
+ await rawAxios.post('/_api/v3/g2g-transfer/attachment', form, this.generateAxiosConfig(tk, { headers: form.getHeaders() }));
|
|
|
+ }
|
|
|
+
|
|
|
+}
|
|
|
+
|
|
|
+/**
|
|
|
+ * G2g transfer receiver
|
|
|
+ */
|
|
|
+export class G2GTransferReceiverService implements Receiver {
|
|
|
+
|
|
|
+ crowi: any;
|
|
|
+
|
|
|
+ constructor(crowi: any) {
|
|
|
+ this.crowi = crowi;
|
|
|
+ }
|
|
|
+
|
|
|
+ public async validateTransferKey(key: string): Promise<void> {
|
|
|
+ const transferKey = await (TransferKeyModel as any).findOne({ key });
|
|
|
+
|
|
|
+ if (transferKey == null) {
|
|
|
+ throw new Error(`Transfer key "${key}" was expired or not found`);
|
|
|
+ }
|
|
|
+
|
|
|
+ try {
|
|
|
+ TransferKey.parse(transferKey.keyString);
|
|
|
+ }
|
|
|
+ catch (err) {
|
|
|
+ logger.error(err);
|
|
|
+ throw new Error(`Transfer key "${key}" is invalid`);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ public async answerGROWIInfo(): Promise<IDataGROWIInfo> {
|
|
|
+ const { version, configManager, fileUploadService } = this.crowi;
|
|
|
+ const userUpperLimit = configManager.getConfig('crowi', 'security:userUpperLimit');
|
|
|
+ const fileUploadDisabled = configManager.getConfig('crowi', 'app:fileUploadDisabled');
|
|
|
+ const fileUploadTotalLimit = configManager.getFileUploadTotalLimit();
|
|
|
+ const isWritable = await fileUploadService.isWritable();
|
|
|
+
|
|
|
+ const attachmentInfo = {
|
|
|
+ type: configManager.getConfig('crowi', 'app:fileUploadType'),
|
|
|
+ writable: isWritable,
|
|
|
+ bucket: undefined,
|
|
|
+ customEndpoint: undefined, // for S3
|
|
|
+ uploadNamespace: undefined, // for GCS
|
|
|
+ };
|
|
|
+
|
|
|
+ // put storage location info to check storage identification
|
|
|
+ switch (attachmentInfo.type) {
|
|
|
+ case 'aws':
|
|
|
+ attachmentInfo.bucket = configManager.getConfig('crowi', 'aws:s3Bucket');
|
|
|
+ attachmentInfo.customEndpoint = configManager.getConfig('crowi', 'aws:s3CustomEndpoint');
|
|
|
+ break;
|
|
|
+ case 'gcs':
|
|
|
+ attachmentInfo.bucket = configManager.getConfig('crowi', 'gcs:bucket');
|
|
|
+ attachmentInfo.uploadNamespace = configManager.getConfig('crowi', 'gcs:uploadNamespace');
|
|
|
+ break;
|
|
|
+ default:
|
|
|
+ }
|
|
|
+
|
|
|
+ return {
|
|
|
+ userUpperLimit,
|
|
|
+ fileUploadDisabled,
|
|
|
+ fileUploadTotalLimit,
|
|
|
+ version,
|
|
|
+ attachmentInfo,
|
|
|
+ };
|
|
|
+ }
|
|
|
+
|
|
|
+ public async createTransferKey(appSiteUrlOrigin: string): Promise<string> {
|
|
|
+ const uuid = new MongooseTypes.ObjectId().toString();
|
|
|
+ const transferKeyString = TransferKey.generateKeyString(uuid, appSiteUrlOrigin);
|
|
|
+
|
|
|
+ // Save TransferKey document
|
|
|
+ let tkd;
|
|
|
+ try {
|
|
|
+ tkd = await TransferKeyModel.create({ _id: uuid, keyString: transferKeyString, key: uuid });
|
|
|
+ }
|
|
|
+ catch (err) {
|
|
|
+ logger.error(err);
|
|
|
+ throw err;
|
|
|
+ }
|
|
|
+
|
|
|
+ return tkd.keyString;
|
|
|
+ }
|
|
|
+
|
|
|
+ public getImportSettingMap(
|
|
|
+ innerFileStats: any[],
|
|
|
+ optionsMap: { [key: string]: GrowiArchiveImportOption; },
|
|
|
+ operatorUserId: string,
|
|
|
+ ): { [key: string]: ImportSettings; } {
|
|
|
+ const { importService } = this.crowi;
|
|
|
+
|
|
|
+ const importSettingsMap = {};
|
|
|
+ innerFileStats.forEach(({ fileName, collectionName }) => {
|
|
|
+ const options = new GrowiArchiveImportOption(null, optionsMap[collectionName]);
|
|
|
+
|
|
|
+ if (collectionName === 'configs' && options.mode !== 'flushAndInsert') {
|
|
|
+ throw new Error('`flushAndInsert` is only available as an import setting for configs collection');
|
|
|
+ }
|
|
|
+ if (collectionName === 'pages' && options.mode === 'insert') {
|
|
|
+ throw new Error('`insert` is not available as an import setting for pages collection');
|
|
|
+ }
|
|
|
+ if (collectionName === 'attachmentFiles.chunks') {
|
|
|
+ throw new Error('`attachmentFiles.chunks` must not be transferred. Please omit it from request body `collections`.');
|
|
|
+ }
|
|
|
+ if (collectionName === 'attachmentFiles.files') {
|
|
|
+ throw new Error('`attachmentFiles.files` must not be transferred. Please omit it from request body `collections`.');
|
|
|
+ }
|
|
|
+
|
|
|
+ const importSettings = importService.generateImportSettings(options.mode);
|
|
|
+ importSettings.jsonFileName = fileName;
|
|
|
+ importSettings.overwriteParams = generateOverwriteParams(collectionName, operatorUserId, options);
|
|
|
+ importSettingsMap[collectionName] = importSettings;
|
|
|
+ });
|
|
|
+
|
|
|
+ return importSettingsMap;
|
|
|
+ }
|
|
|
+
|
|
|
+ public async importCollections(
|
|
|
+ collections: string[],
|
|
|
+ importSettingsMap: { [key: string]: ImportSettings; },
|
|
|
+ sourceGROWIUploadConfigs: FileUploadConfigs,
|
|
|
+ ): Promise<void> {
|
|
|
+ const { configManager, importService, appService } = this.crowi;
|
|
|
+ /** whether to keep current file upload configs */
|
|
|
+ const shouldKeepUploadConfigs = configManager.getConfig('crowi', 'app:fileUploadType') !== 'none';
|
|
|
+
|
|
|
+ if (shouldKeepUploadConfigs) {
|
|
|
+ /** cache file upload configs */
|
|
|
+ const fileUploadConfigs = await this.getFileUploadConfigs();
|
|
|
+
|
|
|
+ // import mongo collections(overwrites file uplaod configs)
|
|
|
+ await importService.import(collections, importSettingsMap);
|
|
|
+
|
|
|
+ // restore file upload config from cache
|
|
|
+ await configManager.removeConfigsInTheSameNamespace('crowi', UPLOAD_CONFIG_KEYS);
|
|
|
+ await configManager.updateConfigsInTheSameNamespace('crowi', fileUploadConfigs);
|
|
|
+ }
|
|
|
+ else {
|
|
|
+ // import mongo collections(overwrites file uplaod configs)
|
|
|
+ await importService.import(collections, importSettingsMap);
|
|
|
+
|
|
|
+ // update file upload config
|
|
|
+ await configManager.updateConfigsInTheSameNamespace('crowi', sourceGROWIUploadConfigs);
|
|
|
+ }
|
|
|
+
|
|
|
+ await this.crowi.setUpFileUpload(true);
|
|
|
+ await appService.setupAfterInstall();
|
|
|
+ }
|
|
|
+
|
|
|
+ public async getFileUploadConfigs(): Promise<FileUploadConfigs> {
|
|
|
+ const { configManager } = this.crowi;
|
|
|
+ const fileUploadConfigs = Object.fromEntries(UPLOAD_CONFIG_KEYS.map((key) => {
|
|
|
+ return [key, configManager.getConfigFromDB('crowi', key)];
|
|
|
+ })) as FileUploadConfigs;
|
|
|
+
|
|
|
+ return fileUploadConfigs;
|
|
|
+ }
|
|
|
+
|
|
|
+ public async updateFileUploadConfigs(fileUploadConfigs: FileUploadConfigs): Promise<void> {
|
|
|
+ const { appService, configManager } = this.crowi;
|
|
|
+
|
|
|
+ await configManager.removeConfigsInTheSameNamespace('crowi', Object.keys(fileUploadConfigs));
|
|
|
+ await configManager.updateConfigsInTheSameNamespace('crowi', fileUploadConfigs);
|
|
|
+ await this.crowi.setUpFileUpload(true);
|
|
|
+ await appService.setupAfterInstall();
|
|
|
+ }
|
|
|
+
|
|
|
+ public async receiveAttachment(content: Readable, attachmentMap): Promise<void> {
|
|
|
+ const { fileUploadService } = this.crowi;
|
|
|
+ return fileUploadService.uploadAttachment(content, attachmentMap);
|
|
|
+ }
|
|
|
+
|
|
|
+}
|