add clean up cron and test

apps/app/src/features/audit-log-bulk-export/server/service/audit-log-bulk-export-job-clean-up-cron.integ.ts

@@ -0,0 +1,235 @@
+import type { IUser } from '@growi/core';
+import mongoose from 'mongoose';
+
+import type Crowi from '~/server/crowi';
+import { configManager } from '~/server/service/config-manager';
+
+import {
+  AuditLogBulkExportFormat,
+  AuditLogBulkExportJobStatus,
+} from '../../interfaces/audit-log-bulk-export';
+import AuditLogBulkExportJob from '../models/audit-log-bulk-export-job';
+
+import instantiateAuditLogBulkExportJobCleanUpCronService, {
+  auditLogBulkExportJobCleanUpCronService,
+} from './audit-log-bulk-export-job-clean-up-cron';
+
+const userSchema = new mongoose.Schema(
+  {
+    name: { type: String },
+    username: { type: String, required: true, unique: true },
+    email: { type: String, unique: true, sparse: true },
+  },
+  {
+    timestamps: true,
+  },
+);
+const User = mongoose.model<IUser>('User', userSchema);
+
+vi.mock('./audit-log-bulk-export-job-cron', () => {
+  return {
+    auditLogBulkExportJobCronService: {
+      cleanUpExportJobResources: vi.fn(() => Promise.resolve()),
+      notifyExportResultAndCleanUp: vi.fn(() => Promise.resolve()),
+    },
+  };
+});
+
+describe('AuditLogBulkExportJobCleanUpCronService', () => {
+  const crowi = {} as Crowi;
+  let user: IUser;
+
+  beforeAll(async () => {
+    await configManager.loadConfigs();
+    user = await User.create({
+      name: 'Example for AuditLogBulkExportJobCleanUpCronService Test',
+      username: 'audit log bulk export job cleanup cron test user',
+      email: 'auditLogBulkExportCleanUpCronTestUser@example.com',
+    });
+    instantiateAuditLogBulkExportJobCleanUpCronService(crowi);
+  });
+
+  beforeEach(async () => {
+    await AuditLogBulkExportJob.deleteMany();
+  });
+
+  describe('deleteExpiredExportJobs', () => {
+    const jobId1 = new mongoose.Types.ObjectId();
+    const jobId2 = new mongoose.Types.ObjectId();
+    const jobId3 = new mongoose.Types.ObjectId();
+    const jobId4 = new mongoose.Types.ObjectId();
+    beforeEach(async () => {
+      await configManager.updateConfig(
+        'app:bulkExportJobExpirationSeconds',
+        86400,
+      );
+
+      await AuditLogBulkExportJob.insertMany([
+        {
+          _id: jobId1,
+          user,
+          filters: {},
+          filterHash: 'hash1',
+          format: AuditLogBulkExportFormat.json,
+          status: AuditLogBulkExportJobStatus.exporting,
+          restartFlag: false,
+          createdAt: new Date(Date.now()),
+        },
+        {
+          _id: jobId2,
+          user,
+          filters: {},
+          filterHash: 'hash2',
+          format: AuditLogBulkExportFormat.json,
+          status: AuditLogBulkExportJobStatus.exporting,
+          restartFlag: false,
+          createdAt: new Date(Date.now() - 86400 * 1000 - 1),
+        },
+        {
+          _id: jobId3,
+          user,
+          filters: {},
+          filterHash: 'hash3',
+          format: AuditLogBulkExportFormat.json,
+          status: AuditLogBulkExportJobStatus.uploading,
+          restartFlag: false,
+          createdAt: new Date(Date.now() - 86400 * 1000 - 2),
+        },
+        {
+          _id: jobId4,
+          user,
+          filters: {},
+          filterHash: 'hash4',
+          format: AuditLogBulkExportFormat.json,
+          status: AuditLogBulkExportJobStatus.failed,
+          restartFlag: false,
+        },
+      ]);
+    });
+
+    test('should delete expired jobs', async () => {
+      expect(await AuditLogBulkExportJob.find()).toHaveLength(4);
+
+      await auditLogBulkExportJobCleanUpCronService?.deleteExpiredExportJobs();
+      const jobs = await AuditLogBulkExportJob.find();
+
+      expect(jobs).toHaveLength(2);
+      expect(jobs.map((job) => job._id).sort()).toStrictEqual(
+        [jobId1, jobId4].sort(),
+      );
+    });
+  });
+
+  describe('deleteDownloadExpiredExportJobs', () => {
+    const jobId1 = new mongoose.Types.ObjectId();
+    const jobId2 = new mongoose.Types.ObjectId();
+    const jobId3 = new mongoose.Types.ObjectId();
+    const jobId4 = new mongoose.Types.ObjectId();
+    beforeEach(async () => {
+      await configManager.updateConfig(
+        'app:bulkExportDownloadExpirationSeconds',
+        86400,
+      );
+
+      await AuditLogBulkExportJob.insertMany([
+        {
+          _id: jobId1,
+          user,
+          filters: {},
+          filterHash: 'hash1',
+          format: AuditLogBulkExportFormat.json,
+          status: AuditLogBulkExportJobStatus.completed,
+          restartFlag: false,
+          completedAt: new Date(Date.now()),
+        },
+        {
+          _id: jobId2,
+          user,
+          filters: {},
+          filterHash: 'hash2',
+          format: AuditLogBulkExportFormat.json,
+          status: AuditLogBulkExportJobStatus.completed,
+          restartFlag: false,
+          completedAt: new Date(Date.now() - 86400 * 1000 - 1),
+        },
+        {
+          _id: jobId3,
+          user,
+          filters: {},
+          filterHash: 'hash3',
+          format: AuditLogBulkExportFormat.json,
+          status: AuditLogBulkExportJobStatus.exporting,
+          restartFlag: false,
+        },
+        {
+          _id: jobId4,
+          user,
+          filters: {},
+          filterHash: 'hash4',
+          format: AuditLogBulkExportFormat.json,
+          status: AuditLogBulkExportJobStatus.failed,
+          restartFlag: false,
+        },
+      ]);
+    });
+
+    test('should delete download expired jobs', async () => {
+      expect(await AuditLogBulkExportJob.find()).toHaveLength(4);
+
+      await auditLogBulkExportJobCleanUpCronService?.deleteDownloadExpiredExportJobs();
+      const jobs = await AuditLogBulkExportJob.find();
+
+      expect(jobs).toHaveLength(3);
+      expect(jobs.map((job) => job._id).sort()).toStrictEqual(
+        [jobId1, jobId3, jobId4].sort(),
+      );
+    });
+  });
+
+  describe('deleteFailedExportJobs', () => {
+    const jobId1 = new mongoose.Types.ObjectId();
+    const jobId2 = new mongoose.Types.ObjectId();
+    const jobId3 = new mongoose.Types.ObjectId();
+    beforeEach(async () => {
+      await AuditLogBulkExportJob.insertMany([
+        {
+          _id: jobId1,
+          user,
+          filters: {},
+          filterHash: 'hash1',
+          format: AuditLogBulkExportFormat.json,
+          status: AuditLogBulkExportJobStatus.failed,
+          restartFlag: false,
+        },
+        {
+          _id: jobId2,
+          user,
+          filters: {},
+          filterHash: 'hash2',
+          format: AuditLogBulkExportFormat.json,
+          status: AuditLogBulkExportJobStatus.exporting,
+          restartFlag: false,
+        },
+        {
+          _id: jobId3,
+          user,
+          filters: {},
+          filterHash: 'hash3',
+          format: AuditLogBulkExportFormat.json,
+          status: AuditLogBulkExportJobStatus.failed,
+          restartFlag: false,
+        },
+      ]);
+    });
+
+    test('should delete failed export jobs', async () => {
+      expect(await AuditLogBulkExportJob.find()).toHaveLength(3);
+
+      await auditLogBulkExportJobCleanUpCronService?.deleteFailedExportJobs();
+      const jobs = await AuditLogBulkExportJob.find();
+
+      expect(jobs).toHaveLength(1);
+      expect(jobs.map((job) => job._id)).toStrictEqual([jobId2]);
+    });
+  });
+});

apps/app/src/features/audit-log-bulk-export/server/service/audit-log-bulk-export-job-clean-up-cron.ts

@@ -0,0 +1,156 @@
+import type { HydratedDocument } from 'mongoose';
+
+import type Crowi from '~/server/crowi';
+import { configManager } from '~/server/service/config-manager';
+import CronService from '~/server/service/cron';
+import loggerFactory from '~/utils/logger';
+
+import {
+  AuditLogBulkExportJobInProgressJobStatus,
+  AuditLogBulkExportJobStatus,
+} from '../../interfaces/audit-log-bulk-export';
+import type { AuditLogBulkExportJobDocument } from '../models/audit-log-bulk-export-job';
+import AuditLogBulkExportJob from '../models/audit-log-bulk-export-job';
+
+import { auditLogBulkExportJobCronService } from './audit-log-bulk-export-job-cron';
+
+const logger = loggerFactory(
+  'growi:service:audit-log-bulk-export-job-clean-up-cron',
+);
+
+/**
+ * Manages cronjob which deletes unnecessary audit log bulk export jobs
+ */
+class AuditLogBulkExportJobCleanUpCronService extends CronService {
+  crowi: Crowi;
+
+  constructor(crowi: Crowi) {
+    super();
+    this.crowi = crowi;
+  }
+
+  override getCronSchedule(): string {
+    return '0 */6 * * *';
+  }
+
+  override async executeJob(): Promise<void> {
+    await this.deleteExpiredExportJobs();
+    await this.deleteDownloadExpiredExportJobs();
+    await this.deleteFailedExportJobs();
+  }
+
+  /**
+   * Delete audit log bulk export jobs which are on-going and has passed the limit time for execution
+   */
+  async deleteExpiredExportJobs() {
+    const exportJobExpirationSeconds = configManager.getConfig(
+      'app:bulkExportJobExpirationSeconds',
+    );
+
+    const thresholdDate = new Date(
+      Date.now() - exportJobExpirationSeconds * 1000,
+    );
+
+    const expiredExportJobs = await AuditLogBulkExportJob.find({
+      $or: Object.values(AuditLogBulkExportJobInProgressJobStatus).map(
+        (status) => ({
+          status,
+        }),
+      ),
+      createdAt: {
+        $lt: thresholdDate,
+      },
+    });
+
+    if (auditLogBulkExportJobCronService != null) {
+      await this.cleanUpAndDeleteBulkExportJobs(
+        expiredExportJobs,
+        auditLogBulkExportJobCronService.cleanUpExportJobResources.bind(
+          auditLogBulkExportJobCronService,
+        ),
+      );
+    }
+  }
+
+  /**
+   * Delete audit log bulk export jobs which have completed but the due time for downloading has passed
+   */
+  async deleteDownloadExpiredExportJobs() {
+    const downloadExpirationSeconds = configManager.getConfig(
+      'app:bulkExportDownloadExpirationSeconds',
+    );
+    const thresholdDate = new Date(
+      Date.now() - downloadExpirationSeconds * 1000,
+    );
+
+    const downloadExpiredExportJobs = await AuditLogBulkExportJob.find({
+      status: AuditLogBulkExportJobStatus.completed,
+      completedAt: { $lt: thresholdDate },
+    });
+
+    const cleanUp = async (job: AuditLogBulkExportJobDocument) => {
+      await auditLogBulkExportJobCronService?.cleanUpExportJobResources(job);
+
+      const hasSameAttachmentAndDownloadNotExpired =
+        await AuditLogBulkExportJob.findOne({
+          attachment: job.attachment,
+          _id: { $ne: job._id },
+          completedAt: { $gte: thresholdDate },
+        });
+      if (hasSameAttachmentAndDownloadNotExpired == null) {
+        await this.crowi.attachmentService?.removeAttachment(job.attachment);
+      }
+    };
+
+    await this.cleanUpAndDeleteBulkExportJobs(
+      downloadExpiredExportJobs,
+      cleanUp,
+    );
+  }
+
+  /**
+   * Delete audit log bulk export jobs which have failed
+   */
+  async deleteFailedExportJobs() {
+    const failedExportJobs = await AuditLogBulkExportJob.find({
+      status: AuditLogBulkExportJobStatus.failed,
+    });
+
+    if (auditLogBulkExportJobCronService != null) {
+      await this.cleanUpAndDeleteBulkExportJobs(
+        failedExportJobs,
+        auditLogBulkExportJobCronService.cleanUpExportJobResources.bind(
+          auditLogBulkExportJobCronService,
+        ),
+      );
+    }
+  }
+
+  async cleanUpAndDeleteBulkExportJobs(
+    auditLogBulkExportJobs: HydratedDocument<AuditLogBulkExportJobDocument>[],
+    cleanUp: (job: AuditLogBulkExportJobDocument) => Promise<void>,
+  ): Promise<void> {
+    const results = await Promise.allSettled(
+      auditLogBulkExportJobs.map((job) => cleanUp(job)),
+    );
+    results.forEach((result) => {
+      if (result.status === 'rejected') logger.error(result.reason);
+    });
+
+    const cleanedUpJobs = auditLogBulkExportJobs.filter(
+      (_, index) => results[index].status === 'fulfilled',
+    );
+    if (cleanedUpJobs.length > 0) {
+      const cleanedUpJobIds = cleanedUpJobs.map((job) => job._id);
+      await AuditLogBulkExportJob.deleteMany({ _id: { $in: cleanedUpJobIds } });
+    }
+  }
+}
+
+export let auditLogBulkExportJobCleanUpCronService:
+  | AuditLogBulkExportJobCleanUpCronService
+  | undefined;
+export default function instantiate(crowi: Crowi): void {
+  auditLogBulkExportJobCleanUpCronService =
+    new AuditLogBulkExportJobCleanUpCronService(crowi);
+}

apps/app/src/server/crowi/index.js

@@ -8,6 +8,9 @@ import lsxRoutes from '@growi/remark-lsx/dist/server/index.cjs';
 import mongoose from 'mongoose';
 import next from 'next';
 
+import instantiateAuditLogBulkExportJobCleanUpCronService, {
+  auditLogBulkExportJobCleanUpCronService,
+} from '~/features/audit-log-bulk-export/server/service/audit-log-bulk-export-job-clean-up-cron';
 import instantiateAuditLogBulkExportJobCronService from '~/features/audit-log-bulk-export/server/service/audit-log-bulk-export-job-cron';
 import { checkAuditLogExportJobInProgressCronService } from '~/features/audit-log-bulk-export/server/service/check-audit-log-bulk-export-job-in-progress-cron';
 import { KeycloakUserGroupSyncService } from '~/features/external-user-group/server/service/keycloak-user-group-sync';
@@ -369,6 +372,9 @@ Crowi.prototype.setupCron = function() {
   instantiateAuditLogBulkExportJobCronService(this);
   checkAuditLogExportJobInProgressCronService.startCron();
 
+  instantiateAuditLogBulkExportJobCleanUpCronService(this);
+  auditLogBulkExportJobCleanUpCronService.startCron();
+
   startOpenaiCronIfEnabled();
   startAccessTokenCron();
 };