Merge branch 'master' into feat/opentelemetry

.github/workflows/ci-app.yml

@@ -17,6 +17,19 @@ on:
       - apps/app/**
       - '!apps/app/docker/**'
       - packages/**
+  pull_request:
+    types: [opened, reopened, synchronize]
+    paths:
+      - .github/mergify.yml
+      - .github/workflows/ci-app.yml
+      - .eslint*
+      - tsconfig.base.json
+      - turbo.json
+      - pnpm-lock.yaml
+      - package.json
+      - apps/app/**
+      - '!apps/app/docker/**'
+      - packages/**
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}

CHANGELOG.md

@@ -1,9 +1,28 @@
 # Changelog
 
-## [Unreleased](https://github.com/weseek/growi/compare/v7.1.1...HEAD)
+## [Unreleased](https://github.com/weseek/growi/compare/v7.1.2...HEAD)
 
 *Please do not manually update this file. We've automated the process.*
 
+## [v7.1.2](https://github.com/weseek/growi/compare/v7.1.1...v7.1.2) - 2024-11-18
+
+### 🚀 Improvement
+
+* imprv(ai): GROWI AI Knowledge Assistant instructions (#9407) @yuki-takei
+* imprv(ai): Knowedge Assistant model configuration by env var (#9410) @yuki-takei
+* imprv(ai): Shorten thread deletion expiredAt (#9419) @yuki-takei
+* imprv(ai): Remove unnecessary strings from markdown when creating VectorStoreFIie (#9411) @miya
+* imprv(ai): Create thead before the first post (#9414) @yuki-takei
+
+### 🐛 Bug Fixes
+
+* fix: Fixed the message when all read (#9405) @Ryosei-Fukushima
+
+### 🧰 Maintenance
+
+* support: Import OpenAI features dynamically (#9413) @yuki-takei
+* support: Welcome back Hufflepuff badger (#9403) @satof3
+
 ## [v7.1.1](https://github.com/weseek/growi/compare/v7.1.0...v7.1.1) - 2024-11-12
 
 ### 💎 Features

apps/app/bin/swagger-jsdoc/definition-apiv3.js

@@ -23,6 +23,11 @@ module.exports = {
         name: 'access_token',
         in: 'query',
       },
+      cookieAuth: {
+        type: 'apiKey',
+        in: 'cookie',
+        name: 'connect.sid',
+      },
     },
   },
   'x-tagGroups': [
@@ -57,6 +62,7 @@ module.exports = {
       name: 'System Management API',
       tags: [
         'Home',
+        'AdminHome',
         'AppSettings',
         'SecuritySetting',
         'MarkDownSetting',

apps/app/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@growi/app",
-  "version": "7.1.2-RC.0",
+  "version": "7.1.3-RC.0",
   "license": "MIT",
   "private": "true",
   "scripts": {

apps/app/src/features/openai/server/models/thread-relation.ts

@@ -1,14 +1,13 @@
+import { addDays } from 'date-fns';
 import type mongoose from 'mongoose';
 import { type Model, type Document, Schema } from 'mongoose';
 
 import { getOrCreateModel } from '~/server/util/mongoose-utils';
 
-const DAYS_UNTIL_EXPIRATION = 30;
+const DAYS_UNTIL_EXPIRATION = 3;
 
 const generateExpirationDate = (): Date => {
-  const currentDate = new Date();
-  const expirationDate = new Date(currentDate.setDate(currentDate.getDate() + DAYS_UNTIL_EXPIRATION));
-  return expirationDate;
+  return addDays(new Date(), DAYS_UNTIL_EXPIRATION);
 };
 
 interface ThreadRelation {

apps/app/src/features/openai/server/routes/index.ts

@@ -1,18 +1,36 @@
+import { ErrorV3 } from '@growi/core/dist/models';
 import express from 'express';
 
-import { postMessageHandlersFactory } from './message';
-import { rebuildVectorStoreHandlersFactory } from './rebuild-vector-store';
-import { createThreadHandlersFactory } from './thread';
+import type Crowi from '~/server/crowi';
+import type { ApiV3Response } from '~/server/routes/apiv3/interfaces/apiv3-response';
+
+import { isAiEnabled } from '../services';
 
 const router = express.Router();
 
-module.exports = (crowi) => {
-  router.post('/rebuild-vector-store', rebuildVectorStoreHandlersFactory(crowi));
 
-  // create thread
-  router.post('/thread', createThreadHandlersFactory(crowi));
-  // post message and return streaming with SSE
-  router.post('/message', postMessageHandlersFactory(crowi));
+export const factory = (crowi: Crowi): express.Router => {
+
+  // disable all routes if AI is not enabled
+  if (!isAiEnabled()) {
+    router.all('*', (req, res: ApiV3Response) => {
+      return res.apiv3Err(new ErrorV3('GROWI AI is not enabled'), 501);
+    });
+  }
+  // enabled
+  else {
+    import('./rebuild-vector-store').then(({ rebuildVectorStoreHandlersFactory }) => {
+      router.post('/rebuild-vector-store', rebuildVectorStoreHandlersFactory(crowi));
+    });
+
+    import('./thread').then(({ createThreadHandlersFactory }) => {
+      router.post('/thread', createThreadHandlersFactory(crowi));
+    });
+
+    import('./message').then(({ postMessageHandlersFactory }) => {
+      router.post('/message', postMessageHandlersFactory(crowi));
+    });
+  }
 
   return router;
 };

apps/app/src/features/openai/server/routes/message.ts

@@ -14,7 +14,7 @@ import type { ApiV3Response } from '~/server/routes/apiv3/interfaces/apiv3-respo
 import loggerFactory from '~/utils/logger';
 
 import { MessageErrorCode, type StreamErrorCode } from '../../interfaces/message-error';
-import { openaiClient } from '../services';
+import { openaiClient } from '../services/client';
 import { getStreamErrorCode } from '../services/getStreamErrorCode';
 import { replaceAnnotationWithPageLink } from '../services/replace-annotation-with-page-link';
 

apps/app/src/features/openai/server/services/cron/index.ts

@@ -0,0 +1,20 @@
+import loggerFactory from '~/utils/logger';
+
+import { isAiEnabled } from '../is-ai-enabled';
+
+
+const logger = loggerFactory('growi:openai:service:cron');
+
+export const startCronIfEnabled = async(): Promise<void> => {
+  if (isAiEnabled()) {
+    logger.info('Starting cron service for thread deletion');
+    const { ThreadDeletionCronService } = await import('./thread-deletion-cron');
+    const threadDeletionCronService = new ThreadDeletionCronService();
+    threadDeletionCronService.startCron();
+
+    logger.info('Starting cron service for vector store file deletion');
+    const { VectorStoreFileDeletionCronService } = await import('./vector-store-file-deletion-cron');
+    const vectorStoreFileDeletionCronService = new VectorStoreFileDeletionCronService();
+    vectorStoreFileDeletionCronService.startCron();
+  }
+};

apps/app/src/features/openai/server/services/thread-deletion-cron.ts → apps/app/src/features/openai/server/services/cron/thread-deletion-cron.ts

@@ -4,11 +4,13 @@ import { configManager } from '~/server/service/config-manager';
 import loggerFactory from '~/utils/logger';
 import { getRandomIntInRange } from '~/utils/rand';
 
-import { getOpenaiService, type IOpenaiService } from './openai';
+import { isAiEnabled } from '../is-ai-enabled';
+import { getOpenaiService, type IOpenaiService } from '../openai';
+
 
 const logger = loggerFactory('growi:service:thread-deletion-cron');
 
-class ThreadDeletionCronService {
+export class ThreadDeletionCronService {
 
   cronJob: nodeCron.ScheduledTask;
 
@@ -25,8 +27,7 @@ class ThreadDeletionCronService {
   sleep = (msec: number): Promise<void> => new Promise(resolve => setTimeout(resolve, msec));
 
   startCron(): void {
-    const isAiEnabled = configManager.getConfig('crowi', 'app:aiEnabled');
-    if (!isAiEnabled) {
+    if (!isAiEnabled()) {
       return;
     }
 
@@ -67,5 +68,3 @@ class ThreadDeletionCronService {
   }
 
 }
-
-export default ThreadDeletionCronService;

apps/app/src/features/openai/server/services/vector-store-file-deletion-cron.ts → apps/app/src/features/openai/server/services/cron/vector-store-file-deletion-cron.ts

@@ -4,11 +4,12 @@ import { configManager } from '~/server/service/config-manager';
 import loggerFactory from '~/utils/logger';
 import { getRandomIntInRange } from '~/utils/rand';
 
-import { getOpenaiService, type IOpenaiService } from './openai';
+import { isAiEnabled } from '../is-ai-enabled';
+import { getOpenaiService, type IOpenaiService } from '../openai';
 
 const logger = loggerFactory('growi:service:vector-store-file-deletion-cron');
 
-class VectorStoreFileDeletionCronService {
+export class VectorStoreFileDeletionCronService {
 
   cronJob: nodeCron.ScheduledTask;
 
@@ -25,8 +26,7 @@ class VectorStoreFileDeletionCronService {
   sleep = (msec: number): Promise<void> => new Promise(resolve => setTimeout(resolve, msec));
 
   startCron(): void {
-    const isAiEnabled = configManager.getConfig('crowi', 'app:aiEnabled');
-    if (!isAiEnabled) {
+    if (!isAiEnabled()) {
       return;
     }
 
@@ -67,5 +67,3 @@ class VectorStoreFileDeletionCronService {
   }
 
 }
-
-export default VectorStoreFileDeletionCronService;

apps/app/src/features/openai/server/services/index.ts

@@ -1,2 +1 @@
-export * from './embeddings';
-export * from './client';
+export * from './is-ai-enabled';

apps/app/src/features/openai/server/services/is-ai-enabled.ts

@@ -0,0 +1,3 @@
+import { configManager } from '~/server/service/config-manager';
+
+export const isAiEnabled = (): boolean => configManager.getConfig('crowi', 'app:aiEnabled');

apps/app/src/features/openai/server/services/normalize-data/index.ts

@@ -0,0 +1 @@
+export * from './normalize-thread-relation-expired-at';

apps/app/src/features/openai/server/services/normalize-data/normalize-thread-relation-expired-at/index.ts

@@ -0,0 +1 @@
+export * from './normalize-thread-relation-expired-at';

apps/app/src/features/openai/server/services/normalize-data/normalize-thread-relation-expired-at/normalize-thread-relation-expired-at.integ.ts

@@ -0,0 +1,70 @@
+import { faker } from '@faker-js/faker';
+import { addDays, subDays } from 'date-fns';
+import { Types } from 'mongoose';
+
+import ThreadRelation from '../../../models/thread-relation';
+
+import { MAX_DAYS_UNTIL_EXPIRATION, normalizeExpiredAtForThreadRelations } from './normalize-thread-relation-expired-at';
+
+describe('normalizeExpiredAtForThreadRelations', () => {
+
+  it('should update expiredAt to 3 days from now for expired thread relations', async() => {
+    // arrange
+    const expiredDays = faker.number.int({ min: MAX_DAYS_UNTIL_EXPIRATION, max: 30 });
+    const expiredDate = addDays(new Date(), expiredDays);
+    const threadRelation = new ThreadRelation({
+      userId: new Types.ObjectId(),
+      threadId: 'test-thread',
+      expiredAt: expiredDate,
+    });
+    await threadRelation.save();
+
+    // act
+    await normalizeExpiredAtForThreadRelations();
+
+    // assert
+    const updatedThreadRelation = await ThreadRelation.findById(threadRelation._id);
+    expect(updatedThreadRelation).not.toBeNull();
+    assert(updatedThreadRelation?.expiredAt != null);
+    expect(updatedThreadRelation.expiredAt < addDays(new Date(), MAX_DAYS_UNTIL_EXPIRATION)).toBeTruthy();
+  });
+
+  it('should not update expiredAt for non-expired thread relations', async() => {
+    // arrange
+    const nonExpiredDays = faker.number.int({ min: 0, max: MAX_DAYS_UNTIL_EXPIRATION });
+    const nonExpiredDate = addDays(new Date(), nonExpiredDays);
+    const threadRelation = new ThreadRelation({
+      userId: new Types.ObjectId(),
+      threadId: 'test-thread-2',
+      expiredAt: nonExpiredDate,
+    });
+    await threadRelation.save();
+
+    // act
+    await normalizeExpiredAtForThreadRelations();
+
+    // assert
+    const updatedThreadRelation = await ThreadRelation.findById(threadRelation._id);
+    expect(updatedThreadRelation).not.toBeNull();
+    expect(updatedThreadRelation?.expiredAt).toEqual(nonExpiredDate);
+  });
+
+  it('should not update expiredAt is before today', async() => {
+    // arrange
+    const nonExpiredDate = subDays(new Date(), 1);
+    const threadRelation = new ThreadRelation({
+      userId: new Types.ObjectId(),
+      threadId: 'test-thread-3',
+      expiredAt: nonExpiredDate,
+    });
+    await threadRelation.save();
+
+    // act
+    await normalizeExpiredAtForThreadRelations();
+
+    // assert
+    const updatedThreadRelation = await ThreadRelation.findById(threadRelation._id);
+    expect(updatedThreadRelation).not.toBeNull();
+    expect(updatedThreadRelation?.expiredAt).toEqual(nonExpiredDate);
+  });
+});

apps/app/src/features/openai/server/services/normalize-data/normalize-thread-relation-expired-at/normalize-thread-relation-expired-at.ts

@@ -0,0 +1,14 @@
+import { addDays } from 'date-fns';
+
+import ThreadRelation from '../../../models/thread-relation';
+
+export const MAX_DAYS_UNTIL_EXPIRATION = 3;
+
+export const normalizeExpiredAtForThreadRelations = async(): Promise<void> => {
+  const maxDaysExpiredAt = addDays(new Date(), MAX_DAYS_UNTIL_EXPIRATION);
+
+  await ThreadRelation.updateMany(
+    { expiredAt: { $gt: maxDaysExpiredAt } },
+    { $set: { expiredAt: maxDaysExpiredAt } },
+  );
+};

apps/app/src/features/openai/server/services/openai.ts

@@ -19,11 +19,13 @@ import { createBatchStream } from '~/server/util/batch-stream';
 import loggerFactory from '~/utils/logger';
 
 import { OpenaiServiceTypes } from '../../interfaces/ai';
+import { sanitizeMarkdown } from '../utils/sanitize-markdown';
 
 import { getClient } from './client-delegator';
 // import { splitMarkdownIntoChunks } from './markdown-splitter/markdown-token-splitter';
 import { oepnaiApiErrorHandler } from './openai-api-error-handler';
 
+
 const BATCH_SIZE = 100;
 
 const logger = loggerFactory('growi:service:openai');
@@ -155,7 +157,8 @@ class OpenaiService implements IOpenaiService {
   // }
 
   private async uploadFile(pageId: Types.ObjectId, body: string): Promise<OpenAI.Files.FileObject> {
-    const file = await toFile(Readable.from(body), `${pageId}.md`);
+    const sanitizedMarkdown = await sanitizeMarkdown(body);
+    const file = await toFile(Readable.from(sanitizedMarkdown), `${pageId}.md`);
     const uploadedFile = await this.client.uploadFile(file);
     return uploadedFile;
   }

apps/app/src/features/openai/server/utils/sanitize-markdown.ts

@@ -0,0 +1,65 @@
+import { dynamicImport } from '@cspell/dynamic-import';
+import type { Root, Code } from 'mdast';
+import type * as RemarkParse from 'remark-parse';
+import type * as RemarkStringify from 'remark-stringify';
+import type * as Unified from 'unified';
+import type * as UnistUtilVisit from 'unist-util-visit';
+
+interface ModuleCache {
+  remarkParse?: typeof RemarkParse.default;
+  remarkStringify?: typeof RemarkStringify.default;
+  unified?: typeof Unified.unified;
+  visit?: typeof UnistUtilVisit.visit;
+}
+
+let moduleCache: ModuleCache = {};
+
+const initializeModules = async(): Promise<void> => {
+  if (moduleCache.remarkParse != null && moduleCache.remarkStringify != null && moduleCache.unified != null && moduleCache.visit != null) {
+    return;
+  }
+
+  const [{ default: remarkParse }, { default: remarkStringify }, { unified }, { visit }] = await Promise.all([
+    dynamicImport<typeof RemarkParse>('remark-parse', __dirname),
+    dynamicImport<typeof RemarkStringify>('remark-stringify', __dirname),
+    dynamicImport<typeof Unified>('unified', __dirname),
+    dynamicImport<typeof UnistUtilVisit>('unist-util-visit', __dirname),
+  ]);
+
+  moduleCache = {
+    remarkParse,
+    remarkStringify,
+    unified,
+    visit,
+  };
+};
+
+export const sanitizeMarkdown = async(markdown: string): Promise<string> => {
+  await initializeModules();
+
+  const {
+    remarkParse, remarkStringify, unified, visit,
+  } = moduleCache;
+
+
+  if (remarkParse == null || remarkStringify == null || unified == null || visit == null) {
+    throw new Error('Failed to initialize required modules');
+  }
+
+  const sanitize = () => {
+    return (tree: Root) => {
+      visit(tree, 'code', (node: Code) => {
+        if (node.lang === 'drawio') {
+          node.value = '<!-- drawio content replaced -->';
+        }
+      });
+    };
+  };
+
+  const processor = unified()
+    .use(remarkParse)
+    .use(sanitize)
+    .use(remarkStringify);
+
+  return processor.processSync(markdown).toString();
+};

apps/app/src/server/crowi/index.js

@@ -12,8 +12,7 @@ import pkg from '^/package.json';
 
 import { KeycloakUserGroupSyncService } from '~/features/external-user-group/server/service/keycloak-user-group-sync';
 import { LdapUserGroupSyncService } from '~/features/external-user-group/server/service/ldap-user-group-sync';
-import OpenaiThreadDeletionCronService from '~/features/openai/server/services/thread-deletion-cron';
-import OpenaiVectorStoreFileDeletionCronService from '~/features/openai/server/services/vector-store-file-deletion-cron';
+import { startCronIfEnabled as startOpenaiCronIfEnabled } from '~/features/openai/server/services/cron';
 import { startInstrumentation } from '~/features/opentelemetry/server';
 import QuestionnaireService from '~/features/questionnaire/server/service/questionnaire';
 import QuestionnaireCronService from '~/features/questionnaire/server/service/questionnaire-cron';
@@ -330,11 +329,7 @@ Crowi.prototype.setupCron = function() {
   this.questionnaireCronService = new QuestionnaireCronService(this);
   this.questionnaireCronService.startCron();
 
-  this.openaiThreadDeletionCronService = new OpenaiThreadDeletionCronService();
-  this.openaiThreadDeletionCronService.startCron();
-
-  this.openaiThreadDeletionCronService = new OpenaiVectorStoreFileDeletionCronService();
-  this.openaiThreadDeletionCronService.startCron();
+  startOpenaiCronIfEnabled();
 };
 
 Crowi.prototype.setupQuestionnaireService = function() {

apps/app/src/server/routes/apiv3/admin-home.js

@@ -14,16 +14,41 @@ const router = express.Router();
  *        properties:
  *          growiVersion:
  *            type: string
- *            description: version of growi
+ *            description: GROWI version or '-'
+ *            example: 7.1.0-RC.0
  *          nodeVersion:
  *            type: string
- *            description: version of node
+ *            description: node version or '-'
+ *            example: 20.2.0
  *          npmVersion:
  *            type: string
- *            description: version of npm
+ *            description: npm version or '-'
+ *            example: 9.6.6
  *          pnpmVersion:
  *            type: string
- *            description: version of pnpm
+ *            description: pnpm version or '-'
+ *            example: 9.12.3
+ *          envVars:
+ *            type: object
+ *            description: environment variables
+ *            additionalProperties:
+ *              type: string
+ *            example:
+ *              "FILE_UPLOAD": "mongodb"
+ *              "APP_SITE_URL": "http://localhost:3000"
+ *              "ELASTICSEARCH_URI": "http://elasticsearch:9200/growi"
+ *              "ELASTICSEARCH_REQUEST_TIMEOUT": 15000
+ *              "ELASTICSEARCH_REJECT_UNAUTHORIZED": true
+ *              "OGP_URI": "http://ogp:8088"
+ *              "QUESTIONNAIRE_SERVER_ORIGIN": "http://host.docker.internal:3003"
+ *          isV5Compatible:
+ *            type: boolean
+ *            description: This value is true if this GROWI is compatible v5.
+ *            example: true
+ *          isMaintenanceMode:
+ *            type: boolean
+ *            description: This value is true if this site is maintenance mode.
+ *            example: false
  *      InstalledPluginsParams:
  *        type: object
  *        properties:
@@ -41,9 +66,11 @@ module.exports = (crowi) => {
    *
    *    /admin-home/:
    *      get:
-   *        tags: [Admin]
+   *        tags: [AdminHome]
    *        operationId: getAdminHome
    *        summary: /admin-home
+   *        security:
+   *          - cookieAuth: []
    *        description: Get adminHome parameters
    *        responses:
    *          200:
@@ -53,8 +80,7 @@ module.exports = (crowi) => {
    *                schema:
    *                  properties:
    *                    adminHomeParams:
-   *                      type: object
-   *                      description: adminHome params
+   *                      $ref: "#/components/schemas/SystemInformationParams"
    */
   router.get('/', loginRequiredStrictly, adminRequired, async(req, res) => {
     const adminHomeParams = {

apps/app/src/server/routes/apiv3/app-settings.js

@@ -19,6 +19,7 @@ const express = require('express');
 
 const router = express.Router();
 
+
 /**
  * @swagger
  *
@@ -28,21 +29,133 @@ const router = express.Router();
  *        description: AppSettingParams
  *        type: object
  *        properties:
+ *          azureReferenceFileWithRelayMode:
+ *            type: boolean
+ *            example: false
+ *          azureUseOnlyEnvVars:
+ *            type: boolean
+ *            example: false
+ *          confidential:
+ *            type: string
+ *            description: confidential show on page header
+ *            example: 'GROWI'
+ *          envAzureClientId:
+ *            type: string
+ *            example: 'AZURE_CLIENT_ID'
+ *          envAzureClientSecret:
+ *            type: string
+ *            example: 'AZURE_CLIENT_SECRET'
+ *          envAzureStorageAccountName:
+ *           type: string
+ *           example: 'AZURE_STORAGE_ACCOUNT_NAME'
+ *          envAzureStorageContainerName:
+ *            type: string
+ *            example: 'AZURE_STORAGE_CONTAINER_NAME'
+ *          envFileUploadType:
+ *            type: string
+ *            example: 'mongodb'
+ *          envGcsApiKeyJsonPath:
+ *            type: string
+ *            example: 'GCS_API_KEY_JSON_PATH'
+ *          envGcsBucket:
+ *            type: string
+ *            example: 'GCS_BUCKET'
+ *          envGcsUploadNamespace:
+ *            type: string
+ *            example: 'GCS_UPLOAD_NAMESPACE'
+ *          envSiteUrl:
+ *            type: string
+ *            example: 'http://localhost:3000'
+ *          fileUpload:
+ *            type: boolean
+ *            example: true
+ *          fileUploadType:
+ *            type: string
+ *            example: 'local'
+ *          fromAddress:
+ *            type: string
+ *            example: info@growi.org
+ *          gcsApiKeyJsonPath:
+ *            type: string
+ *            example: 'GCS_API_KEY_JSON_PATH'
+ *          gcsBucket:
+ *            type: string
+ *            example: 'GCS_BUCKET'
+ *          gcsReferenceFileWithRelayMode:
+ *            type: boolean
+ *            example: false
+ *          gcsUploadNamespace:
+ *            type: string
+ *            example: 'GCS_UPLOAD_NAMESPACE'
+ *          gcsUseOnlyEnvVars:
+ *            type: boolean
+ *            example: false
+ *          globalLang:
+ *            type: string
+ *            example: 'ja_JP'
+ *          isAppSiteUrlHashed:
+ *            type: boolean
+ *            example: false
+ *          isEmailPublishedForNewUser:
+ *            type: boolean
+ *            example: true
+ *          isMaintenanceMode:
+ *            type: boolean
+ *            example: false
+ *          isQuestionnaireEnabled:
+ *            type: boolean
+ *            example: true
+ *          isV5Compatible:
+ *            type: boolean
+ *            example: true
+ *          s3AccessKeyId:
+ *            type: string
+ *          s3Bucket:
+ *            type: string
+ *          s3CustomEndpoint:
+ *            type: string
+ *          s3ReferenceFileWithRelayMode:
+ *            type: boolean
+ *          s3Region:
+ *            type: string
+ *          siteUrl:
+ *            type: string
+ *          siteUrlUseOnlyEnvVars:
+ *            type: boolean
+ *          smtpHost:
+ *            type: string
+ *          smtpPassword:
+ *            type: string
+ *          smtpPort:
+ *            type: string
+ *          smtpUser:
+ *            type: string
+ *          useOnlyEnvVarForFileUploadType:
+ *            type: boolean
+ *      AppSettingPutParams:
+ *        description: AppSettingPutParams
+ *        type: object
+ *        properties:
  *          title:
  *            type: string
- *            description: site name show on page header and tilte of HTML
+ *            description: title of the site
+ *            example: 'GROWI'
  *          confidential:
  *            type: string
  *            description: confidential show on page header
+ *            example: 'GROWI'
  *          globalLang:
  *            type: string
- *            description: language set when create user
+ *            description: global language
+ *            example: 'ja_JP'
  *          isEmailPublishedForNewUser:
  *            type: boolean
- *            description: default email show/hide setting when create user
+ *            description: is email published for new user, or not
+ *            example: true
  *          fileUpload:
  *            type: boolean
- *            description: enable upload file except image file
+ *            description: is file upload enabled, or not
+ *            example: true
  *      SiteUrlSettingParams:
  *        description: SiteUrlSettingParams
  *        type: object
@@ -53,40 +166,96 @@ const router = express.Router();
  *          envSiteUrl:
  *            type: string
  *            description: environment variable 'APP_SITE_URL'
- *      MailSetting:
- *        description: MailSettingParams
+ *      SmtpSettingParams:
+ *        description: SmtpSettingParams
  *        type: object
  *        properties:
- *          fromAddress:
+ *          smtpHost:
  *            type: string
- *            description: e-mail address used as from address of mail which sent from GROWI app
- *          transmissionMethod:
+ *            description: host name of client's smtp server
+ *            example: 'smtp.example.com'
+ *          smtpPort:
  *            type: string
- *            description: transmission method
- *      SmtpSettingParams:
- *        description: SmtpSettingParams
+ *            description: port of client's smtp server
+ *            example: '587'
+ *          smtpUser:
+ *            type: string
+ *            description: user name of client's smtp server
+ *            example: 'USER'
+ *          smtpPassword:
+ *            type: string
+ *            description: password of client's smtp server
+ *            example: 'PASSWORD'
+ *          fromAddress:
+ *            type: string
+ *            description: e-mail address
+ *            example: 'info@example.com'
+ *      SmtpSettingResponseParams:
+ *        description: SmtpSettingResponseParams
  *        type: object
  *        properties:
+ *          isMailerSetup:
+ *            type: boolean
+ *            description: is mailer setup, or not
+ *            example: true
  *          smtpHost:
  *            type: string
  *            description: host name of client's smtp server
+ *            example: 'smtp.example.com'
  *          smtpPort:
  *            type: string
  *            description: port of client's smtp server
+ *            example: '587'
  *          smtpUser:
  *            type: string
  *            description: user name of client's smtp server
+ *            example: 'USER'
  *          smtpPassword:
  *            type: string
  *            description: password of client's smtp server
+ *            example: 'PASSWORD'
+ *          fromAddress:
+ *            type: string
+ *            description: e-mail address
+ *            example: 'info@example.com'
  *      SesSettingParams:
  *        description: SesSettingParams
  *        type: object
  *        properties:
- *          accessKeyId:
+ *          from:
+ *            type: string
+ *            description: e-mail address used as from address of mail which sent from GROWI app
+ *            example: 'info@growi.org'
+ *          transmissionMethod:
+ *            type: string
+ *            description: transmission method
+ *            example: 'ses'
+ *          sesAccessKeyId:
+ *            type: string
+ *            description: accesskey id for authentification of AWS
+ *          sesSecretAccessKey:
+ *            type: string
+ *            description: secret key for authentification of AWS
+ *      SesSettingResponseParams:
+ *        description: SesSettingParams
+ *        type: object
+ *        properties:
+ *          isMailerSetup:
+ *            type: boolean
+ *            description: is mailer setup, or not
+ *            example: true
+ *          from:
+ *            type: string
+ *            description: e-mail address used as from address of mail which sent from GROWI app
+ *            example: 'info@growi.org'
+ *          transmissionMethod:
+ *            type: string
+ *            description: transmission method
+ *            example: 'ses'
+ *          sesAccessKeyId:
  *            type: string
  *            description: accesskey id for authentification of AWS
- *          secretAccessKey:
+ *          sesSecretAccessKey:
  *            type: string
  *            description: secret key for authentification of AWS
  *      FileUploadSettingParams:
@@ -126,22 +295,35 @@ const router = express.Router();
  *          gcsReferenceFileWithRelayMode:
  *            type: boolean
  *            description: is enable internal stream system for gcs file request
- *          envGcsApiKeyJsonPath:
+ *          azureTenantId:
  *            type: string
- *            description: Path of the JSON file that contains service account key to authenticate to GCP API
- *          envGcsBucket:
+ *            description: tenant id of azure
+ *          azureClientId:
  *            type: string
- *            description: Name of the GCS bucket
- *          envGcsUploadNamespace:
+ *            description: client id of azure
+ *          azureClientSecret:
  *            type: string
- *            description: Directory name to create in the bucket
- *      PluginSettingParams:
- *        description: PluginSettingParams
+ *            description: client secret of azure
+ *          azureStorageAccountName:
+ *            type: string
+ *            description: storage account name of azure
+ *          azureStorageContainerName:
+ *            type: string
+ *            description: storage container name of azure
+ *          azureReferenceFileWithRelayMode:
+ *            type: boolean
+ *            description: is enable internal stream system for azure file request
+ *      QuestionnaireSettingParams:
+ *        description: QuestionnaireSettingParams
  *        type: object
  *        properties:
- *          isEnabledPlugins:
- *            type: string
- *            description: enable use plugins
+ *          isQuestionnaireEnabled:
+ *            type: boolean
+ *            description: is questionnaire enabled, or not
+ *            example: true
+ *          isAppSiteUrlHashed:
+ *            type: boolean
+ *            description: is app site url hashed, or not
  */
 
 module.exports = (crowi) => {
@@ -231,6 +413,8 @@ module.exports = (crowi) => {
    *      get:
    *        tags: [AppSettings]
    *        operationId: getAppSettings
+   *        security:
+   *          - api_key: []
    *        summary: /app-settings
    *        description: get app setting params
    *        responses:
@@ -242,7 +426,7 @@ module.exports = (crowi) => {
    *                  properties:
    *                    appSettingsParams:
    *                      type: object
-   *                      description: app settings params
+   *                      $ref: '#/components/schemas/AppSettingParams'
    */
   router.get('/', accessTokenParser, loginRequiredStrictly, adminRequired, async(req, res) => {
     const appSettingsParams = {
@@ -318,22 +502,28 @@ module.exports = (crowi) => {
    *    /app-settings/app-setting:
    *      put:
    *        tags: [AppSettings]
-   *        summary: /app-settings/app-setting
    *        operationId: updateAppSettings
+   *        security:
+   *          - cookieAuth: []
+   *        summary: /app-settings/app-setting
    *        description: Update app setting
    *        requestBody:
    *          required: true
    *          content:
    *            application/json:
    *              schema:
-   *                $ref: '#/components/schemas/AppSettingParams'
+   *                $ref: '#/components/schemas/AppSettingPutParams'
    *        responses:
    *          200:
    *            description: Succeeded to update app setting
    *            content:
    *              application/json:
    *                schema:
-   *                  $ref: '#/components/schemas/AppSettingParams'
+   *                  type: object
+   *                  properties:
+   *                    appSettingParams:
+   *                      type: object
+   *                      $ref: '#/components/schemas/AppSettingPutParams'
    */
   router.put('/app-setting', loginRequiredStrictly, adminRequired, addActivity, validator.appSetting, apiV3FormValidator, async(req, res) => {
     const requestAppSettingParams = {
@@ -374,6 +564,8 @@ module.exports = (crowi) => {
    *      put:
    *        tags: [AppSettings]
    *        operationId: updateAppSettingSiteUrlSetting
+   *        security:
+   *          - cookieAuth: []
    *        summary: /app-settings/site-url-setting
    *        description: Update site url setting
    *        requestBody:
@@ -388,7 +580,15 @@ module.exports = (crowi) => {
    *            content:
    *              application/json:
    *                schema:
-   *                  $ref: '#/components/schemas/SiteUrlSettingParams'
+   *                  type: object
+   *                  properties:
+   *                    siteUrlSettingParams:
+   *                      type: object
+   *                      properties:
+   *                        siteUrl:
+   *                          type: string
+   *                          description: Site URL. e.g. https://example.com, https://example.com:3000
+   *                          example: 'http://localhost:3000'
    */
   router.put('/site-url-setting', loginRequiredStrictly, adminRequired, addActivity, validator.siteUrlSetting, apiV3FormValidator, async(req, res) => {
 
@@ -516,6 +716,8 @@ module.exports = (crowi) => {
    *      put:
    *        tags: [AppSettings]
    *        operationId: updateAppSettingSmtpSetting
+   *        security:
+   *          - cookieAuth: []
    *        summary: /app-settings/smtp-setting
    *        description: Update smtp setting
    *        requestBody:
@@ -530,7 +732,11 @@ module.exports = (crowi) => {
    *            content:
    *              application/json:
    *                schema:
-   *                  $ref: '#/components/schemas/SmtpSettingParams'
+   *                  type: object
+   *                  properties:
+   *                    mailSettingParams:
+   *                      type: object
+   *                      $ref: '#/components/schemas/SmtpSettingResponseParams'
    */
   router.put('/smtp-setting', loginRequiredStrictly, adminRequired, addActivity, validator.smtpSetting, apiV3FormValidator, async(req, res) => {
     const requestMailSettingParams = {
@@ -562,11 +768,18 @@ module.exports = (crowi) => {
    *      post:
    *        tags: [AppSettings]
    *        operationId: postSmtpTest
+   *        security:
+   *          - cookieAuth: []
    *        summary: /app-settings/smtp-setting
    *        description: Send test mail for smtp
    *        responses:
    *          200:
    *            description: Succeeded to send test mail for smtp
+   *            content:
+   *              application/json:
+   *                schema:
+   *                  type: object
+   *                  description: Empty object
    */
   router.post('/smtp-test', loginRequiredStrictly, adminRequired, addActivity, async(req, res) => {
     const { t } = await getTranslation({ lang: req.user.lang });
@@ -592,6 +805,8 @@ module.exports = (crowi) => {
    *      put:
    *        tags: [AppSettings]
    *        operationId: updateAppSettingSesSetting
+   *        security:
+   *          - cookieAuth: []
    *        summary: /app-settings/ses-setting
    *        description: Update ses setting
    *        requestBody:
@@ -606,7 +821,7 @@ module.exports = (crowi) => {
    *            content:
    *              application/json:
    *                schema:
-   *                  $ref: '#/components/schemas/SesSettingParams'
+   *                  $ref: '#/components/schemas/SesSettingResponseParams'
    */
   router.put('/ses-setting', loginRequiredStrictly, adminRequired, addActivity, validator.sesSetting, apiV3FormValidator, async(req, res) => {
     const { mailService } = crowi;
@@ -642,6 +857,8 @@ module.exports = (crowi) => {
    *      put:
    *        tags: [AppSettings]
    *        operationId: updateAppSettingFileUploadSetting
+   *        security:
+   *          - cookieAuth: []
    *        summary: /app-settings/file-upload-setting
    *        description: Update fileUploadSetting
    *        requestBody:
@@ -656,7 +873,11 @@ module.exports = (crowi) => {
    *            content:
    *              application/json:
    *                schema:
-   *                  $ref: '#/components/schemas/FileUploadSettingParams'
+   *                  type: object
+   *                  properties:
+   *                    responseParams:
+   *                      type: object
+   *                      $ref: '#/components/schemas/FileUploadSettingParams'
    */
   //  eslint-disable-next-line max-len
   router.put('/file-upload-setting', loginRequiredStrictly, adminRequired, addActivity, validator.fileUploadSetting, apiV3FormValidator, async(req, res) => {
@@ -740,6 +961,35 @@ module.exports = (crowi) => {
 
   });
 
+  /**
+   * @swagger
+   *
+   *    /app-settings/questionnaire-settings:
+   *      put:
+   *        tags: [AppSettings]
+   *        operationId: updateAppSettingQuestionnaireSettings
+   *        security:
+   *          - cookieAuth: []
+   *        summary: /app-settings/questionnaire-settings
+   *        description: Update QuestionnaireSetting
+   *        requestBody:
+   *          required: true
+   *          content:
+   *            application/json:
+   *              schema:
+   *                $ref: '#/components/schemas/QuestionnaireSettingParams'
+   *        responses:
+   *          200:
+   *            description: Succeeded to update QuestionnaireSetting
+   *            content:
+   *              application/json:
+   *                schema:
+   *                  type: object
+   *                  properties:
+   *                    responseParams:
+   *                      type: object
+   *                      $ref: '#/components/schemas/QuestionnaireSettingParams'
+   */
   // eslint-disable-next-line max-len
   router.put('/questionnaire-settings', loginRequiredStrictly, adminRequired, addActivity, validator.questionnaireSettings, apiV3FormValidator, async(req, res) => {
     const { isQuestionnaireEnabled, isAppSiteUrlHashed } = req.body;
@@ -769,6 +1019,30 @@ module.exports = (crowi) => {
 
   });
 
+  /**
+   * @swagger
+   *
+   *    /app-settings/v5-schema-migration:
+   *      post:
+   *        tags: [AppSettings]
+   *        operationId: updateAppSettingV5SchemaMigration
+   *        security:
+   *          - api_key: []
+   *        summary: AccessToken supported.
+   *        description: Update V5SchemaMigration
+   *        responses:
+   *          200:
+   *            description: Succeeded to get V5SchemaMigration
+   *            content:
+   *              application/json:
+   *                schema:
+   *                  type: object
+   *                  properties:
+   *                    isV5Compatible:
+   *                      type: boolean
+   *                      description: is V5 compatible, or not
+   *                      example: true
+   */
   router.post('/v5-schema-migration', accessTokenParser, loginRequiredStrictly, adminRequired, async(req, res) => {
     const isMaintenanceMode = crowi.appService.isMaintenanceMode();
     if (!isMaintenanceMode) {
@@ -790,6 +1064,39 @@ module.exports = (crowi) => {
     return res.apiv3({ isV5Compatible });
   });
 
+  /**
+   * @swagger
+   *
+   *    /app-settings/maintenance-mode:
+   *      post:
+   *        tags: [AppSettings]
+   *        operationId: updateAppSettingMaintenanceMode
+   *        security:
+   *          - api_key: []
+   *        summary: AccessToken supported.
+   *        description: Update MaintenanceMode
+   *        requestBody:
+   *          content:
+   *            application/json:
+   *              schema:
+   *                type: object
+   *                properties:
+   *                  flag:
+   *                    type: boolean
+   *                    description: flag for maintenance mode
+   *        responses:
+   *          200:
+   *            description: Succeeded to update MaintenanceMode
+   *            content:
+   *              application/json:
+   *                schema:
+   *                  type: object
+   *                  properties:
+   *                    flag:
+   *                      type: boolean
+   *                      description: true if maintenance mode is enabled
+   *                      example: true
+   */
   // eslint-disable-next-line max-len
   router.post('/maintenance-mode', accessTokenParser, loginRequiredStrictly, adminRequired, addActivity, validator.maintenanceMode, apiV3FormValidator, async(req, res) => {
     const { flag } = req.body;

apps/app/src/server/routes/apiv3/index.js

@@ -1,5 +1,5 @@
 import growiPlugin from '~/features/growi-plugin/server/routes/apiv3/admin';
-import openai from '~/features/openai/server/routes';
+import { factory as openaiRouteFactory } from '~/features/openai/server/routes';
 import loggerFactory from '~/utils/logger';
 
 import { generateAddActivityMiddleware } from '../../middlewares/add-activity';
@@ -120,7 +120,7 @@ module.exports = (crowi, app) => {
   router.use('/questionnaire', require('~/features/questionnaire/server/routes/apiv3/questionnaire')(crowi));
   router.use('/templates', require('~/features/templates/server/routes/apiv3')(crowi));
 
-  router.use('/openai', openai(crowi));
+  router.use('/openai', openaiRouteFactory(crowi));
 
   return [router, routerForAdmin, routerForAuth];
 };

apps/app/src/server/routes/apiv3/page/create-page.ts

@@ -11,7 +11,7 @@ import { body } from 'express-validator';
 import type { HydratedDocument } from 'mongoose';
 import mongoose from 'mongoose';
 
-import { getOpenaiService } from '~/features/openai/server/services/openai';
+import { isAiEnabled } from '~/features/openai/server/services';
 import { SupportedAction, SupportedTargetModel } from '~/interfaces/activity';
 import type { IApiv3PageCreateParams } from '~/interfaces/apiv3';
 import { subscribeRuleNames } from '~/interfaces/in-app-notification';
@@ -202,12 +202,15 @@ export const createPageHandlersFactory: CreatePageHandlersFactory = (crowi) => {
     }
 
     // Rebuild vector store file
-    try {
-      const openaiService = getOpenaiService();
-      await openaiService?.rebuildVectorStore(createdPage);
-    }
-    catch (err) {
-      logger.error('Rebuild vector store failed', err);
+    if (isAiEnabled()) {
+      const { getOpenaiService } = await import('~/features/openai/server/services/openai');
+      try {
+        const openaiService = getOpenaiService();
+        await openaiService?.rebuildVectorStore(createdPage);
+      }
+      catch (err) {
+        logger.error('Rebuild vector store failed', err);
+      }
     }
   }
 

apps/app/src/server/routes/apiv3/page/update-page.ts

@@ -11,7 +11,7 @@ import { body } from 'express-validator';
 import type { HydratedDocument } from 'mongoose';
 import mongoose from 'mongoose';
 
-import { getOpenaiService } from '~/features/openai/server/services/openai';
+import { isAiEnabled } from '~/features/openai/server/services';
 import { SupportedAction, SupportedTargetModel } from '~/interfaces/activity';
 import { type IApiv3PageUpdateParams, PageUpdateErrorCode } from '~/interfaces/apiv3';
 import type { IOptionsForUpdate } from '~/interfaces/page';
@@ -118,12 +118,15 @@ export const updatePageHandlersFactory: UpdatePageHandlersFactory = (crowi) => {
     }
 
     // Rebuild vector store file
-    try {
-      const openaiService = getOpenaiService();
-      await openaiService?.rebuildVectorStore(updatedPage);
-    }
-    catch (err) {
-      logger.error('Rebuild vector store failed', err);
+    if (isAiEnabled()) {
+      const { getOpenaiService } = await import('~/features/openai/server/services/openai');
+      try {
+        const openaiService = getOpenaiService();
+        await openaiService?.rebuildVectorStore(updatedPage);
+      }
+      catch (err) {
+        logger.error('Rebuild vector store failed', err);
+      }
     }
   }
 

apps/app/src/server/service/config-loader.ts

@@ -839,7 +839,7 @@ Guideline as a RAG:
     ns: 'crowi',
     key: 'app:openaiThreadDeletionCronMaxMinutesUntilRequest',
     type: ValueType.NUMBER,
-    default: 60,
+    default: 30,
   },
   OPENAI_THREAD_DELETION_BARCH_SIZE: {
     ns: 'crowi',
@@ -863,7 +863,7 @@ Guideline as a RAG:
     ns: 'crowi',
     key: 'app:openaiVectorStoreFileDeletionCronMaxMinutesUntilRequest',
     type: ValueType.NUMBER,
-    default: 60,
+    default: 30,
   },
   OPENAI_VECTOR_STORE_FILE_DELETION_BARCH_SIZE: {
     ns: 'crowi',

apps/app/src/server/service/normalize-data/index.ts

@@ -1,3 +1,4 @@
+import { normalizeExpiredAtForThreadRelations } from '~/features/openai/server/services/normalize-data';
 import loggerFactory from '~/utils/logger';
 
 import { convertRevisionPageIdToObjectId } from './convert-revision-page-id-to-objectid';
@@ -8,6 +9,7 @@ const logger = loggerFactory('growi:service:NormalizeData');
 export const normalizeData = async(): Promise<void> => {
   await renameDuplicateRootPages();
   await convertRevisionPageIdToObjectId();
+  await normalizeExpiredAtForThreadRelations();
 
   logger.info('normalizeData has been executed');
   return;

apps/app/src/server/service/page/index.ts

@@ -23,7 +23,7 @@ import streamToPromise from 'stream-to-promise';
 import { Comment } from '~/features/comment/server';
 import type { ExternalUserGroupDocument } from '~/features/external-user-group/server/models/external-user-group';
 import ExternalUserGroupRelation from '~/features/external-user-group/server/models/external-user-group-relation';
-import { getOpenaiService } from '~/features/openai/server/services/openai';
+import { isAiEnabled } from '~/features/openai/server/services';
 import { SupportedAction } from '~/interfaces/activity';
 import { V5ConversionErrCode } from '~/interfaces/errors/v5-conversion-error';
 import type { IOptionsForCreate, IOptionsForUpdate } from '~/interfaces/page';
@@ -1175,9 +1175,13 @@ class PageService implements IPageService {
         newPagePath, populatedPage?.revision?.body ?? '', user, options,
       );
 
-      // Do not await because communication with OpenAI takes time
-      const openaiService = getOpenaiService();
-      openaiService?.createVectorStoreFile([duplicatedTarget]);
+      if (isAiEnabled()) {
+        const { getOpenaiService } = await import('~/features/openai/server/services/openai');
+
+        // Do not await because communication with OpenAI takes time
+        const openaiService = getOpenaiService();
+        openaiService?.createVectorStoreFile([duplicatedTarget]);
+      }
     }
     this.pageEvent.emit('duplicate', page, user);
 
@@ -1412,9 +1416,13 @@ class PageService implements IPageService {
     const duplicatedPagesWithPopulatedToShowRevison = await Page
       .find({ _id: { $in: duplicatedPageIds }, grant: PageGrant.GRANT_PUBLIC }).populate('revision') as PageDocument[];
 
-    // Do not await because communication with OpenAI takes time
-    const openaiService = getOpenaiService();
-    openaiService?.createVectorStoreFile(duplicatedPagesWithPopulatedToShowRevison);
+    if (isAiEnabled()) {
+      const { getOpenaiService } = await import('~/features/openai/server/services/openai');
+
+      // Do not await because communication with OpenAI takes time
+      const openaiService = getOpenaiService();
+      openaiService?.createVectorStoreFile(duplicatedPagesWithPopulatedToShowRevison);
+    }
   }
 
   private async duplicateDescendantsV4(pages, user, oldPagePathPrefix, newPagePathPrefix) {
@@ -1901,11 +1909,15 @@ class PageService implements IPageService {
       // Leave bookmarks without deleting -- 2024.05.17 Yuki Takei
     ]);
 
-    const openaiService = getOpenaiService();
-    if (openaiService != null) {
-      const vectorStore = await openaiService.getOrCreateVectorStoreForPublicScope();
-      const deleteVectorStoreFilePromises = pageIds.map(pageId => openaiService.deleteVectorStoreFile(vectorStore._id, pageId));
-      await Promise.allSettled(deleteVectorStoreFilePromises);
+    if (isAiEnabled()) {
+      const { getOpenaiService } = await import('~/features/openai/server/services/openai');
+
+      const openaiService = getOpenaiService();
+      if (openaiService != null) {
+        const vectorStore = await openaiService.getOrCreateVectorStoreForPublicScope();
+        const deleteVectorStoreFilePromises = pageIds.map(pageId => openaiService.deleteVectorStoreFile(vectorStore._id, pageId));
+        await Promise.allSettled(deleteVectorStoreFilePromises);
+      }
     }
   }
 

apps/slackbot-proxy/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@growi/slackbot-proxy",
-  "version": "7.1.2-slackbot-proxy.0",
+  "version": "7.1.3-slackbot-proxy.0",
   "license": "MIT",
   "private": "true",
   "scripts": {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "growi",
-  "version": "7.1.2-RC.0",
+  "version": "7.1.3-RC.0",
   "description": "Team collaboration software using markdown",
   "license": "MIT",
   "private": "true",