Merge pull request #9486 from weseek/master

Release v7.1.6

apps/app/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@growi/app",
-  "version": "7.1.5",
+  "version": "7.1.6-RC.0",
   "license": "MIT",
   "private": "true",
   "scripts": {
@@ -157,7 +157,7 @@
     "multer": "~1.4.0",
     "multer-autoreap": "^1.0.3",
     "mustache": "^4.2.0",
-    "next": "^14.2.13",
+    "next": "^14.2.15",
     "next-dynamic-loading-props": "^0.1.1",
     "next-i18next": "^15.3.1",
     "next-superjson": "^0.0.4",
@@ -197,9 +197,11 @@
     "reconnecting-websocket": "^4.4.0",
     "redis": "^3.0.2",
     "rehype-katex": "^7.0.1",
+    "rehype-meta": "^4.0.1",
     "rehype-raw": "^7.0.0",
     "rehype-sanitize": "^6.0.0",
     "rehype-slug": "^6.0.0",
+    "rehype-stringify": "^10.0.1",
     "rehype-toc": "^3.0.2",
     "remark-breaks": "^4.0.0",
     "remark-directive": "^3.0.0",

apps/app/public/static/locales/en_US/translation.json

@@ -117,7 +117,7 @@
   "Create under": "Create page under below:",
   "V5 Page Migration": "Convert To V5 Compatibility",
   "GROWI.5.0_new_schema": "GROWI.5.0 new schema",
-  "See_more_detail_on_new_schema": "See more detail on <a href='https://docs.growi.org/en/admin-guide/upgrading/50x.html#about-the-new-v5-compatible-format' target='_blank'>{{title}}</a> <span className='growi-custom-icons'>external_link</span> ",
+  "See_more_detail_on_new_schema": "See more detail on <a href='https://docs.growi.org/en/admin-guide/upgrading/50x.html#about-the-new-v5-compatible-format' target='_blank'>{{title}}</a> <span class='growi-custom-icons'>external_link</span> ",
   "external_account_management": "External Account Management",
   "UserGroup": "UserGroup",
   "Basic Settings": "Basic Settings",
@@ -157,7 +157,7 @@
   "duplicated_path": "Duplicated path",
   "Link sharing is disabled": "Link sharing is disabled",
   "successfully_saved_the_page": "Successfully saved the page",
-  "you_can_not_create_page_with_this_name": "You can not create page with this name",
+  "you_can_not_create_page_with_this_name_or_hierarchy": "You can not create page with this name or page hierarchy",
   "not_allowed_to_see_this_page": "You cannot see this page",
   "Confirm": "Confirm",
   "Successfully requested": "Successfully requested.",
@@ -615,7 +615,7 @@
     "alert_desc1": "On this page, you can select pages with the checkbox and batch convert to the new v5 compatible format from the \"Bulk operation\" button at the top of the screen.",
     "nopages_title": "Congratulations. Ready to use GROWI v5!",
     "nopages_desc1": "Now all the pages you can manage seem to be in v5 compatible format.",
-    "detail_info": "See the detail information from <a href='https://docs.growi.org/en/admin-guide/upgrading/50x.html' target='_blank' class='alert-link'>Upgrading GROWI to v5.0.x <span className='growi-custom-icons'>external_link</span></a>.",
+    "detail_info": "See the detail information from <a href='https://docs.growi.org/en/admin-guide/upgrading/50x.html' target='_blank' class='alert-link'>Upgrading GROWI to v5.0.x <span class='growi-custom-icons'>external_link</span></a>.",
     "modal": {
       "title": "Convert to new v5 compatible format",
       "converting_pages": "Converting pages",

apps/app/public/static/locales/fr_FR/translation.json

@@ -117,7 +117,7 @@
   "Create under": "Créer la page sous:",
   "V5 Page Migration": "Convertir vers la V5",
   "GROWI.5.0_new_schema": "Nouveau schéma GROWI.5.0",
-  "See_more_detail_on_new_schema": "Plus de détails sur <a href='https://docs.growi.org/en/admin-guide/upgrading/50x.html#about-the-new-v5-compatible-format' target='_blank'>{{title}}</a> <i class='icon-share-alt'></i> ",
+  "See_more_detail_on_new_schema": "Plus de détails sur <a href='https://docs.growi.org/en/admin-guide/upgrading/50x.html#about-the-new-v5-compatible-format' target='_blank'>{{title}}</a><span class='growi-custom-icons'>external_link</span> ",
   "external_account_management": "Gestion des comptes externes",
   "UserGroup": "Groupe utilisateur",
   "Basic Settings": "Paramètres de base",
@@ -157,7 +157,7 @@
   "duplicated_path": "Chemin dupliqué",
   "Link sharing is disabled": "Le partage est désactivé",
   "successfully_saved_the_page": "Page sauvegardée",
-  "you_can_not_create_page_with_this_name": "Vous ne pouvez pas créer cette page",
+  "you_can_not_create_page_with_this_name_or_hierarchy": "Vous ne pouvez pas créer de page avec ce nom ou cette hiérarchie de pages",
   "not_allowed_to_see_this_page": "Vous ne pouvez pas voir cette page",
   "Confirm": "Confirmer",
   "Successfully requested": "Demande envoyée.",
@@ -608,7 +608,7 @@
     "alert_desc1": "Sélectionner les pages à convertir vers le format V5 avec le bouton \"Opération de masse\".",
     "nopages_title": "GROWI V5 est maintenant utilisable!",
     "nopages_desc1": "Toutes les pages ont été converties au format V5.",
-    "detail_info": "Pour plus de détails, voir <a href='https://docs.growi.org/en/admin-guide/upgrading/50x.html' target='_blank' class='alert-link'>Convertir vers GROWI v5.0.x <span className='growi-custom-icons'>external_link</span></a>.",
+    "detail_info": "Pour plus de détails, voir <a href='https://docs.growi.org/en/admin-guide/upgrading/50x.html' target='_blank' class='alert-link'>Convertir vers GROWI v5.0.x <span class='growi-custom-icons'>external_link</span></a>.",
     "modal": {
       "title": "Convertir au format V5",
       "converting_pages": "Conversion des pages",

apps/app/public/static/locales/ja_JP/translation.json

@@ -116,7 +116,7 @@
   "Create under": "ページを以下に作成",
   "V5 Page Migration": "V5 互換形式 への変換",
   "GROWI.5.0_new_schema": "GROWI.5.0における新スキーマについて",
-  "See_more_detail_on_new_schema": "詳しくは<a href='https://docs.growi.org/ja/admin-guide/upgrading/50x.html#新しい-v5-互換形式について' target='_blank'>{{title}}</a><span className='growi-custom-icons'>external_link</span>を参照ください。",
+  "See_more_detail_on_new_schema": "詳しくは<a href='https://docs.growi.org/ja/admin-guide/upgrading/50x.html#新しい-v5-互換形式について' target='_blank'>{{title}}</a><span class='growi-custom-icons'>external_link</span>を参照ください。",
   "external_account_management": "外部アカウント管理",
   "UserGroup": "グループ",
   "Basic Settings": "基本設定",
@@ -158,7 +158,7 @@
   "duplicated_path": "重複したパス",
   "Link sharing is disabled": "リンクのシェアは無効化されています",
   "successfully_saved_the_page": "ページが正常に保存されました",
-  "you_can_not_create_page_with_this_name": "この名前でページを作成することはできません",
+  "you_can_not_create_page_with_this_name_or_hierarchy": "この名前、または階層でページを作成することはできません",
   "not_allowed_to_see_this_page": "このページは閲覧できません",
   "Confirm": "確認",
   "Successfully requested": "正常に処理を受け付けました",
@@ -647,7 +647,7 @@
     "alert_desc1": "このページでは、チェックボックスでページを選択し、画面上部の「一括操作」ボタンから新しい v5 互換形式に一括変換できます。",
     "nopages_title": "おめでとうございます。GROWI v5 を使う準備が完了しました！",
     "nopages_desc1": "今あなたが管理可能なページはすべて v5 互換形式になっているようです。",
-    "detail_info": "詳しくは <a href='https://docs.growi.org/ja/admin-guide/upgrading/50x.html' target='_blank' class='alert-link'>GROWI v5.0.x へのアップグレード <span className='growi-custom-icons'>external_link</span></a> を参照ください。",
+    "detail_info": "詳しくは <a href='https://docs.growi.org/ja/admin-guide/upgrading/50x.html' target='_blank' class='alert-link'>GROWI v5.0.x へのアップグレード <span class='growi-custom-icons'>external_link</span></a> を参照ください。",
     "modal": {
       "title": "新しい v5 互換形式への変換",
       "converting_pages": "以下のページを変換します",

apps/app/public/static/locales/zh_CN/translation.json

@@ -122,7 +122,7 @@
   "Create under": "Create page under below:",
   "V5 Page Migration": "转换为V5的兼容性",
   "GROWI.5.0_new_schema": "GROWI.5.0 new schema",
-  "See_more_detail_on_new_schema": "更多详情请见<a href='https://docs.growi.org/en/admin-guide/upgrading/50x.html#about-the-new-v5-compatible-format' target='_blank'> {{title}}</a> <span className='growi-custom-icons'>external_link</span> ",
+  "See_more_detail_on_new_schema": "更多详情请见<a href='https://docs.growi.org/en/admin-guide/upgrading/50x.html#about-the-new-v5-compatible-format' target='_blank'> {{title}}</a> <span class='growi-custom-icons'>external_link</span> ",
 	"Markdown Settings": "Markdown设置",
 	"external_account_management": "外部账户管理",
   "UserGroup": "用户组",
@@ -163,7 +163,7 @@
   "duplicated_path": "Duplicated path",
   "Link sharing is disabled": "你不允许分享该链接",
   "successfully_saved_the_page": "成功地保存了该页面",
-  "you_can_not_create_page_with_this_name": "您无法使用此名称创建页面",
+  "you_can_not_create_page_with_this_name_or_hierarchy": "您無法使用此名稱或頁面層級建立頁面",
   "not_allowed_to_see_this_page": "你不能看到这个页面",
   "Confirm": "确定",
   "Successfully requested": "进程成功接受",
@@ -617,7 +617,7 @@
     "alert_desc1": "在这一页，你可以用复选框选择页面，并通过屏幕上方的批量操作按钮批量转换为新的v5兼容格式。",
     "nopages_title": "恭喜你。准备使用GROWI v5!",
     "nopages_desc1": "现在你能管理的所有页面似乎都是v5兼容的格式。",
-    "detail_info": "请参见 <a href='https://docs.growi.org/en/admin-guide/upgrading/50x.html' target='_blank' class='alert-link'>升级GROWI到v5.0.x <span className='growi-custom-icons'>external_link</span></a>.的详细内容。",
+    "detail_info": "请参见 <a href='https://docs.growi.org/en/admin-guide/upgrading/50x.html' target='_blank' class='alert-link'>升级GROWI到v5.0.x <span class='growi-custom-icons'>external_link</span></a>.的详细内容。",
     "modal": {
       "title": "转换为新的v5兼容格式",
       "converting_pages": "转换页面",

apps/app/src/client/components/TreeItem/NewPageInput/use-new-page-input.tsx

@@ -99,7 +99,7 @@ export const useNewPageInput = (): UseNewPageInput => {
       const isCreatable = pagePathUtils.isCreatablePage(newPagePath);
 
       if (!isCreatable) {
-        toastWarning(t('you_can_not_create_page_with_this_name'));
+        toastWarning(t('you_can_not_create_page_with_this_name_or_hierarchy'));
         return;
       }
 

apps/app/src/components/Common/PagePathNav/PagePathNav.module.scss

@@ -4,7 +4,7 @@
 
 .grw-page-path-nav-layout :global {
   .grw-page-path-nav-copydropdown {
-    display: none;
+    visibility: hidden;
     @include bs.media-breakpoint-down(md) {
       display: block;
     }
@@ -15,7 +15,7 @@
   &:global {
     &:hover {
       .grw-page-path-nav-copydropdown {
-        display: block;
+        visibility: visible;
       }
     }
   }

apps/app/src/features/openai/server/services/openai.ts

@@ -2,6 +2,7 @@ import assert from 'node:assert';
 import { Readable, Transform } from 'stream';
 import { pipeline } from 'stream/promises';
 
+import type { IPagePopulatedToShowRevision } from '@growi/core';
 import { PageGrant, isPopulated } from '@growi/core';
 import type { HydratedDocument, Types } from 'mongoose';
 import mongoose from 'mongoose';
@@ -20,7 +21,7 @@ import { createBatchStream } from '~/server/util/batch-stream';
 import loggerFactory from '~/utils/logger';
 
 import { OpenaiServiceTypes } from '../../interfaces/ai';
-import { sanitizeMarkdown } from '../utils/sanitize-markdown';
+import { convertMarkdownToHtml } from '../utils/convert-markdown-to-html';
 
 import { getClient } from './client-delegator';
 // import { splitMarkdownIntoChunks } from './markdown-splitter/markdown-token-splitter';
@@ -157,9 +158,9 @@ class OpenaiService implements IOpenaiService {
   //   }
   // }
 
-  private async uploadFile(pageId: Types.ObjectId, body: string): Promise<OpenAI.Files.FileObject> {
-    const sanitizedMarkdown = await sanitizeMarkdown(body);
-    const file = await toFile(Readable.from(sanitizedMarkdown), `${pageId}.md`);
+  private async uploadFile(pageId: Types.ObjectId, pagePath: string, revisionBody: string): Promise<OpenAI.Files.FileObject> {
+    const convertedHtml = await convertMarkdownToHtml({ pagePath, revisionBody });
+    const file = await toFile(Readable.from(convertedHtml), `${pageId}.html`);
     const uploadedFile = await this.client.uploadFile(file);
     return uploadedFile;
   }
@@ -183,17 +184,17 @@ class OpenaiService implements IOpenaiService {
   async createVectorStoreFile(pages: Array<HydratedDocument<PageDocument>>): Promise<void> {
     const vectorStore = await this.getOrCreateVectorStoreForPublicScope();
     const vectorStoreFileRelationsMap: VectorStoreFileRelationsMap = new Map();
-    const processUploadFile = async(page: PageDocument) => {
+    const processUploadFile = async(page: HydratedDocument<PageDocument>) => {
       if (page._id != null && page.grant === PageGrant.GRANT_PUBLIC && page.revision != null) {
         if (isPopulated(page.revision) && page.revision.body.length > 0) {
-          const uploadedFile = await this.uploadFile(page._id, page.revision.body);
+          const uploadedFile = await this.uploadFile(page._id, page.path, page.revision.body);
           prepareVectorStoreFileRelations(vectorStore._id, page._id, uploadedFile.id, vectorStoreFileRelationsMap);
           return;
         }
 
         const pagePopulatedToShowRevision = await page.populateDataToShowRevision();
         if (pagePopulatedToShowRevision.revision != null && pagePopulatedToShowRevision.revision.body.length > 0) {
-          const uploadedFile = await this.uploadFile(page._id, pagePopulatedToShowRevision.revision.body);
+          const uploadedFile = await this.uploadFile(page._id, page.path, pagePopulatedToShowRevision.revision.body);
           prepareVectorStoreFileRelations(vectorStore._id, page._id, uploadedFile.id, vectorStoreFileRelationsMap);
         }
       }

apps/app/src/features/openai/server/utils/convert-markdown-to-html.ts

@@ -0,0 +1,89 @@
+import { dynamicImport } from '@cspell/dynamic-import';
+import type { Root, Code } from 'mdast';
+import type * as RehypeMeta from 'rehype-meta';
+import type * as RehypeStringify from 'rehype-stringify';
+import type * as RemarkParse from 'remark-parse';
+import type * as RemarkRehype from 'remark-rehype';
+import type * as Unified from 'unified';
+import type * as UnistUtilVisit from 'unist-util-visit';
+
+interface ModuleCache {
+  unified?: typeof Unified.unified;
+  visit?: typeof UnistUtilVisit.visit;
+  remarkParse?: typeof RemarkParse.default;
+  remarkRehype?: typeof RemarkRehype.default;
+  rehypeMeta?: typeof RehypeMeta.default;
+  rehypeStringify?: typeof RehypeStringify.default;
+}
+
+let moduleCache: ModuleCache = {};
+
+const initializeModules = async(): Promise<void> => {
+  if (moduleCache.unified != null
+    && moduleCache.visit != null
+    && moduleCache.remarkParse != null
+    && moduleCache.remarkRehype != null
+    && moduleCache.rehypeMeta != null
+    && moduleCache.rehypeStringify != null
+  ) {
+    return;
+  }
+
+  const [
+    { unified },
+    { visit },
+    { default: remarkParse },
+    { default: remarkRehype },
+    { default: rehypeMeta },
+    { default: rehypeStringify },
+  ] = await Promise.all([
+    dynamicImport<typeof Unified>('unified', __dirname),
+    dynamicImport<typeof UnistUtilVisit>('unist-util-visit', __dirname),
+    dynamicImport<typeof RemarkParse>('remark-parse', __dirname),
+    dynamicImport<typeof RemarkRehype>('remark-rehype', __dirname),
+    dynamicImport<typeof RehypeMeta>('rehype-meta', __dirname),
+    dynamicImport<typeof RehypeStringify>('rehype-stringify', __dirname),
+  ]);
+
+  moduleCache = {
+    unified,
+    visit,
+    remarkParse,
+    remarkRehype,
+    rehypeMeta,
+    rehypeStringify,
+  };
+};
+
+export const convertMarkdownToHtml = async({ pagePath, revisionBody }: { pagePath: string, revisionBody: string }): Promise<string> => {
+  await initializeModules();
+
+  const {
+    unified, visit, remarkParse, remarkRehype, rehypeMeta, rehypeStringify,
+  } = moduleCache;
+
+  if (unified == null || visit == null || remarkParse == null || remarkRehype == null || rehypeMeta == null || rehypeStringify == null) {
+    throw new Error('Failed to initialize required modules');
+  }
+
+  const sanitizeMarkdown = () => {
+    return (tree: Root) => {
+      visit(tree, 'code', (node: Code) => {
+        if (node.lang === 'drawio') {
+          node.value = '<!-- drawio content replaced -->';
+        }
+      });
+    };
+  };
+
+  const processor = unified()
+    .use(remarkParse)
+    .use(sanitizeMarkdown)
+    .use(remarkRehype)
+    .use(rehypeMeta, {
+      title: pagePath,
+    })
+    .use(rehypeStringify);
+
+  return processor.processSync(revisionBody).toString();
+};

apps/app/src/features/openai/server/utils/sanitize-markdown.ts

@@ -1,65 +0,0 @@
-import { dynamicImport } from '@cspell/dynamic-import';
-import type { Root, Code } from 'mdast';
-import type * as RemarkParse from 'remark-parse';
-import type * as RemarkStringify from 'remark-stringify';
-import type * as Unified from 'unified';
-import type * as UnistUtilVisit from 'unist-util-visit';
-
-interface ModuleCache {
-  remarkParse?: typeof RemarkParse.default;
-  remarkStringify?: typeof RemarkStringify.default;
-  unified?: typeof Unified.unified;
-  visit?: typeof UnistUtilVisit.visit;
-}
-
-let moduleCache: ModuleCache = {};
-
-const initializeModules = async(): Promise<void> => {
-  if (moduleCache.remarkParse != null && moduleCache.remarkStringify != null && moduleCache.unified != null && moduleCache.visit != null) {
-    return;
-  }
-
-  const [{ default: remarkParse }, { default: remarkStringify }, { unified }, { visit }] = await Promise.all([
-    dynamicImport<typeof RemarkParse>('remark-parse', __dirname),
-    dynamicImport<typeof RemarkStringify>('remark-stringify', __dirname),
-    dynamicImport<typeof Unified>('unified', __dirname),
-    dynamicImport<typeof UnistUtilVisit>('unist-util-visit', __dirname),
-  ]);
-
-  moduleCache = {
-    remarkParse,
-    remarkStringify,
-    unified,
-    visit,
-  };
-};
-
-export const sanitizeMarkdown = async(markdown: string): Promise<string> => {
-  await initializeModules();
-
-  const {
-    remarkParse, remarkStringify, unified, visit,
-  } = moduleCache;
-
-
-  if (remarkParse == null || remarkStringify == null || unified == null || visit == null) {
-    throw new Error('Failed to initialize required modules');
-  }
-
-  const sanitize = () => {
-    return (tree: Root) => {
-      visit(tree, 'code', (node: Code) => {
-        if (node.lang === 'drawio') {
-          node.value = '<!-- drawio content replaced -->';
-        }
-      });
-    };
-  };
-
-  const processor = unified()
-    .use(remarkParse)
-    .use(sanitize)
-    .use(remarkStringify);
-
-  return processor.processSync(markdown).toString();
-};

apps/slackbot-proxy/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@growi/slackbot-proxy",
-  "version": "7.1.5-slackbot-proxy.0",
+  "version": "7.1.6-slackbot-proxy.0",
   "license": "MIT",
   "private": "true",
   "scripts": {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "growi",
-  "version": "7.1.5",
+  "version": "7.1.6-RC.0",
   "description": "Team collaboration software using markdown",
   "license": "MIT",
   "private": "true",

packages/core/src/utils/page-path-utils/index.ts

@@ -1,6 +1,6 @@
 import escapeStringRegexp from 'escape-string-regexp';
 
-import { IUser } from '~/interfaces';
+import type { IUser } from '~/interfaces';
 
 import { isValidObjectId } from '../objectid-utils';
 import { addTrailingSlash } from '../path-utils';
@@ -117,6 +117,7 @@ const restrictedPatternsToCreate: Array<RegExp> = [
   /^\/(_search|_private-legacy-pages)(\/.*|$)/,
   /^\/(installer|register|login|logout|admin|me|files|trash|paste|comments|tags|share|attachment)(\/.*|$)/,
   /^\/user(?:\/[^/]+)?$/, // https://regex101.com/r/9Eh2S1/1
+  /^(\/.+){130,}$/, // avoid deep layer path. see: https://regex101.com/r/L0kzOD/1
 ];
 export const isCreatablePage = (path: string): boolean => {
   return !restrictedPatternsToCreate.some(pattern => path.match(pattern));

packages/remark-lsx/src/server/index.ts

@@ -14,8 +14,8 @@ const filterXSS = new FilterXSS();
 
 const lsxValidator = [
   query('pagePath').notEmpty().isString(),
-  query('offset').optional().isInt(),
-  query('limit').optional().isInt(),
+  query('offset').optional().isInt().toInt(),
+  query('limit').optional().isInt().toInt(),
   query('options')
     .optional()
     .customSanitizer((options) => {