Merge pull request #4173 from weseek/feat/7075-add-textlint-rules

7075 7134 7191 Use Japanese Textlint Rules in NO_CDN mode

packages/app/.gitignore

@@ -6,6 +6,7 @@
 /dist/
 /transpiled/
 /public/static/js
+/public/static/dict
 /public/static/styles
 /public/uploads
 /tmp/

packages/app/bin/cdn/cdn-resources-downloader.ts

@@ -4,7 +4,9 @@ import urljoin from 'url-join';
 import { Transform } from 'stream';
 import replaceStream from 'replacestream';
 
-import { cdnLocalScriptRoot, cdnLocalStyleRoot, cdnLocalStyleWebRoot } from '^/config/cdn';
+import {
+  cdnLocalScriptRoot, cdnLocalDictRoot, cdnLocalStyleRoot, cdnLocalStyleWebRoot,
+} from '^/config/cdn';
 import * as cdnManifests from '^/resource/cdn-manifests';
 
 import { CdnResource, CdnManifest } from '~/interfaces/cdn';
@@ -19,6 +21,15 @@ export default class CdnResourcesDownloader {
     const cdnScriptResources: CdnResource[] = cdnManifests.js.map((manifest: CdnManifest) => {
       return { manifest, outDir: cdnLocalScriptRoot };
     });
+
+    const cdnDictResources: CdnResource[] = cdnManifests.dict.map((manifest: CdnManifest) => {
+      return { manifest, outDir: cdnLocalDictRoot };
+    });
+
+    const dictExtensionOptions = {
+      ext: 'gz',
+    };
+
     const cdnStyleResources: CdnResource[] = cdnManifests.style.map((manifest) => {
       return { manifest, outDir: cdnLocalStyleRoot };
     });
@@ -31,6 +42,7 @@ export default class CdnResourcesDownloader {
 
     return Promise.all([
       this.downloadScripts(cdnScriptResources),
+      this.downloadScripts(cdnDictResources, dictExtensionOptions),
       this.downloadStyles(cdnStyleResources, dlStylesOptions),
     ]);
   }

packages/app/config/cdn.js

@@ -4,5 +4,6 @@ import { projectRoot } from '~/utils/project-dir-utils';
 
 export const cdnLocalScriptRoot = path.join(projectRoot, 'public/static/js/cdn');
 export const cdnLocalScriptWebRoot = '/static/js/cdn';
+export const cdnLocalDictRoot = path.join(projectRoot, 'public/static/dict/cdn');
 export const cdnLocalStyleRoot = path.join(projectRoot, 'public/static/styles/cdn');
 export const cdnLocalStyleWebRoot = '/static/styles/cdn';

packages/app/package.json

@@ -56,8 +56,8 @@
     "@browser-bunyan/console-formatted-stream": "^1.6.2",
     "@google-cloud/storage": "^5.8.5",
     "@growi/plugin-attachment-refs": "^4.3.3-RC",
-    "@growi/plugin-pukiwiki-like-linker": "^4.3.3-RC",
     "@growi/plugin-lsx": "^4.3.3-RC",
+    "@growi/plugin-pukiwiki-like-linker": "^4.3.3-RC",
     "@growi/slack": "^4.3.3-RC",
     "@kobalab/socket.io-session": "^1.0.3",
     "@promster/express": "^5.0.1",
@@ -97,7 +97,6 @@
     "graceful-fs": "^4.1.11",
     "growi-commons": "^5.0.4",
     "helmet": "^4.6.0",
-    "nocache": "^3.0.1",
     "http-errors": "~1.6.2",
     "i18next": "^20.3.2",
     "i18next-express-middleware": "^2.0.0",
@@ -115,6 +114,7 @@
     "mongoose-unique-validator": "^2.0.3",
     "multer": "~1.4.0",
     "multer-autoreap": "^1.0.3",
+    "nocache": "^3.0.1",
     "nodemailer": "^6.6.2",
     "nodemailer-ses-transport": "~1.5.0",
     "openid-client": "=2.5.0",
@@ -236,6 +236,7 @@
     "textlint-plugin-markdown": "^4.0.6",
     "textlint-rule-common-misspellings": "^1.0.1",
     "textlint-rule-max-comma": "^2.0.2",
+    "textlint-rule-no-dropping-the-ra": "^3.0.0",
     "throttle-debounce": "^2.0.0",
     "toastr": "^2.1.2",
     "ts-loader": "^8.3.0",

packages/app/resource/cdn-manifests.js

@@ -89,6 +89,92 @@ module.exports = {
       },
     },
   ],
+  dict: [
+    {
+      name: 'base.dat',
+      url: 'https://cdn.jsdelivr.net/npm/kuromoji@0.1.2/dict/base.dat.gz',
+      args: {
+        integrity: '',
+      },
+    },
+    {
+      name: 'cc.dat',
+      url: 'https://cdn.jsdelivr.net/npm/kuromoji@0.1.2/dict/cc.dat.gz',
+      args: {
+        integrity: '',
+      },
+    },
+    {
+      name: 'check.dat',
+      url: 'https://cdn.jsdelivr.net/npm/kuromoji@0.1.2/dict/check.dat.gz',
+      args: {
+        integrity: '',
+      },
+    },
+    {
+      name: 'tid_map.dat',
+      url: 'https://cdn.jsdelivr.net/npm/kuromoji@0.1.2/dict/tid_map.dat.gz',
+      args: {
+        integrity: '',
+      },
+    },
+    {
+      name: 'tid_pos.dat',
+      url: 'https://cdn.jsdelivr.net/npm/kuromoji@0.1.2/dict/tid_pos.dat.gz',
+      args: {
+        integrity: '',
+      },
+    },
+    {
+      name: 'tid.dat',
+      url: 'https://cdn.jsdelivr.net/npm/kuromoji@0.1.2/dict/tid.dat.gz',
+      args: {
+        integrity: '',
+      },
+    },
+    {
+      name: 'unk_char.dat',
+      url: 'https://cdn.jsdelivr.net/npm/kuromoji@0.1.2/dict/unk_char.dat.gz',
+      args: {
+        integrity: '',
+      },
+    },
+    {
+      name: 'unk_compat.dat',
+      url: 'https://cdn.jsdelivr.net/npm/kuromoji@0.1.2/dict/unk_compat.dat.gz',
+      args: {
+        integrity: '',
+      },
+    },
+    {
+      name: 'unk_invoke.dat',
+      url: 'https://cdn.jsdelivr.net/npm/kuromoji@0.1.2/dict/unk_invoke.dat.gz',
+      args: {
+        integrity: '',
+      },
+    },
+    {
+      name: 'unk_map.dat',
+      url: 'https://cdn.jsdelivr.net/npm/kuromoji@0.1.2/dict/unk_map.dat.gz',
+      args: {
+        integrity: '',
+      },
+    },
+    {
+      name: 'unk_pos.dat',
+      url: 'https://cdn.jsdelivr.net/npm/kuromoji@0.1.2/dict/unk_pos.dat.gz',
+      args: {
+        integrity: '',
+      },
+    },
+    {
+      name: 'unk.dat',
+      url: 'https://cdn.jsdelivr.net/npm/kuromoji@0.1.2/dict/unk.dat.gz',
+      args: {
+        integrity: '',
+      },
+    },
+  ],
   style: [
     {
       name: 'lato',

packages/app/src/client/util/codemirror/codemirror-textlint.ts

@@ -2,6 +2,7 @@ import { TextlintKernel, TextlintKernelRule, TextlintRuleOptions } from '@textli
 import textlintToCodeMirror from 'textlint-message-to-codemirror';
 import textlintRuleMaxComma from 'textlint-rule-max-comma';
 import textlintRuleCommonMisspellings from 'textlint-rule-common-misspellings';
+import textlintRuleNoDroppingTheRa from 'textlint-rule-no-dropping-the-ra';
 import { AsyncLinter, Annotation } from 'codemirror/addon/lint/lint';
 import loggerFactory from '../../../utils/logger';
 
@@ -17,6 +18,7 @@ type RuleExtension = {
 const ruleModulesList = {
   'max-comma': textlintRuleMaxComma,
   'common-misspellings': textlintRuleCommonMisspellings,
+  'no-dropping-the-ra': textlintRuleNoDroppingTheRa,
 };
 
 const logger = loggerFactory('growi:codemirror:codemirror-textlint');

packages/app/src/components/PageEditor/CodeMirrorEditor.jsx

@@ -152,8 +152,11 @@ export default class CodeMirrorEditor extends AbstractEditor {
 
   init() {
     this.cmCdnRoot = 'https://cdn.jsdelivr.net/npm/codemirror@5.42.0';
-    this.cmNoCdnScriptRoot = '/js/cdn';
-    this.cmNoCdnStyleRoot = '/styles/cdn';
+    this.cmNoCdnScriptRoot = '/static/js/cdn';
+    this.cmNoCdnStyleRoot = '/static/styles/cdn';
+    window.kuromojin = this.props.noCdn
+      ? { dicPath: '/static/dict/cdn' }
+      : { dicPath: 'https://cdn.jsdelivr.net/npm/kuromoji@0.1.2/dict' };
 
     // TODO: Get configs from db
     this.isLintEnabled = true;
@@ -162,6 +165,9 @@ export default class CodeMirrorEditor extends AbstractEditor {
       {
         name: 'max-comma',
       },
+      {
+        name: 'no-dropping-the-ra',
+      },
       {
         name: 'common-misspellings',
         options: {

packages/app/src/services/cdn-resources-service.js

@@ -6,10 +6,10 @@ const urljoin = require('url-join');
 
 const { envUtils } = require('growi-commons');
 
-const cdnLocalScriptRoot = 'public/js/cdn';
-const cdnLocalScriptWebRoot = '/js/cdn';
-const cdnLocalStyleRoot = 'public/styles/cdn';
-const cdnLocalStyleWebRoot = '/styles/cdn';
+const cdnLocalScriptRoot = 'public/static/js/cdn';
+const cdnLocalScriptWebRoot = '/static/js/cdn';
+const cdnLocalStyleRoot = 'public/static/styles/cdn';
+const cdnLocalStyleWebRoot = '/static/styles/cdn';
 
 
 class CdnResourcesService {

yarn.lock

@@ -2524,6 +2524,13 @@
     "@textlint/types" "^12.0.2"
     debug "^4.3.2"
 
+"@textlint/types@^1.5.5":
+  version "1.5.5"
+  resolved "https://registry.yarnpkg.com/@textlint/types/-/types-1.5.5.tgz#9c82dbcbf4e00116573f05c6739c6c8ec3b35304"
+  integrity sha512-80P6fcqgsG9bP6JgR6W/E/oIx+71pplaicYCvvB4vMIeGk0OnWls4Q21kCpDYmq/C/ABtZ/Gy/Ov/8ExQPeQ7A==
+  dependencies:
+    "@textlint/ast-node-types" "^4.4.3"
+
 "@textlint/types@^12.0.2":
   version "12.0.2"
   resolved "https://registry.yarnpkg.com/@textlint/types/-/types-12.0.2.tgz#991a0e500e4577eddd8f1c7a2f3dd7016525e48f"
@@ -4026,7 +4033,7 @@ async@3.2.0, async@>=3.2.0, async@^3.2.0:
   resolved "https://registry.yarnpkg.com/async/-/async-3.2.0.tgz#b3a2685c5ebb641d3de02d161002c60fc9f85720"
   integrity sha512-TR2mEZFVOj2pLStYxLht7TyfuRzaydfpxr3k9RpHIzMgw7A64dzsdqCxH1WJyQdoe8T10nDXd9wnEigmiuHIZw==
 
-async@^2.6.2:
+async@^2.0.1, async@^2.6.2:
   version "2.6.3"
   resolved "https://registry.yarnpkg.com/async/-/async-2.6.3.tgz#d72625e2344a3656e3a3ad4fa749fa83299d82ff"
   integrity sha512-zflvls11DCy+dQWzTW2dzuilv8Z5X/pjfmZOWba6TNIVDm+2UDaJmXSOXlasHKfNBs8oo3M0aT50fDEWfKZjXg==
@@ -7071,6 +7078,11 @@ dotenv@>=8.2.0, dotenv@^8.0.0, dotenv@^8.2.0:
   resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-8.2.0.tgz#97e619259ada750eea3e4ea3e26bceea5424b16a"
   integrity sha512-8sJ78ElpbDJBHNeBzUbUVLsqKdccaa/BXF1uPTw3GrvQTBgrQrtObr2mUrE38vzYd8cEv+m/JBfDLioYcfXoaw==
 
+doublearray@0.0.2:
+  version "0.0.2"
+  resolved "https://registry.yarnpkg.com/doublearray/-/doublearray-0.0.2.tgz#63186fe8d34413276d3621f6aa0ec5f79e227ef9"
+  integrity sha1-Yxhv6NNEEydtNiH2qg7F954ifvk=
+
 dtrace-provider@~0.8:
   version "0.8.8"
   resolved "https://registry.yarnpkg.com/dtrace-provider/-/dtrace-provider-0.8.8.tgz#2996d5490c37e1347be263b423ed7b297fb0d97e"
@@ -11808,6 +11820,23 @@ kruptein@^3.0.0:
   dependencies:
     asn1.js "^5.4.1"
 
+kuromoji@0.1.2:
+  version "0.1.2"
+  resolved "https://registry.yarnpkg.com/kuromoji/-/kuromoji-0.1.2.tgz#293f0d6706df006112137980588d5daac26d0790"
+  integrity sha512-V0dUf+C2LpcPEXhoHLMAop/bOht16Dyr+mDiIE39yX3vqau7p80De/koFqpiTcL1zzdZlc3xuHZ8u5gjYRfFaQ==
+  dependencies:
+    async "^2.0.1"
+    doublearray "0.0.2"
+    zlibjs "^0.3.1"
+
+kuromojin@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/kuromojin/-/kuromojin-3.0.0.tgz#54a1a6643110f49f741c4beb82fef400d1cd498b"
+  integrity sha512-3h3qnn/NVVhqoKFP4oc7e6apO2B01Atc056oiVlIY7Uoup4rhrnBe28g3y9lK1HTmLDQEejvXB+3I3qxAneF7A==
+  dependencies:
+    kuromoji "0.1.2"
+    lru_map "^0.4.1"
+
 last-call-webpack-plugin@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/last-call-webpack-plugin/-/last-call-webpack-plugin-3.0.0.tgz#9742df0e10e3cf46e5c0381c2de90d3a7a2d7555"
@@ -12324,6 +12353,11 @@ lru-cache@^5.1.1:
   dependencies:
     yallist "^3.0.2"
 
+lru_map@^0.4.1:
+  version "0.4.1"
+  resolved "https://registry.yarnpkg.com/lru_map/-/lru_map-0.4.1.tgz#f7b4046283c79fb7370c36f8fca6aee4324b0a98"
+  integrity sha512-I+lBvqMMFfqaV8CJCISjI3wbjmwVu/VyOoU7+qtu9d7ioW5klMgsTTiUOUp+DJvfTTzKXoPbyC6YfgkNcyPSOg==
+
 lucene-query-parser@^1.2.0:
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/lucene-query-parser/-/lucene-query-parser-1.2.0.tgz#46dad5b4ddc59abbf27f9df4c519d959c2033432"
@@ -19302,6 +19336,16 @@ textlint-rule-helper@^1.1.5:
   dependencies:
     unist-util-visit "^1.1.0"
 
+textlint-rule-helper@^2.1.1:
+  version "2.2.0"
+  resolved "https://registry.yarnpkg.com/textlint-rule-helper/-/textlint-rule-helper-2.2.0.tgz#30522ba904a03849d57ea3e5ebd5920027cd8da3"
+  integrity sha512-9S5CsgQuQwPjM2wvr4JGdpkLf+pR9gOjedSQFa/Dkrbh+D9MXt1LIR4Jvx1RujKtt2nq42prmEX2q3xOxyUcIQ==
+  dependencies:
+    "@textlint/ast-node-types" "^4.4.3"
+    "@textlint/types" "^1.5.5"
+    structured-source "^3.0.2"
+    unist-util-visit "^1.1.0"
+
 textlint-rule-max-comma@^2.0.2:
   version "2.0.2"
   resolved "https://registry.yarnpkg.com/textlint-rule-max-comma/-/textlint-rule-max-comma-2.0.2.tgz#c204ae840b436d572d5c24e3a3056776992a882b"
@@ -19311,6 +19355,14 @@ textlint-rule-max-comma@^2.0.2:
     textlint-util-to-string "^3.1.1"
     unist-util-filter "^2.0.3"
 
+textlint-rule-no-dropping-the-ra@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/textlint-rule-no-dropping-the-ra/-/textlint-rule-no-dropping-the-ra-3.0.0.tgz#cc408b719302818e52917f5dee537e1007c206e2"
+  integrity sha512-KbSIlcxj1kLs4sGSMSLGA8OmgRoaPAWtodJaGEyEUiy7uiZM/VPqYALpuD8vf16N1OR5SM/bXXeZFME65r8ZgQ==
+  dependencies:
+    kuromojin "^3.0.0"
+    textlint-rule-helper "^2.1.1"
+
 textlint-util-to-string@^3.1.1:
   version "3.1.1"
   resolved "https://registry.yarnpkg.com/textlint-util-to-string/-/textlint-util-to-string-3.1.1.tgz#666c8b0f2e00a92b29c8b168b453a9b8aeb48381"
@@ -21358,3 +21410,8 @@ zip-stream@^4.1.0:
     archiver-utils "^2.1.0"
     compress-commons "^4.1.0"
     readable-stream "^3.6.0"
+
+zlibjs@^0.3.1:
+  version "0.3.1"
+  resolved "https://registry.yarnpkg.com/zlibjs/-/zlibjs-0.3.1.tgz#50197edb28a1c42ca659cc8b4e6a9ddd6d444554"
+  integrity sha1-UBl+2yihxCymWcyLTmqd3W1ERVQ=