Merge pull request #6053 from weseek/feat/95777-issue-configurable-api-rate-limit-by-flexible

feat: Rate Limit by rate-limit-flexible

packages/app/config/rate-limiter.ts

@@ -0,0 +1,83 @@
+export type IApiRateLimitConfig = {
+  method: string,
+  maxRequests: number,
+  usersPerIpProspection?: number,
+}
+export type IApiRateLimitEndpointMap = {
+  [endpoint: string]: IApiRateLimitConfig
+}
+
+export const DEFAULT_MAX_REQUESTS = 500;
+export const DEFAULT_DURATION_SEC = 60;
+export const DEFAULT_USERS_PER_IP_PROSPECTION = 5;
+
+const MAX_REQUESTS_TIER_1 = 5;
+const MAX_REQUESTS_TIER_2 = 20;
+const MAX_REQUESTS_TIER_3 = 50;
+const MAX_REQUESTS_TIER_4 = 100;
+
+// default config without reg exp
+export const defaultConfig: IApiRateLimitEndpointMap = {
+  '/_api/v3/healthcheck': {
+    method: 'GET',
+    maxRequests: 60,
+    usersPerIpProspection: 1,
+  },
+  '/installer': {
+    method: 'POST',
+    maxRequests: MAX_REQUESTS_TIER_1,
+    usersPerIpProspection: 1,
+  },
+  '/login': {
+    method: 'POST',
+    maxRequests: MAX_REQUESTS_TIER_1,
+    usersPerIpProspection: 100,
+  },
+  '/login/activateInvited': {
+    method: 'POST',
+    maxRequests: MAX_REQUESTS_TIER_2,
+  },
+  '/register': {
+    method: 'POST',
+    maxRequests: MAX_REQUESTS_TIER_1,
+    usersPerIpProspection: 20,
+  },
+  '/user-activation/register': {
+    method: 'POST',
+    maxRequests: MAX_REQUESTS_TIER_1,
+    usersPerIpProspection: 20,
+  },
+  '/_api/login/testLdap': {
+    method: 'POST',
+    maxRequests: MAX_REQUESTS_TIER_2,
+    usersPerIpProspection: 1,
+  },
+  '/_api/check_username': {
+    method: 'GET',
+    maxRequests: MAX_REQUESTS_TIER_3,
+  },
+};
+
+// default config with reg exp
+export const defaultConfigWithRegExp = {
+  '/forgot-password/.*': {
+    method: 'ALL',
+    maxRequests: MAX_REQUESTS_TIER_1,
+  },
+  '/user-activation/.*': {
+    method: 'GET',
+    maxRequests: MAX_REQUESTS_TIER_1,
+  },
+  '/attachment/[0-9a-z]{24}': {
+    method: 'GET',
+    maxRequests: MAX_REQUESTS_TIER_4,
+  },
+  '/download/[0-9a-z]{24}': {
+    method: 'GET',
+    maxRequests: MAX_REQUESTS_TIER_4,
+  },
+  '/share/[0-9a-z]{24}': {
+    method: 'GET',
+    maxRequests: MAX_REQUESTS_TIER_4,
+  },
+};

packages/app/package.json

@@ -103,7 +103,6 @@
     "express": "^4.16.1",
     "express-bunyan-logger": "^1.3.3",
     "express-mongo-sanitize": "^2.1.0",
-    "express-rate-limit": "^5.3.0",
     "express-session": "^1.16.1",
     "express-validator": "^6.14.0",
     "express-webpack-assets": "^0.1.0",
@@ -141,6 +140,7 @@
     "passport-saml": "^3.2.0",
     "passport-twitter": "^1.0.4",
     "prom-client": "^13.0.0",
+    "rate-limiter-flexible": "^2.3.7",
     "react-card-flip": "^1.0.10",
     "react-datepicker": "^4.7.0",
     "react-dnd": "^14.0.5",

packages/app/src/server/middlewares/rate-limiter.ts

@@ -0,0 +1,138 @@
+import { NextFunction, Request, Response } from 'express';
+import md5 from 'md5';
+import mongoose from 'mongoose';
+import { IRateLimiterMongoOptions, RateLimiterMongo } from 'rate-limiter-flexible';
+
+import {
+  DEFAULT_DURATION_SEC, DEFAULT_MAX_REQUESTS, DEFAULT_USERS_PER_IP_PROSPECTION, IApiRateLimitConfig,
+} from '^/config/rate-limiter';
+
+import { IUserHasId } from '~/interfaces/user';
+import loggerFactory from '~/utils/logger';
+
+import { generateApiRateLimitConfig } from '../util/rate-limiter';
+
+
+const logger = loggerFactory('growi:middleware:api-rate-limit');
+
+// config sample
+// API_RATE_LIMIT_010_FOO_ENDPOINT=/_api/v3/foo
+// API_RATE_LIMIT_010_FOO_METHODS=GET,POST
+// API_RATE_LIMIT_010_FOO_MAX_REQUESTS=10
+
+const POINTS_THRESHOLD = 100;
+
+const opts: IRateLimiterMongoOptions = {
+  storeClient: mongoose.connection,
+  points: POINTS_THRESHOLD, // set default value
+  duration: DEFAULT_DURATION_SEC, // set default value
+};
+const rateLimiter = new RateLimiterMongo(opts);
+
+// generate ApiRateLimitConfig for api rate limiter
+const apiRateLimitConfig = generateApiRateLimitConfig();
+const configWithoutRegExp = apiRateLimitConfig.withoutRegExp;
+const configWithRegExp = apiRateLimitConfig.withRegExp;
+const allRegExp = new RegExp(Object.keys(configWithRegExp).join('|'));
+const keysWithRegExp = Object.keys(configWithRegExp).map(key => new RegExp(`^${key}`));
+const valuesWithRegExp = Object.values(configWithRegExp);
+
+
+const _consumePoints = async(
+    method: string, key: string | null, customizedConfig?: IApiRateLimitConfig, maxRequestsMultiplier?: number,
+) => {
+  if (key == null) {
+    return;
+  }
+
+  let maxRequests = DEFAULT_MAX_REQUESTS;
+
+  // use customizedConfig
+  if (customizedConfig != null && (customizedConfig.method.includes(method) || customizedConfig.method === 'ALL')) {
+    maxRequests = customizedConfig.maxRequests;
+  }
+
+  // multiply
+  if (maxRequestsMultiplier != null) {
+    maxRequests *= maxRequestsMultiplier;
+  }
+
+  // because the maximum request is reduced by 1 if it is divisible by
+  // https://github.com/weseek/growi/pull/6225
+  const consumePoints = (POINTS_THRESHOLD + 0.0001) / maxRequests;
+  await rateLimiter.consume(key, consumePoints);
+};
+
+/**
+ * consume per user per endpoint
+ * @param method
+ * @param key
+ * @param customizedConfig
+ * @returns
+ */
+const consumePointsByUser = async(method: string, key: string | null, customizedConfig?: IApiRateLimitConfig) => {
+  return _consumePoints(method, key, customizedConfig);
+};
+
+/**
+ * consume per ip per endpoint
+ * @param method
+ * @param key
+ * @param customizedConfig
+ * @returns
+ */
+const consumePointsByIp = async(method: string, key: string | null, customizedConfig?: IApiRateLimitConfig) => {
+  const maxRequestsMultiplier = customizedConfig?.usersPerIpProspection ?? DEFAULT_USERS_PER_IP_PROSPECTION;
+  return _consumePoints(method, key, customizedConfig, maxRequestsMultiplier);
+};
+
+
+module.exports = () => {
+
+  return async(req: Request & { user?: IUserHasId }, res: Response, next: NextFunction) => {
+
+    const endpoint = req.path;
+
+    // determine keys
+    const keyForUser: string | null = req.user != null
+      ? md5(`${req.user._id}_${endpoint}_${req.method}`)
+      : null;
+    const keyForIp: string = md5(`${req.ip}_${endpoint}_${req.method}`);
+
+    // determine customized config
+    let customizedConfig: IApiRateLimitConfig | undefined;
+    const configForEndpoint = configWithoutRegExp[endpoint];
+    if (configForEndpoint) {
+      customizedConfig = configForEndpoint;
+    }
+    else if (allRegExp.test(endpoint)) {
+      keysWithRegExp.forEach((key, index) => {
+        if (key.test(endpoint)) {
+          customizedConfig = valuesWithRegExp[index];
+        }
+      });
+    }
+
+    // check for the current user
+    if (req.user != null) {
+      try {
+        await consumePointsByUser(req.method, keyForUser, customizedConfig);
+      }
+      catch {
+        logger.error(`${req.user._id}: too many request at ${endpoint}`);
+        return res.sendStatus(429);
+      }
+    }
+
+    // check for ip
+    try {
+      await consumePointsByIp(req.method, keyForIp, customizedConfig);
+    }
+    catch {
+      logger.error(`${req.ip}: too many request at ${endpoint}`);
+      return res.sendStatus(429);
+    }
+
+    return next();
+  };
+};

packages/app/src/server/routes/apiv3/activity.ts

@@ -1,6 +1,5 @@
 import { parseISO, addMinutes, isValid } from 'date-fns';
 import express, { Request, Router } from 'express';
-import rateLimit from 'express-rate-limit';
 import { query } from 'express-validator';
 
 import { ISearchFilter } from '~/interfaces/activity';
@@ -24,13 +23,6 @@ const validator = {
   ],
 };
 
-const apiLimiter = rateLimit({
-  windowMs: 15 * 60 * 1000, // 15 minutes
-  max: 30, // limit each IP to 30 requests per windowMs
-  message:
-    'Too many requests sent from this IP, please try again after 15 minutes.',
-});
-
 module.exports = (crowi: Crowi): Router => {
   const adminRequired = require('../../middlewares/admin-required')(crowi);
   const accessTokenParser = require('../../middlewares/access-token-parser')(crowi);
@@ -39,7 +31,7 @@ module.exports = (crowi: Crowi): Router => {
   const router = express.Router();
 
   // eslint-disable-next-line max-len
-  router.get('/', apiLimiter, accessTokenParser, loginRequiredStrictly, adminRequired, validator.list, apiV3FormValidator, async(req: Request, res: ApiV3Response) => {
+  router.get('/', accessTokenParser, loginRequiredStrictly, adminRequired, validator.list, apiV3FormValidator, async(req: Request, res: ApiV3Response) => {
     const auditLogEnabled = crowi.configManager?.getConfig('crowi', 'app:auditLogEnabled') || false;
     if (!auditLogEnabled) {
       const msg = 'AuditLog is not enabled';

packages/app/src/server/routes/apiv3/forgot-password.js

@@ -1,5 +1,4 @@
 import { format, subSeconds } from 'date-fns';
-import rateLimit from 'express-rate-limit';
 
 import injectResetOrderByTokenMiddleware from '~/server/middlewares/inject-reset-order-by-token-middleware';
 import PasswordResetOrder from '~/server/models/password-reset-order';
@@ -41,13 +40,6 @@ module.exports = (crowi) => {
     ],
   };
 
-  const apiLimiter = rateLimit({
-    windowMs: 1 * 60 * 1000, // 1 minutes
-    max: 30, // limit each IP to 30 requests per windowMs
-    message:
-    'Too many requests were sent from this IP. Please try a password reset request again on the password reset request form',
-  });
-
   const checkPassportStrategyMiddleware = checkForgotPasswordEnabledMiddlewareFactory(crowi, true);
 
   async function sendPasswordResetEmail(txtFileName, i18n, email, url, expiredAt) {
@@ -95,7 +87,7 @@ module.exports = (crowi) => {
   });
 
   // eslint-disable-next-line max-len
-  router.put('/', apiLimiter, checkPassportStrategyMiddleware, injectResetOrderByTokenMiddleware, csrf, validator.password, apiV3FormValidator, async(req, res) => {
+  router.put('/', checkPassportStrategyMiddleware, injectResetOrderByTokenMiddleware, csrf, validator.password, apiV3FormValidator, async(req, res) => {
     const { passwordResetOrder } = req;
     const { email } = passwordResetOrder;
     const grobalLang = configManager.getConfig('crowi', 'app:globalLang');

packages/app/src/server/routes/index.js

@@ -10,23 +10,14 @@ import {
   generateUnavailableWhenMaintenanceModeMiddleware, generateUnavailableWhenMaintenanceModeMiddlewareForApi,
 } from '../middlewares/unavailable-when-maintenance-mode';
 
-
 import * as allInAppNotifications from './all-in-app-notifications';
 import * as forgotPassword from './forgot-password';
 import * as privateLegacyPages from './private-legacy-pages';
 import * as userActivation from './user-activation';
 
-const rateLimit = require('express-rate-limit');
 const multer = require('multer');
 const autoReap = require('multer-autoreap');
 
-const apiLimiter = rateLimit({
-  windowMs: 1 * 60 * 1000, // 1 minutes
-  max: 60, // limit each IP to 60 requests per windowMs
-  message:
-    'Too many requests sent from this IP, please try again after 1 minute',
-});
-
 autoReap.options.reapOnError = true; // continue reaping the file even if an error occurs
 
 module.exports = function(crowi, app) {
@@ -40,6 +31,7 @@ module.exports = function(crowi, app) {
   const certifySharedFile = require('../middlewares/certify-shared-file')(crowi);
   const csrf = require('../middlewares/csrf')(crowi);
   const injectUserUISettings = require('../middlewares/inject-user-ui-settings-to-localvars')();
+  const rateLimiter = require('../middlewares/rate-limiter')();
   const addActivity = generateAddActivityMiddleware(crowi);
 
   const uploads = multer({ dest: `${crowi.tmpDir}uploads` });
@@ -67,6 +59,9 @@ module.exports = function(crowi, app) {
 
   app.use('/api-docs', require('./apiv3/docs')(crowi));
 
+  // Rate limiter
+  app.use(rateLimiter);
+
   // API v3 for admin
   app.use('/_api/v3', apiV3AdminRouter);
 
@@ -78,10 +73,10 @@ module.exports = function(crowi, app) {
   app.get('/login/error/:reason'      , applicationInstalled, login.error);
   app.get('/login'                    , applicationInstalled, login.preLogin, login.login);
   app.get('/login/invited'            , applicationInstalled, login.invited);
-  app.post('/login/activateInvited'   , apiLimiter , applicationInstalled, loginFormValidator.inviteRules(), loginFormValidator.inviteValidation, csrf, login.invited);
-  app.post('/login'                   , apiLimiter , applicationInstalled, loginFormValidator.loginRules(), loginFormValidator.loginValidation, csrf,  addActivity, loginPassport.loginWithLocal, loginPassport.loginWithLdap, loginPassport.loginFailure);
+  app.post('/login/activateInvited'   , applicationInstalled, loginFormValidator.inviteRules(), loginFormValidator.inviteValidation, csrf, login.invited);
+  app.post('/login'                   , applicationInstalled, loginFormValidator.loginRules(), loginFormValidator.loginValidation, csrf,  addActivity, loginPassport.loginWithLocal, loginPassport.loginWithLdap, loginPassport.loginFailure);
 
-  app.post('/register'                , apiLimiter , applicationInstalled, registerFormValidator.registerRules(), registerFormValidator.registerValidation, csrf, addActivity, login.register);
+  app.post('/register'                , applicationInstalled, registerFormValidator.registerRules(), registerFormValidator.registerValidation, csrf, addActivity, login.register);
   app.get('/register'                 , applicationInstalled, login.preLogin, login.register);
 
   app.get('/admin'                    , applicationInstalled, loginRequiredStrictly , adminRequired , admin.index);
@@ -91,7 +86,7 @@ module.exports = function(crowi, app) {
   if (!isInstalled) {
     const installer = require('./installer')(crowi);
     app.get('/installer'              , applicationNotInstalled , installer.index);
-    app.post('/installer'             , apiLimiter , applicationNotInstalled , registerFormValidator.registerRules(), registerFormValidator.registerValidation, csrf, addActivity, installer.install);
+    app.post('/installer'             , applicationNotInstalled , registerFormValidator.registerRules(), registerFormValidator.registerValidation, csrf, addActivity, installer.install);
     return;
   }
 
@@ -108,7 +103,7 @@ module.exports = function(crowi, app) {
   app.get('/passport/oidc/callback'               , loginPassport.loginPassportOidcCallback     , loginPassport.loginFailure);
   app.post('/passport/saml/callback'              , addActivity, loginPassport.loginPassportSamlCallback, loginPassport.loginFailure);
 
-  app.post('/_api/login/testLdap'    , apiLimiter , loginRequiredStrictly , loginFormValidator.loginRules() , loginFormValidator.loginValidation , loginPassport.testLdapCredentials);
+  app.post('/_api/login/testLdap'    , loginRequiredStrictly , loginFormValidator.loginRules() , loginFormValidator.loginValidation , loginPassport.testLdapCredentials);
 
   // security admin
   app.get('/admin/security'          , loginRequiredStrictly , adminRequired , admin.security.index);
@@ -230,15 +225,15 @@ module.exports = function(crowi, app) {
   app.use('/forgot-password', express.Router()
     .use(forgotPassword.checkForgotPasswordEnabledMiddlewareFactory(crowi))
     .get('/', forgotPassword.forgotPassword)
-    .get('/:token', apiLimiter, injectResetOrderByTokenMiddleware, forgotPassword.resetPassword)
+    .get('/:token', injectResetOrderByTokenMiddleware, forgotPassword.resetPassword)
     .use(forgotPassword.handleErrosMiddleware));
 
   app.use('/_private-legacy-pages', express.Router()
     .get('/', injectUserUISettings, privateLegacyPages.renderPrivateLegacyPages));
   app.use('/user-activation', express.Router()
-    .get('/:token', apiLimiter, applicationInstalled, injectUserRegistrationOrderByTokenMiddleware, userActivation.form)
+    .get('/:token', applicationInstalled, injectUserRegistrationOrderByTokenMiddleware, userActivation.form)
     .use(userActivation.tokenErrorHandlerMiddeware));
-  app.post('/user-activation/register', apiLimiter, applicationInstalled, csrf, userActivation.registerRules(), userActivation.validateRegisterForm, userActivation.registerAction(crowi));
+  app.post('/user-activation/register', applicationInstalled, csrf, userActivation.registerRules(), userActivation.validateRegisterForm, userActivation.registerAction(crowi));
 
   app.get('/share/:linkId', page.showSharedPage);
 

packages/app/src/server/util/rate-limiter.ts

@@ -0,0 +1,84 @@
+import {
+  defaultConfig, defaultConfigWithRegExp, IApiRateLimitEndpointMap,
+} from '^/config/rate-limiter';
+
+const envVar = process.env;
+
+// https://regex101.com/r/aNDjmI/1
+const regExp = /^API_RATE_LIMIT_(\w+)_ENDPOINT(_WITH_REGEXP)?$/;
+
+const generateApiRateLimitConfigFromEndpoint = (envVar: NodeJS.ProcessEnv, targets: string[], withRegExp: boolean): IApiRateLimitEndpointMap => {
+  const apiRateLimitConfig: IApiRateLimitEndpointMap = {};
+  targets.forEach((target) => {
+
+    const endpointKey = withRegExp ? `API_RATE_LIMIT_${target}_ENDPOINT_WITH_REGEXP` : `API_RATE_LIMIT_${target}_ENDPOINT`;
+
+    const endpoint = envVar[endpointKey];
+
+    if (endpoint == null) {
+      return;
+    }
+    const methodKey = `API_RATE_LIMIT_${target}_METHODS`;
+    const maxRequestsKey = `API_RATE_LIMIT_${target}_MAX_REQUESTS`;
+    const method = envVar[methodKey] ?? 'ALL';
+    const maxRequests = Number(envVar[maxRequestsKey]);
+
+    if (endpoint == null || maxRequests == null) {
+      return;
+    }
+
+    const config = {
+      method,
+      maxRequests,
+    };
+
+    apiRateLimitConfig[endpoint] = config;
+  });
+
+  return apiRateLimitConfig;
+};
+
+type ApiRateLimitConfigResult = {
+  'withoutRegExp': IApiRateLimitEndpointMap,
+  'withRegExp': IApiRateLimitEndpointMap
+}
+
+export const generateApiRateLimitConfig = (): ApiRateLimitConfigResult => {
+
+  const apiRateConfigTargets: string[] = [];
+  const apiRateConfigTargetsWithRegExp: string[] = [];
+  Object.keys(envVar).forEach((key) => {
+    const result = key.match(regExp);
+
+    if (result == null) { return null }
+
+    const target = result[1];
+    const isWithRegExp = result[2] != null;
+
+    if (isWithRegExp) {
+      apiRateConfigTargetsWithRegExp.push(target);
+    }
+    else {
+      apiRateConfigTargets.push(target);
+    }
+  });
+
+  // sort priority
+  apiRateConfigTargets.sort();
+  apiRateConfigTargetsWithRegExp.sort();
+
+  // get config
+  const apiRateLimitConfig = generateApiRateLimitConfigFromEndpoint(envVar, apiRateConfigTargets, false);
+  const apiRateLimitConfigWithRegExp = generateApiRateLimitConfigFromEndpoint(envVar, apiRateConfigTargetsWithRegExp, true);
+
+  const config = { ...defaultConfig, ...apiRateLimitConfig };
+  const configWithRegExp = { ...defaultConfigWithRegExp, ...apiRateLimitConfigWithRegExp };
+
+  const result: ApiRateLimitConfigResult = {
+    withoutRegExp: config,
+    withRegExp: configWithRegExp,
+  };
+
+
+  return result;
+};

packages/app/tsconfig.build.server.json

@@ -15,6 +15,7 @@
     }
   },
   "exclude": [
+    "resouce",
     "src/client",
     "src/components",
     "src/linter-checker",

packages/core/src/utils/page-utils.ts

@@ -1,12 +1,12 @@
 import { isTopPage } from './page-path-utils';
 
-const GRANT_PUBLIC = 1;
+// const GRANT_PUBLIC = 1;
 const GRANT_RESTRICTED = 2;
 const GRANT_SPECIFIED = 3; // DEPRECATED
-const GRANT_OWNER = 4;
-const GRANT_USER_GROUP = 5;
-const PAGE_GRANT_ERROR = 1;
-const STATUS_PUBLISHED = 'published';
+// const GRANT_OWNER = 4;
+// const GRANT_USER_GROUP = 5;
+// const PAGE_GRANT_ERROR = 1;
+// const STATUS_PUBLISHED = 'published';
 const STATUS_DELETED = 'deleted';
 
 /**

yarn.lock

@@ -9483,11 +9483,6 @@ express-mongo-sanitize@^2.1.0:
   resolved "https://registry.yarnpkg.com/express-mongo-sanitize/-/express-mongo-sanitize-2.1.0.tgz#a8c647787c25ded6e97b5e864d113e7687c5d471"
   integrity sha512-ELGeH/Tx+kJGn3klCzSmOewfN1ezJQrkqzq83dl/K3xhd5PUbvLtiD5CiuYRmQfoZPL4rUEVjANf/YjE2BpTWQ==
 
-express-rate-limit@^5.3.0:
-  version "5.3.0"
-  resolved "https://registry.yarnpkg.com/express-rate-limit/-/express-rate-limit-5.3.0.tgz#e7b9d3c2e09ece6e0406a869b2ce00d03fe48aea"
-  integrity sha512-qJhfEgCnmteSeZAeuOKQ2WEIFTX5ajrzE0xS6gCOBCoRQcU+xEzQmgYQQTpzCcqUAAzTEtu4YEih4pnLfvNtew==
-
 express-session@^1.16.1:
   version "1.16.1"
   resolved "https://registry.yarnpkg.com/express-session/-/express-session-1.16.1.tgz#251ff9776c59382301de6c8c33411af357ed439c"
@@ -17509,6 +17504,11 @@ range-parser@~1.2.1:
   resolved "https://registry.yarnpkg.com/range-parser/-/range-parser-1.2.1.tgz#3cf37023d199e1c24d1a55b84800c2f3e6468031"
   integrity sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==
 
+rate-limiter-flexible@^2.3.7:
+  version "2.3.7"
+  resolved "https://registry.yarnpkg.com/rate-limiter-flexible/-/rate-limiter-flexible-2.3.7.tgz#c23e1f818a1575f1de1fd173437f4072125e1615"
+  integrity sha512-dmc+J/IffVBvHlqq5/XClsdLdkOdQV/tjrz00cwneHUbEDYVrf4aUDAyR4Jybcf2+Vpn4NwoVrnnAyt/D0ciWw==
+
 raw-body@2.3.2:
   version "2.3.2"
   resolved "https://registry.yarnpkg.com/raw-body/-/raw-body-2.3.2.tgz#bcd60c77d3eb93cde0050295c3f379389bc88f89"