Merge pull request #6225 from weseek/imprv/refactor-api-rate-limiter

imprv: API Rate Limiter

packages/app/config/api-rate-limiter.ts

@@ -0,0 +1,83 @@
+export type IApiRateLimitConfig = {
+  method: string,
+  maxRequests: number,
+  usersPerIpProspection?: number,
+}
+export type IApiRateLimitEndpointMap = {
+  [endpoint: string]: IApiRateLimitConfig
+}
+
+export const DEFAULT_MAX_REQUESTS = 500;
+export const DEFAULT_DURATION_SEC = 60;
+export const DEFAULT_USERS_PER_IP_PROSPECTION = 5;
+
+const MAX_REQUESTS_TIER_1 = 5;
+const MAX_REQUESTS_TIER_2 = 20;
+const MAX_REQUESTS_TIER_3 = 50;
+const MAX_REQUESTS_TIER_4 = 100;
+
+// default config without reg exp
+export const defaultConfig: IApiRateLimitEndpointMap = {
+  '/_api/v3/healthcheck': {
+    method: 'GET',
+    maxRequests: 60,
+    usersPerIpProspection: 1,
+  },
+  '/installer': {
+    method: 'POST',
+    maxRequests: MAX_REQUESTS_TIER_1,
+    usersPerIpProspection: 1,
+  },
+  '/login': {
+    method: 'POST',
+    maxRequests: MAX_REQUESTS_TIER_1,
+    usersPerIpProspection: 100,
+  },
+  '/login/activateInvited': {
+    method: 'POST',
+    maxRequests: MAX_REQUESTS_TIER_2,
+  },
+  '/register': {
+    method: 'POST',
+    maxRequests: MAX_REQUESTS_TIER_1,
+    usersPerIpProspection: 20,
+  },
+  '/user-activation/register': {
+    method: 'POST',
+    maxRequests: MAX_REQUESTS_TIER_1,
+    usersPerIpProspection: 20,
+  },
+  '/_api/login/testLdap': {
+    method: 'POST',
+    maxRequests: MAX_REQUESTS_TIER_2,
+    usersPerIpProspection: 1,
+  },
+  '/_api/check_username': {
+    method: 'GET',
+    maxRequests: MAX_REQUESTS_TIER_3,
+  },
+};
+
+// default config with reg exp
+export const defaultConfigWithRegExp = {
+  '/forgot-password/.*': {
+    method: 'ALL',
+    maxRequests: MAX_REQUESTS_TIER_1,
+  },
+  '/user-activation/.*': {
+    method: 'GET',
+    maxRequests: MAX_REQUESTS_TIER_1,
+  },
+  '/attachment/[0-9a-z]{24}': {
+    method: 'GET',
+    maxRequests: MAX_REQUESTS_TIER_4,
+  },
+  '/download/[0-9a-z]{24}': {
+    method: 'GET',
+    maxRequests: MAX_REQUESTS_TIER_4,
+  },
+  '/share/[0-9a-z]{24}': {
+    method: 'GET',
+    maxRequests: MAX_REQUESTS_TIER_4,
+  },
+};

packages/app/src/server/interfaces/api-rate-limit-config.ts

@@ -1,6 +0,0 @@
-export type IApiRateLimitConfig = {
-  [endpoint: string]: {
-    method: string,
-    maxRequests: number
-  }
-}

packages/app/src/server/middlewares/api-rate-limiter.ts

@@ -1,11 +1,16 @@
 import { NextFunction, Request, Response } from 'express';
 import md5 from 'md5';
 import mongoose from 'mongoose';
-import { RateLimiterMongo } from 'rate-limiter-flexible';
+import { IRateLimiterMongoOptions, RateLimiterMongo } from 'rate-limiter-flexible';
 
+import {
+  DEFAULT_DURATION_SEC, DEFAULT_MAX_REQUESTS, DEFAULT_USERS_PER_IP_PROSPECTION, IApiRateLimitConfig,
+} from '^/config/api-rate-limiter';
+
+import { IUserHasId } from '~/interfaces/user';
 import loggerFactory from '~/utils/logger';
 
-import { generateApiRateLimitConfig } from '../util/api-rate-limit-config/generateApiRateLimitConfig';
+import { generateApiRateLimitConfig } from '../util/api-rate-limiter';
 
 
 const logger = loggerFactory('growi:middleware:api-rate-limit');
@@ -15,13 +20,12 @@ const logger = loggerFactory('growi:middleware:api-rate-limit');
 // API_RATE_LIMIT_010_FOO_METHODS=GET,POST
 // API_RATE_LIMIT_010_FOO_MAX_REQUESTS=10
 
-const defaultMaxPoints = 100;
-const defaultMaxRequests = 10;
-const defaultDuration = 1;
-const opts = {
+const POINTS_THRESHOLD = 100;
+
+const opts: IRateLimiterMongoOptions = {
   storeClient: mongoose.connection,
-  points: defaultMaxPoints, // set default value
-  duration: defaultDuration, // set default value
+  points: POINTS_THRESHOLD, // set default value
+  duration: DEFAULT_DURATION_SEC, // set default value
 };
 const rateLimiter = new RateLimiterMongo(opts);
 
@@ -30,22 +34,73 @@ const apiRateLimitConfig = generateApiRateLimitConfig();
 const configWithoutRegExp = apiRateLimitConfig.withoutRegExp;
 const configWithRegExp = apiRateLimitConfig.withRegExp;
 const allRegExp = new RegExp(Object.keys(configWithRegExp).join('|'));
-const keysWithRegExp = Object.keys(configWithRegExp).map(key => new RegExp(key));
+const keysWithRegExp = Object.keys(configWithRegExp).map(key => new RegExp(`^${key}`));
 const valuesWithRegExp = Object.values(configWithRegExp);
 
-const consumePoints = async(rateLimiter: RateLimiterMongo, key: string, maxRequests: number) => {
-  const consumePoints = Math.floor(defaultMaxPoints / maxRequests);
+
+const _consumePoints = async(
+    method: string, key: string | null, customizedConfig?: IApiRateLimitConfig, maxRequestsMultiplier?: number,
+) => {
+  if (key == null) {
+    return;
+  }
+
+  let maxRequests = DEFAULT_MAX_REQUESTS;
+
+  // use customizedConfig
+  if (customizedConfig != null && (customizedConfig.method.includes(method) || customizedConfig.method === 'ALL')) {
+    maxRequests = customizedConfig.maxRequests;
+  }
+
+  // multiply
+  if (maxRequestsMultiplier != null) {
+    maxRequests *= maxRequestsMultiplier;
+  }
+
+  // because the maximum request is reduced by 1 if it is divisible by
+  // https://github.com/weseek/growi/pull/6225
+  const consumePoints = (POINTS_THRESHOLD + 0.0001) / maxRequests;
   await rateLimiter.consume(key, consumePoints);
 };
 
+/**
+ * consume per user per endpoint
+ * @param method
+ * @param key
+ * @param customizedConfig
+ * @returns
+ */
+const consumePointsByUser = async(method: string, key: string | null, customizedConfig?: IApiRateLimitConfig) => {
+  return _consumePoints(method, key, customizedConfig);
+};
+
+/**
+ * consume per ip per endpoint
+ * @param method
+ * @param key
+ * @param customizedConfig
+ * @returns
+ */
+const consumePointsByIp = async(method: string, key: string | null, customizedConfig?: IApiRateLimitConfig) => {
+  const maxRequestsMultiplier = customizedConfig?.usersPerIpProspection ?? DEFAULT_USERS_PER_IP_PROSPECTION;
+  return _consumePoints(method, key, customizedConfig, maxRequestsMultiplier);
+};
+
+
 module.exports = () => {
 
-  return async(req: Request, res: Response, next: NextFunction) => {
+  return async(req: Request & { user?: IUserHasId }, res: Response, next: NextFunction) => {
 
     const endpoint = req.path;
-    const key = md5(`${req.ip}_${endpoint}_${req.method}`);
 
-    let customizedConfig;
+    // determine keys
+    const keyForUser: string | null = req.user != null
+      ? md5(`${req.user._id}_${endpoint}_${req.method}`)
+      : null;
+    const keyForIp: string = md5(`${req.ip}_${endpoint}_${req.method}`);
+
+    // determine customized config
+    let customizedConfig: IApiRateLimitConfig | undefined;
     const configForEndpoint = configWithoutRegExp[endpoint];
     if (configForEndpoint) {
       customizedConfig = configForEndpoint;
@@ -58,23 +113,26 @@ module.exports = () => {
       });
     }
 
-    try {
-      if (customizedConfig === undefined) {
-        await consumePoints(rateLimiter, key, defaultMaxRequests);
-        return next();
+    // check for the current user
+    if (req.user != null) {
+      try {
+        await consumePointsByUser(req.method, keyForUser, customizedConfig);
       }
-
-      if (customizedConfig.method.includes(req.method) || customizedConfig.method === 'ALL') {
-        await consumePoints(rateLimiter, key, customizedConfig.maxRequests);
-        return next();
+      catch {
+        logger.error(`${req.user._id}: too many request at ${endpoint}`);
+        return res.sendStatus(429);
       }
+    }
 
-      await consumePoints(rateLimiter, key, defaultMaxRequests);
-      return next();
+    // check for ip
+    try {
+      await consumePointsByIp(req.method, keyForIp, customizedConfig);
     }
     catch {
       logger.error(`${req.ip}: too many request at ${endpoint}`);
       return res.sendStatus(429);
     }
+
+    return next();
   };
 };

packages/app/src/server/routes/index.js

@@ -57,15 +57,15 @@ module.exports = function(crowi, app) {
 
   app.use('/api-docs', require('./apiv3/docs')(crowi));
 
+  // API rate limiter
+  app.use(apiRateLimiter);
+
   // API v3 for admin
   app.use('/_api/v3', apiV3AdminRouter);
 
   // API v3 for auth
   app.use('/_api/v3', apiV3AuthRouter);
 
-  // API rate limiter
-  app.use(apiRateLimiter);
-
   app.get('/'                         , applicationInstalled, unavailableWhenMaintenanceMode, loginRequired, autoReconnectToSearch, injectUserUISettings, page.showTopPage);
 
   app.get('/login/error/:reason'      , applicationInstalled, login.error);

packages/app/src/server/util/api-rate-limit-config/defaultApiRateLimitConfig.ts

@@ -1,62 +0,0 @@
-import { IApiRateLimitConfig } from '../../interfaces/api-rate-limit-config';
-
-// strict config
-const defaultStrictMaxRequests = 1; // per second
-const defaultStrictConfig: IApiRateLimitConfig = {
-  '/login/activateInvited': {
-    method: 'POST',
-    maxRequests: defaultStrictMaxRequests,
-  },
-  '/login': {
-    method: 'POST',
-    maxRequests: defaultStrictMaxRequests,
-  },
-  '/register': {
-    method: 'POST',
-    maxRequests: defaultStrictMaxRequests,
-  },
-  '/installer': {
-    method: 'POST',
-    maxRequests: defaultStrictMaxRequests,
-  },
-  '/_api/login/testLdap': {
-    method: 'POST',
-    maxRequests: defaultStrictMaxRequests,
-  },
-  '/user-activation/register': {
-    method: 'POST',
-    maxRequests: defaultStrictMaxRequests,
-  },
-};
-
-
-// infinity config
-const defaultInfinityConfig: IApiRateLimitConfig = {
-  '/_api/v3/healthcheck': {
-    method: 'GET',
-    maxRequests: Infinity,
-  },
-};
-
-// default config without reg exp
-export const defaultConfig = { ...defaultStrictConfig, ...defaultInfinityConfig };
-
-// default config with reg exp
-export const defaultConfigWithRegExp = {
-  '/forgot-password/.*': {
-    method: 'GET',
-    maxRequests: defaultStrictMaxRequests,
-  },
-  '/user-activation/.*': {
-    method: 'GET',
-    maxRequests: defaultStrictMaxRequests,
-  },
-  '/download/[0-9a-z]{24}': {
-    method: 'GET',
-    maxRequests: defaultStrictMaxRequests,
-  },
-  '/share/[0-9a-z]{24}': {
-    method: 'GET',
-    maxRequests: defaultStrictMaxRequests,
-  },
-};

packages/app/src/server/util/api-rate-limit-config/generateApiRateLimitConfig.ts → packages/app/src/server/util/api-rate-limiter.ts

@@ -1,14 +1,14 @@
-import { IApiRateLimitConfig } from '../../interfaces/api-rate-limit-config';
-
-import { defaultConfig, defaultConfigWithRegExp } from './defaultApiRateLimitConfig';
+import {
+  defaultConfig, defaultConfigWithRegExp, IApiRateLimitEndpointMap,
+} from '^/config/api-rate-limiter';
 
 const envVar = process.env;
 
 // https://regex101.com/r/aNDjmI/1
 const regExp = /^API_RATE_LIMIT_(\w+)_ENDPOINT(_WITH_REGEXP)?$/;
 
-const generateApiRateLimitConfigFromEndpoint = (envVar: NodeJS.ProcessEnv, targets: string[], withRegExp: boolean): IApiRateLimitConfig => {
-  const apiRateLimitConfig: IApiRateLimitConfig = {};
+const generateApiRateLimitConfigFromEndpoint = (envVar: NodeJS.ProcessEnv, targets: string[], withRegExp: boolean): IApiRateLimitEndpointMap => {
+  const apiRateLimitConfig: IApiRateLimitEndpointMap = {};
   targets.forEach((target) => {
 
     const endpointKey = withRegExp ? `API_RATE_LIMIT_${target}_ENDPOINT_WITH_REGEXP` : `API_RATE_LIMIT_${target}_ENDPOINT`;
@@ -39,8 +39,8 @@ const generateApiRateLimitConfigFromEndpoint = (envVar: NodeJS.ProcessEnv, targe
 };
 
 type ApiRateLimitConfigResult = {
-  'withoutRegExp': IApiRateLimitConfig,
-  'withRegExp': IApiRateLimitConfig
+  'withoutRegExp': IApiRateLimitEndpointMap,
+  'withRegExp': IApiRateLimitEndpointMap
 }
 
 export const generateApiRateLimitConfig = (): ApiRateLimitConfigResult => {