refs 124384: impl ldap group tree sync

apps/app/src/interfaces/external-user-group.ts

@@ -1,12 +1,18 @@
-import { IUserGroupRelation, Ref } from '@growi/core';
+import { HasObjectId, IUserGroupRelation, Ref } from '@growi/core';
 
 import { IUserGroup } from './user';
 
+export const ExternalGroupProviderType = { ldap: 'ldap' } as const;
+export type ExternalGroupProviderType = typeof ExternalGroupProviderType[keyof typeof ExternalGroupProviderType];
+
 export interface IExternalUserGroup extends Omit<IUserGroup, 'parent'> {
   parent: Ref<IExternalUserGroup> | null
   externalId: string // identifier used in external app/server
+  provider: ExternalGroupProviderType
 }
 
+export type IExternalUserGroupHasId = IExternalUserGroup & HasObjectId;
+
 export interface IExternalUserGroupRelation extends Omit<IUserGroupRelation, 'relatedGroup'> {
   relatedGroup: Ref<IExternalUserGroup>
 }
@@ -22,11 +28,11 @@ export interface LdapGroupSyncSettings {
   ldapGroupDescriptionAttribute?: string
 }
 
-// interface for objects before they are converted into ExternalUserGroup
-export interface LdapGroup {
-  dn: string
-  users: string[] // DN or UID
-  childGroups: string[] // DN
+// Data structure to express the tree structure of external groups, before converting to ExternalUserGroup model
+export interface ExternalUserGroupTreeNode {
+  id: string
+  externalUserIds: string[]
+  childGroupNodes: ExternalUserGroupTreeNode[]
   name: string
   description?: string
 }

apps/app/src/server/models/external-user-group.ts

@@ -16,14 +16,15 @@ const schema = new Schema<ExternalUserGroupDocument, ExternalUserGroupModel>({
   parent: { type: Schema.Types.ObjectId, ref: 'ExternalUserGroup', index: true },
   description: { type: String, default: '' },
   externalId: { type: String, required: true, unique: true },
+  provider: { type: String, required: true },
 }, {
   timestamps: true,
 });
 
-schema.statics.createGroup = async function(name, description, externalId, parentId) {
+schema.statics.findAndUpdateOrCreateGroup = async function(name, description, externalId, provider, parentId) {
   // create without parent
   if (parentId == null) {
-    return this.create({ name, description, externalId });
+    return this.findOneAndUpdate({ name }, { description, externalId, provider }, { upsert: true });
   }
 
   // create with parent
@@ -31,13 +32,9 @@ schema.statics.createGroup = async function(name, description, externalId, paren
   if (parent == null) {
     throw Error('Parent does not exist.');
   }
-  return this.create({
-    name, description, externalId, parent,
-  });
-};
-
-schema.statics.getByExternalIdOrCreateGroup = async function(name, description, externalId, parentId) {
-  return this.createGroup(name, description, externalId, parentId);
+  return this.findOneAndUpdate({ name }, {
+    description, externalId, provider, parent,
+  }, { upsert: true });
 };
 
 export default getOrCreateModel<ExternalUserGroupDocument, ExternalUserGroupModel>('ExternalUserGroup', schema);

apps/app/src/server/routes/apiv3/external-user-group.ts

@@ -81,12 +81,12 @@ module.exports = (crowi: Crowi): Router => {
 
   router.put('/ldap/sync', loginRequiredStrictly, adminRequired, async(req: AuthorizedRequest, res: ApiV3Response) => {
     try {
-      const ldapUserGroupSyncService = new LdapUserGroupSyncService(req.user.name, req.body.password);
-      console.log(await ldapUserGroupSyncService.getMemberUser('cn=customuser2,ou=users,dc=example,dc=org'));
-      // console.log(schema.statics);
+      const ldapUserGroupSyncService = new LdapUserGroupSyncService(crowi, req.user.name, req.body.password);
+      await ldapUserGroupSyncService.syncExternalUserGroups();
     }
-    catch (e) {
-      res.apiv3Err(e, 500);
+    catch (err) {
+      logger.error(err);
+      return res.apiv3Err(err, 500);
     }
 
     return res.apiv3({}, 204);

apps/app/src/server/service/external-group/external-user-group-sync-service.ts

@@ -1,35 +1,86 @@
-import { IExternalUserGroup } from '~/interfaces/external-user-group';
-import { IUser } from '~/interfaces/user';
+import { ExternalGroupProviderType, ExternalUserGroupTreeNode, IExternalUserGroupHasId } from '~/interfaces/external-user-group';
+import { IUserHasId } from '~/interfaces/user';
+import ExternalUserGroup from '~/server/models/external-user-group';
+import ExternalUserGroupRelation from '~/server/models/external-user-group-relation';
+
+import { configManager } from '../config-manager';
 
 abstract class ExternalUserGroupSyncService {
 
-  // 全グループ同期メソッド
-  /* 継承先の実メソッドイメージ
-     1. 読み込まれたパラメータを元に外部グループを全て取得する
-     2. 各親グループについて、createUpdateExternalUserGroup を呼び出す
-     3. 子についても同様に呼び出し、返却された子グループを親グループと紐付ける
-     4. 2, 3 を再起的に行う
-         - 木探索アルゴリズムはなんでも良いが、実クラスで実装が容易になるように上手く抽象化したい
-     5. 「外部サービスから削除されたグループを GROWI に残すか」が false の場合、木探索の過程で見つからなかった ExternalUserGroup は削除する
-    */
-  abstract syncExternalUserGroups(): void
-
-  // グループ生成/更新メソッド
-  /* 継承先の実メソッドイメージ
-     1. 読み込まれたパラメータを元に外部グループ情報をリクエストする
-     2. 読み込まれたパラメータと 1 で返却された外部グループ情報を元に ExternalUserGroup を生成/更新する
-     3. 外部グループ情報にある各ユーザ情報を元に、ExternalUserGroup に所属していないメンバーについて getMemberUser を呼び出し、返却されたユーザを ExternalUserGroup に所属させる (ExternalUserGroupRelation を生成する)
-     4. ExternalUserGroup を返却する
-    */
-  // abstract createUpdateExternalUserGroup(): IExternalUserGroup
-
-  // ユーザ検索メソッド
-  /* 継承先の実メソッドイメージ
-     1. 読み込まれたパラメータパラメータを元に外部ユーザ情報をリクエストする
-     2. 読み込まれたパラメータと 1 で返却された外部ユーザ情報を元に GROWI User を検索し、返却する
-       - 「作成されていない GROWI アカウントを自動生成するか」が true の場合、検索して見つからなければ生成して返却する
-    */
-  // abstract getMemberUser(): IUser
+  provider: ExternalGroupProviderType; // name of external service that contains user group info (e.g: ldap)
+
+  constructor(provider: ExternalGroupProviderType) {
+    this.provider = provider;
+  }
+
+  /** External user group tree sync method
+   * 1. Generate external user group tree
+   * 2. Use createUpdateExternalUserGroup on each node in the tree using DFS
+   * 3. If preserveDeletedLDAPGroups is false、delete all ExternalUserGroups that were not found during tree search
+  */
+  async syncExternalUserGroups(): Promise<void> {
+    const trees = await this.generateExternalUserGroupTrees();
+
+    const existingExternalUserGroupIds: string[] = [];
+
+    const syncNode = async(node: ExternalUserGroupTreeNode, parentId?: string) => {
+      const externalUserGroup = await this.createUpdateExternalUserGroup(node, parentId);
+      existingExternalUserGroupIds.push(externalUserGroup._id);
+      node.childGroupNodes.forEach((childNode) => {
+        syncNode(childNode, externalUserGroup._id);
+      });
+    };
+
+    await Promise.all(trees.map((root) => {
+      return syncNode(root);
+    }));
+
+    const preserveDeletedLdapGroups: boolean = configManager?.getConfig('crowi', `external-user-group:${this.provider}:preserveDeletedGroups`);
+    if (!preserveDeletedLdapGroups) {
+      await ExternalUserGroup.deleteMany({ _id: { $nin: existingExternalUserGroupIds }, provider: this.provider });
+    }
+  }
+
+  /** External user group node sync method
+   * 1. Create/Update ExternalUserGroup from using information of ExternalUserGroupTreeNode
+   * 2. For every element in node.users, call getMemberUser and create an ExternalUserGroupRelation with ExternalUserGroup if it does not have one
+   * 3. Retrun ExternalUserGroup
+   * @param {string} node Node of external group tree
+   * @param {string} parentId Parent group id (id in GROWI) of the group we wan't to create/update
+   * @returns {Promise<IExternalUserGroupHasId>} ExternalUserGroup that was created/updated
+  */
+  async createUpdateExternalUserGroup(node: ExternalUserGroupTreeNode, parentId?: string): Promise<IExternalUserGroupHasId> {
+    const externalUserGroup = await ExternalUserGroup.findAndUpdateOrCreateGroup(
+      node.name, node.description, node.id, this.provider, parentId,
+    );
+    await Promise.all(node.externalUserIds.map((externalUserId) => {
+      return (async() => {
+        const user = await this.getMemberUser(externalUserId);
+        // TODO: create relations for parent groups also
+        if (user != null) {
+          await ExternalUserGroupRelation.findOrCreateRelation(externalUserGroup, user);
+        }
+      })();
+    }));
+
+    return externalUserGroup;
+  }
+
+  /** Method to get group member GROWI user
+   * 1. Execute search on external app/server for user info using externalUserId
+   * 2. Search for GROWI user based on user info of 1, and return user
+   *   - if autoGenerateUserOnHogeGroupSync is true and GROWI user is not found, create new GROWI user
+   * @param {string} externalUserId Search LDAP server using this identifier (DN or UID)
+   * @returns {Promise<IUserHasId | null>} User when found or created, null when neither
+   */
+  abstract getMemberUser(externalUserId: string): Promise<IUserHasId | null>
+
+  /** Method to generate external group tree structure
+   * 1. Fetch user group info from external app/server
+   * 2. Convert each group tree structure to ExternalUserGroupTreeNode
+   * 3. Return the root node of each tree
+  */
+  abstract generateExternalUserGroupTrees(): Promise<ExternalUserGroupTreeNode[]>;
 
 }
 

apps/app/src/server/service/external-group/ldap-user-group-sync-service.ts

@@ -1,7 +1,5 @@
-import { IExternalUserGroup, LdapGroup } from '~/interfaces/external-user-group';
+import { ExternalGroupProviderType, ExternalUserGroupTreeNode } from '~/interfaces/external-user-group';
 import { IUserHasId } from '~/interfaces/user';
-import ExternalUserGroup from '~/server/models/external-user-group';
-import ExternalUserGroupRelation from '~/server/models/external-user-group-relation';
 
 import { configManager } from '../config-manager';
 import ExternalAccountService from '../external-account';
@@ -9,9 +7,7 @@ import LdapService, { SearchResultEntry } from '../ldap';
 
 import ExternalUserGroupSyncService from './external-user-group-sync-service';
 
-class LdapUserGroupSyncService {
-
-  ldapGroups: LdapGroup[];
+class LdapUserGroupSyncService extends ExternalUserGroupSyncService {
 
   ldapService: LdapService;
 
@@ -21,72 +17,21 @@ class LdapUserGroupSyncService {
 
   // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types
   constructor(crowi: any, userBindUsername?: string, userBindPassword?: string) {
+    super(ExternalGroupProviderType.ldap);
     this.crowi = crowi;
     this.ldapService = new LdapService(userBindUsername, userBindPassword);
     this.externalAccountService = new ExternalAccountService(crowi);
   }
 
-  async fetchLdapGroups(): Promise<void> {
-    const isUserBind = configManager.getConfig('crowi', 'security:passport-ldap:isUserBind');
-    const getGroupDirData = async() => {
-      if (isUserBind) {
-        return this.ldapService.searchGroupDir();
-      }
-      return this.ldapService.searchGroupDir();
-    };
-
-    const groupDirData = await getGroupDirData();
-
-    this.ldapGroups = groupDirData.map(data => this.convertSearchResultEntryToLdapGroup(data))
-      .filter((group): group is NonNullable<LdapGroup> => group != null);
-  }
-
-  // 全グループ同期メソッド
-  syncExternalUserGroups(): void {
-    /*
-     1. ldapGroupSearchBase を使って LDAP から全てのグループを取得する
-         - 設定値を元に LDAPGroup に変換する
-     2. 各親グループについて、createUpdateExternalUserGroup を呼び出す
-     3. 子についても同様に呼び出し、返却された子グループを親グループと紐付ける
-     4. 2, 3 を再起的に行う
-     5. preserveDeletedLDAPGroups が false の場合、木探索の過程で見つからなかった ExternalUserGroup は削除する
-    */
-  }
-
-  /**
-   * 1. Create/Update ExternalUserGroup from ldapGroup
-   * 2. For every element in ldapGroup.users, call getMemberUser and create an ExternalUserGroupRelation with ExternalUserGroup if it does not have one
-   * 3. Retrun ExternalUserGroup
-  */
-  async createUpdateExternalUserGroup(ldapGroup: LdapGroup, parentId: string): Promise<IExternalUserGroup> {
-    const externalUserGroup = await ExternalUserGroup.createGroup(ldapGroup.name, ldapGroup.description, ldapGroup.dn, parentId);
-
-    await Promise.all(ldapGroup.users.map((userIdentifier) => {
-      return (async() => {
-        const user = await this.getMemberUser(userIdentifier);
-        await ExternalUserGroupRelation.findOrCreateRelation(externalUserGroup, user);
-      })();
-    }));
-
-    return externalUserGroup;
-  }
-
-  /**
-   * 1. Execute search on LDAP server for user using useridentifier
-   * 2. Search for GROWI user based on LDAP user info, and return
-   *   - if autoGenerateUserOnLDAPGroupSync is true and GROWI user is not found, create new GROWI user
-   * @param {string} userIdentifier Search LDAP server using this identifier (DN or UID)
-   * @returns {Promise<IUser | null>} IUser when found or created, null when neither
-   */
-  async getMemberUser(userIdentifier: string): Promise<IUserHasId | null> {
+  async getMemberUser(externalUserId: string): Promise<IUserHasId | null> {
     const groupMembershipAttributeType = configManager?.getConfig('crowi', 'external-user-group:ldap:groupMembershipAttributeType');
 
     const getUser = async() => {
       if (groupMembershipAttributeType === 'DN') {
-        return this.ldapService.search(undefined, userIdentifier, 'one');
+        return this.ldapService.search(undefined, externalUserId, 'base');
       }
       if (groupMembershipAttributeType === 'UID') {
-        return this.ldapService.search(`(uid=${userIdentifier})`, undefined, 'one');
+        return this.ldapService.search(`(uid=${externalUserId})`, undefined);
       }
     };
 
@@ -106,25 +51,62 @@ class LdapUserGroupSyncService {
     return null;
   }
 
-  private convertSearchResultEntryToLdapGroup(groupEntry: SearchResultEntry): LdapGroup | null {
+  async generateExternalUserGroupTrees(): Promise<ExternalUserGroupTreeNode[]> {
     const groupChildGroupAttribute: string = configManager.getConfig('crowi', 'external-user-group:ldap:groupChildGroupAttribute');
     const groupMembershipAttribute: string = configManager.getConfig('crowi', 'external-user-group:ldap:groupMembershipAttribute');
-    const groupNameAttribute = configManager.getConfig('crowi', 'external-user-group:ldap:groupNameAttribute');
+    const groupNameAttribute: string = configManager.getConfig('crowi', 'external-user-group:ldap:groupNameAttribute');
     const groupDescriptionAttribute: string = configManager.getConfig('crowi', 'external-user-group:ldap:groupDescriptionAttribute');
+    const groupBase: string = this.ldapService.getGroupSearchBase();
+
+    const groupEntries = await this.ldapService.searchGroupDir();
+
+    const getChildGroupDnsFromGroupEntry = (groupEntry: SearchResultEntry) => {
+      // groupChildGroupAttribute and groupMembershipAttribute may be the same,
+      // so filter values of groupChildGroupAttribute to ones that include groupBase
+      return this.ldapService.getArrayValFromSearchResultEntry(groupEntry, groupChildGroupAttribute).filter(attr => attr.includes(groupBase));
+    };
+    const getExternalUserIdsFromGroupEntry = (groupEntry: SearchResultEntry) => {
+      // groupChildGroupAttribute and groupMembershipAttribute may be the same,
+      // so filter values of groupMembershipAttribute to ones that does not include groupBase
+      return this.ldapService.getArrayValFromSearchResultEntry(groupEntry, groupMembershipAttribute).filter(attr => !attr.includes(groupBase));
+    };
 
-    const childGroups = this.ldapService.getArrayValFromSearchResultEntry(groupEntry, groupChildGroupAttribute);
-    const users = this.ldapService.getArrayValFromSearchResultEntry(groupEntry, groupMembershipAttribute);
+    const convert = (entry: SearchResultEntry, converted: string[]): ExternalUserGroupTreeNode | null => {
+      if (converted.includes(entry.objectName)) {
+        throw Error('There is a possible circular reference in your LDAP group tree structure');
+      }
+      converted.push(entry.objectName);
+
+      const externalUserIds = getExternalUserIdsFromGroupEntry(entry);
+      const name = this.ldapService.getStringValFromSearchResultEntry(entry, groupNameAttribute);
+      const description = this.ldapService.getStringValFromSearchResultEntry(entry, groupDescriptionAttribute);
+      const childGroupDNs = getChildGroupDnsFromGroupEntry(entry);
+
+      const childGroupNodes: ExternalUserGroupTreeNode[] = childGroupDNs.map((dn) => {
+        const childEntry = groupEntries.find(ge => ge.objectName === dn);
+        return childEntry != null ? convert(childEntry, converted) : null;
+      }).filter((node): node is NonNullable<ExternalUserGroupTreeNode> => node != null);
+
+      return name != null ? {
+        id: entry.objectName,
+        externalUserIds,
+        childGroupNodes,
+        name,
+        description,
+      } : null;
+    };
+
+    // all the DNs of groups that are not a root of a tree
+    const allChildGroupDNs = new Set(groupEntries.flatMap((entry) => {
+      return getChildGroupDnsFromGroupEntry(entry);
+    }));
 
-    const name = this.ldapService.getStringValFromSearchResultEntry(groupEntry, groupNameAttribute);
-    const description = this.ldapService.getStringValFromSearchResultEntry(groupEntry, groupDescriptionAttribute);
+    // root of every tree
+    const rootEntries = groupEntries.filter((entry) => {
+      return !allChildGroupDNs.has(entry.objectName);
+    });
 
-    return name != null ? {
-      dn: groupEntry.objectName || '',
-      childGroups,
-      users,
-      name,
-      description,
-    } : null;
+    return rootEntries.map(entry => convert(entry, [])).filter((node): node is NonNullable<ExternalUserGroupTreeNode> => node != null);
   }
 
   private async getExternalAccount(uid: string, userEntry: SearchResultEntry) {

apps/app/src/server/service/ldap.ts

@@ -1,6 +1,6 @@
 import assert from 'assert';
 
-import ldap from 'ldapjs';
+import ldap, { NoSuchObjectError } from 'ldapjs';
 
 import loggerFactory from '~/utils/logger';
 
@@ -11,7 +11,8 @@ const logger = loggerFactory('growi:service:ldap-service');
 
 // @types/ldapjs is outdated, and SearchResultEntry does not exist.
 // Declare it manually in the meantime.
-export interface SearchResultEntry extends Omit<ldap.SearchEntry, 'attributes'> {
+export interface SearchResultEntry {
+  objectName: string // DN
   attributes: {
     type: string,
     values: string | string[]
@@ -94,7 +95,12 @@ class LdapService {
           searchResults.push(pojo);
         });
         res.on('error', (err) => {
-          reject(err);
+          if (err instanceof NoSuchObjectError) {
+            resolve([]);
+          }
+          else {
+            reject(err);
+          }
         });
         res.on('end', (result) => {
           if (result?.status === 0) {
@@ -109,10 +115,7 @@ class LdapService {
   }
 
   searchGroupDir(): Promise<SearchResultEntry[]> {
-    const groupSearchBase = configManager?.getConfig('crowi', 'external-user-group:ldap:groupSearchBase')
-    || configManager?.getConfig('crowi', 'security:passport-ldap:groupSearchBase');
-
-    return this.search(undefined, groupSearchBase);
+    return this.search(undefined, this.getGroupSearchBase());
   }
 
   getArrayValFromSearchResultEntry(entry: SearchResultEntry, attributeType: string): string[] {
@@ -131,6 +134,11 @@ class LdapService {
     return undefined;
   }
 
+  getGroupSearchBase(): string {
+    return configManager?.getConfig('crowi', 'external-user-group:ldap:groupSearchBase')
+    || configManager?.getConfig('crowi', 'security:passport-ldap:groupSearchBase');
+  }
+
 }
 
 export default LdapService;