Merge pull request #9361 from weseek/imprv/155078-156769-close-write-stream

imprv: Use stream.pipeline

apps/app/package.json

@@ -211,7 +211,6 @@
     "remark-stringify": "^11.0.0",
     "sanitize-filename": "^1.6.3",
     "socket.io": "^4.7.5",
-    "stream-to-promise": "^3.0.0",
     "string-width": "=4.2.2",
     "superjson": "^1.9.1",
     "swagger-jsdoc": "^6.2.8",

apps/app/src/features/growi-plugin/server/services/growi-plugin/growi-plugin.integ.ts

@@ -11,7 +11,7 @@ describe('Installing a GROWI template plugin', () => {
   it('install() should success', async() => {
     // when
     const result = await growiPluginService.install({
-      url: 'https://github.com/weseek/growi-plugin-templates-for-office',
+      url: 'https://github.com/growilabs/growi-plugin-templates-for-office',
     });
     const count = await GrowiPlugin.count({ 'meta.name': 'growi-plugin-templates-for-office' });
 
@@ -20,7 +20,7 @@ describe('Installing a GROWI template plugin', () => {
     expect(count).toBe(1);
     expect(fs.existsSync(path.join(
       PLUGIN_STORING_PATH,
-      'weseek',
+      'growilabs',
       'growi-plugin-templates-for-office',
     ))).toBeTruthy();
   });
@@ -33,7 +33,7 @@ describe('Installing a GROWI template plugin', () => {
     // setup
     const dummyFilePath = path.join(
       PLUGIN_STORING_PATH,
-      'weseek',
+      'growilabs',
       'growi-plugin-templates-for-office',
       'dummy.txt',
     );
@@ -42,7 +42,7 @@ describe('Installing a GROWI template plugin', () => {
 
     // when
     const result = await growiPluginService.install({
-      url: 'https://github.com/weseek/growi-plugin-templates-for-office',
+      url: 'https://github.com/growilabs/growi-plugin-templates-for-office',
     });
     const count2 = await GrowiPlugin.count({ 'meta.name': 'growi-plugin-templates-for-office' });
 
@@ -59,7 +59,7 @@ describe('Installing a GROWI theme plugin', () => {
   it('install() should success', async() => {
     // when
     const result = await growiPluginService.install({
-      url: 'https://github.com/weseek/growi-plugin-theme-vivid-internet',
+      url: 'https://github.com/growilabs/growi-plugin-theme-vivid-internet',
     });
     const count = await GrowiPlugin.count({ 'meta.name': 'growi-plugin-theme-vivid-internet' });
 
@@ -68,7 +68,7 @@ describe('Installing a GROWI theme plugin', () => {
     expect(count).toBe(1);
     expect(fs.existsSync(path.join(
       PLUGIN_STORING_PATH,
-      'weseek',
+      'growilabs',
       'growi-plugin-theme-vivid-internet',
     ))).toBeTruthy();
   });
@@ -88,7 +88,7 @@ describe('Installing a GROWI theme plugin', () => {
     expect(results.themeMetadata).not.toBeNull();
     expect(results.themeHref).not.toBeNull();
     expect(results.themeHref
-      .startsWith('/static/plugins/weseek/growi-plugin-theme-vivid-internet/dist/assets/style-')).toBeTruthy();
+      .startsWith('/static/plugins/growilabs/growi-plugin-theme-vivid-internet/dist/assets/style-')).toBeTruthy();
   });
 
 });

apps/app/src/features/growi-plugin/server/services/growi-plugin/growi-plugin.ts

@@ -1,5 +1,6 @@
 import fs, { readFileSync } from 'fs';
 import path from 'path';
+import { pipeline } from 'stream/promises';
 
 import { GrowiPluginType } from '@growi/core';
 import type { GrowiThemeMetadata, ViteManifest } from '@growi/core';
@@ -8,7 +9,6 @@ import { importPackageJson, validateGrowiDirective } from '@growi/pluginkit/dist
 // eslint-disable-next-line no-restricted-imports
 import axios from 'axios';
 import type mongoose from 'mongoose';
-import streamToPromise from 'stream-to-promise';
 import unzipStream from 'unzip-stream';
 
 import loggerFactory from '~/utils/logger';
@@ -209,10 +209,8 @@ export class GrowiPluginService implements IGrowiPluginService {
 
   private async unzip(zipFilePath: fs.PathLike, destPath: fs.PathLike): Promise<void> {
     try {
-      const stream = fs.createReadStream(zipFilePath);
-      const unzipFileStream = stream.pipe(unzipStream.Extract({ path: destPath.toString() }));
-
-      await streamToPromise(unzipFileStream);
+      const readZipStream = fs.createReadStream(zipFilePath);
+      await pipeline(readZipStream, unzipStream.Extract({ path: destPath.toString() }));
     }
     catch (err) {
       logger.error(err);

apps/app/src/features/openai/server/services/openai.ts

@@ -1,5 +1,6 @@
 import assert from 'node:assert';
 import { Readable, Transform } from 'stream';
+import { pipeline } from 'stream/promises';
 
 import { PageGrant, isPopulated } from '@growi/core';
 import type { HydratedDocument, Types } from 'mongoose';
@@ -345,9 +346,7 @@ class OpenaiService implements IOpenaiService {
       },
     });
 
-    pagesStream
-      .pipe(batchStrem)
-      .pipe(createVectorStoreFileStream);
+    await pipeline(pagesStream, batchStrem, createVectorStoreFileStream);
   }
 
   async rebuildVectorStore(page: HydratedDocument<PageDocument>) {

apps/app/src/migrations/20211227060705-revision-path-to-page-id-schema-migration--fixed-7549.js

@@ -1,7 +1,7 @@
 import { Writable } from 'stream';
+import { pipeline } from 'stream/promises';
 
 import mongoose from 'mongoose';
-import streamToPromise from 'stream-to-promise';
 
 import getPageModel from '~/server/models/page';
 import { Revision } from '~/server/models/revision';
@@ -56,11 +56,7 @@ module.exports = {
       },
     });
 
-    pagesStream
-      .pipe(batchStrem)
-      .pipe(migratePagesStream);
-
-    await streamToPromise(migratePagesStream);
+    await pipeline(pagesStream, batchStrem, migratePagesStream);
 
     logger.info('Migration has successfully applied');
   },
@@ -107,11 +103,7 @@ module.exports = {
       },
     });
 
-    pagesStream
-      .pipe(batchStrem)
-      .pipe(migratePagesStream);
-
-    await streamToPromise(migratePagesStream);
+    await pipeline(pagesStream, batchStrem, migratePagesStream);
 
     logger.info('Migration down has successfully applied');
   },

apps/app/src/server/routes/apiv3/page/index.ts

@@ -1,4 +1,5 @@
 import path from 'path';
+import { pipeline, type Readable } from 'stream';
 
 import type { IPage } from '@growi/core';
 import {
@@ -735,7 +736,7 @@ module.exports = (crowi) => {
       fileName = '_top';
     }
 
-    let stream;
+    let stream: Readable;
 
     try {
       stream = exportService.getReadStreamFromRevision(revision, format);
@@ -760,7 +761,7 @@ module.exports = (crowi) => {
     };
     await crowi.activityService.createActivity(parameters);
 
-    return stream.pipe(res);
+    return pipeline(stream, res);
   });
 
   /**

apps/app/src/server/service/export.js

@@ -8,10 +8,10 @@ const logger = loggerFactory('growi:services:ExportService'); // eslint-disable-
 const fs = require('fs');
 const path = require('path');
 const { Transform } = require('stream');
+const { pipeline } = require('stream/promises');
 
 const archiver = require('archiver');
 const mongoose = require('mongoose');
-const streamToPromise = require('stream-to-promise');
 
 const CollectionProgressingStatus = require('../models/vo/collection-progressing-status');
 
@@ -107,7 +107,7 @@ class ExportService {
     writeStream.write(JSON.stringify(metaData));
     writeStream.close();
 
-    await streamToPromise(writeStream);
+    await pipeline([writeStream]);
 
     return metaJson;
   }
@@ -196,12 +196,7 @@ class ExportService {
     const jsonFileToWrite = path.join(this.baseDir, `${collectionName}.json`);
     const writeStream = fs.createWriteStream(jsonFileToWrite, { encoding: this.growiBridgeService.getEncoding() });
 
-    readStream
-      .pipe(logStream)
-      .pipe(transformStream)
-      .pipe(writeStream);
-
-    await streamToPromise(writeStream);
+    await pipeline(readStream, logStream, transformStream, writeStream);
 
     return writeStream.path;
   }
@@ -355,13 +350,12 @@ class ExportService {
     const output = fs.createWriteStream(zipFile);
 
     // pipe archive data to the file
-    archive.pipe(output);
+    const stream = pipeline(archive, output);
 
     // finalize the archive (ie we are done appending files but streams have to finish yet)
     // 'close', 'end' or 'finish' may be fired right after calling this method so register to them beforehand
     archive.finalize();
-
-    await streamToPromise(archive);
+    await stream;
 
     logger.info(`zipped GROWI data into ${zipFile} (${archive.pointer()} bytes)`);
 

apps/app/src/server/service/file-uploader/local.ts

@@ -1,5 +1,7 @@
 import type { ReadStream } from 'fs';
+import type { Writable } from 'stream';
 import { Readable } from 'stream';
+import { pipeline } from 'stream/promises';
 
 import type { Response } from 'express';
 
@@ -24,7 +26,6 @@ const fsPromises = require('fs/promises');
 const path = require('path');
 
 const mkdir = require('mkdirp');
-const streamToPromise = require('stream-to-promise');
 const urljoin = require('url-join');
 
 
@@ -163,8 +164,9 @@ module.exports = function(crowi) {
     // mkdir -p
     mkdir.sync(dirpath);
 
-    const stream = fileStream.pipe(fs.createWriteStream(filePath));
-    return streamToPromise(stream);
+    const writeStream: Writable = fs.createWriteStream(filePath);
+
+    return pipeline(fileStream, writeStream);
   };
 
   lib.saveFile = async function({ filePath, contentType, data }) {
@@ -177,8 +179,8 @@ module.exports = function(crowi) {
     const fileStream = new Readable();
     fileStream.push(data);
     fileStream.push(null); // EOF
-    const stream = fileStream.pipe(fs.createWriteStream(absFilePath));
-    return streamToPromise(stream);
+    const writeStream: Writable = fs.createWriteStream(absFilePath);
+    return pipeline(fileStream, writeStream);
   };
 
   /**

apps/app/src/server/service/growi-bridge/index.ts

@@ -1,7 +1,8 @@
 import fs from 'fs';
 import path from 'path';
+import { pipeline } from 'stream';
+import { pipeline as pipelinePromise } from 'stream/promises';
 
-import streamToPromise from 'stream-to-promise';
 import unzipStream, { type Entry } from 'unzip-stream';
 
 import loggerFactory from '~/utils/logger';
@@ -78,10 +79,12 @@ class GrowiBridgeService {
     let meta = {};
 
     const readStream = fs.createReadStream(zipFile);
-    const unzipStreamPipe = readStream.pipe(unzipStream.Parse());
+    const parseStream = unzipStream.Parse();
+    const unzipEntryStream = pipeline(readStream, parseStream);
+
     let tapPromise;
 
-    const unzipEntryStream = unzipStreamPipe.on('entry', (entry: Entry) => {
+    unzipEntryStream.on('entry', (entry: Entry) => {
       const fileName = entry.path;
       const size = entry.size; // might be undefined in some archives
       if (fileName === this.getMetaFileName()) {
@@ -100,7 +103,7 @@ class GrowiBridgeService {
     });
 
     try {
-      await streamToPromise(unzipEntryStream);
+      await pipelinePromise([unzipEntryStream]);
       await tapPromise;
     }
     // if zip is broken

apps/app/src/server/service/import/import.ts

@@ -1,7 +1,8 @@
 import fs from 'fs';
 import path from 'path';
 import type { EventEmitter } from 'stream';
-import { Writable, Transform } from 'stream';
+import { Writable, Transform, pipeline } from 'stream';
+import { pipeline as pipelinePromise } from 'stream/promises';
 
 import JSONStream from 'JSONStream';
 import gc from 'expose-gc/function';
@@ -10,7 +11,6 @@ import type {
 } from 'mongodb';
 import type { Document } from 'mongoose';
 import mongoose from 'mongoose';
-import streamToPromise from 'stream-to-promise';
 import unzipStream from 'unzip-stream';
 
 import { ImportMode } from '~/models/admin/import-mode';
@@ -267,13 +267,7 @@ export class ImportService {
         },
       });
 
-      readStream
-        .pipe(jsonStream)
-        .pipe(convertStream)
-        .pipe(batchStream)
-        .pipe(writeStream);
-
-      await streamToPromise(writeStream);
+      await pipelinePromise(readStream, jsonStream, convertStream, batchStream, writeStream);
 
       // clean up tmp directory
       fs.unlinkSync(jsonFile);
@@ -349,10 +343,11 @@ export class ImportService {
    */
   async unzip(zipFile) {
     const readStream = fs.createReadStream(zipFile);
-    const unzipStreamPipe = readStream.pipe(unzipStream.Parse());
+    const parseStream = unzipStream.Parse();
+    const unzipStreamPipe = pipeline(readStream, parseStream);
     const files: string[] = [];
 
-    unzipStreamPipe.on('entry', (/** @type {Entry} */ entry) => {
+    const unzipEntryStream = unzipStreamPipe.on('entry', (/** @type {Entry} */ entry) => {
       const fileName = entry.path;
       // https://regex101.com/r/mD4eZs/6
       // prevent from unexpecting attack doing unzip file (path traversal attack)
@@ -370,12 +365,12 @@ export class ImportService {
       else {
         const jsonFile = path.join(this.baseDir, fileName);
         const writeStream = fs.createWriteStream(jsonFile, { encoding: this.growiBridgeService.getEncoding() });
-        entry.pipe(writeStream);
+        pipeline(entry, writeStream);
         files.push(jsonFile);
       }
     });
 
-    await streamToPromise(unzipStreamPipe);
+    await pipelinePromise([unzipEntryStream]);
 
     return files;
   }

apps/app/src/server/service/page/delete-completely-user-home-by-system.ts

@@ -1,11 +1,11 @@
 import { Writable } from 'stream';
+import { pipeline } from 'stream/promises';
 
 import { getIdForRef } from '@growi/core';
 import type { IPage, Ref } from '@growi/core';
 import { isUsersHomepage } from '@growi/core/dist/utils/page-path-utils';
 import type { HydratedDocument } from 'mongoose';
 import mongoose from 'mongoose';
-import streamToPromise from 'stream-to-promise';
 
 import type { PageDocument, PageModel } from '~/server/models/page';
 import { createBatchStream } from '~/server/util/batch-stream';
@@ -87,8 +87,9 @@ export const deleteCompletelyUserHomeBySystem = async(userHomepagePath: string,
       .lean()
       .cursor({ batchSize: BULK_REINDEX_SIZE });
 
-    let count = 0;
+    const batchStream = createBatchStream(BULK_REINDEX_SIZE);
 
+    let count = 0;
     const writeStream = new Writable({
       objectMode: true,
       async write(batch, encoding, callback) {
@@ -109,11 +110,7 @@ export const deleteCompletelyUserHomeBySystem = async(userHomepagePath: string,
       },
     });
 
-    readStream
-      .pipe(createBatchStream(BULK_REINDEX_SIZE))
-      .pipe(writeStream);
-
-    await streamToPromise(writeStream);
+    await pipeline(readStream, batchStream, writeStream);
     // ────────┤ end │─────────
   }
   catch (err) {

apps/app/src/server/service/page/index.ts

@@ -1,6 +1,7 @@
 import type EventEmitter from 'events';
 import pathlib from 'path';
 import { Readable, Writable } from 'stream';
+import { pipeline } from 'stream/promises';
 
 import {
   PageStatus, YDocStatus, getIdForRef,
@@ -18,7 +19,6 @@ import {
 import escapeStringRegexp from 'escape-string-regexp';
 import type { Cursor, HydratedDocument } from 'mongoose';
 import mongoose from 'mongoose';
-import streamToPromise from 'stream-to-promise';
 
 import { Comment } from '~/features/comment/server';
 import type { ExternalUserGroupDocument } from '~/features/external-user-group/server/models/external-user-group';
@@ -1006,6 +1006,8 @@ class PageService implements IPageService {
     const factory = new PageCursorsForDescendantsFactory(user, targetPage, true);
     const readStream = await factory.generateReadable();
 
+    const batchStream = createBatchStream(BULK_REINDEX_SIZE);
+
     const newPagePathPrefix = newPagePath;
     const pathRegExp = new RegExp(`^${escapeStringRegexp(targetPage.path)}`, 'i');
 
@@ -1043,16 +1045,13 @@ class PageService implements IPageService {
       },
     });
 
-    readStream
-      .pipe(createBatchStream(BULK_REINDEX_SIZE))
-      .pipe(writeStream);
-
-    await streamToPromise(writeStream);
+    await pipeline(readStream, batchStream, writeStream);
   }
 
   private async renameDescendantsWithStreamV4(targetPage, newPagePath, user, options = {}) {
 
     const readStream = await this.generateReadStreamToOperateOnlyDescendants(targetPage.path, user);
+    const batchStream = createBatchStream(BULK_REINDEX_SIZE);
 
     const newPagePathPrefix = newPagePath;
     const pathRegExp = new RegExp(`^${escapeStringRegexp(targetPage.path)}`, 'i');
@@ -1083,11 +1082,7 @@ class PageService implements IPageService {
       },
     });
 
-    readStream
-      .pipe(createBatchStream(BULK_REINDEX_SIZE))
-      .pipe(writeStream);
-
-    await streamToPromise(writeStream);
+    await pipeline(readStream, batchStream, writeStream);
   }
 
   /*
@@ -1477,6 +1472,7 @@ class PageService implements IPageService {
 
     const iterableFactory = new PageCursorsForDescendantsFactory(user, page, true);
     const readStream = await iterableFactory.generateReadable();
+    const batchStream = createBatchStream(BULK_REINDEX_SIZE);
 
     const newPagePathPrefix = newPagePath;
     const pathRegExp = new RegExp(`^${escapeStringRegexp(page.path)}`, 'i');
@@ -1509,17 +1505,14 @@ class PageService implements IPageService {
       },
     });
 
-    readStream
-      .pipe(createBatchStream(BULK_REINDEX_SIZE))
-      .pipe(writeStream);
-
-    await streamToPromise(writeStream);
+    await pipeline(readStream, batchStream, writeStream);
 
     return nNonEmptyDuplicatedPages;
   }
 
   private async duplicateDescendantsWithStreamV4(page, newPagePath, user, onlyDuplicateUserRelatedResources: boolean) {
     const readStream = await this.generateReadStreamToOperateOnlyDescendants(page.path, user);
+    const batchStream = createBatchStream(BULK_REINDEX_SIZE);
 
     const newPagePathPrefix = newPagePath;
     const pathRegExp = new RegExp(`^${escapeStringRegexp(page.path)}`, 'i');
@@ -1550,11 +1543,7 @@ class PageService implements IPageService {
       },
     });
 
-    readStream
-      .pipe(createBatchStream(BULK_REINDEX_SIZE))
-      .pipe(writeStream);
-
-    await streamToPromise(writeStream);
+    await pipeline(readStream, batchStream, writeStream);
 
     return count;
   }
@@ -1849,6 +1838,7 @@ class PageService implements IPageService {
       readStream = await factory.generateReadable();
     }
 
+    const batchStream = createBatchStream(BULK_REINDEX_SIZE);
 
     const deleteDescendants = this.deleteDescendants.bind(this);
     let count = 0;
@@ -1881,11 +1871,7 @@ class PageService implements IPageService {
       },
     });
 
-    readStream
-      .pipe(createBatchStream(BULK_REINDEX_SIZE))
-      .pipe(writeStream);
-
-    await streamToPromise(writeStream);
+    await pipeline(readStream, batchStream, writeStream);
 
     return nDeletedNonEmptyPages;
   }
@@ -2117,6 +2103,8 @@ class PageService implements IPageService {
       readStream = await factory.generateReadable();
     }
 
+    const batchStream = createBatchStream(BULK_REINDEX_SIZE);
+
     let count = 0;
     let nDeletedNonEmptyPages = 0; // used for updating descendantCount
 
@@ -2148,11 +2136,7 @@ class PageService implements IPageService {
       },
     });
 
-    readStream
-      .pipe(createBatchStream(BULK_REINDEX_SIZE))
-      .pipe(writeStream);
-
-    await streamToPromise(writeStream);
+    await pipeline(readStream, batchStream, writeStream);
 
     return nDeletedNonEmptyPages;
   }
@@ -2428,7 +2412,7 @@ class PageService implements IPageService {
     );
 
     const childPagesReadableStream = builder.query.cursor({ batchSize: BULK_REINDEX_SIZE });
-
+    const batchStream = createBatchStream(BULK_REINDEX_SIZE);
     const childPagesWritable = new Writable({
       objectMode: true,
       write: async(batch, encoding, callback) => {
@@ -2437,10 +2421,8 @@ class PageService implements IPageService {
       },
     });
 
-    childPagesReadableStream
-      .pipe(createBatchStream(BULK_REINDEX_SIZE))
-      .pipe(childPagesWritable);
-    await streamToPromise(childPagesWritable);
+    await pipeline(childPagesReadableStream, batchStream, childPagesWritable);
+
   }
 
   async updateChildPagesGrant(
@@ -2477,6 +2459,7 @@ class PageService implements IPageService {
     }
 
     const readStream = await this.generateReadStreamToOperateOnlyDescendants(targetPage.path, user);
+    const batchStream = createBatchStream(BULK_REINDEX_SIZE);
 
     const revertDeletedDescendants = this.revertDeletedDescendants.bind(this);
     let count = 0;
@@ -2505,17 +2488,14 @@ class PageService implements IPageService {
       },
     });
 
-    readStream
-      .pipe(createBatchStream(BULK_REINDEX_SIZE))
-      .pipe(writeStream);
-
-    await streamToPromise(writeStream);
+    await pipeline(readStream, batchStream, writeStream);
 
     return count;
   }
 
   private async revertDeletedDescendantsWithStreamV4(targetPage, user, options = {}) {
     const readStream = await this.generateReadStreamToOperateOnlyDescendants(targetPage.path, user);
+    const batchStream = createBatchStream(BULK_REINDEX_SIZE);
 
     const revertDeletedDescendants = this.revertDeletedDescendants.bind(this);
     let count = 0;
@@ -2540,11 +2520,7 @@ class PageService implements IPageService {
       },
     });
 
-    readStream
-      .pipe(createBatchStream(BULK_REINDEX_SIZE))
-      .pipe(writeStream);
-
-    await streamToPromise(readStream);
+    await pipeline(readStream, batchStream, writeStream);
 
     return count;
   }
@@ -3390,11 +3366,7 @@ class PageService implements IPageService {
       },
     });
 
-    pagesStream
-      .pipe(batchStream)
-      .pipe(migratePagesStream);
-
-    await streamToPromise(migratePagesStream);
+    await pipeline(pagesStream, batchStream, migratePagesStream);
 
     if (await Page.exists(matchFilter) && shouldContinue) {
       return this._normalizeParentRecursively(
@@ -3495,6 +3467,7 @@ class PageService implements IPageService {
    */
   async recountAndUpdateDescendantCountOfPages(pageCursor: Cursor<any>, batchSize:number): Promise<void> {
     const Page = this.crowi.model('Page');
+    const batchStream = createBatchStream(batchSize);
     const recountWriteStream = new Writable({
       objectMode: true,
       async write(pageDocuments, encoding, callback) {
@@ -3508,11 +3481,8 @@ class PageService implements IPageService {
         callback();
       },
     });
-    pageCursor
-      .pipe(createBatchStream(batchSize))
-      .pipe(recountWriteStream);
 
-    await streamToPromise(recountWriteStream);
+    await pipeline(pageCursor, batchStream, recountWriteStream);
   }
 
   // update descendantCount of all pages that are ancestors of a provided pageId by count

apps/app/src/server/service/search-delegator/elasticsearch.ts

@@ -1,10 +1,10 @@
 import { Writable, Transform } from 'stream';
+import { pipeline } from 'stream/promises';
 import { URL } from 'url';
 
 import { getIdStringForRef, type IPage } from '@growi/core';
 import gc from 'expose-gc/function';
 import mongoose from 'mongoose';
-import streamToPromise from 'stream-to-promise';
 
 import { SearchDelegatorName } from '~/interfaces/named-query';
 import type { ISearchResult, ISearchResultData } from '~/interfaces/search';
@@ -553,14 +553,15 @@ class ElasticsearchDelegator implements SearchDelegator<Data, ESTermsKey, ESQuer
       },
     });
 
-    readStream
-      .pipe(batchStream)
-      .pipe(appendTagNamesStream)
-      // .pipe(appendEmbeddingStream)
-      // .pipe(appendFileUploadedStream)
-      .pipe(writeStream);
 
-    return streamToPromise(writeStream);
+    return pipeline(
+      readStream,
+      batchStream,
+      appendTagNamesStream,
+      // appendEmbeddingStream,
+      // appendFileUploadedStream,
+      writeStream,
+    );
   }
 
   deletePages(pages) {

pnpm-lock.yaml

@@ -639,9 +639,6 @@ importers:
       socket.io:
         specifier: ^4.7.5
         version: 4.8.1
-      stream-to-promise:
-        specifier: ^3.0.0
-        version: 3.0.0
       string-width:
         specifier: '=4.2.2'
         version: 4.2.2
@@ -10816,13 +10813,6 @@ packages:
     engines: {node: '>= 0.10.0'}
     hasBin: true
 
-  stream-to-array@2.3.0:
-    resolution: {integrity: sha512-UsZtOYEn4tWU2RGLOXr/o/xjRBftZRlG3dEWoaHr8j4GuypJ3isitGbVyjQKAuMu+xbiop8q224TjiZWc4XTZA==}
-
-  stream-to-promise@3.0.0:
-    resolution: {integrity: sha512-h+7wLeFiYegOdgTfTxjRsrT7/Op7grnKEIHWgaO1RTHwcwk7xRreMr3S8XpDfDMesSxzgM2V4CxNCFAGo6ssnA==}
-    engines: {node: '>= 10'}
-
   stream-transform@2.1.3:
     resolution: {integrity: sha512-9GHUiM5hMiCi6Y03jD2ARC1ettBXkQBoQAe7nJsPknnI0ow10aXjTnew8QtYQmLjzn974BnmWEAJgCY6ZP1DeQ==}
 
@@ -13399,7 +13389,7 @@ snapshots:
       '@babel/traverse': 7.24.6
       '@babel/types': 7.25.6
       convert-source-map: 2.0.0
-      debug: 4.3.7(supports-color@5.5.0)
+      debug: 4.3.7
       gensync: 1.0.0-beta.2
       json5: 2.2.3
       semver: 6.3.1
@@ -13590,7 +13580,7 @@ snapshots:
       '@babel/helper-split-export-declaration': 7.24.6
       '@babel/parser': 7.25.6
       '@babel/types': 7.25.6
-      debug: 4.3.7(supports-color@5.5.0)
+      debug: 4.3.7
       globals: 11.12.0
     transitivePeerDependencies:
       - supports-color
@@ -14095,7 +14085,7 @@ snapshots:
 
   '@elastic/elasticsearch@7.17.13':
     dependencies:
-      debug: 4.3.7(supports-color@5.5.0)
+      debug: 4.3.7
       hpagent: 0.1.2
       ms: 2.1.3
       secure-json-parse: 2.7.0
@@ -14111,7 +14101,7 @@ snapshots:
 
   '@elastic/transport@8.6.1':
     dependencies:
-      debug: 4.3.7(supports-color@5.5.0)
+      debug: 4.3.7
       hpagent: 1.2.0
       ms: 2.1.3
       secure-json-parse: 2.7.0
@@ -14222,7 +14212,7 @@ snapshots:
   '@eslint/eslintrc@2.0.3':
     dependencies:
       ajv: 6.12.6
-      debug: 4.3.7(supports-color@5.5.0)
+      debug: 4.3.7
       espree: 9.6.1
       globals: 13.24.0
       ignore: 5.3.1
@@ -14295,7 +14285,7 @@ snapshots:
   '@humanwhocodes/config-array@0.11.8':
     dependencies:
       '@humanwhocodes/object-schema': 1.2.1
-      debug: 4.3.7(supports-color@5.5.0)
+      debug: 4.3.7
       minimatch: 3.1.2
     transitivePeerDependencies:
       - supports-color
@@ -14311,7 +14301,7 @@ snapshots:
       '@antfu/install-pkg': 0.4.1
       '@antfu/utils': 0.7.10
       '@iconify/types': 2.0.0
-      debug: 4.3.7(supports-color@5.5.0)
+      debug: 4.3.7
       kolorist: 1.8.0
       local-pkg: 0.5.0
       mlly: 1.7.1
@@ -15770,7 +15760,7 @@ snapshots:
       '@swc-node/sourcemap-support': 0.5.0
       '@swc/core': 1.5.25(@swc/helpers@0.5.11)
       colorette: 2.0.20
-      debug: 4.3.7(supports-color@5.5.0)
+      debug: 4.3.7
       pirates: 4.0.6
       tslib: 2.8.0
       typescript: 5.4.2
@@ -16371,7 +16361,7 @@ snapshots:
       '@typescript-eslint/scope-manager': 5.59.7
       '@typescript-eslint/type-utils': 5.59.7(eslint@8.41.0)(typescript@5.4.2)
       '@typescript-eslint/utils': 5.59.7(eslint@8.41.0)(typescript@5.4.2)
-      debug: 4.3.7(supports-color@5.5.0)
+      debug: 4.3.7
       eslint: 8.41.0
       grapheme-splitter: 1.0.4
       ignore: 5.3.1
@@ -16401,7 +16391,7 @@ snapshots:
       '@typescript-eslint/scope-manager': 5.59.7
       '@typescript-eslint/types': 5.59.7
       '@typescript-eslint/typescript-estree': 5.59.7(typescript@5.4.2)
-      debug: 4.3.7(supports-color@5.5.0)
+      debug: 4.3.7
       eslint: 8.41.0
     optionalDependencies:
       typescript: 5.4.2
@@ -16430,7 +16420,7 @@ snapshots:
     dependencies:
       '@typescript-eslint/typescript-estree': 5.59.7(typescript@5.4.2)
       '@typescript-eslint/utils': 5.59.7(eslint@8.41.0)(typescript@5.4.2)
-      debug: 4.3.7(supports-color@5.5.0)
+      debug: 4.3.7
       eslint: 8.41.0
       tsutils: 3.21.0(typescript@5.4.2)
     optionalDependencies:
@@ -16459,7 +16449,7 @@ snapshots:
     dependencies:
       '@typescript-eslint/types': 5.59.7
       '@typescript-eslint/visitor-keys': 5.59.7
-      debug: 4.3.7(supports-color@5.5.0)
+      debug: 4.3.7
       globby: 11.1.0
       is-glob: 4.0.3
       semver: 7.6.3
@@ -16845,13 +16835,13 @@ snapshots:
 
   agent-base@6.0.2:
     dependencies:
-      debug: 4.3.7(supports-color@5.5.0)
+      debug: 4.3.7
     transitivePeerDependencies:
       - supports-color
 
   agent-base@7.1.1:
     dependencies:
-      debug: 4.3.7(supports-color@5.5.0)
+      debug: 4.3.7
     transitivePeerDependencies:
       - supports-color
 
@@ -17925,7 +17915,7 @@ snapshots:
 
   connect-mongo@4.6.0(express-session@1.18.0)(mongodb@4.17.2(@aws-sdk/client-sso-oidc@3.600.0)):
     dependencies:
-      debug: 4.3.7(supports-color@5.5.0)
+      debug: 4.3.7
       express-session: 1.18.0
       kruptein: 3.0.6
       mongodb: 4.17.2(@aws-sdk/client-sso-oidc@3.600.0)
@@ -18405,6 +18395,10 @@ snapshots:
     dependencies:
       ms: 2.1.3
 
+  debug@4.3.7:
+    dependencies:
+      ms: 2.1.3
+
   debug@4.3.7(supports-color@5.5.0):
     dependencies:
       ms: 2.1.3
@@ -18699,7 +18693,7 @@ snapshots:
   engine.io-client@6.6.2:
     dependencies:
       '@socket.io/component-emitter': 3.1.2
-      debug: 4.3.7(supports-color@5.5.0)
+      debug: 4.3.7
       engine.io-parser: 5.2.3
       ws: 8.17.1
       xmlhttprequest-ssl: 2.1.2
@@ -18719,7 +18713,7 @@ snapshots:
       base64id: 2.0.0
       cookie: 0.7.2
       cors: 2.8.5
-      debug: 4.3.7(supports-color@5.5.0)
+      debug: 4.3.7
       engine.io-parser: 5.2.3
       ws: 8.17.1
     transitivePeerDependencies:
@@ -19165,7 +19159,7 @@ snapshots:
       ajv: 6.12.6
       chalk: 4.1.2
       cross-spawn: 7.0.3
-      debug: 4.3.7(supports-color@5.5.0)
+      debug: 4.3.7
       doctrine: 3.0.0
       escape-string-regexp: 4.0.0
       eslint-scope: 7.2.0
@@ -19505,7 +19499,7 @@ snapshots:
 
   follow-redirects@1.15.9(debug@4.3.7):
     optionalDependencies:
-      debug: 4.3.7(supports-color@5.5.0)
+      debug: 4.3.7
 
   follow-redirects@1.5.10:
     dependencies:
@@ -20153,14 +20147,14 @@ snapshots:
     dependencies:
       '@tootallnate/once': 2.0.0
       agent-base: 6.0.2
-      debug: 4.3.7(supports-color@5.5.0)
+      debug: 4.3.7
     transitivePeerDependencies:
       - supports-color
 
   http-proxy-agent@7.0.2:
     dependencies:
       agent-base: 7.1.1
-      debug: 4.3.7(supports-color@5.5.0)
+      debug: 4.3.7
     transitivePeerDependencies:
       - supports-color
 
@@ -20183,14 +20177,14 @@ snapshots:
   https-proxy-agent@5.0.1:
     dependencies:
       agent-base: 6.0.2
-      debug: 4.3.7(supports-color@5.5.0)
+      debug: 4.3.7
     transitivePeerDependencies:
       - supports-color
 
   https-proxy-agent@7.0.5:
     dependencies:
       agent-base: 7.1.1
-      debug: 4.3.7(supports-color@5.5.0)
+      debug: 4.3.7
     transitivePeerDependencies:
       - supports-color
 
@@ -20549,7 +20543,7 @@ snapshots:
 
   istanbul-lib-source-maps@4.0.1:
     dependencies:
-      debug: 4.3.7(supports-color@5.5.0)
+      debug: 4.3.7
       istanbul-lib-coverage: 3.2.2
       source-map: 0.6.1
     transitivePeerDependencies:
@@ -21948,7 +21942,7 @@ snapshots:
   micromark@4.0.0:
     dependencies:
       '@types/debug': 4.1.7
-      debug: 4.3.7(supports-color@5.5.0)
+      debug: 4.3.7
       decode-named-character-reference: 1.0.2
       devlop: 1.1.0
       micromark-core-commonmark: 2.0.1
@@ -22111,7 +22105,7 @@ snapshots:
     dependencies:
       async-mutex: 0.4.1
       camelcase: 6.3.0
-      debug: 4.3.7(supports-color@5.5.0)
+      debug: 4.3.7
       find-cache-dir: 3.3.2
       follow-redirects: 1.15.9(debug@4.3.7)
       https-proxy-agent: 7.0.5
@@ -22219,7 +22213,7 @@ snapshots:
 
   mquery@4.0.3:
     dependencies:
-      debug: 4.3.7(supports-color@5.5.0)
+      debug: 4.3.7
     transitivePeerDependencies:
       - supports-color
 
@@ -22309,7 +22303,7 @@ snapshots:
 
   new-find-package-json@2.0.0:
     dependencies:
-      debug: 4.3.7(supports-color@5.5.0)
+      debug: 4.3.7
     transitivePeerDependencies:
       - supports-color
 
@@ -22840,7 +22834,7 @@ snapshots:
   passport-saml@3.2.4:
     dependencies:
       '@xmldom/xmldom': 0.7.13
-      debug: 4.3.7(supports-color@5.5.0)
+      debug: 4.3.7
       passport-strategy: 1.0.0
       xml-crypto: 2.1.5
       xml-encryption: 2.0.0
@@ -23827,7 +23821,7 @@ snapshots:
 
   retry-request@4.2.2:
     dependencies:
-      debug: 4.3.7(supports-color@5.5.0)
+      debug: 4.3.7
       extend: 3.0.2
     transitivePeerDependencies:
       - supports-color
@@ -24217,7 +24211,7 @@ snapshots:
 
   socket.io-adapter@2.5.5:
     dependencies:
-      debug: 4.3.7(supports-color@5.5.0)
+      debug: 4.3.7
       ws: 8.17.1
     transitivePeerDependencies:
       - bufferutil
@@ -24227,7 +24221,7 @@ snapshots:
   socket.io-client@4.8.1:
     dependencies:
       '@socket.io/component-emitter': 3.1.2
-      debug: 4.3.7(supports-color@5.5.0)
+      debug: 4.3.7
       engine.io-client: 6.6.2
       socket.io-parser: 4.2.4
     transitivePeerDependencies:
@@ -24238,7 +24232,7 @@ snapshots:
   socket.io-parser@4.2.4:
     dependencies:
       '@socket.io/component-emitter': 3.1.2
-      debug: 4.3.7(supports-color@5.5.0)
+      debug: 4.3.7
     transitivePeerDependencies:
       - supports-color
 
@@ -24247,7 +24241,7 @@ snapshots:
       accepts: 1.3.8
       base64id: 2.0.0
       cors: 2.8.5
-      debug: 4.3.7(supports-color@5.5.0)
+      debug: 4.3.7
       engine.io: 6.6.2
       socket.io-adapter: 2.5.5
       socket.io-parser: 4.2.4
@@ -24392,16 +24386,6 @@ snapshots:
       commander: 2.20.3
       limiter: 1.1.5
 
-  stream-to-array@2.3.0:
-    dependencies:
-      any-promise: 1.3.0
-
-  stream-to-promise@3.0.0:
-    dependencies:
-      any-promise: 1.3.0
-      end-of-stream: 1.4.4
-      stream-to-array: 2.3.0
-
   stream-transform@2.1.3:
     dependencies:
       mixme: 0.5.10