enable back pressuring from archiver

apps/app/package.json

@@ -83,6 +83,7 @@
     "@keycloak/keycloak-admin-client": "^18.0.0",
     "@slack/web-api": "^6.2.4",
     "@slack/webhook": "^6.0.0",
+    "@types/async": "^3.2.24",
     "@types/jest": "^29.5.2",
     "@types/ldapjs": "^2.2.5",
     "JSONStream": "^1.3.5",
@@ -229,9 +230,9 @@
     "@next/bundle-analyzer": "^14.1.3",
     "@swc-node/jest": "^1.6.2",
     "@swc/jest": "^0.2.24",
-    "@types/archiver": "^6.0.2",
     "@testing-library/react": "^14.1.2",
     "@testing-library/user-event": "^14.5.2",
+    "@types/archiver": "^6.0.2",
     "@types/express": "^4.17.11",
     "@types/jest": "^29.5.2",
     "@types/react-scroll": "^1.8.4",

apps/app/src/features/page-bulk-export/server/service/page-bulk-export.ts

@@ -1,4 +1,3 @@
-import { createGzip } from 'node:zlib';
 import type { Readable } from 'stream';
 import { Writable } from 'stream';
 
@@ -6,6 +5,7 @@ import { type IPage, isPopulated } from '@growi/core';
 import { normalizePath } from '@growi/core/dist/utils/path-utils';
 import type { Archiver } from 'archiver';
 import archiver from 'archiver';
+import type { QueueObject } from 'async';
 import mongoose from 'mongoose';
 
 import type { PageModel, PageDocument } from '~/server/models/page';
@@ -17,6 +17,11 @@ const logger = loggerFactory('growi:services:PageBulkExportService');
 
 const streamToPromise = require('stream-to-promise');
 
+// Custom type for back pressure workaround
+interface ArchiverWithQueue extends Archiver {
+  _queue?: QueueObject<any>;
+}
+
 class PageBulkExportService {
 
   crowi: any;
@@ -80,9 +85,16 @@ class PageBulkExportService {
 
           if (revision != null && isPopulated(revision)) {
             const markdownBody = revision.body;
-            // write to zip
             const pathNormalized = normalizePath(page.path);
-            zipArchiver.append(markdownBody, { name: `${pathNormalized}.md` });
+            // Since archiver does not provide a proper way to back pressure at the moment, use the _queue property as a workaround
+            // ref: https://github.com/archiverjs/node-archiver/issues/611
+            const { _queue } = zipArchiver.append(markdownBody, { name: `${pathNormalized}.md` }) as ArchiverWithQueue;
+            if (_queue == null) {
+              throw Error('Cannot back pressure the export pipeline. Aborting the export.');
+            }
+            if (_queue.length() > 100) {
+              await _queue.drain();
+            }
           }
         }
         catch (err) {

yarn.lock

@@ -3812,6 +3812,11 @@
   resolved "https://registry.yarnpkg.com/@types/aria-query/-/aria-query-5.0.4.tgz#1a31c3d378850d2778dabb6374d036dcba4ba708"
   integrity sha512-rfT93uj5s0PRL7EzccGMs3brplhcrghnDoV26NqKhCAS1hVo+WdNsPvE/yb6ilfr5hi2MEk6d5EWJTKdxg8jVw==
 
+"@types/async@^3.2.24":
+  version "3.2.24"
+  resolved "https://registry.yarnpkg.com/@types/async/-/async-3.2.24.tgz#3a96351047575bbcf2340541b2d955a35339608f"
+  integrity sha512-8iHVLHsCCOBKjCF2KwFe0p9Z3rfM9mL+sSP8btyR5vTjJRAqpBYD28/ZLgXPf0pjG1VxOvtCV/BgXkQbpSe8Hw==
+
 "@types/babel__core@^7.1.14", "@types/babel__core@^7.20.5":
   version "7.20.5"
   resolved "https://registry.yarnpkg.com/@types/babel__core/-/babel__core-7.20.5.tgz#3df15f27ba85319caa07ba08d0721889bb39c017"