Merge pull request #271 from crowi/fix-bulk-operation

Fix search bulk operation

lib/util/search.js

@@ -211,30 +211,49 @@ SearchClient.prototype.addAllPages = function()
   var Page = this.crowi.model('Page');
   var cursor = Page.getStreamOfFindAll();
   var body = [];
+  var sent = 0;
+  var skipped = 0;
 
   var counter = 0;
 
   return new Promise(function(resolve, reject) {
     cursor.on('data', function (doc) {
       if (!doc.creator || !doc.revision || !self.shouldIndexed(doc)) {
-        debug('Skipped', doc.path);
+        //debug('Skipped', doc.path);
+        skipped++;
         return ;
       }
 
       self.prepareBodyForCreate(body, doc);
+      //debug(body.length);
+      if (body.length > 2000) {
+        sent++;
+        debug('Sending request (seq, skipped)', sent, skipped);
+        self.client.bulk({
+          body: body,
+          requestTimeout: Infinity,
+        }).then(res => {
+          debug('addAllPages add anyway (items, errors, took): ', (res.items || []).length, res.errors, res.took)
+        }).catch(err => {
+          debug('addAllPages error on add anyway: ', err)
+        });
+
+        body = [];
+      }
     }).on('error', function (err) {
       // TODO: handle err
       debug('Error cursor:', err);
     }).on('close', function () {
       // all done
 
-      // 最後に送信
+      // 最後にすべてを送信
+      debug('Sending last body of bulk operation:', body.length)
       self.client.bulk({
         body: body,
         requestTimeout: Infinity,
       })
       .then(function(res) {
-        debug('Reponse from es:', res);
+        debug('Reponse from es (item length, errros, took):', (res.items || []).length, res.errors, res.took);
         return resolve(res);
       }).catch(function(err) {
         debug('Err from es:', err);