Add SearchClient

bin/search.js

@@ -1,19 +1,18 @@
 
 var program = require('commander')
   , debug = require('debug')('debug:console:search-util')
-  , crowi = new (require('../lib/crowi'))(__dirname, process.env)
+  , crowi = new (require('../lib/crowi'))(__dirname + '/../', process.env)
   ;
 
 crowi.init()
   .then(function(app) {
-    var search = require('../lib/util/search')(crowi);
-
     program
       .version(crowi.version);
 
     program
-      .command('create-index [name]')
+      .command('create-index')
       .action(function (cmd, env) {
+        var search = crowi.getSearcher();
 
         search.buildIndex()
           .then(function(data) {
@@ -24,6 +23,20 @@ crowi.init()
           });
       });
 
+    program
+      .command('add-pages')
+      .action(function (cmd, env) {
+        var search = crowi.getSearcher();
+
+        search.addAllPages()
+          .then(function(data) {
+            console.log(data);
+          })
+          .catch(function(err) {
+            console.log("Error", err);
+          });
+      });
+
     program
       .command('rebuild-index [name]')
       .action(function (cmd, env) {
@@ -39,6 +52,7 @@ crowi.init()
       });
 
     program.parse(process.argv);
+
   }).catch(crowi.exitOnError);
 
 
@@ -47,30 +61,3 @@ crowi.init()
 //  .command('list', 'list packages installed', {isDefault: true})
 
 
-/*
-crowi.init()
-  .then(function(app) {
-    var search = require('./lib/util/search')(crowi);
-
-    search.buildIndex()
-      .then(function(data) {
-        console.log(data);
-      })
-      .catch(function(err) {
-        console.log("Error", err);
-      });
-  }).catch(crowi.exitOnError);
-
-cli.parse({
-    seed:      [false, 'Password seed', 'string', ''],
-    password:  [false, 'Password raw string', 'string'],
-});
-
-cli.main(function(args, options)
-{
-  console.log("args", args);
-  console.log("options", options);
-
-  this.output();
-});
-*/

lib/crowi/index.js

@@ -206,7 +206,7 @@ Crowi.prototype.setupSearcher = function() {
 
   return new Promise(function(resolve, reject) {
     if (searcherUri) {
-      self.searcher = require('../util/searcher')(self);
+      self.searcher = new (require(path.join(self.libDir, 'util', 'search')))(self, searcherUri);
     }
     resolve();
   });

lib/util/search.js

@@ -2,24 +2,133 @@
  * Search
  */
 
-
-//module.exports = SearchClient;
-
-/*
 var elasticsearch = require('elasticsearch'),
-  debug = require('debug')('crowi:lib:search'),
+  debug = require('debug')('crowi:lib:search');
 
-function SearchClient(crowi) {
+function SearchClient(crowi, esUri) {
+  this.esUri = esUri;
   this.crowi = crowi;
-  this.Page = crowi.model('Page');
-  this.Config = crowi.model('Config');
-  this.config = crowi.getConfig();
+
+  var uri = this.parseUri(this.esUri);
+  this.host = uri.host;
+  this.index_name = uri.index_name;
+
+  this.client = new elasticsearch.Client({
+    host: this.host,
+  });
+
+  this.mappingFile = crowi.resourceDir + 'search/mappings.json';
+  //this.Page = crowi.model('Page');
+  //this.Config = crowi.model('Config');
+  //this.config = crowi.getConfig();
 }
 
+SearchClient.prototype.parseUri = function(uri) {
+  if (!(m = uri.match(/^elasticsearch:\/\/([^:]+):([^\/]+)\/(.+)$/))) {
+    throw new Error('Invalid ELASTICSEARCH_URI format. Should be elasticsearch://host:port/index_name');
+  }
+
+  return {
+    host: m[1] + ':' + m[2],
+    index_name: m[3],
+  };
+};
+
+SearchClient.prototype.buildIndex = function(uri) {
+  return this.client.indices.create({
+    index: this.index_name,
+    body: require(this.mappingFile)
+  });
+};
+
+SearchClient.prototype.prepareBodyForCreate = function(body, page) {
+  if (!Array.isArray(body)) {
+    throw new Error('Body must be an array.');
+  }
+
+  var command = {
+    index: {
+      _index: this.index_name,
+      _type: 'page',
+      _id: page._id.toString(),
+    }
+  };
+
+  var document = {
+    path: page.path,
+    body: page.revision.body,
+    username: page.creator.username,
+    comment_count: page.commentCount,
+    like_count: page.liker.length || 0,
+    created_at: page.createdAt,
+    updated_at: page.updatedAt,
+  };
+
+  body.push(command);
+  body.push(document);
+};
+
+SearchClient.prototype.addPages = function(pages)
+{
+  var self = this;
+  var body = [];
+
+  pages.map(function(page) {
+    self.prepareBodyForCreate(body, page);
+  });
+
+  return this.client.bulk({
+    body: body,
+  });
+};
+
+SearchClient.prototype.addAllPages = function()
+{
+  var self = this;
+  var offset = 0;
+  var Page = this.crowi.model('Page');
+  var stream = Page.getStreamOfFindAll();
+  var body = [];
+
+  stream.on('data', function (doc) {
+    if (!doc.creator || !doc.revision) {
+      debug('Skipped', doc.path);
+      return ;
+    }
+
+    debug('Prepare', doc);
+    self.prepareBodyForCreate(body, doc);
+    //debug('Data received: ', doc.path, doc.liker.length, doc.revision.body);
+  }).on('error', function (err) {
+    debug('Error stream:', err);
+    // handle err
+  }).on('close', function () {
+    // all done
+    debug('Close');
+
+    debug('SEnd', body);
+    // 最後に送信
+    self.client.bulk({ body: body, })
+    .then(function(res) {
+      debug('Reponse from es:', res);
+    }).catch(function(err) {
+      debug('Err from es:', err);
+    });
+  });
+
+  return Promise.resolve();
+};
+
+module.exports = SearchClient;
+
+/*
+
+
 SearchClient.prototype.deleteIndex = function() {
 };
 */
 
+/*
 module.exports = function(crowi) {
   var elasticsearch = require('elasticsearch'),
     debug = require('debug')('crowi:lib:search'),
@@ -94,10 +203,6 @@ module.exports = function(crowi) {
     });
   };
 
-
-  /**
-   * @return Promise
-   */
   lib.addPage = function(id, path, body, creator, likeCount, bookmarkCount, updated, is_public) {
     var self = this;
 
@@ -139,40 +244,6 @@ module.exports = function(crowi) {
       body: queryBody
     });
 
-  /*
-     {
-     "query": {
-     "bool": {
-     "should": [
-     {"term": {
-     "path": {
-     "term": "php",
-     "boost": 2.0
-     }
-     }},
-     {"term": {
-     "body": {
-     "term": "php"
-     }
-     }}
-     ]
-     }
-     },
-     "highlight" : {
-     "fields" : {
-     "body" : {}
-     }
-     },
-     "sort": [
-     {
-     "updated": {
-     "order": "desc"
-     }
-     }
-     ]
-     }
-     */
-
   };
 
   lib.searchPageByLikeCount = function() {
@@ -181,3 +252,4 @@ module.exports = function(crowi) {
   return lib;
 };
 
+*/