Searchable apis

bin/search.js

@@ -1,7 +1,8 @@
 
 var program = require('commander')
   , sprintf = require('sprintf')
-  , debug = require('debug')('debug:console:search-util')
+  , debug = require('debug')('crowi:console:search-util')
+  , colors = require('colors')
   , crowi = new (require('../lib/crowi'))(__dirname + '/../', process.env)
   ;
 
@@ -36,8 +37,17 @@ crowi.init()
         search.addAllPages()
           .then(function(data) {
             if (data.errors) {
-              console.error(data);
-              console.error('Failed to index.');
+              console.error(colors.red.underline('Failed to index all pages.'));
+              console.error("");
+              data.items.forEach(function(item, i) {
+                var index = item.index || null;
+                if (index && index.status != 200) {
+                  console.error(colors.red('Error item: id=%s'), index._id)
+                  console.error('error.type=%s, error.reason=%s', index.error.type, index.error.reason);
+                  console.error(index.error.caused_by);
+                }
+                //debug('Item', i, item);
+              });
             } else {
               console.log('Data is successfully indexed.');
             }
@@ -77,6 +87,32 @@ crowi.init()
           });
       });
 
+    program
+      .command('search')
+      .action(function (cmd, env) {
+        var Page = crowi.model('Page');
+        var search = crowi.getSearcher();
+        var keyword = "";
+
+        debug(cmd, env);
+
+        search.searchKeyword(keyword, {})
+          .then(function(data) {
+            console.log(colors.green('Search result: %d of %d total.'), data.meta.results, data.meta.total);
+
+            return Page.findListByPageIds(data.data.map(function(p) { return p._id; }));
+          }).then(function(pages) {
+            pages.map(function(page) {
+              console.log(page.path);
+            });
+          })
+          .catch(function(err) {
+            console.error('Error', err);
+
+            process.exit(0);
+          });
+      });
+
 
     program.parse(process.argv);
 

lib/util/search.js

@@ -6,6 +6,9 @@ var elasticsearch = require('elasticsearch'),
   debug = require('debug')('crowi:lib:search');
 
 function SearchClient(crowi, esUri) {
+  this.DEFAULT_OFFSET = 0;
+  this.DEFAULT_LIMIT = 50;
+
   this.esUri = esUri;
   this.crowi = crowi;
 
@@ -163,114 +166,155 @@ SearchClient.prototype.addAllPages = function()
   });
 };
 
-SearchClient.prototype.createSearchQuerySortedByUpdatedAt = function()
+/**
+ * search returning type:
+ * {
+ *   meta: { total: Integer, results: Integer},
+ *   data: [ pages ...],
+ * }
+ */
+SearchClient.prototype.search = function(query)
+{
+  var self = this;
+
+  return new Promise(function(resolve, reject) {
+    self.client.search(query)
+    .then(function(data) {
+      var result = {};
+      result.total = data.hits.total;
+      result.results = data.hits.hits.length;
+      result.data = data.hits.hits.map(function(elm) {
+        return {_id: elm._id, _score: elm._score};
+      });
+
+      resolve(result);
+    }).catch(function(err) {
+      reject(err);
+    });
+  });
+};
+
+SearchClient.prototype.createSearchQuerySortedByUpdatedAt = function(option)
 {
+  // getting path by default is almost for debug
+  var fields = ['path', '_id'];
+  if (option) {
+    fields = option.fields || fields;
+  }
+
   // default is only id field, sorted by updated_at
-  return {
+  var query = {
     index: this.index_name,
     type: 'pages',
     body: {
-      fields: ['_id'],
+      fields: fields,
       sort: [{ updated_at: { order: 'desc'}}],
       query: {}, // query
     }
   };
+  this.appendResultSize(query);
+
+  return query;
 };
 
-SearchClient.prototype.createSearchQuerySortedByScore = function()
+SearchClient.prototype.createSearchQuerySortedByScore = function(option)
 {
+  var fields = ['path', '_id'];
+  if (option) {
+    fields = option.fields || fields;
+  }
+
   // sort by score
-  return {
+  var query = {
     index: this.index_name,
     type: 'pages',
     body: {
-      fields: ['_id'],
+      fields: fields,
       sort: [ {_score: { order: 'desc'} }],
       query: {}, // query
     }
   };
+  this.appendResultSize(query);
+
+  return query;
 };
 
-SearchClient.prototype.searchSuggest = function(keyword)
+SearchClient.prototype.appendResultSize = function(query, from, size)
 {
-  var query = this.createSearchQuerySortedByScore();
+  query.from = from || this.DEFAULT_OFFSET;
+  query.size = size || this.DEFAULT_LIMIT;
+};
 
-  query.body.query = {
-    bool: {
-      must: [
-        {
-          bool: {
-            should: [
-              {
-                match: {
-                  'path.raw': {
-                    query: sprintf('*%s*', keyword),
-                    operator: 'or'
-                  }
-                }
-              },
-              {
-                match: {
-                  'body.ja': {
-                    query: keyword,
-                    operator: 'or'
-                  }
-                }
-              }
-            ]
-          }
-        }
-      ]
-    }
-  };
+SearchClient.prototype.appendCriteriaForKeywordContains = function(query, keyword)
+{
+  // query is created by createSearchQuerySortedByScore() or createSearchQuerySortedByUpdatedAt()
+  if (!query.body.query.bool) {
+    query.body.query.bool = {};
+  }
+
+  if (!query.body.query.bool.must || !Array.isArray(query.body.query.must)) {
+    query.body.query.bool.must = [];
+  }
 
-  return this.client.search(query);
+  query.body.query.bool.must.push({
+    multi_match: {
+      query: keyword,
+      fields: [
+        "path.ja",
+        "body.ja"
+      ],
+      operator: "and"
+    }
+  });
 };
 
-SearchClient.prototype.searchKeyword = function(keyword)
+SearchClient.prototype.appendCriteriaForPathFilter = function(query, path)
 {
-  var query = this.createSearchQuerySortedByUpdatedAt();
-
-  query.body.query = {
-    bool: {
-      must: [
-        {
-          bool: {
-            should: [
-              {
-                match: {
-                  'path.raw': {
-                    query: sprintf('*%s*', keyword),
-                    operator: 'or'
-                  }
-                }
-              },
-              {
-                match: {
-                  'body.ja': {
-                    query: keyword,
-                    operator: 'or'
-                  }
-                }
-              }
-            ]
-          }
-        }
-      ]
+  // query is created by createSearchQuerySortedByScore() or createSearchQuerySortedByUpdatedAt()
+  if (!query.body.query.bool) {
+    query.body.query.bool = {};
+  }
+
+  if (!query.body.query.bool.filter || !Array.isArray(query.body.query.bool.filter)) {
+    query.body.query.bool.filter = [];
+  }
+
+  if (path.match(/\/$/)) {
+    path = path.substr(0, path.length - 1);
+  }
+  query.body.query.bool.filter.push({
+    wildcard: {
+      "path.raw": path + "/*"
     }
-  };
+  });
+};
+
+SearchClient.prototype.searchKeyword = function(keyword, option)
+{
+  var from = option.offset || null;
+  var query = this.createSearchQuerySortedByScore();
+  this.appendCriteriaForKeywordContains(query, keyword);
 
-  return this.client.search(query);
+  return this.search(query);
 };
 
 SearchClient.prototype.searchByPath = function(keyword, prefix)
 {
-  var query = this.createSearchQuerySortedByUpdatedAt();
+  // TODO path 名だけから検索
 };
 
-SearchClient.prototype.searchKeywordUnderPath = function(keyword, path)
+SearchClient.prototype.searchKeywordUnderPath = function(keyword, path, option)
 {
-  var query = this.createSearchQuerySortedByUpdatedAt();
+  var from = option.offset || null;
+  var query = this.createSearchQuerySortedByScore();
+  this.appendCriteriaForKeywordContains(query, keyword);
+  this.appendCriteriaForPathFilter(query, path);
+
+  if (from) {
+    this.appendResultSize(query, from);
+  }
+
+  return this.search(query);
 };
 
 module.exports = SearchClient;