Add configuration option search_tokenizer_separator

The default is for hyphens to separate tokens in search terms: `gem-based` is equivalent to `gem based`.

This adds `search_tokenizer_separator` as a site configuation option, to support search for hyphenated words.

_config.yml

@@ -24,6 +24,11 @@ exclude: ["node_modules/", "*.gemspec", "*.gem", "Gemfile", "Gemfile.lock", "pac
 # Enable or disable the site search
 search_enabled: true
 
+# Set the search token separator
+search_tokenizer_separator: /[\s\-/]+/
+# For hyphenated-word search:
+# search_tokenizer_separator: /[\s/]+/
+
 # Enable or disable heading anchors
 heading_anchors: true
 

assets/js/just-the-docs.js

@@ -53,7 +53,7 @@ function initSearch() {
       // Success!
       var data = JSON.parse(request.responseText);
 
-      lunr.tokenizer.separator = /[\s\-/]+/
+      lunr.tokenizer.separator = {{ site.search_tokenizer_separator }}
       var index = lunr(function () {
         this.ref('id');
         this.field('title', { boost: 200 });

docs/search.md

@@ -59,6 +59,15 @@ In your site's `_config.yml`, enable search:
 search_enabled: true
 ```
 
+The default is for hyphens to separate tokens in search terms:
+`gem-based` is equivalent to `gem based`, matching either word.
+To allow search for hyphenated words:
+
+```yaml
+# Set the search token separator
+search_tokenizer_separator: /[\s/]+/
+```
+
 ## Hiding pages from search
 
 Sometimes you might have a page that you don't want to be indexed for the search nor to show up in search results, e.g, a 404 page. To exclude a page from search, add the `search_exclude: true` parameter to the page's YAML front matter: