diff options
| -rw-r--r-- | Gemfile | 2 | ||||
| -rw-r--r-- | Gemfile.lock | 2 | ||||
| -rw-r--r-- | app/repositories/package_repository.rb | 26 | ||||
| -rw-r--r-- | app/repositories/query_parser/search_query_parser.rb | 145 | ||||
| -rw-r--r-- | app/views/index/index.html.erb | 187 |
5 files changed, 344 insertions, 18 deletions
@@ -33,6 +33,8 @@ gem 'sidekiq', require: false gem 'rdiscount' +gem 'parslet' + # UI gem 'octicons_helper' diff --git a/Gemfile.lock b/Gemfile.lock index 71c12c2..f959a48 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -130,6 +130,7 @@ GEM parallel (1.17.0) parser (2.6.4.1) ast (~> 2.4.0) + parslet (1.8.2) rack (2.0.7) rack-protection (2.0.7) rack @@ -247,6 +248,7 @@ DEPENDENCIES listen nokogiri octicons_helper + parslet rails (~> 6.0.0) rails-controller-testing rdiscount diff --git a/app/repositories/package_repository.rb b/app/repositories/package_repository.rb index dd516e9..2a2e891 100644 --- a/app/repositories/package_repository.rb +++ b/app/repositories/package_repository.rb @@ -1,5 +1,6 @@ require 'forwardable' require 'singleton' +require_relative './query_parser/search_query_parser' class PackageRepository < BaseRepository include Singleton @@ -62,16 +63,7 @@ class PackageRepository < BaseRepository end def suggest(q) - PackageRepository.search( - size: 20, - query: { - wildcard: { - name_sort: { - wildcard: q.downcase + '*' - } - } - } - ) + search(build_query(q, 20, 0)) end # Tries to resolve a query atom to one or more packages @@ -114,22 +106,20 @@ class PackageRepository < BaseRepository def default_search(q, offset) return [] if q.nil? || q.empty? - part1, part2 = q.split('/', 2) + search(build_query(q, default_search_size, offset)) - if part2.nil? - search(build_query(part1, nil, default_search_size, offset)) - else - search(build_query(part2, part1, default_search_size, offset)) - end end - def build_query(q, category, size, offset) + def build_query(q, size, offset) + parser = Object.const_get("SearchQueryParser::QueryParser").new + transformer = Object.const_get("SearchQueryParser::QueryTransformer").new + { size: size, from: offset, query: { function_score: { - query: { bool: bool_query_parts(q, category) }, + query: { bool: transformer.apply(parser.parse(q)).to_elasticsearch }, functions: scoring_functions } } diff --git a/app/repositories/query_parser/search_query_parser.rb b/app/repositories/query_parser/search_query_parser.rb new file mode 100644 index 0000000..f3e67c6 --- /dev/null +++ b/app/repositories/query_parser/search_query_parser.rb @@ -0,0 +1,145 @@ +require 'parslet' + +module SearchQueryParser + + class QueryParser < Parslet::Parser + rule(:term) { match('[^\s"]').repeat(1).as(:term) } + rule(:quote) { str('"') } + rule(:operator) { (str('+') | str('-')).as(:operator) } + + rule(:fieldname) { match('[^\s:"]').repeat(1).as(:fieldname) } + rule(:field) { (fieldname >> str(':')).as(:field) } + + rule(:phrase) do + (quote >> (term >> space.maybe).repeat >> quote).as(:phrase) + end + rule(:clause) { (operator.maybe >> field.maybe >> (phrase | term)).as(:clause) } + rule(:space) { match('\s').repeat(1) } + rule(:query) { (clause >> space.maybe).repeat.as(:query) } + root(:query) + end + + class QueryTransformer < Parslet::Transform + rule(:clause => subtree(:clause)) do + if clause[:term] + TermClause.new(clause[:operator]&.to_s, clause[:field], clause[:term].to_s) + elsif clause[:phrase] + phrase = clause[:phrase].map { |p| p[:term].to_s }.join(" ") + PhraseClause.new(clause[:operator]&.to_s, clause[:field], phrase) + else + raise "Unexpected clause type: '#{clause}'" + end + end + rule(:query => sequence(:clauses)) { Query.new(clauses) } + end + + class Operator + def self.symbol(str) + case str + when '+' + :must + when '-' + :must_not + when nil + :should + else + raise "Unknown operator: #{str}" + end + end + end + + class TermClause + attr_accessor :operator, :field, :term + + def initialize(operator, field, term) + self.operator = Operator.symbol(operator) + self.field = field + self.term = term + end + end + + class PhraseClause + attr_accessor :operator, :field, :phrase + + def initialize(operator, field, phrase) + self.operator = Operator.symbol(operator) + self.field = field + self.phrase = phrase + end + end + + class Query + attr_accessor :should_clauses, :must_not_clauses, :must_clauses + + def initialize(clauses) + grouped = clauses.chunk { |c| c.operator }.to_h + self.should_clauses = grouped.fetch(:should, []) + self.must_not_clauses = grouped.fetch(:must_not, []) + self.must_clauses = grouped.fetch(:must, []) + end + + def to_elasticsearch + query = { } + + if should_clauses.any? + query[:should] = should_clauses.map do |clause| + clause_to_query(clause) + end + end + + if must_clauses.any? + query[:must] = must_clauses.map do |clause| + clause_to_query(clause) + end + end + + if must_not_clauses.any? + query[:must_not] = must_not_clauses.map do |clause| + clause_to_query(clause) + end + end + + query + end + + def clause_to_query(clause) + case clause + when TermClause + match(clause.field, clause.term) + when PhraseClause + match_phrase(clause.field, clause.phrase) + else + raise "Unknown clause type: #{clause}" + end + end + + def match(field, term) + if field + { + :match => { + field[:fieldname].to_s.to_sym => { + :query => term + } + } + } + else + { + :multi_match => { + :query => term, + :fields => ["atom^3", "name^2"] + } + } + end + end + + def match_phrase(field, phrase) + { + :match_phrase => { + field ? field[:fieldname].to_s.to_sym : :name => { + :query => phrase + } + } + } + end + end +end diff --git a/app/views/index/index.html.erb b/app/views/index/index.html.erb index af86c9e..e54eb74 100644 --- a/app/views/index/index.html.erb +++ b/app/views/index/index.html.erb @@ -8,6 +8,11 @@ <input id="q" name="q" type="search" autocomplete="off" placeholder="<%= t :find_packages %>" aria-label="<%= t :find_packages %>" autofocus> </span> <span class="typeahead-button"> + <button type="button" onclick="$('#searchHelp').modal('show')" title="Help" aria-label="<%= "Help" %>"> + <span class="fa fa-question" style="font-size: 15px;"></span><span class="sr-only"><%= "Help" %></span> + </button> + </span> + <span class="typeahead-button"> <button type="submit" title="<%= t :find %>" aria-label="<%= t :find %>"> <span class="typeahead-search-icon"></span><span class="sr-only"><%= t :find %></span> </button> @@ -50,4 +55,186 @@ </div> <% end %> +<div class="modal fade" id="searchHelp" tabindex="-1" role="dialog" aria-labelledby="searchHelpTitle"> + <div class="modal-dialog modal-lg" role="document"> + <div class="modal-content"> + <div class="modal-header"> + <button type="button" class="close" data-dismiss="modal" aria-label="Close"><span aria-hidden="true">×</span></button> + <h3 class="modal-title" id="searchHelpTitle"><span class="fa fa-info" style="font-size: 15px;"></span> Package Search Syntax</h3> + </div> + <div class="modal-body"> + The search can be used to find Gentoo packages. You can search in two different ways: + <ul > + <li>Keywords</li> + <li>Field/Value pairs</li> + </ul> + <h4 style="margin-top:35px;">Keywords</h4> + You can use simple keywords to find packages. For instance, if you search for <i>gentoo-sources</i> you will find the package <i>sys-kernel/gentoo-sources</i>. + When searching for keywords, the + <ul > + <li><b>name</b></li> + <li> and the <b>atom</b> + </ul> + fields are searched for the keyword. + <h4 style="margin-top:35px;">Field/Value pairs</h4> + If you, however, like to run advanced queries, you can use field/value pairs combined with operators. The possible fields and operators are summarized in the following tables. + <table class="table"> + <thead> + <tr> + <th scope="col">Field</th> + <th scope="col">Description</th> + </tr> + </thead> + <tbody> + <tr> + <th scope="row">atom</th> + <td>The unique identifier of a package <br> <i style="padding-left:2em">e.g. sys-kernel/gentoo-sources</i></td> + </tr> + <tr> + <th scope="row">category</th> + <td>The category of a package <br> <i style="padding-left:2em">e.g. sys-kernel</i></td> + </tr> + <tr> + <th scope="row">name</th> + <td>The name of a package <br> <i style="padding-left:2em">e.g. gentoo-sources</i></td> + </tr> + <tr> + <th scope="row">description</th> + <td>The description of a package <br> <i style="padding-left:2em">e.g. A tiling window manager</i> </td> + </tr> + <tr> + <th scope="row">longdescription</th> + <td>The full descripiton of a package <br> <i style="padding-left:2em">e.g. xmonad is a tiling window manager for [...]</i></td> + </tr> + <tr> + <th scope="row">homepage</th> + <td>The homepage of a package <br> <i style="padding-left:2em">e.g. http://xmonad.org</i></td> + </tr> + <tr> + <th scope="row">license</th> + <td>The license of a package <br> <i style="padding-left:2em">e.g. BSD</i></td> + </tr> + <tr> + <th scope="row">Maintainers</th> + <td></td> + </tr> + <tr> + <th scope="row" style="padding-left:2em">maintainers.name</th> + <td>The name of the maintainer <br> <i style="padding-left:2em">e.g. Gentoo Haskell</i></td> + </tr> + <tr> + <th scope="row" style="padding-left:2em">maintainers.description</th> + <td>The description of the maintainers</td> + </tr> + <tr> + <th scope="row" style="padding-left:2em">maintainers.type</th> + <td>The type of maintainter <br> <i style="padding-left:2em">e.g. project</i></td> + </tr> + <tr> + <th scope="row" style="padding-left:2em">maintainers.restrict</th> + <td></td> + </tr> + <tr> + <th scope="row" style="padding-left:2em">maintainers.email</th> + <td>The email of the maintainer <br> <i style="padding-left:2em">e.g. haskell@gentoo.org</i></td> + </tr> + <tr> + <th scope="row">Useflag</th> + <td></td> + </tr> + <tr> + <th scope="row" style="padding-left:1em">global</th> + <td></td> + </tr> + <tr> + <th scope="row" style="padding-left:2em">useflags.global.name</th> + <td>The name of the global useflag <br> <i style="padding-left:2em">e.g. hscolour</i></td> + </tr> + <tr> + <th scope="row" style="padding-left:2em">useflags.global.description</th> + <td>The description of the global useflag <br> <i style="padding-left:2em">e.g. Include coloured haskell sources to [...]</i></td> + </tr> + <tr> + <th scope="row" style="padding-left:1em">local</th> + <td></td> + </tr> + <tr> + <th scope="row" style="padding-left:2em">useflags.local.name</th> + <td>The name of the local useflag</td> + </tr> + <tr> + <th scope="row" style="padding-left:2em">useflags.local.description</th> + <td>The description of the local useflag</td> + </tr> + <tr> + <th scope="row" style="padding-left:1em">use_expand</th> + <td></td> + </tr> + <tr> + <th scope="row" style="padding-left:2em">useflags.use_expand.name</th> + <td>The name of the local use_expand</td> + </tr> + <tr> + <th scope="row" style="padding-left:2em">useflags.use_expand.description</th> + <td>The description of the use_expand</td> + </tr> + <tr> + <th scope="row" style="padding-left:2em">useflags.use_expand.use_expand_prefix</th> + <td>The use_expand prefix <br> <i style="padding-left:2em">e.g. python_targets</i></td> + </tr> + <tr> + <th scope="row">metadata_hash</th> + <td>The hash of the metadata <br> <i style="padding-left:2em">e.g. 5cd76e098f966b4edcd1848866dd9099</i></td> + </tr> + </tbody> + </table> + The following operators can be used to combine multiple field/value pairs: + <table class="table"> + <thead> + <tr> + <th scope="col">Operator</th> + <th scope="col">Description</th> + </tr> + </thead> + <tbody> + <tr> + <th scope="row"></th> + <td>The term <b>should</b> appear (default)</td> + </tr> + <tr> + <th scope="row">+</th> + <td>The term <b>must</b> appear</td> + </tr> + <tr> + <th scope="row">-</th> + <td>The term <b>must not</b> appear</td> + </tr> + <tr> + <th scope="row">"..."</th> + <td>Can be used to <b>group</b> phrases <br> <i style="padding-left:2em">e.g. +description:"window manager"</i></td> + </tr> + </tbody> + </table> + + <h4>Examples</h4> + + <ul> + <li>Find all packages named git: <br><code style="margin-left:2em">+name:git</code></li> + <li>Find all packages in the category sys-kernel: <br><code style="margin-left:2em">+category:sys-kernel</code></li> + <li>Find all packages with a BSD license: <br><code style="margin-left:2em">+license:BSD</code></li> + <li>Find all packages that neither have a BSD license nor a MIT license: <br><code style="margin-left:2em">-license:BSD -license:MIT</code></li> + <li>Find all packages maintained by the Haskell Team: <br><code style="margin-left:2em">+maintainer.email:haskell@gentoo.org</code></li> + <li>Find all packages maintained by the Haskell Team but that aren't in the 'dev-haskell' category: <br><code style="margin-left:2em">+maintainer.email:haskell@gentoo.org -category:dev-haskell</code></li> + <li>Find all packages those description contains 'window manager': <br><code style="margin-left:2em">+description:"window manager"</code></li> + <li>Find all packages that contain the use_expand 'python_targets': <br><code style="margin-left:2em">+useflags.use_expand.use_expand_prefix:python_targets</code></li> + </ul> + + </div> + <div class="modal-footer"> + <button type="button" class="btn btn-primary" data-dismiss="modal">Close</button> + </div> + </div> + </div> +</div> + <%= javascript_include_tag 'index/typeahead.js' %> |