diff --git a/app/elastic/application_search.rb b/app/elastic/application_search.rb deleted file mode 100644 index cb5776b5dde73a465a95519e4b731c98452a76c1..0000000000000000000000000000000000000000 --- a/app/elastic/application_search.rb +++ /dev/null @@ -1,103 +0,0 @@ -module ApplicationSearch - extend ActiveSupport::Concern - - included do - include Elasticsearch::Model - - self.__elasticsearch__.client = Elasticsearch::Client.new( - host: Gitlab.config.elasticsearch.host, - port: Gitlab.config.elasticsearch.port - ) - - index_name [Rails.application.class.parent_name.downcase, self.name.downcase, Rails.env].join('-') - - settings \ - index: { - analysis: { - analyzer: { - default:{ - tokenizer: "standard", - filter: ["standard", "lowercase", "my_stemmer"] - } - }, - filter: { - my_stemmer: { - type: "stemmer", - name: "light_english" - } - } - } - } - - if Gitlab.config.elasticsearch.enabled - after_commit on: :create do - ElasticIndexerWorker.perform_async(:index, self.class.to_s, self.id) - end - - after_commit on: :update do - ElasticIndexerWorker.perform_async(:update, self.class.to_s, self.id) - end - - after_commit on: :destroy do - ElasticIndexerWorker.perform_async(:delete, self.class.to_s, self.id) - end - end - end - - module ClassMethods - def highlight_options(fields) - es_fields = fields.map { |field| field.split('^').first }.inject({}) do |memo, field| - memo[field.to_sym] = {} - memo - end - - { fields: es_fields } - end - - def basic_query_hash(fields, query) - query_hash = if query.present? - { - query: { - filtered: { - query: { - multi_match: { - fields: fields, - query: query, - operator: :and - } - }, - }, - } - } - else - { - query: { - filtered: { - query: { match_all: {} } - } - }, - track_scores: true - } - end - - query_hash[:sort] = [ - { updated_at_sort: { order: :desc, mode: :min } }, - :_score - ] - - query_hash[:highlight] = highlight_options(fields) - - query_hash - end - - def project_ids_filter(query_hash, project_ids) - if project_ids - query_hash[:query][:filtered][:filter] = { - and: [ { terms: { project_id: project_ids } } ] - } - end - - query_hash - end - end -end diff --git a/app/elastic/issues_search.rb b/app/elastic/issues_search.rb deleted file mode 100644 index f152993d18e4f182a94397a342a48fd177738fa7..0000000000000000000000000000000000000000 --- a/app/elastic/issues_search.rb +++ /dev/null @@ -1,47 +0,0 @@ -module IssuesSearch - extend ActiveSupport::Concern - - included do - include ApplicationSearch - - mappings do - indexes :id, type: :integer - - indexes :iid, type: :integer, index: :not_analyzed - indexes :title, type: :string, - index_options: 'offsets' - indexes :description, type: :string, - index_options: 'offsets' - indexes :created_at, type: :date - indexes :updated_at, type: :date - indexes :state, type: :string - - indexes :project_id, type: :integer - indexes :author_id, type: :integer - - indexes :project, type: :nested - indexes :author, type: :nested - - indexes :updated_at_sort, type: :date, index: :not_analyzed - end - - def as_indexed_json(options = {}) - as_json( - include: { - project: { only: :id }, - author: { only: :id } - } - ).merge({ updated_at_sort: updated_at }) - end - - def self.elastic_search(query, options: {}) - options[:in] = %w(title^2 description) - - query_hash = basic_query_hash(options[:in], query) - - query_hash = project_ids_filter(query_hash, options[:projects_ids]) - - self.__elasticsearch__.search(query_hash) - end - end -end diff --git a/app/elastic/merge_requests_search.rb b/app/elastic/merge_requests_search.rb deleted file mode 100644 index c01e41778abaccf59a5a40e92181b9dac9edeecb..0000000000000000000000000000000000000000 --- a/app/elastic/merge_requests_search.rb +++ /dev/null @@ -1,63 +0,0 @@ -module MergeRequestsSearch - extend ActiveSupport::Concern - - included do - include ApplicationSearch - - mappings do - indexes :id, type: :integer - - indexes :iid, type: :integer - indexes :target_branch, type: :string, - index_options: 'offsets' - indexes :source_branch, type: :string, - index_options: 'offsets' - indexes :title, type: :string, - index_options: 'offsets' - indexes :description, type: :string, - index_options: 'offsets' - indexes :created_at, type: :date - indexes :updated_at, type: :date - indexes :state, type: :string - indexes :merge_status, type: :string - - indexes :source_project_id, type: :integer - indexes :target_project_id, type: :integer - indexes :author_id, type: :integer - - indexes :source_project, type: :nested - indexes :target_project, type: :nested - indexes :author, type: :nested - - indexes :updated_at_sort, type: :string, index: 'not_analyzed' - end - - def as_indexed_json(options = {}) - as_json( - include: { - source_project: { only: :id }, - target_project: { only: :id }, - author: { only: :id } - } - ).merge({ updated_at_sort: updated_at }) - end - - def self.elastic_search(query, options: {}) - query_hash = basic_query_hash(%w(title^2 description), query) - - if options[:projects_ids] - query_hash[:query][:filtered][:filter] = { - and: [ - { - terms: { - target_project_id: [options[:projects_ids]].flatten - } - } - ] - } - end - - self.__elasticsearch__.search(query_hash) - end - end -end diff --git a/app/elastic/milestones_search.rb b/app/elastic/milestones_search.rb deleted file mode 100644 index 843bcbac87441f1c7ccbf4992b40c0892663cec8..0000000000000000000000000000000000000000 --- a/app/elastic/milestones_search.rb +++ /dev/null @@ -1,33 +0,0 @@ -module MilestonesSearch - extend ActiveSupport::Concern - - included do - include ApplicationSearch - - mappings do - indexes :id, type: :integer - indexes :title, type: :string, - index_options: 'offsets' - indexes :description, type: :string, - index_options: 'offsets' - indexes :project_id, type: :integer - indexes :created_at, type: :date - - indexes :updated_at_sort, type: :string, index: 'not_analyzed' - end - - def as_indexed_json(options = {}) - as_json.merge({ updated_at_sort: updated_at }) - end - - def self.elastic_search(query, options: {}) - options[:in] = %w(title^2 description) - - query_hash = basic_query_hash(options[:in], query) - - query_hash = project_ids_filter(query_hash, options[:projects_ids]) - - self.__elasticsearch__.search(query_hash) - end - end -end diff --git a/app/elastic/notes_search.rb b/app/elastic/notes_search.rb deleted file mode 100644 index 8ec2a7bbac6bd0dd08e30e38f8f069a211616d98..0000000000000000000000000000000000000000 --- a/app/elastic/notes_search.rb +++ /dev/null @@ -1,49 +0,0 @@ -module NotesSearch - extend ActiveSupport::Concern - - included do - include ApplicationSearch - - mappings do - indexes :id, type: :integer - indexes :note, type: :string, - index_options: 'offsets' - indexes :project_id, type: :integer - indexes :created_at, type: :date - - indexes :updated_at_sort, type: :string, index: 'not_analyzed' - end - - def as_indexed_json(options = {}) - as_json.merge({ updated_at_sort: updated_at }) - end - - def self.elastic_search(query, options: {}) - options[:in] = ["note"] - - query_hash = { - query: { - filtered: { - query: { match: { note: query } }, - }, - } - } - - if query.blank? - query_hash[:query][:filtered][:query] = { match_all: {} } - query_hash[:track_scores] = true - end - - query_hash = project_ids_filter(query_hash, options[:projects_ids]) - - query_hash[:sort] = [ - { updated_at_sort: { order: :desc, mode: :min } }, - :_score - ] - - query_hash[:highlight] = highlight_options(options[:in]) - - self.__elasticsearch__.search(query_hash) - end - end -end diff --git a/app/elastic/projects_search.rb b/app/elastic/projects_search.rb deleted file mode 100644 index bee7126fc51825f26904a9d4454ca145ebc4438d..0000000000000000000000000000000000000000 --- a/app/elastic/projects_search.rb +++ /dev/null @@ -1,116 +0,0 @@ -module ProjectsSearch - extend ActiveSupport::Concern - - included do - include ApplicationSearch - - mappings do - indexes :id, type: :integer - - indexes :name, type: :string, - index_options: 'offsets' - indexes :path, type: :string, - index_options: 'offsets' - indexes :name_with_namespace, type: :string, - index_options: 'offsets' - indexes :path_with_namespace, type: :string, - index_options: 'offsets' - indexes :description, type: :string, - index_options: 'offsets' - - indexes :namespace_id, type: :integer - - indexes :created_at, type: :date - indexes :archived, type: :boolean - indexes :visibility_level, type: :integer - indexes :last_activity_at, type: :date - indexes :last_pushed_at, type: :date - end - - def as_indexed_json(options = {}) - as_json.merge({ - name_with_namespace: name_with_namespace, - path_with_namespace: path_with_namespace - }) - end - - def self.elastic_search(query, options: {}) - options[:in] = %w(name^10 name_with_namespace^2 path_with_namespace path^9) - - query_hash = basic_query_hash(options[:in], query) - - filters = [] - - if options[:abandoned] - filters << { - range: { - last_pushed_at: { - lte: "now-6M/m" - } - } - } - end - - if options[:with_push] - filters << { - not: { - missing: { - field: :last_pushed_at, - existence: true, - null_value: true - } - } - } - end - - if options[:namespace_id] - filters << { - terms: { - namespace_id: [options[:namespace_id]].flatten - } - } - end - - if options[:non_archived] - filters << { - terms: { - archived: [!options[:non_archived]].flatten - } - } - end - - if options[:visibility_levels] - filters << { - terms: { - visibility_level: [options[:visibility_levels]].flatten - } - } - end - - if !options[:owner_id].blank? - filters << { - nested: { - path: :owner, - filter: { - term: { "owner.id" => options[:owner_id] } - } - } - } - end - - if options[:pids] - filters << { - ids: { - values: options[:pids] - } - } - end - - query_hash[:query][:filtered][:filter] = { and: filters } - - query_hash[:sort] = [:_score] - - self.__elasticsearch__.search(query_hash) - end - end -end diff --git a/app/elastic/repositories_search.rb b/app/elastic/repositories_search.rb deleted file mode 100644 index 93ffb7000e216d02ddbd2608b236e7b636b9820f..0000000000000000000000000000000000000000 --- a/app/elastic/repositories_search.rb +++ /dev/null @@ -1,35 +0,0 @@ -module RepositoriesSearch - extend ActiveSupport::Concern - - included do - include Elasticsearch::Git::Repository - - self.__elasticsearch__.client = Elasticsearch::Client.new( - host: Gitlab.config.elasticsearch.host, - port: Gitlab.config.elasticsearch.port - ) - - def repository_id - project.id - end - - def self.repositories_count - Project.count - end - - def client_for_indexing - self.__elasticsearch__.client - end - - def self.import - Repository.__elasticsearch__.create_index! - - Project.find_each do |project| - if project.repository.exists? && !project.repository.empty? - project.repository.index_commits - project.repository.index_blobs - end - end - end - end -end diff --git a/app/elastic/snippets_search.rb b/app/elastic/snippets_search.rb deleted file mode 100644 index 72e20bb023c02910722134f795377630073351f1..0000000000000000000000000000000000000000 --- a/app/elastic/snippets_search.rb +++ /dev/null @@ -1,77 +0,0 @@ -module SnippetsSearch - extend ActiveSupport::Concern - - included do - include ApplicationSearch - - mappings do - indexes :id, type: :integer - - indexes :title, type: :string, - index_options: 'offsets' - indexes :file_name, type: :string, - index_options: 'offsets' - indexes :content, type: :string, - index_options: 'offsets' - indexes :created_at, type: :date - indexes :updated_at, type: :date - indexes :state, type: :string - - indexes :project_id, type: :integer - indexes :author_id, type: :integer - - indexes :project, type: :nested - indexes :author, type: :nested - - indexes :updated_at_sort, type: :date, index: :not_analyzed - end - - def as_indexed_json(options = {}) - as_json( - include: { - project: { only: :id }, - author: { only: :id } - } - ) - end - - def self.elastic_search(query, options: {}) - query_hash = basic_query_hash(%w(title file_name), query) - - query_hash = limit_ids(query_hash, options[:ids]) - - self.__elasticsearch__.search(query_hash) - end - - def self.elastic_search_code(query, options: {}) - query_hash = { - query: { - filtered: { - query: { match: { content: query } }, - }, - } - } - - query_hash = limit_ids(query_hash, options[:ids]) - - query_hash[:sort] = [ - { updated_at_sort: { order: :desc, mode: :min } }, - :_score - ] - - query_hash[:highlight] = { fields: { content: {} } } - - self.__elasticsearch__.search(query_hash) - end - - def self.limit_ids(query_hash, ids) - if ids - query_hash[:query][:filtered][:filter] = { - and: [ { terms: { id: ids } } ] - } - end - - query_hash - end - end -end diff --git a/app/elastic/wiki_repositories_search.rb b/app/elastic/wiki_repositories_search.rb deleted file mode 100644 index af462a1b6bbbef324a66e24d37eb5bd5554a626e..0000000000000000000000000000000000000000 --- a/app/elastic/wiki_repositories_search.rb +++ /dev/null @@ -1,34 +0,0 @@ -module WikiRepositoriesSearch - extend ActiveSupport::Concern - - included do - include Elasticsearch::Git::Repository - - self.__elasticsearch__.client = Elasticsearch::Client.new( - host: Gitlab.config.elasticsearch.host, - port: Gitlab.config.elasticsearch.port - ) - - def repository_id - "wiki_#{project.id}" - end - - def self.repositories_count - Project.where(wiki_enabled: true).count - end - - def client_for_indexing - self.__elasticsearch__.client - end - - def self.import - ProjectWiki.__elasticsearch__.create_index! - - Project.where(wiki_enabled: true).find_each do |project| - unless project.wiki.empty? - project.wiki.index_blobs - end - end - end - end -end diff --git a/app/models/issue.rb b/app/models/issue.rb index f6a0c9a79e28e08526cab99c42bb54edd33ffe47..7774bfee0634dad308e3477bd47116523ad4c6b2 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -27,7 +27,7 @@ class Issue < ActiveRecord::Base include Referable include Sortable include Taskable - include IssuesSearch + include Elastic::IssuesSearch WEIGHT_RANGE = 1..9 diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index 943a941ab5f4056e06b77939d8d26e679e3986a7..72f39b31ced5b8c4b39d159724ab3acf2d387f0b 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -35,7 +35,7 @@ class MergeRequest < ActiveRecord::Base include Referable include Sortable include Taskable - include MergeRequestsSearch + include Elastic::MergeRequestsSearch belongs_to :target_project, foreign_key: :target_project_id, class_name: "Project" belongs_to :source_project, foreign_key: :source_project_id, class_name: "Project" diff --git a/app/models/milestone.rb b/app/models/milestone.rb index ae7b7b93f6b1d65064a032e5f1e36c5272c2b48f..e780d7b5b608ae2a5b89bd5be2954eaa1e267b42 100644 --- a/app/models/milestone.rb +++ b/app/models/milestone.rb @@ -24,7 +24,7 @@ class Milestone < ActiveRecord::Base include Sortable include Referable include StripAttribute - include MilestonesSearch + include Elastic::MilestonesSearch belongs_to :project has_many :issues diff --git a/app/models/note.rb b/app/models/note.rb index 2eb9dd7ba83711ee241a55cdc8bebe064a46dc2c..ac47729dd890ca161cb7dbec6011393602f4482f 100644 --- a/app/models/note.rb +++ b/app/models/note.rb @@ -26,7 +26,7 @@ class Note < ActiveRecord::Base include Gitlab::CurrentSettings include Participable include Mentionable - include NotesSearch + include Elastic::NotesSearch default_value_for :system, false diff --git a/app/models/project.rb b/app/models/project.rb index 3fddd48d0665f3c4e125f817075a20032fa78115..f99674e36e930977d6f27c725898ea06c1ddf93d 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -52,7 +52,7 @@ class Project < ActiveRecord::Base include AfterCommitQueue include CaseSensitivity include TokenAuthenticatable - include ProjectsSearch + include Elastic::ProjectsSearch extend Gitlab::ConfigHelper diff --git a/app/models/project_wiki.rb b/app/models/project_wiki.rb index 43abd25fb22773a649da73cb54c5e89dec1b5f68..93e7c567bd7ff4beb5ca5840be96dd3336d9d622 100644 --- a/app/models/project_wiki.rb +++ b/app/models/project_wiki.rb @@ -1,6 +1,6 @@ class ProjectWiki include Gitlab::ShellAdapter - include WikiRepositoriesSearch + include Elastic::WikiRepositoriesSearch MARKUPS = { 'Markdown' => :md, diff --git a/app/models/repository.rb b/app/models/repository.rb index c19e23d4f97d9620435a6975a2af8b1f38d09731..0918a712858596549fd6d6ba609a9893c5da9ffc 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -1,7 +1,7 @@ require 'securerandom' class Repository - include RepositoriesSearch + include Elastic::RepositoriesSearch class CommitError < StandardError; end diff --git a/app/models/snippet.rb b/app/models/snippet.rb index cd27c8bb1e0706514d47cd28344a8a0b8a6d76a2..e18d198ea7c2d0e8a2cd31e315c7248355b69dba 100644 --- a/app/models/snippet.rb +++ b/app/models/snippet.rb @@ -21,7 +21,7 @@ class Snippet < ActiveRecord::Base include Participable include Referable include Sortable - include SnippetsSearch + include Elastic::SnippetsSearch default_value_for :visibility_level, Snippet::PRIVATE diff --git a/config/application.rb b/config/application.rb index 85da29b09e2892437fe4aad533a301db58bc2169..d255ff0719f52405d2d0d9eac1868d6bb4027d03 100644 --- a/config/application.rb +++ b/config/application.rb @@ -16,8 +16,7 @@ module Gitlab #{config.root}/app/models/hooks #{config.root}/app/models/concerns #{config.root}/app/models/project_services - #{config.root}/app/models/members - #{config.root}/app/elastic)) + #{config.root}/app/models/members)) # Only load the plugins named here, in the order given (default is alphabetical). # :all can be used as a placeholder for all plugins not explicitly named. diff --git a/lib/elastic/application_search.rb b/lib/elastic/application_search.rb new file mode 100644 index 0000000000000000000000000000000000000000..87d51f41d2ba36b6d680ab0b85b99b21c907880a --- /dev/null +++ b/lib/elastic/application_search.rb @@ -0,0 +1,105 @@ +module Elastic + module ApplicationSearch + extend ActiveSupport::Concern + + included do + include Elasticsearch::Model + + self.__elasticsearch__.client = Elasticsearch::Client.new( + host: Gitlab.config.elasticsearch.host, + port: Gitlab.config.elasticsearch.port + ) + + index_name [Rails.application.class.parent_name.downcase, self.name.downcase, Rails.env].join('-') + + settings \ + index: { + analysis: { + analyzer: { + default:{ + tokenizer: "standard", + filter: ["standard", "lowercase", "my_stemmer"] + } + }, + filter: { + my_stemmer: { + type: "stemmer", + name: "light_english" + } + } + } + } + + if Gitlab.config.elasticsearch.enabled + after_commit on: :create do + ElasticIndexerWorker.perform_async(:index, self.class.to_s, self.id) + end + + after_commit on: :update do + ElasticIndexerWorker.perform_async(:update, self.class.to_s, self.id) + end + + after_commit on: :destroy do + ElasticIndexerWorker.perform_async(:delete, self.class.to_s, self.id) + end + end + end + + module ClassMethods + def highlight_options(fields) + es_fields = fields.map { |field| field.split('^').first }.inject({}) do |memo, field| + memo[field.to_sym] = {} + memo + end + + { fields: es_fields } + end + + def basic_query_hash(fields, query) + query_hash = if query.present? + { + query: { + filtered: { + query: { + multi_match: { + fields: fields, + query: query, + operator: :and + } + }, + }, + } + } + else + { + query: { + filtered: { + query: { match_all: {} } + } + }, + track_scores: true + } + end + + query_hash[:sort] = [ + { updated_at_sort: { order: :desc, mode: :min } }, + :_score + ] + + query_hash[:highlight] = highlight_options(fields) + + query_hash + end + + def project_ids_filter(query_hash, project_ids) + if project_ids + query_hash[:query][:filtered][:filter] = { + and: [ { terms: { project_id: project_ids } } ] + } + end + + query_hash + end + end + end +end diff --git a/lib/elastic/issues_search.rb b/lib/elastic/issues_search.rb new file mode 100644 index 0000000000000000000000000000000000000000..e7d3d6ecbc87e6911e07bd3614795f6e6c614fa7 --- /dev/null +++ b/lib/elastic/issues_search.rb @@ -0,0 +1,49 @@ +module Elastic + module IssuesSearch + extend ActiveSupport::Concern + + included do + include ApplicationSearch + + mappings do + indexes :id, type: :integer + + indexes :iid, type: :integer, index: :not_analyzed + indexes :title, type: :string, + index_options: 'offsets' + indexes :description, type: :string, + index_options: 'offsets' + indexes :created_at, type: :date + indexes :updated_at, type: :date + indexes :state, type: :string + + indexes :project_id, type: :integer + indexes :author_id, type: :integer + + indexes :project, type: :nested + indexes :author, type: :nested + + indexes :updated_at_sort, type: :date, index: :not_analyzed + end + + def as_indexed_json(options = {}) + as_json( + include: { + project: { only: :id }, + author: { only: :id } + } + ).merge({ updated_at_sort: updated_at }) + end + + def self.elastic_search(query, options: {}) + options[:in] = %w(title^2 description) + + query_hash = basic_query_hash(options[:in], query) + + query_hash = project_ids_filter(query_hash, options[:projects_ids]) + + self.__elasticsearch__.search(query_hash) + end + end + end +end diff --git a/lib/elastic/merge_requests_search.rb b/lib/elastic/merge_requests_search.rb new file mode 100644 index 0000000000000000000000000000000000000000..ec7384ea84ebb6c2872ec8e254797ae8f86033ff --- /dev/null +++ b/lib/elastic/merge_requests_search.rb @@ -0,0 +1,65 @@ +module Elastic + module MergeRequestsSearch + extend ActiveSupport::Concern + + included do + include ApplicationSearch + + mappings do + indexes :id, type: :integer + + indexes :iid, type: :integer + indexes :target_branch, type: :string, + index_options: 'offsets' + indexes :source_branch, type: :string, + index_options: 'offsets' + indexes :title, type: :string, + index_options: 'offsets' + indexes :description, type: :string, + index_options: 'offsets' + indexes :created_at, type: :date + indexes :updated_at, type: :date + indexes :state, type: :string + indexes :merge_status, type: :string + + indexes :source_project_id, type: :integer + indexes :target_project_id, type: :integer + indexes :author_id, type: :integer + + indexes :source_project, type: :nested + indexes :target_project, type: :nested + indexes :author, type: :nested + + indexes :updated_at_sort, type: :string, index: 'not_analyzed' + end + + def as_indexed_json(options = {}) + as_json( + include: { + source_project: { only: :id }, + target_project: { only: :id }, + author: { only: :id } + } + ).merge({ updated_at_sort: updated_at }) + end + + def self.elastic_search(query, options: {}) + query_hash = basic_query_hash(%w(title^2 description), query) + + if options[:projects_ids] + query_hash[:query][:filtered][:filter] = { + and: [ + { + terms: { + target_project_id: [options[:projects_ids]].flatten + } + } + ] + } + end + + self.__elasticsearch__.search(query_hash) + end + end + end +end diff --git a/lib/elastic/milestones_search.rb b/lib/elastic/milestones_search.rb new file mode 100644 index 0000000000000000000000000000000000000000..0c9340d94a30f8dbec01d0d166c9223b6f0eb54d --- /dev/null +++ b/lib/elastic/milestones_search.rb @@ -0,0 +1,35 @@ +module Elastic + module MilestonesSearch + extend ActiveSupport::Concern + + included do + include ApplicationSearch + + mappings do + indexes :id, type: :integer + indexes :title, type: :string, + index_options: 'offsets' + indexes :description, type: :string, + index_options: 'offsets' + indexes :project_id, type: :integer + indexes :created_at, type: :date + + indexes :updated_at_sort, type: :string, index: 'not_analyzed' + end + + def as_indexed_json(options = {}) + as_json.merge({ updated_at_sort: updated_at }) + end + + def self.elastic_search(query, options: {}) + options[:in] = %w(title^2 description) + + query_hash = basic_query_hash(options[:in], query) + + query_hash = project_ids_filter(query_hash, options[:projects_ids]) + + self.__elasticsearch__.search(query_hash) + end + end + end +end diff --git a/lib/elastic/notes_search.rb b/lib/elastic/notes_search.rb new file mode 100644 index 0000000000000000000000000000000000000000..bd72741bebb4c72da57cea8500b70031092f862d --- /dev/null +++ b/lib/elastic/notes_search.rb @@ -0,0 +1,51 @@ +module Elastic + module NotesSearch + extend ActiveSupport::Concern + + included do + include ApplicationSearch + + mappings do + indexes :id, type: :integer + indexes :note, type: :string, + index_options: 'offsets' + indexes :project_id, type: :integer + indexes :created_at, type: :date + + indexes :updated_at_sort, type: :string, index: 'not_analyzed' + end + + def as_indexed_json(options = {}) + as_json.merge({ updated_at_sort: updated_at }) + end + + def self.elastic_search(query, options: {}) + options[:in] = ["note"] + + query_hash = { + query: { + filtered: { + query: { match: { note: query } }, + }, + } + } + + if query.blank? + query_hash[:query][:filtered][:query] = { match_all: {} } + query_hash[:track_scores] = true + end + + query_hash = project_ids_filter(query_hash, options[:projects_ids]) + + query_hash[:sort] = [ + { updated_at_sort: { order: :desc, mode: :min } }, + :_score + ] + + query_hash[:highlight] = highlight_options(options[:in]) + + self.__elasticsearch__.search(query_hash) + end + end + end +end diff --git a/lib/elastic/projects_search.rb b/lib/elastic/projects_search.rb new file mode 100644 index 0000000000000000000000000000000000000000..3e8c42a8982c9699298b08ef4c2bffba95b103c5 --- /dev/null +++ b/lib/elastic/projects_search.rb @@ -0,0 +1,118 @@ +module Elastic + module ProjectsSearch + extend ActiveSupport::Concern + + included do + include ApplicationSearch + + mappings do + indexes :id, type: :integer + + indexes :name, type: :string, + index_options: 'offsets' + indexes :path, type: :string, + index_options: 'offsets' + indexes :name_with_namespace, type: :string, + index_options: 'offsets' + indexes :path_with_namespace, type: :string, + index_options: 'offsets' + indexes :description, type: :string, + index_options: 'offsets' + + indexes :namespace_id, type: :integer + + indexes :created_at, type: :date + indexes :archived, type: :boolean + indexes :visibility_level, type: :integer + indexes :last_activity_at, type: :date + indexes :last_pushed_at, type: :date + end + + def as_indexed_json(options = {}) + as_json.merge({ + name_with_namespace: name_with_namespace, + path_with_namespace: path_with_namespace + }) + end + + def self.elastic_search(query, options: {}) + options[:in] = %w(name^10 name_with_namespace^2 path_with_namespace path^9) + + query_hash = basic_query_hash(options[:in], query) + + filters = [] + + if options[:abandoned] + filters << { + range: { + last_pushed_at: { + lte: "now-6M/m" + } + } + } + end + + if options[:with_push] + filters << { + not: { + missing: { + field: :last_pushed_at, + existence: true, + null_value: true + } + } + } + end + + if options[:namespace_id] + filters << { + terms: { + namespace_id: [options[:namespace_id]].flatten + } + } + end + + if options[:non_archived] + filters << { + terms: { + archived: [!options[:non_archived]].flatten + } + } + end + + if options[:visibility_levels] + filters << { + terms: { + visibility_level: [options[:visibility_levels]].flatten + } + } + end + + if !options[:owner_id].blank? + filters << { + nested: { + path: :owner, + filter: { + term: { "owner.id" => options[:owner_id] } + } + } + } + end + + if options[:pids] + filters << { + ids: { + values: options[:pids] + } + } + end + + query_hash[:query][:filtered][:filter] = { and: filters } + + query_hash[:sort] = [:_score] + + self.__elasticsearch__.search(query_hash) + end + end + end +end diff --git a/lib/elastic/repositories_search.rb b/lib/elastic/repositories_search.rb new file mode 100644 index 0000000000000000000000000000000000000000..0a42d754d24970b1949079fb81bf6943fe5b2909 --- /dev/null +++ b/lib/elastic/repositories_search.rb @@ -0,0 +1,37 @@ +module Elastic + module RepositoriesSearch + extend ActiveSupport::Concern + + included do + include Elasticsearch::Git::Repository + + self.__elasticsearch__.client = Elasticsearch::Client.new( + host: Gitlab.config.elasticsearch.host, + port: Gitlab.config.elasticsearch.port + ) + + def repository_id + project.id + end + + def self.repositories_count + Project.count + end + + def client_for_indexing + self.__elasticsearch__.client + end + + def self.import + Repository.__elasticsearch__.create_index! + + Project.find_each do |project| + if project.repository.exists? && !project.repository.empty? + project.repository.index_commits + project.repository.index_blobs + end + end + end + end + end +end diff --git a/lib/elastic/snippets_search.rb b/lib/elastic/snippets_search.rb new file mode 100644 index 0000000000000000000000000000000000000000..e49435d34999405b5b342fc48d0c3aaca3193628 --- /dev/null +++ b/lib/elastic/snippets_search.rb @@ -0,0 +1,79 @@ +module Elastic + module SnippetsSearch + extend ActiveSupport::Concern + + included do + include ApplicationSearch + + mappings do + indexes :id, type: :integer + + indexes :title, type: :string, + index_options: 'offsets' + indexes :file_name, type: :string, + index_options: 'offsets' + indexes :content, type: :string, + index_options: 'offsets' + indexes :created_at, type: :date + indexes :updated_at, type: :date + indexes :state, type: :string + + indexes :project_id, type: :integer + indexes :author_id, type: :integer + + indexes :project, type: :nested + indexes :author, type: :nested + + indexes :updated_at_sort, type: :date, index: :not_analyzed + end + + def as_indexed_json(options = {}) + as_json( + include: { + project: { only: :id }, + author: { only: :id } + } + ) + end + + def self.elastic_search(query, options: {}) + query_hash = basic_query_hash(%w(title file_name), query) + + query_hash = limit_ids(query_hash, options[:ids]) + + self.__elasticsearch__.search(query_hash) + end + + def self.elastic_search_code(query, options: {}) + query_hash = { + query: { + filtered: { + query: { match: { content: query } }, + }, + } + } + + query_hash = limit_ids(query_hash, options[:ids]) + + query_hash[:sort] = [ + { updated_at_sort: { order: :desc, mode: :min } }, + :_score + ] + + query_hash[:highlight] = { fields: { content: {} } } + + self.__elasticsearch__.search(query_hash) + end + + def self.limit_ids(query_hash, ids) + if ids + query_hash[:query][:filtered][:filter] = { + and: [ { terms: { id: ids } } ] + } + end + + query_hash + end + end + end +end diff --git a/lib/elastic/wiki_repositories_search.rb b/lib/elastic/wiki_repositories_search.rb new file mode 100644 index 0000000000000000000000000000000000000000..979605cce2eb21ab792da6ab7f6e75c78ddf64d1 --- /dev/null +++ b/lib/elastic/wiki_repositories_search.rb @@ -0,0 +1,36 @@ +module Elastic + module WikiRepositoriesSearch + extend ActiveSupport::Concern + + included do + include Elasticsearch::Git::Repository + + self.__elasticsearch__.client = Elasticsearch::Client.new( + host: Gitlab.config.elasticsearch.host, + port: Gitlab.config.elasticsearch.port + ) + + def repository_id + "wiki_#{project.id}" + end + + def self.repositories_count + Project.where(wiki_enabled: true).count + end + + def client_for_indexing + self.__elasticsearch__.client + end + + def self.import + ProjectWiki.__elasticsearch__.create_index! + + Project.where(wiki_enabled: true).find_each do |project| + unless project.wiki.empty? + project.wiki.index_blobs + end + end + end + end + end +end