Commit 8d32ead9 authored by mbergeron's avatar mbergeron Committed by Micael Bergeron

Make `gitlab:elastic:index` enable `elasticsearch_indexing`

Before this change, the `gitlab:elastic:index` command would always fail
the on the first run, as it needed a setting to be set in the UI in the
middle of the execution.

Now this setting will be set automatically, thus making this rake task
the easiest way to populate the Elasticsearch index for a small GitLab
instance.
parent 81be8aed
...@@ -245,7 +245,7 @@ This will delete your existing indexes. ...@@ -245,7 +245,7 @@ This will delete your existing indexes.
If the database size is less than 500 MiB, and the size of all hosted repos is less than 5 GiB: If the database size is less than 500 MiB, and the size of all hosted repos is less than 5 GiB:
1. [Enable **Elasticsearch indexing** and configure your host and port](#enabling-elasticsearch). 1. [Configure your Elasticsearch host and port](#enabling-elasticsearch).
1. Index your data: 1. Index your data:
```shell ```shell
...@@ -424,7 +424,7 @@ The following are some available Rake tasks: ...@@ -424,7 +424,7 @@ The following are some available Rake tasks:
| Task | Description | | Task | Description |
|:--------------------------------------------------------------------------------------------------------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| |:--------------------------------------------------------------------------------------------------------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| [`sudo gitlab-rake gitlab:elastic:index`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Wrapper task for `gitlab:elastic:create_empty_index`, `gitlab:elastic:clear_index_status`, `gitlab:elastic:index_projects`, and `gitlab:elastic:index_snippets`. | | [`sudo gitlab-rake gitlab:elastic:index`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Enables Elasticsearch Indexing and run `gitlab:elastic:create_empty_index`, `gitlab:elastic:clear_index_status`, `gitlab:elastic:index_projects`, and `gitlab:elastic:index_snippets`. |
| [`sudo gitlab-rake gitlab:elastic:index_projects`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Iterates over all projects and queues Sidekiq jobs to index them in the background. | | [`sudo gitlab-rake gitlab:elastic:index_projects`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Iterates over all projects and queues Sidekiq jobs to index them in the background. |
| [`sudo gitlab-rake gitlab:elastic:index_projects_status`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Determines the overall status of the indexing. It is done by counting the total number of indexed projects, dividing by a count of the total number of projects, then multiplying by 100. | | [`sudo gitlab-rake gitlab:elastic:index_projects_status`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Determines the overall status of the indexing. It is done by counting the total number of indexed projects, dividing by a count of the total number of projects, then multiplying by 100. |
| [`sudo gitlab-rake gitlab:elastic:clear_index_status`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Deletes all instances of IndexStatus for all projects. | | [`sudo gitlab-rake gitlab:elastic:clear_index_status`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Deletes all instances of IndexStatus for all projects. |
......
---
title: Automatically enable Elasticsearch indexing with 'gitlab:elastic:index'
merge_request: 35342
author:
type: changed
...@@ -9,6 +9,18 @@ namespace :gitlab do ...@@ -9,6 +9,18 @@ namespace :gitlab do
Rake::Task["gitlab:elastic:recreate_index"].invoke Rake::Task["gitlab:elastic:recreate_index"].invoke
Rake::Task["gitlab:elastic:clear_index_status"].invoke Rake::Task["gitlab:elastic:clear_index_status"].invoke
# enable `elasticsearch_indexing` if it isn't
unless Gitlab::CurrentSettings.elasticsearch_indexing?
ApplicationSettings::UpdateService.new(
Gitlab::CurrentSettings.current_application_settings,
nil,
{ elasticsearch_indexing: true }
).execute
puts "Setting `elasticsearch_indexing` has been enabled."
end
Rake::Task["gitlab:elastic:index_projects"].invoke Rake::Task["gitlab:elastic:index_projects"].invoke
Rake::Task["gitlab:elastic:index_snippets"].invoke Rake::Task["gitlab:elastic:index_snippets"].invoke
end end
......
...@@ -5,78 +5,87 @@ require 'rake_helper' ...@@ -5,78 +5,87 @@ require 'rake_helper'
RSpec.describe 'gitlab:elastic namespace rake tasks', :elastic do RSpec.describe 'gitlab:elastic namespace rake tasks', :elastic do
before do before do
Rake.application.rake_require 'tasks/gitlab/elastic' Rake.application.rake_require 'tasks/gitlab/elastic'
stub_ee_application_setting(elasticsearch_indexing: true)
end end
describe 'index' do context "with elasticsearch_indexing enabled" do
it 'calls all indexing tasks in order' do before do
expect(Rake::Task['gitlab:elastic:recreate_index']).to receive(:invoke).ordered stub_ee_application_setting(elasticsearch_indexing: true)
expect(Rake::Task['gitlab:elastic:clear_index_status']).to receive(:invoke).ordered
expect(Rake::Task['gitlab:elastic:index_projects']).to receive(:invoke).ordered
expect(Rake::Task['gitlab:elastic:index_snippets']).to receive(:invoke).ordered
run_rake_task 'gitlab:elastic:index'
end end
end
describe 'index_projects' do describe 'index' do
let(:project1) { create :project } it 'calls all indexing tasks in order' do
let(:project2) { create :project } expect(Rake::Task['gitlab:elastic:recreate_index']).to receive(:invoke).ordered
let(:project3) { create :project } expect(Rake::Task['gitlab:elastic:clear_index_status']).to receive(:invoke).ordered
expect(Rake::Task['gitlab:elastic:index_projects']).to receive(:invoke).ordered
expect(Rake::Task['gitlab:elastic:index_snippets']).to receive(:invoke).ordered
before do run_rake_task 'gitlab:elastic:index'
Sidekiq::Testing.disable! do
project1
project2
end end
end end
it 'queues jobs for each project batch' do describe 'index_projects' do
expect(Elastic::ProcessInitialBookkeepingService).to receive(:backfill_projects!).with( let(:project1) { create :project }
project1, project2 let(:project2) { create :project }
) let(:project3) { create :project }
run_rake_task 'gitlab:elastic:index_projects'
end
context 'with limited indexing enabled' do
before do before do
Sidekiq::Testing.disable! do Sidekiq::Testing.disable! do
project1 project1
project2 project2
project3 end
end
it 'queues jobs for each project batch' do
expect(Elastic::ProcessInitialBookkeepingService).to receive(:backfill_projects!).with(project1, project2)
create :elasticsearch_indexed_project, project: project1 run_rake_task 'gitlab:elastic:index_projects'
create :elasticsearch_indexed_namespace, namespace: project3.namespace end
context 'with limited indexing enabled' do
before do
Sidekiq::Testing.disable! do
project1
project2
project3
create :elasticsearch_indexed_project, project: project1
create :elasticsearch_indexed_namespace, namespace: project3.namespace
end
stub_ee_application_setting(elasticsearch_limit_indexing: true)
end end
stub_ee_application_setting(elasticsearch_limit_indexing: true) it 'does not queue jobs for projects that should not be indexed' do
expect(Elastic::ProcessInitialBookkeepingService).to receive(:backfill_projects!).with(project1, project3)
run_rake_task 'gitlab:elastic:index_projects'
end
end end
end
it 'does not queue jobs for projects that should not be indexed' do describe 'index_snippets' do
expect(Elastic::ProcessInitialBookkeepingService).to receive(:backfill_projects!).with( it 'indexes snippets' do
project1, project3 expect(Snippet).to receive(:es_import)
)
run_rake_task 'gitlab:elastic:index_projects' run_rake_task 'gitlab:elastic:index_snippets'
end end
end end
end
describe 'index_snippets' do describe 'recreate_index' do
it 'indexes snippets' do it 'calls all related subtasks in order' do
expect(Snippet).to receive(:es_import) expect(Rake::Task['gitlab:elastic:delete_index']).to receive(:invoke).ordered
expect(Rake::Task['gitlab:elastic:create_empty_index']).to receive(:invoke).ordered
run_rake_task 'gitlab:elastic:index_snippets' run_rake_task 'gitlab:elastic:recreate_index'
end
end end
end end
describe 'recreate_index' do context "with elasticsearch_indexing is disabled" do
it 'calls all related subtasks in order' do it 'enables `elasticsearch_indexing`' do
expect(Rake::Task['gitlab:elastic:delete_index']).to receive(:invoke).ordered expect { run_rake_task 'gitlab:elastic:index' }.to change {
expect(Rake::Task['gitlab:elastic:create_empty_index']).to receive(:invoke).ordered Gitlab::CurrentSettings.elasticsearch_indexing?
}.from(false).to(true)
run_rake_task 'gitlab:elastic:recreate_index'
end end
end end
end end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment