Commit 8d32ead9 authored by mbergeron's avatar mbergeron Committed by Micael Bergeron

Make `gitlab:elastic:index` enable `elasticsearch_indexing`

Before this change, the `gitlab:elastic:index` command would always fail
the on the first run, as it needed a setting to be set in the UI in the
middle of the execution.

Now this setting will be set automatically, thus making this rake task
the easiest way to populate the Elasticsearch index for a small GitLab
instance.
parent 81be8aed
......@@ -245,7 +245,7 @@ This will delete your existing indexes.
If the database size is less than 500 MiB, and the size of all hosted repos is less than 5 GiB:
1. [Enable **Elasticsearch indexing** and configure your host and port](#enabling-elasticsearch).
1. [Configure your Elasticsearch host and port](#enabling-elasticsearch).
1. Index your data:
```shell
......@@ -424,7 +424,7 @@ The following are some available Rake tasks:
| Task | Description |
|:--------------------------------------------------------------------------------------------------------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| [`sudo gitlab-rake gitlab:elastic:index`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Wrapper task for `gitlab:elastic:create_empty_index`, `gitlab:elastic:clear_index_status`, `gitlab:elastic:index_projects`, and `gitlab:elastic:index_snippets`. |
| [`sudo gitlab-rake gitlab:elastic:index`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Enables Elasticsearch Indexing and run `gitlab:elastic:create_empty_index`, `gitlab:elastic:clear_index_status`, `gitlab:elastic:index_projects`, and `gitlab:elastic:index_snippets`. |
| [`sudo gitlab-rake gitlab:elastic:index_projects`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Iterates over all projects and queues Sidekiq jobs to index them in the background. |
| [`sudo gitlab-rake gitlab:elastic:index_projects_status`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Determines the overall status of the indexing. It is done by counting the total number of indexed projects, dividing by a count of the total number of projects, then multiplying by 100. |
| [`sudo gitlab-rake gitlab:elastic:clear_index_status`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Deletes all instances of IndexStatus for all projects. |
......
---
title: Automatically enable Elasticsearch indexing with 'gitlab:elastic:index'
merge_request: 35342
author:
type: changed
......@@ -9,6 +9,18 @@ namespace :gitlab do
Rake::Task["gitlab:elastic:recreate_index"].invoke
Rake::Task["gitlab:elastic:clear_index_status"].invoke
# enable `elasticsearch_indexing` if it isn't
unless Gitlab::CurrentSettings.elasticsearch_indexing?
ApplicationSettings::UpdateService.new(
Gitlab::CurrentSettings.current_application_settings,
nil,
{ elasticsearch_indexing: true }
).execute
puts "Setting `elasticsearch_indexing` has been enabled."
end
Rake::Task["gitlab:elastic:index_projects"].invoke
Rake::Task["gitlab:elastic:index_snippets"].invoke
end
......
......@@ -5,78 +5,87 @@ require 'rake_helper'
RSpec.describe 'gitlab:elastic namespace rake tasks', :elastic do
before do
Rake.application.rake_require 'tasks/gitlab/elastic'
stub_ee_application_setting(elasticsearch_indexing: true)
end
describe 'index' do
it 'calls all indexing tasks in order' do
expect(Rake::Task['gitlab:elastic:recreate_index']).to receive(:invoke).ordered
expect(Rake::Task['gitlab:elastic:clear_index_status']).to receive(:invoke).ordered
expect(Rake::Task['gitlab:elastic:index_projects']).to receive(:invoke).ordered
expect(Rake::Task['gitlab:elastic:index_snippets']).to receive(:invoke).ordered
run_rake_task 'gitlab:elastic:index'
context "with elasticsearch_indexing enabled" do
before do
stub_ee_application_setting(elasticsearch_indexing: true)
end
end
describe 'index_projects' do
let(:project1) { create :project }
let(:project2) { create :project }
let(:project3) { create :project }
describe 'index' do
it 'calls all indexing tasks in order' do
expect(Rake::Task['gitlab:elastic:recreate_index']).to receive(:invoke).ordered
expect(Rake::Task['gitlab:elastic:clear_index_status']).to receive(:invoke).ordered
expect(Rake::Task['gitlab:elastic:index_projects']).to receive(:invoke).ordered
expect(Rake::Task['gitlab:elastic:index_snippets']).to receive(:invoke).ordered
before do
Sidekiq::Testing.disable! do
project1
project2
run_rake_task 'gitlab:elastic:index'
end
end
it 'queues jobs for each project batch' do
expect(Elastic::ProcessInitialBookkeepingService).to receive(:backfill_projects!).with(
project1, project2
)
run_rake_task 'gitlab:elastic:index_projects'
end
describe 'index_projects' do
let(:project1) { create :project }
let(:project2) { create :project }
let(:project3) { create :project }
context 'with limited indexing enabled' do
before do
Sidekiq::Testing.disable! do
project1
project2
project3
end
end
it 'queues jobs for each project batch' do
expect(Elastic::ProcessInitialBookkeepingService).to receive(:backfill_projects!).with(project1, project2)
create :elasticsearch_indexed_project, project: project1
create :elasticsearch_indexed_namespace, namespace: project3.namespace
run_rake_task 'gitlab:elastic:index_projects'
end
context 'with limited indexing enabled' do
before do
Sidekiq::Testing.disable! do
project1
project2
project3
create :elasticsearch_indexed_project, project: project1
create :elasticsearch_indexed_namespace, namespace: project3.namespace
end
stub_ee_application_setting(elasticsearch_limit_indexing: true)
end
stub_ee_application_setting(elasticsearch_limit_indexing: true)
it 'does not queue jobs for projects that should not be indexed' do
expect(Elastic::ProcessInitialBookkeepingService).to receive(:backfill_projects!).with(project1, project3)
run_rake_task 'gitlab:elastic:index_projects'
end
end
end
it 'does not queue jobs for projects that should not be indexed' do
expect(Elastic::ProcessInitialBookkeepingService).to receive(:backfill_projects!).with(
project1, project3
)
describe 'index_snippets' do
it 'indexes snippets' do
expect(Snippet).to receive(:es_import)
run_rake_task 'gitlab:elastic:index_projects'
run_rake_task 'gitlab:elastic:index_snippets'
end
end
end
describe 'index_snippets' do
it 'indexes snippets' do
expect(Snippet).to receive(:es_import)
describe 'recreate_index' do
it 'calls all related subtasks in order' do
expect(Rake::Task['gitlab:elastic:delete_index']).to receive(:invoke).ordered
expect(Rake::Task['gitlab:elastic:create_empty_index']).to receive(:invoke).ordered
run_rake_task 'gitlab:elastic:index_snippets'
run_rake_task 'gitlab:elastic:recreate_index'
end
end
end
describe 'recreate_index' do
it 'calls all related subtasks in order' do
expect(Rake::Task['gitlab:elastic:delete_index']).to receive(:invoke).ordered
expect(Rake::Task['gitlab:elastic:create_empty_index']).to receive(:invoke).ordered
run_rake_task 'gitlab:elastic:recreate_index'
context "with elasticsearch_indexing is disabled" do
it 'enables `elasticsearch_indexing`' do
expect { run_rake_task 'gitlab:elastic:index' }.to change {
Gitlab::CurrentSettings.elasticsearch_indexing?
}.from(false).to(true)
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment