Commit 8dd0c74d authored by Adrien Kohlbecker's avatar Adrien Kohlbecker Committed by Natalia Tepluhina

Add full text search to pod logs

Uses ES querying capabilities to search
parent c56c0620
---
title: Add full text search to pod logs
merge_request: 21656
author:
type: added
......@@ -92,7 +92,7 @@ export default {
* @param {string=} params.containerName - Container name, if not set the backend assumes a default one
* @returns {Promise} Axios promise for the result of a GET request of logs
*/
getPodLogs({ projectPath, environmentName, podName, containerName }) {
getPodLogs({ projectPath, environmentName, podName, containerName, search }) {
const url = this.buildUrl(this.podLogsPath).replace(':project_full_path', projectPath);
const params = {
......@@ -105,6 +105,9 @@ export default {
if (containerName) {
params.container_name = containerName;
}
if (search) {
params.search = search;
}
return axios.get(url, { params });
},
......
<script>
import { mapActions, mapState, mapGetters } from 'vuex';
import { GlDropdown, GlDropdownItem, GlFormGroup } from '@gitlab/ui';
import { GlDropdown, GlDropdownItem, GlFormGroup, GlSearchBoxByClick } from '@gitlab/ui';
import { scrollDown } from '~/lib/utils/scroll_utils';
import LogControlButtons from './log_control_buttons.vue';
......@@ -9,6 +9,7 @@ export default {
GlDropdown,
GlDropdownItem,
GlFormGroup,
GlSearchBoxByClick,
LogControlButtons,
},
props: {
......@@ -32,12 +33,20 @@ export default {
default: '',
},
},
data() {
return {
searchQuery: '',
};
},
computed: {
...mapState('environmentLogs', ['environments', 'logs', 'pods']),
...mapGetters('environmentLogs', ['trace']),
showLoader() {
return this.logs.isLoading || !this.logs.isComplete;
},
featureElasticEnabled() {
return gon.features && gon.features.enableClusterApplicationElasticStack;
},
},
watch: {
trace(val) {
......@@ -61,6 +70,7 @@ export default {
methods: {
...mapActions('environmentLogs', [
'setInitData',
'setSearch',
'showPodLogs',
'showEnvironment',
'fetchEnvironments',
......@@ -77,7 +87,7 @@ export default {
:label="s__('Environments|Environment')"
label-size="sm"
label-for="environments-dropdown"
class="col-6"
:class="featureElasticEnabled ? 'col-4' : 'col-6'"
>
<gl-dropdown
id="environments-dropdown"
......@@ -96,11 +106,11 @@ export default {
</gl-dropdown>
</gl-form-group>
<gl-form-group
id="environments-dropdown-fg"
id="pods-dropdown-fg"
:label="s__('Environments|Pod logs from')"
label-size="sm"
label-for="pods-dropdown"
class="col-6"
:class="featureElasticEnabled ? 'col-4' : 'col-6'"
>
<gl-dropdown
id="pods-dropdown"
......@@ -118,6 +128,24 @@ export default {
</gl-dropdown-item>
</gl-dropdown>
</gl-form-group>
<gl-form-group
v-if="featureElasticEnabled"
id="search-fg"
:label="s__('Environments|Search')"
label-size="sm"
label-for="search"
class="col-4"
>
<gl-search-box-by-click
v-model.trim="searchQuery"
:disabled="environments.isLoading"
:placeholder="s__('Environments|Search')"
class="js-logs-search"
type="search"
autofocus
@submit="setSearch(searchQuery)"
/>
</gl-form-group>
</div>
<log-control-buttons
......
......@@ -6,9 +6,9 @@ import flash from '~/flash';
import { s__ } from '~/locale';
import * as types from './mutation_types';
const requestLogsUntilData = ({ projectPath, environmentName, podName }) =>
const requestLogsUntilData = params =>
backOff((next, stop) => {
Api.getPodLogs({ projectPath, environmentName, podName })
Api.getPodLogs(params)
.then(res => {
if (res.status === httpStatusCodes.ACCEPTED) {
next();
......@@ -33,6 +33,11 @@ export const showPodLogs = ({ dispatch, commit }, podName) => {
dispatch('fetchLogs');
};
export const setSearch = ({ dispatch, commit }, searchQuery) => {
commit(types.SET_SEARCH, searchQuery);
dispatch('fetchLogs');
};
export const showEnvironment = ({ dispatch, commit }, environmentName) => {
commit(types.SET_PROJECT_ENVIRONMENT, environmentName);
commit(types.SET_CURRENT_POD_NAME, null);
......@@ -58,6 +63,7 @@ export const fetchLogs = ({ commit, state }) => {
projectPath: state.projectPath,
environmentName: state.environments.current,
podName: state.pods.current,
search: state.search,
};
commit(types.REQUEST_PODS_DATA);
......
export const SET_PROJECT_PATH = 'SET_PROJECT_PATH';
export const SET_PROJECT_ENVIRONMENT = 'SET_PROJECT_ENVIRONMENT';
export const SET_SEARCH = 'SET_SEARCH';
export const REQUEST_ENVIRONMENTS_DATA = 'REQUEST_ENVIRONMENTS_DATA';
export const RECEIVE_ENVIRONMENTS_DATA_SUCCESS = 'RECEIVE_ENVIRONMENTS_DATA_SUCCESS';
......
......@@ -6,6 +6,11 @@ export default {
state.projectPath = projectPath;
},
/** Search data */
[types.SET_SEARCH](state, searchQuery) {
state.search = searchQuery;
},
/** Environments data */
[types.SET_PROJECT_ENVIRONMENT](state, environmentName) {
state.environments.current = environmentName;
......
......@@ -4,6 +4,11 @@ export default () => ({
*/
projectPath: '',
/**
* Full text search
*/
search: '',
/**
* Environments list information
*/
......
......@@ -5,7 +5,7 @@ module Projects
before_action :authorize_read_pod_logs!
before_action :environment
before_action do
push_frontend_feature_flag(:environment_logs_use_vue_ui)
push_frontend_feature_flag(:enable_cluster_application_elastic_stack)
end
def index
......@@ -38,7 +38,7 @@ module Projects
end
def filter_params
params.permit(:container_name, :pod_name)
params.permit(:container_name, :pod_name, :search)
end
def environment
......
......@@ -29,7 +29,7 @@ module EE
::Gitlab::Kubernetes::RolloutStatus.from_deployments(*deployments, pods: pods, legacy_deployments: legacy_deployments)
end
def read_pod_logs(environment_id, pod_name, namespace, container: nil)
def read_pod_logs(environment_id, pod_name, namespace, container: nil, search: nil)
# environment_id is required for use in reactive_cache_updated(),
# to invalidate the ETag cache.
with_reactive_cache(
......@@ -37,7 +37,8 @@ module EE
'environment_id' => environment_id,
'pod_name' => pod_name,
'namespace' => namespace,
'container' => container
'container' => container,
'search' => search
) do |result|
result
end
......@@ -49,11 +50,12 @@ module EE
container = opts['container']
pod_name = opts['pod_name']
namespace = opts['namespace']
search = opts['search']
handle_exceptions(_('Pod not found'), pod_name: pod_name, container_name: container) do
handle_exceptions(_('Pod not found'), pod_name: pod_name, container_name: container, search: search) do
container ||= container_names_of(pod_name, namespace).first
pod_logs(pod_name, namespace, container: container)
pod_logs(pod_name, namespace, container: container, search: search)
end
end
end
......@@ -82,9 +84,9 @@ module EE
private
def pod_logs(pod_name, namespace, container: nil)
def pod_logs(pod_name, namespace, container: nil, search: nil)
logs = if ::Feature.enabled?(:enable_cluster_application_elastic_stack) && elastic_stack_client
elastic_stack_pod_logs(namespace, pod_name, container)
elastic_stack_pod_logs(namespace, pod_name, container, search)
else
platform_pod_logs(namespace, pod_name, container)
end
......@@ -113,11 +115,11 @@ module EE
end
end
def elastic_stack_pod_logs(namespace, pod_name, container_name)
def elastic_stack_pod_logs(namespace, pod_name, container_name, search)
client = elastic_stack_client
return [] if client.nil?
::Gitlab::Elasticsearch::Logs.new(client).pod_logs(namespace, pod_name, container_name)
::Gitlab::Elasticsearch::Logs.new(client).pod_logs(namespace, pod_name, container_name, search)
end
def elastic_stack_client
......
......@@ -7,7 +7,7 @@ class PodLogsService < ::BaseService
K8S_NAME_MAX_LENGTH = 253
PARAMS = %w(pod_name container_name).freeze
PARAMS = %w(pod_name container_name search).freeze
SUCCESS_RETURN_KEYS = [:status, :logs, :pod_name, :container_name, :pods].freeze
......@@ -77,7 +77,8 @@ class PodLogsService < ::BaseService
environment.id,
result[:pod_name],
namespace,
container: result[:container_name]
container: result[:container_name],
search: params['search']
)
return { status: :processing } unless response
......
......@@ -10,7 +10,7 @@ module Gitlab
@client = client
end
def pod_logs(namespace, pod_name, container_name = nil)
def pod_logs(namespace, pod_name, container_name = nil, search = nil)
query = {
bool: {
must: [
......@@ -44,6 +44,16 @@ module Gitlab
}
end
unless search.nil?
query[:bool][:must] << {
simple_query_string: {
query: search,
fields: [:message],
default_operator: :and
}
}
end
body = {
query: query,
# reverse order so we can query N-most recent records
......
{
"query": {
"bool": {
"must": [
{
"match_phrase": {
"kubernetes.pod.name": {
"query": "production-6866bc8974-m4sk4"
}
}
},
{
"match_phrase": {
"kubernetes.namespace": {
"query": "autodevops-deploy-9-production"
}
}
}
]
}
},
"sort": [
{
"@timestamp": {
"order": "desc"
}
},
{
"offset": {
"order": "desc"
}
}
],
"_source": [
"@timestamp",
"message"
],
"size": 500
}
{
"query": {
"bool": {
"must": [
{
"match_phrase": {
"kubernetes.pod.name": {
"query": "production-6866bc8974-m4sk4"
}
}
},
{
"match_phrase": {
"kubernetes.namespace": {
"query": "autodevops-deploy-9-production"
}
}
},
{
"match_phrase": {
"kubernetes.container.name": {
"query": "auto-deploy-app"
}
}
}
]
}
},
"sort": [
{
"@timestamp": {
"order": "desc"
}
},
{
"offset": {
"order": "desc"
}
}
],
"_source": [
"@timestamp",
"message"
],
"size": 500
}
{
"query": {
"bool": {
"must": [
{
"match_phrase": {
"kubernetes.pod.name": {
"query": "production-6866bc8974-m4sk4"
}
}
},
{
"match_phrase": {
"kubernetes.namespace": {
"query": "autodevops-deploy-9-production"
}
}
},
{
"simple_query_string": {
"query": "foo +bar ",
"fields": [
"message"
],
"default_operator": "and"
}
}
]
}
},
"sort": [
{
"@timestamp": {
"order": "desc"
}
},
{
"offset": {
"order": "desc"
}
}
],
"_source": [
"@timestamp",
"message"
],
"size": 500
}
......@@ -169,6 +169,7 @@ describe('Api', () => {
const environmentName = 'production';
const podName = 'pod';
const containerName = 'container';
const search = 'foo +bar';
const getRequest = () => mock.history.get[0];
......@@ -224,6 +225,22 @@ describe('Api', () => {
.then(done)
.catch(done.fail);
});
it('calls `axios.get` with pod_name and search', done => {
const expectedUrl = `${dummyUrlRoot}/${projectPath}/-/logs/k8s.json`;
Api.getPodLogs({ projectPath, environmentName, podName, search })
.then(() => {
expect(getRequest().url).toBe(expectedUrl);
expect(getRequest().params).toEqual({
environment_name: environmentName,
pod_name: podName,
search,
});
})
.then(done)
.catch(done.fail);
});
});
describe('packages', () => {
......
......@@ -41,6 +41,7 @@ describe('EnvironmentLogs', () => {
const findEnvironmentsDropdown = () => wrapper.find('.js-environments-dropdown');
const findPodsDropdown = () => wrapper.find('.js-pods-dropdown');
const findSearchBar = () => wrapper.find('.js-logs-search');
const findLogControlButtons = () => wrapper.find({ name: 'log-control-buttons-stub' });
const findLogTrace = () => wrapper.find('.js-log-trace');
......@@ -118,6 +119,9 @@ describe('EnvironmentLogs', () => {
state.environments.options = [];
state.environments.isLoading = true;
gon.features = gon.features || {};
gon.features.enableClusterApplicationElasticStack = true;
initWrapper();
});
......@@ -131,6 +135,11 @@ describe('EnvironmentLogs', () => {
expect(findPodsDropdown().findAll(GlDropdownItem).length).toBe(0);
});
it('displays a disabled search bar', () => {
expect(findSearchBar().exists()).toEqual(true);
expect(findSearchBar().attributes('disabled')).toEqual('true');
});
it('does not update buttons state', () => {
expect(updateControlBtnsMock).not.toHaveBeenCalled();
});
......@@ -145,6 +154,21 @@ describe('EnvironmentLogs', () => {
});
});
describe('elastic stack disabled', () => {
beforeEach(() => {
gon.features = gon.features || {};
gon.features.enableClusterApplicationElasticStack = false;
initWrapper();
});
it("doesn't display the search bar", () => {
expect(findSearchBar().exists()).toEqual(false);
expect(wrapper.find('#environments-dropdown-fg').attributes('class')).toEqual('col-6');
expect(wrapper.find('#pods-dropdown-fg').attributes('class')).toEqual('col-6');
});
});
describe('state with data', () => {
beforeEach(() => {
actionMocks.setInitData.mockImplementation(() => {
......@@ -166,6 +190,9 @@ describe('EnvironmentLogs', () => {
state.environments.options = mockEnvironments;
});
gon.features = gon.features || {};
gon.features.enableClusterApplicationElasticStack = true;
initWrapper();
});
......@@ -186,6 +213,7 @@ describe('EnvironmentLogs', () => {
const item = items.at(i);
expect(item.text()).toBe(env.name);
});
expect(wrapper.find('#environments-dropdown-fg').attributes('class')).toEqual('col-4');
});
it('populates pods dropdown', () => {
......@@ -197,6 +225,7 @@ describe('EnvironmentLogs', () => {
const item = items.at(i);
expect(item.text()).toBe(pod);
});
expect(wrapper.find('#pods-dropdown-fg').attributes('class')).toEqual('col-4');
});
it('populates logs trace', () => {
......@@ -205,6 +234,12 @@ describe('EnvironmentLogs', () => {
expect(trace.text().split('\n')).toEqual(mockTrace);
});
it('displays the search bar', () => {
expect(findSearchBar().exists()).toEqual(true);
expect(findSearchBar().attributes('disabled')).toEqual(undefined);
expect(wrapper.find('#search-fg').attributes('class')).toEqual('col-4');
});
it('update control buttons state', () => {
expect(updateControlBtnsMock).toHaveBeenCalledTimes(1);
});
......
......@@ -78,3 +78,5 @@ export const mockTrace = [
'Dec 13 13:43:48.324Z | 10.36.0.1 - - [16/Oct/2019:06:30:18 UTC] "GET / HTTP/1.1" 200 13',
'Dec 13 13:43:48.325Z | - -> /',
];
export const mockSearch = 'foo +bar';
......@@ -3,7 +3,13 @@ import MockAdapter from 'axios-mock-adapter';
import testAction from 'helpers/vuex_action_helper';
import * as types from 'ee/logs/stores/mutation_types';
import logsPageState from 'ee/logs/stores/state';
import { setInitData, showPodLogs, fetchEnvironments, fetchLogs } from 'ee/logs/stores/actions';
import {
setInitData,
setSearch,
showPodLogs,
fetchEnvironments,
fetchLogs,
} from 'ee/logs/stores/actions';
import axios from '~/lib/utils/axios_utils';
import flash from '~/flash';
......@@ -16,6 +22,7 @@ import {
mockPods,
mockLogsResult,
mockEnvName,
mockSearch,
} from '../mock_data';
jest.mock('~/flash');
......@@ -49,6 +56,19 @@ describe('Logs Store actions', () => {
});
});
describe('setSearch', () => {
it('should commit search mutation', done => {
testAction(
setSearch,
mockSearch,
state,
[{ type: types.SET_SEARCH, payload: mockSearch }],
[{ type: 'fetchLogs' }],
done,
);
});
});
describe('showPodLogs', () => {
it('should commit pod name', done => {
testAction(
......@@ -143,6 +163,43 @@ describe('Logs Store actions', () => {
);
});
it('should commit logs and pod data when there is pod name and search', done => {
state.projectPath = mockProjectPath;
state.environments.current = mockEnvName;
state.pods.current = mockPodName;
state.advancedFeaturesEnabled = true;
state.search = mockSearch;
const endpoint = `/${mockProjectPath}/-/logs/k8s.json`;
mock
.onGet(endpoint, {
params: { environment_name: mockEnvName, pod_name: mockPodName, search: mockSearch },
})
.reply(200, {
pod_name: mockPodName,
pods: mockPods,
logs: mockLogsResult,
});
mock.onGet(endpoint).replyOnce(202); // mock reactive cache
testAction(
fetchLogs,
null,
state,
[
{ type: types.REQUEST_PODS_DATA },
{ type: types.REQUEST_LOGS_DATA },
{ type: types.SET_CURRENT_POD_NAME, payload: mockPodName },
{ type: types.RECEIVE_PODS_DATA_SUCCESS, payload: mockPods },
{ type: types.RECEIVE_LOGS_DATA_SUCCESS, payload: mockLogsResult },
],
[],
done,
);
});
it('should commit logs and pod data when no pod name defined', done => {
state.projectPath = mockProjectPath;
state.environments.current = mockEnvName;
......
......@@ -9,6 +9,7 @@ import {
mockPods,
mockPodName,
mockLogsResult,
mockSearch,
} from '../mock_data';
describe('Logs Store Mutations', () => {
......@@ -36,6 +37,13 @@ describe('Logs Store Mutations', () => {
});
});
describe('SET_SEARCH', () => {
it('sets the search', () => {
mutations[types.SET_SEARCH](state, mockSearch);
expect(state.search).toEqual(mockSearch);
});
});
describe('REQUEST_ENVIRONMENTS_DATA', () => {
it('inits data', () => {
mutations[types.REQUEST_ENVIRONMENTS_DATA](state);
......
......@@ -17,111 +17,38 @@ describe Gitlab::Elasticsearch::Logs do
let(:namespace) { "autodevops-deploy-9-production" }
let(:pod_name) { "production-6866bc8974-m4sk4" }
let(:container_name) { "auto-deploy-app" }
let(:search) { "foo +bar "}
let(:body) do
{
query: {
bool: {
must: [
{
match_phrase: {
"kubernetes.pod.name" => {
query: pod_name
}
}
},
{
match_phrase: {
"kubernetes.namespace" => {
query: namespace
}
}
}
]
}
},
sort: [
{
:@timestamp => {
order: :desc
}
},
{
offset: {
order: :desc
}
}
],
_source: [
"@timestamp",
"message"
],
size: 500
}
end
let(:body) { JSON.parse(fixture_file('lib/elasticsearch/query.json', dir: 'ee')) }
let(:body_with_container) { JSON.parse(fixture_file('lib/elasticsearch/query_with_container.json', dir: 'ee')) }
let(:body_with_search) { JSON.parse(fixture_file('lib/elasticsearch/query_with_search.json', dir: 'ee')) }
let(:body_with_container) do
{
query: {
bool: {
must: [
{
match_phrase: {
"kubernetes.pod.name" => {
query: pod_name
}
}
},
{
match_phrase: {
"kubernetes.namespace" => {
query: namespace
}
}
},
{
match_phrase: {
"kubernetes.container.name" => {
query: container_name
}
}
}
]
}
},
sort: [
{
:@timestamp => {
order: :desc
}
},
{
offset: {
order: :desc
}
}
],
_source: [
"@timestamp",
"message"
],
size: 500
}
RSpec::Matchers.define :a_hash_equal_to_json do |expected|
match do |actual|
actual.as_json == expected
end
end
describe '#pod_logs' do
it 'returns the logs as an array' do
expect(client).to receive(:search).with(body: body).and_return(es_response)
expect(client).to receive(:search).with(body: a_hash_equal_to_json(body)).and_return(es_response)
result = subject.pod_logs(namespace, pod_name)
expect(result).to eq([es_message_4, es_message_3, es_message_2, es_message_1])
end
it 'can further filter the logs by container name' do
expect(client).to receive(:search).with(body: body_with_container).and_return(es_response)
expect(client).to receive(:search).with(body: a_hash_equal_to_json(body_with_container)).and_return(es_response)
result = subject.pod_logs(namespace, pod_name, container_name)
expect(result).to eq([es_message_4, es_message_3, es_message_2, es_message_1])
end
it 'can further filter the logs by search' do
expect(client).to receive(:search).with(body: a_hash_equal_to_json(body_with_search)).and_return(es_response)
result = subject.pod_logs(namespace, pod_name, nil, search)
expect(result).to eq([es_message_4, es_message_3, es_message_2, es_message_1])
end
end
end
......@@ -264,7 +264,8 @@ describe Clusters::Platforms::Kubernetes do
'environment_id' => environment.id,
'pod_name' => pod_name,
'namespace' => namespace,
'container' => container
'container' => container,
'search' => nil
}
]
end
......
......@@ -13,6 +13,7 @@ describe PodLogsService do
let(:response_pod_name) { pod_name }
let(:pods) { [pod_name] }
let(:container_name) { 'container-1' }
let(:search) { nil }
let(:logs) { ['Log 1', 'Log 2', 'Log 3'] }
let(:result) { subject.execute }
......@@ -20,7 +21,8 @@ describe PodLogsService do
ActionController::Parameters.new(
{
'pod_name' => pod_name,
'container_name' => container_name
'container_name' => container_name,
'search' => search
}
).permit!
end
......@@ -54,7 +56,7 @@ describe PodLogsService do
shared_context 'return error' do |message|
before do
allow_any_instance_of(EE::Clusters::Platforms::Kubernetes).to receive(:read_pod_logs)
.with(environment.id, pod_name, environment.deployment_namespace, container: container_name)
.with(environment.id, pod_name, environment.deployment_namespace, container: container_name, search: search)
.and_return({
status: :error,
error: message,
......@@ -67,7 +69,7 @@ describe PodLogsService do
shared_context 'return success' do
before do
allow_any_instance_of(EE::Clusters::Platforms::Kubernetes).to receive(:read_pod_logs)
.with(environment.id, response_pod_name, environment.deployment_namespace, container: container_name)
.with(environment.id, response_pod_name, environment.deployment_namespace, container: container_name, search: search)
.and_return({
status: :success,
logs: ["Log 1", "Log 2", "Log 3"],
......@@ -151,7 +153,7 @@ describe PodLogsService do
it 'returns logs of first pod' do
expect_any_instance_of(EE::Clusters::Platforms::Kubernetes).to receive(:read_pod_logs)
.with(environment.id, first_pod_name, environment.deployment_namespace, container: nil)
.with(environment.id, first_pod_name, environment.deployment_namespace, container: nil, search: search)
subject.execute
end
......@@ -177,6 +179,16 @@ describe PodLogsService do
end
end
context 'when search is specified' do
let(:pod_name) { 'some-pod' }
let(:container_name) { nil }
let(:search) { 'foo +bar' }
include_context 'return success'
it_behaves_like 'success'
end
context 'when error is returned' do
include_context 'return error', 'Kubernetes API returned status code: 400'
......@@ -188,7 +200,7 @@ describe PodLogsService do
context 'when nil is returned' do
before do
allow_any_instance_of(EE::Clusters::Platforms::Kubernetes).to receive(:read_pod_logs)
.with(environment.id, pod_name, environment.deployment_namespace, container: container_name)
.with(environment.id, pod_name, environment.deployment_namespace, container: container_name, search: search)
.and_return(nil)
end
......
......@@ -6896,6 +6896,9 @@ msgstr ""
msgid "Environments|Rollback environment %{name}?"
msgstr ""
msgid "Environments|Search"
msgstr ""
msgid "Environments|Show all"
msgstr ""
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment