Commit b5ab9368 authored by Igor Drozdov's avatar Igor Drozdov

Code Nav PoC: API for fetching code-nav info

This MR provides API for fetching code navigation
info from the stored LSIF data
parent 9ac70062
......@@ -23,6 +23,7 @@ module Ci
belongs_to :trigger_request
belongs_to :erased_by, class_name: 'User'
belongs_to :resource_group, class_name: 'Ci::ResourceGroup', inverse_of: :builds
belongs_to :pipeline, class_name: 'Ci::Pipeline', foreign_key: :commit_id
RUNNER_FEATURES = {
upload_multiple_artifacts: -> (build) { build.publishes_artifacts_reports? },
......
......@@ -28,7 +28,7 @@ module Ci
license_scanning: 'gl-license-scanning-report.json',
performance: 'performance.json',
metrics: 'metrics.txt',
lsif: 'lsif.sqlite3'
lsif: 'lsif.json'
}.freeze
INTERNAL_TYPES = {
......@@ -74,6 +74,7 @@ module Ci
scope :with_files_stored_locally, -> { where(file_store: [nil, ::JobArtifactUploader::Store::LOCAL]) }
scope :with_files_stored_remotely, -> { where(file_store: ::JobArtifactUploader::Store::REMOTE) }
scope :for_sha, ->(sha) { joins(job: :pipeline).where(ci_pipelines: { sha: sha }) }
scope :with_file_types, -> (file_types) do
types = self.file_types.select { |file_type| file_types.include?(file_type) }.values
......@@ -117,7 +118,7 @@ module Ci
metrics: 12, ## EE-specific
metrics_referee: 13, ## runner referees
network_referee: 14, ## runner referees
lsif: 15 # LSIF dump for code navigation
lsif: 15 # LSIF data for code navigation
}
enum file_format: {
......
# frozen_string_literal: true
module Projects
class LsifDataService
attr_reader :file, :project, :path, :commit_id
CACHE_EXPIRE_IN = 1.hour
def initialize(file, project, params)
@file = file
@project = project
@path = params[:path]
@commit_id = params[:commit_id]
end
def execute
docs, doc_ranges, ranges =
fetch_data.values_at('docs', 'doc_ranges', 'ranges')
doc_id = doc_id_from(docs)
doc_ranges[doc_id]&.map do |range_id|
line_data, column_data = ranges[range_id]['loc']
{
start_line: line_data.first,
end_line: line_data.last,
start_char: column_data.first,
end_char: column_data.last
}
end
end
private
def fetch_data
Rails.cache.fetch("project:#{project.id}:lsif:#{commit_id}", expires_in: CACHE_EXPIRE_IN) do
data = nil
file.open do |stream|
Zlib::GzipReader.wrap(stream) do |gz_stream|
data = JSON.parse(gz_stream.read)
end
end
data
end
end
def doc_id_from(docs)
docs.reduce(nil) do |doc_id, (id, doc_path)|
next doc_id unless doc_path =~ /#{path}$/
if doc_id.nil? || docs[doc_id].size > doc_path.size
doc_id = id
end
doc_id
end
end
end
end
......@@ -142,6 +142,7 @@ module API
mount ::API::Keys
mount ::API::Labels
mount ::API::Lint
mount ::API::LsifData
mount ::API::Markdown
mount ::API::Members
mount ::API::MergeRequestDiffs
......
# frozen_string_literal: true
module API
class LsifData < Grape::API
MAX_FILE_SIZE = 10.megabytes
before do
not_found! if Feature.disabled?(:code_navigation, user_project)
end
params do
requires :id, type: String, desc: 'The ID of a project'
requires :commit_id, type: String, desc: 'The ID of a commit'
end
resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
segment ':id/commits/:commit_id' do
params do
requires :path, type: String, desc: 'The path of a file'
end
get 'lsif/info' do
authorize! :download_code, user_project
artifact =
@project.job_artifacts
.with_file_types(['lsif'])
.for_sha(params[:commit_id])
.last
not_found! unless artifact
authorize! :read_pipeline, artifact.job.pipeline
file_too_large! if artifact.file.cached_size > MAX_FILE_SIZE
::Projects::LsifDataService.new(artifact.file, @project, params).execute
end
end
end
end
end
......@@ -139,6 +139,16 @@ FactoryBot.define do
end
end
trait :lsif do
file_type { :lsif }
file_format { :raw }
after(:build) do |artifact, evaluator|
artifact.file = fixture_file_upload(
Rails.root.join('spec/fixtures/lsif.json.gz'), 'application/octet-stream')
end
end
trait :correct_checksum do
after(:build) do |artifact, evaluator|
artifact.file_sha256 = Digest::SHA256.file(artifact.file.path).hexdigest
......
......@@ -43,7 +43,7 @@ describe Gitlab::Ci::Config::Entry::Reports do
:license_management | 'gl-license-management-report.json'
:license_scanning | 'gl-license-scanning-report.json'
:performance | 'performance.json'
:lsif | 'lsif.sqlite3'
:lsif | 'lsif.json'
end
with_them do
......
......@@ -111,6 +111,18 @@ describe Ci::JobArtifact do
end
end
describe '.for_sha' do
it 'returns job artifacts for a given pipeline sha' do
first_pipeline = create(:ci_pipeline)
second_pipeline = create(:ci_pipeline, sha: Digest::SHA1.hexdigest(SecureRandom.hex))
first_artifact = create(:ci_job_artifact, job: create(:ci_build, pipeline: first_pipeline))
second_artifact = create(:ci_job_artifact, job: create(:ci_build, pipeline: second_pipeline))
expect(described_class.for_sha(first_pipeline.sha)).to eq([first_artifact])
expect(described_class.for_sha(second_pipeline.sha)).to eq([second_artifact])
end
end
describe 'callbacks' do
subject { create(:ci_job_artifact, :archive) }
......
# frozen_string_literal: true
require "spec_helper"
describe API::LsifData do
let_it_be(:user) { create(:user) }
let_it_be(:project) { create(:project, :repository) }
let(:commit) { project.commit }
describe 'GET lsif/info' do
let(:endpoint_path) { "/projects/#{project.id}/commits/#{commit.id}/lsif/info" }
context 'user does not have access to the project' do
before do
project.add_guest(user)
end
it 'returns 403' do
get api(endpoint_path, user), params: { path: 'main.go' }
expect(response).to have_gitlab_http_status(:forbidden)
end
end
context 'user has access to the project' do
before do
project.add_reporter(user)
end
context 'code_navigation feature is disabled' do
before do
stub_feature_flags(code_navigation: false)
end
it 'returns 404' do
get api(endpoint_path, user)
expect(response).to have_gitlab_http_status(:not_found)
end
end
context 'there is no job artifact for the passed commit' do
it 'returns 404' do
get api(endpoint_path, user), params: { path: 'main.go' }
expect(response).to have_gitlab_http_status(:not_found)
end
end
context 'lsif data is stored as a job artifact' do
let!(:pipeline) { create(:ci_pipeline, project: project, sha: commit.id) }
let!(:artifact) { create(:ci_job_artifact, :lsif, job: create(:ci_build, pipeline: pipeline)) }
it 'returns code navigation info for a given path' do
get api(endpoint_path, user), params: { path: 'main.go' }
expect(response).to have_gitlab_http_status(:ok)
expect(response.parsed_body.last).to eq({
'end_char' => 18,
'end_line' => 8,
'start_char' => 13,
'start_line' => 8
})
end
context 'the stored file is too large' do
it 'returns 413' do
allow_any_instance_of(JobArtifactUploader).to receive(:cached_size).and_return(20.megabytes)
get api(endpoint_path, user), params: { path: 'main.go' }
expect(response).to have_gitlab_http_status(:payload_too_large)
end
end
context 'the user does not have access to the pipeline' do
let(:project) { create(:project, :repository, builds_access_level: ProjectFeature::DISABLED) }
it 'returns 403' do
get api(endpoint_path, user), params: { path: 'main.go' }
expect(response).to have_gitlab_http_status(:forbidden)
end
end
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
describe Projects::LsifDataService do
let(:artifact) { create(:ci_job_artifact, :lsif) }
let(:project) { build_stubbed(:project) }
let(:path) { 'main.go' }
let(:commit_id) { Digest::SHA1.hexdigest(SecureRandom.hex) }
let(:params) { { path: path, commit_id: commit_id } }
let(:service) { described_class.new(artifact.file, project, params) }
describe '#execute' do
context 'fetched lsif file', :use_clean_rails_memory_store_caching do
it 'is cached' do
service.execute
cached_data = Rails.cache.fetch("project:#{project.id}:lsif:#{commit_id}")
expect(cached_data.keys).to eq(%w[def_refs doc_ranges docs hover_refs ranges])
end
end
context 'for main.go' do
it 'returns lsif ranges for the file' do
expect(service.execute).to eq([
{
end_char: 9,
end_line: 6,
start_char: 5,
start_line: 6
},
{
end_char: 36,
end_line: 3,
start_char: 1,
start_line: 3
},
{
end_char: 12,
end_line: 7,
start_char: 1,
start_line: 7
},
{
end_char: 20,
end_line: 7,
start_char: 13,
start_line: 7
},
{
end_char: 12,
end_line: 8,
start_char: 1,
start_line: 8
},
{
end_char: 18,
end_line: 8,
start_char: 13,
start_line: 8
}
])
end
end
context 'for morestring/reverse.go' do
let(:path) { 'morestrings/reverse.go' }
it 'returns lsif ranges for the file' do
expect(service.execute.first).to eq({
end_char: 2,
end_line: 11,
start_char: 1,
start_line: 11
})
end
end
context 'for an unknown file' do
let(:path) { 'unknown.go' }
it 'returns nil' do
expect(service.execute).to eq(nil)
end
end
end
describe '#doc_id_from' do
context 'when the passed path matches multiple files' do
let(:path) { 'check/main.go' }
let(:docs) do
{
1 => 'cmd/check/main.go',
2 => 'cmd/command.go',
3 => 'check/main.go',
4 => 'cmd/nested/check/main.go'
}
end
it 'fetches the document with the shortest absolute path' do
expect(service.__send__(:doc_id_from, docs)).to eq(3)
end
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment