Support SendURL for performing indirect download of artifacts if clients does...

Support SendURL for performing indirect download of artifacts if clients does not specify that it supports that

Use Workhorse 3.6.0 as this is the first version that implements send-url:.

GITLAB_WORKHORSE_VERSION

-3.5.1
+3.6.0

changelogs/unreleased-ee/use-send-url-for-incompatible-runners.yml

+---
+title: Support SendURL for performing indirect download of artifacts if clients does
+  not specify that it supports that
+merge_request:
+author:
+type: fixed

lib/api/helpers.rb

@@ -439,13 +439,17 @@ module API
       end
     end
 
-    def present_artifacts!(artifacts_file)
+    def present_artifacts!(artifacts_file, direct_download: true)
       return not_found! unless artifacts_file.exists?
 
       if artifacts_file.file_storage?
         present_file!(artifacts_file.path, artifacts_file.filename)
-      else
+      elsif direct_download
         redirect(artifacts_file.url)
+      else
+        header(*Gitlab::Workhorse.send_url(artifacts_file.url))
+        status :ok
+        body
       end
     end
 

lib/api/runner.rb

@@ -244,11 +244,12 @@ module API
       params do
         requires :id, type: Integer, desc: %q(Job's ID)
         optional :token, type: String, desc: %q(Job's authentication token)
+        optional :direct_download, default: false, type: Boolean, desc: %q(Perform direct download from remote storage instead of proxying artifacts)
       end
       get '/:id/artifacts' do
         job = authenticate_job!
 
-        present_artifacts!(job.artifacts_file)
+        present_artifacts!(job.artifacts_file, direct_download: params[:direct_download])
       end
     end
   end

lib/gitlab/workhorse.rb

@@ -161,6 +161,18 @@ module Gitlab
         ]
       end
 
+      def send_url(url, allow_redirects: false)
+        params = {
+          'URL' => url,
+          'AllowRedirects' => allow_redirects
+        }
+
+        [
+          SEND_DATA_HEADER,
+          "send-url:#{encode(params)}"
+        ]
+      end
+
       def terminal_websocket(terminal)
         details = {
           'Terminal' => {

spec/lib/gitlab/workhorse_spec.rb

@@ -465,4 +465,21 @@ describe Gitlab::Workhorse do
       end
     end
   end
+
+  describe '.send_url' do
+    let(:url) { 'http://example.com' }
+
+    subject { described_class.send_url(url) }
+
+    it 'sets the header correctly' do
+      key, command, params = decode_workhorse_header(subject)
+
+      expect(key).to eq("Gitlab-Workhorse-Send-Data")
+      expect(command).to eq("send-url")
+      expect(params).to eq({
+        'URL' => url,
+        'AllowRedirects' => false
+      }.deep_stringify_keys)
+    end
+  end
 end

spec/requests/api/runner_spec.rb

@@ -1159,8 +1159,6 @@ describe API::Runner do
 
           before do
             create(:ci_job_artifact, :archive, file_store: store, job: job)
-
-            download_artifact
           end
 
           context 'when using job token' do
@@ -1170,6 +1168,10 @@ describe API::Runner do
                   'Content-Disposition' => 'attachment; filename=ci_build_artifacts.zip' }
               end
 
+              before do
+                download_artifact
+              end
+
               it 'download artifacts' do
                 expect(response).to have_gitlab_http_status(200)
                 expect(response.headers).to include download_headers
@@ -1180,8 +1182,27 @@ describe API::Runner do
               let(:store) { JobArtifactUploader::Store::REMOTE }
               let!(:job) { create(:ci_build) }
 
-              it 'download artifacts' do
+              context 'when proxy download is being used' do
+                before do
+                  download_artifact(direct_download: false)
+                end
+
+                it 'uses workhorse send-url' do
+                  expect(response).to have_gitlab_http_status(200)
+                  expect(response.headers).to include(
+                    'Gitlab-Workhorse-Send-Data' => /send-url:/)
+                end
+              end
+
+              context 'when direct download is being used' do
+                before do
+                  download_artifact(direct_download: true)
+                end
+
+                it 'receive redirect for downloading artifacts' do
                   expect(response).to have_gitlab_http_status(302)
+                  expect(response.headers).to include('Location')
+                end
               end
             end
           end
@@ -1189,6 +1210,10 @@ describe API::Runner do
           context 'when using runnners token' do
             let(:token) { job.project.runners_token }
 
+            before do
+              download_artifact
+            end
+
             it 'responds with forbidden' do
               expect(response).to have_gitlab_http_status(403)
             end