Merge branch 'docker-registry-work-with-redirect' into 'master'

Make docker registry work with location redirects when external storage is used

## What does this MR do?

Honor `Location:` header when working with local registry.
Location makes it possible to download manifests from external storage.

## What are the relevant issue numbers?

Fixed https://gitlab.com/gitlab-org/gitlab-ce/issues/18477

## Remark

Adding `FollowRedirects` makes that we leak `Authorization:` in followed requests.

That is why it is implemented manually and we are explicitly removing `Authorization` header.

cc @marin @twk3 

See merge request !4677

CHANGELOG

@@ -91,6 +91,7 @@ v 8.10.0 (unreleased)
   - Handle custom Git hook result in GitLab UI
   - Allow to access Container Registry for Public and Internal projects
   - Allow '?', or '&' for label names
+  - Support redirected blobs for Container Registry integration
   - Fix importer for GitHub Pull Requests when a branch was reused across Pull Requests
   - Add date when user joined the team on the member page
   - Fix 404 redirect after validation fails importing a GitLab project

lib/container_registry/client.rb

@@ -7,62 +7,91 @@ module ContainerRegistry
 
     MANIFEST_VERSION = 'application/vnd.docker.distribution.manifest.v2+json'
 
+    # Taken from: FaradayMiddleware::FollowRedirects
+    REDIRECT_CODES = Set.new [301, 302, 303, 307]
+
     def initialize(base_uri, options = {})
       @base_uri = base_uri
-      @faraday = Faraday.new(@base_uri) do |conn|
-        initialize_connection(conn, options)
-      end
+      @options = options
     end
 
     def repository_tags(name)
-      response_body @faraday.get("/v2/#{name}/tags/list")
+      response_body faraday.get("/v2/#{name}/tags/list")
     end
 
     def repository_manifest(name, reference)
-      response_body @faraday.get("/v2/#{name}/manifests/#{reference}")
+      response_body faraday.get("/v2/#{name}/manifests/#{reference}")
     end
 
     def repository_tag_digest(name, reference)
-      response = @faraday.head("/v2/#{name}/manifests/#{reference}")
+      response = faraday.head("/v2/#{name}/manifests/#{reference}")
       response.headers['docker-content-digest'] if response.success?
     end
 
     def delete_repository_tag(name, reference)
-      @faraday.delete("/v2/#{name}/manifests/#{reference}").success?
+      faraday.delete("/v2/#{name}/manifests/#{reference}").success?
     end
 
     def blob(name, digest, type = nil)
-      headers = {}
-      headers['Accept'] = type if type
-      response_body @faraday.get("/v2/#{name}/blobs/#{digest}", nil, headers)
+      type ||= 'application/octet-stream'
+      response_body faraday_blob.get("/v2/#{name}/blobs/#{digest}", nil, 'Accept' => type), allow_redirect: true
     end
 
     def delete_blob(name, digest)
-      @faraday.delete("/v2/#{name}/blobs/#{digest}").success?
+      faraday.delete("/v2/#{name}/blobs/#{digest}").success?
     end
-    
+
     private
-    
+
     def initialize_connection(conn, options)
       conn.request :json
+
+      if options[:user] && options[:password]
+        conn.request(:basic_auth, options[:user].to_s, options[:password].to_s)
+      elsif options[:token]
+        conn.request(:authorization, :bearer, options[:token].to_s)
+      end
+
+      conn.adapter :net_http
+    end
+
+    def accept_manifest(conn)
       conn.headers['Accept'] = MANIFEST_VERSION
 
       conn.response :json, content_type: 'application/json'
       conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v1+prettyjws'
       conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v1+json'
       conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v2+json'
+    end
 
-      if options[:user] && options[:password]
-        conn.request(:basic_auth, options[:user].to_s, options[:password].to_s)
-      elsif options[:token]
-        conn.request(:authorization, :bearer, options[:token].to_s)
+    def response_body(response, allow_redirect: false)
+      if allow_redirect && REDIRECT_CODES.include?(response.status)
+        response = redirect_response(response.headers['location'])
       end
 
-      conn.adapter :net_http
+      response.body if response && response.success?
+    end
+
+    def redirect_response(location)
+      return unless location
+
+      # We explicitly remove authorization token
+      faraday_blob.get(location) do |req|
+        req['Authorization'] = ''
+      end
     end
 
-    def response_body(response)
-      response.body if response.success?
+    def faraday
+      @faraday ||= Faraday.new(@base_uri) do |conn|
+        initialize_connection(conn, @options)
+        accept_manifest(conn)
+      end
+    end
+
+    def faraday_blob
+      @faraday_blob ||= Faraday.new(@base_uri) do |conn|
+        initialize_connection(conn, @options)
+      end
     end
   end
 end

lib/container_registry/tag.rb

@@ -53,7 +53,7 @@ module ContainerRegistry
     def config
       return unless config_blob
 
-      @config ||= ContainerRegistry::Config.new(self, config_blob)
+      @config ||= ContainerRegistry::Config.new(self, config_blob) if config_blob.data
     end
 
     def created_at

spec/lib/container_registry/blob_spec.rb

@@ -9,8 +9,9 @@ describe ContainerRegistry::Blob do
       'size' => 1000
     }
   end
+  let(:token) { 'authorization-token' }
   
-  let(:registry) { ContainerRegistry::Registry.new('http://example.com') }
+  let(:registry) { ContainerRegistry::Registry.new('http://example.com', token: token) }
   let(:repository) { registry.repository('group/test') }
   let(:blob) { repository.blob(config) }
 
@@ -58,4 +59,53 @@ describe ContainerRegistry::Blob do
 
     it { is_expected.to be_truthy }
   end
+
+  context '#data' do
+    let(:data) { '{"key":"value"}' }
+
+    subject { blob.data }
+
+    context 'when locally stored' do
+      before do
+        stub_request(:get, 'http://example.com/v2/group/test/blobs/sha256:0123456789012345').
+          to_return(
+            status: 200,
+            headers: { 'Content-Type' => 'application/json' },
+            body: data)
+      end
+
+      it { is_expected.to eq(data) }
+    end
+
+    context 'when externally stored' do
+      before do
+        stub_request(:get, 'http://example.com/v2/group/test/blobs/sha256:0123456789012345').
+          with(headers: { 'Authorization' => "bearer #{token}" }).
+          to_return(
+            status: 307,
+            headers: { 'Location' => location })
+      end
+
+      context 'for a valid address' do
+        let(:location) { 'http://external.com/blob/file' }
+
+        before do
+          stub_request(:get, location).
+            with(headers: { 'Authorization' => nil }).
+            to_return(
+              status: 200,
+              headers: { 'Content-Type' => 'application/json' },
+              body: data)
+        end
+
+        it { is_expected.to eq(data) }
+      end
+
+      context 'for invalid file' do
+        let(:location) { 'file:///etc/passwd' }
+
+        it { expect{ subject }.to raise_error(ArgumentError, 'invalid address') }
+      end
+    end
+  end
 end

spec/lib/container_registry/tag_spec.rb

@@ -77,24 +77,47 @@ describe ContainerRegistry::Tag do
       end
 
       context 'config processing' do
-        before do
-          stub_request(:get, 'http://example.com/v2/group/test/blobs/sha256:d7a513a663c1a6dcdba9ed832ca53c02ac2af0c333322cd6ca92936d1d9917ac').
-            with(headers: { 'Accept' => 'application/octet-stream' }).
-            to_return(
-              status: 200,
-              body: File.read(Rails.root + 'spec/fixtures/container_registry/config_blob.json'))
-        end
+        shared_examples 'a processable' do
+          context '#config' do
+            subject { tag.config }
 
-        context '#config' do
-          subject { tag.config }
+            it { is_expected.not_to be_nil }
+          end
+
+          context '#created_at' do
+            subject { tag.created_at }
 
-          it { is_expected.not_to be_nil }
+            it { is_expected.not_to be_nil }
+          end
         end
 
-        context '#created_at' do
-          subject { tag.created_at }
+        context 'when locally stored' do
+          before do
+            stub_request(:get, 'http://example.com/v2/group/test/blobs/sha256:d7a513a663c1a6dcdba9ed832ca53c02ac2af0c333322cd6ca92936d1d9917ac').
+              with(headers: { 'Accept' => 'application/octet-stream' }).
+              to_return(
+                status: 200,
+                body: File.read(Rails.root + 'spec/fixtures/container_registry/config_blob.json'))
+          end
+
+          it_behaves_like 'a processable'
+        end
 
-          it { is_expected.not_to be_nil }
+        context 'when externally stored' do
+          before do
+            stub_request(:get, 'http://example.com/v2/group/test/blobs/sha256:d7a513a663c1a6dcdba9ed832ca53c02ac2af0c333322cd6ca92936d1d9917ac').
+              with(headers: { 'Accept' => 'application/octet-stream' }).
+              to_return(
+                status: 307,
+                headers: { 'Location' => 'http://external.com/blob/file' })
+
+            stub_request(:get, 'http://external.com/blob/file').
+              to_return(
+                status: 200,
+                body: File.read(Rails.root + 'spec/fixtures/container_registry/config_blob.json'))
+          end
+
+          it_behaves_like 'a processable'
         end
       end
     end