Add GraphQL API endpoint access from primary to secondary Geo nodes

This includes an authentication method that sends a Geo signed
token from the primary using the current session's user ID, that
is then authenticated on the secondary.

Changelog: added
EE: true

config/feature_flags/development/geo_token_user_authentication.yml

+---
+name: geo_token_user_authentication
+introduced_by_url:
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/351450
+milestone: '14.8'
+type: development
+group: group::geo
+default_enabled: false

ee/app/models/geo_node.rb

@@ -200,6 +200,10 @@ class GeoNode < ApplicationRecord
     api_url("geo_nodes/#{node.id}")
   end
 
+  def graphql_url
+    geo_api_url('graphql')
+  end
+
   def snapshot_url(repository)
     url = api_url("projects/#{repository.project.id}/snapshot")
     url += "?wiki=1" if repository.repo_type.wiki?

ee/app/services/geo/graphql_request_service.rb

+# frozen_string_literal: true
+
+module Geo
+  class GraphqlRequestService < RequestService
+    include Gitlab::Geo::LogHelpers
+
+    attr_reader :node, :user
+
+    def initialize(node, user)
+      @node = node
+      @user = user
+    end
+
+    def execute(body)
+      super(graphql_url, body, with_response: true)
+    end
+
+    private
+
+    def graphql_url
+      node&.graphql_url
+    end
+
+    def headers
+      return super unless user.present?
+
+      Gitlab::Geo::JsonRequest.new(scope: ::Gitlab::Geo::API_SCOPE, authenticating_user_id: user.id).headers
+    end
+  end
+end

ee/app/services/geo/request_service.rb

@@ -4,7 +4,7 @@ module Geo
   class RequestService
     private
 
-    def execute(url, body, method: Net::HTTP::Post)
+    def execute(url, body, method: Net::HTTP::Post, with_response: false)
       return false if url.nil?
 
       response = Gitlab::HTTP.perform_request(method, url, body: body, allow_local_requests: true, headers: headers, timeout: timeout)
@@ -14,6 +14,8 @@ module Geo
         return false
       end
 
+      return response.parsed_response if with_response
+
       true
     rescue Gitlab::HTTP::Error, Timeout::Error, SocketError, SystemCallError, OpenSSL::SSL::SSLError => e
       log_error("Failed to #{method} to primary url: #{url}", e)

ee/config/routes/api.rb

+# frozen_string_literal: true
+
+# Needed for Geo with unified URLs, regular requests to /api/graphql get proxied to
+# the primary, while these are local requests that bypass the proxy
+match '/api/v4/geo/graphql', via: [:get, :post], to: 'graphql#execute'

ee/lib/ee/api/geo.rb

@@ -202,6 +202,34 @@ module EE
               response.body
             end
           end
+
+          resource 'node_proxy' do
+            before do
+              authenticated_as_admin!
+            end
+
+            route_param :id, type: Integer, desc: 'The ID of the Geo node' do
+              helpers do
+                def geo_node
+                  strong_memoize(:geo_node) { GeoNode.find(params[:id]) }
+                end
+              end
+
+              # Query the graphql endpoint of an existing Geo node
+              #
+              # Example request:
+              #   POST /geo/node_proxy/:id/graphql
+              desc 'Query the graphql endpoint of a specific Geo node'
+              post 'graphql' do
+                not_found!('GeoNode') unless geo_node
+
+                body = env['api.request.input']
+
+                status 200
+                ::Geo::GraphqlRequestService.new(geo_node, current_user).execute(body) || {}
+              end
+            end
+          end
         end
       end
     end

ee/lib/ee/gitlab/auth/auth_finders.rb

@@ -7,6 +7,30 @@ module EE
         extend ActiveSupport::Concern
         extend ::Gitlab::Utils::Override
 
+        def find_user_from_geo_token
+          return unless ::Feature.enabled?(:geo_token_user_authentication, default_enabled: :yaml)
+          return unless geo_api_request?
+
+          token = current_request.authorization
+          return unless ::Gitlab::Geo::JwtRequestDecoder.geo_auth_attempt?(token)
+
+          token_data = parse_geo_token(token)
+          raise(::Gitlab::Auth::UnauthorizedError) unless
+            token_data.is_a?(Hash) && token_data[:scope] == ::Gitlab::Geo::API_SCOPE
+
+          ::User.find(token_data[:authenticating_user_id])
+        rescue ::ActiveRecord::RecordNotFound
+          raise(::Gitlab::Auth::UnauthorizedError)
+        end
+
+        def parse_geo_token(token)
+          geo_jwt_decoder = ::Gitlab::Geo::JwtRequestDecoder.new(token)
+          geo_jwt_decoder.decode
+        rescue ::Gitlab::Geo::InvalidDecryptionKeyError, ::Gitlab::Geo::InvalidSignatureTimeError => e
+          ::Gitlab::ErrorTracking.track_exception(e)
+          nil
+        end
+
         override :find_oauth_access_token
         def find_oauth_access_token
           return if scim_request?
@@ -17,6 +41,10 @@ module EE
         def scim_request?
           current_request.path.starts_with?("/api/scim/")
         end
+
+        def geo_api_request?
+          current_request.path.starts_with?("/api/#{::API::API.version}/geo/")
+        end
       end
     end
   end

ee/lib/ee/gitlab/auth/request_authenticator.rb

+# frozen_string_literal: true
+
+module EE
+  module Gitlab
+    module Auth
+      module RequestAuthenticator
+        extend ::Gitlab::Utils::Override
+
+        override :find_sessionless_user
+        def find_sessionless_user(request_format)
+          find_user_from_geo_token || super(request_format)
+        rescue ::Gitlab::Auth::AuthenticationError
+          nil
+        end
+      end
+    end
+  end
+end

ee/lib/ee/gitlab/middleware/read_only/controller.rb

@@ -55,7 +55,13 @@ module EE
           #
           # @return [Boolean] true whether current route is in allow list.
           def allowlisted_routes
-            allowed = super || geo_node_update_route? || geo_resync_designs_route? || geo_sign_out_route? || admin_settings_update? || geo_node_status_update_route?
+            allowed = super ||
+                      geo_node_update_route? ||
+                      geo_resync_designs_route? ||
+                      geo_sign_out_route? ||
+                      admin_settings_update? ||
+                      geo_node_status_update_route? ||
+                      geo_graphql_query?
 
             return true if allowed
             return sign_in_route? if ::Gitlab.maintenance_mode?
@@ -119,6 +125,12 @@ module EE
             end
           end
 
+          def geo_graphql_query?
+            request.post? && ::Gitlab::Middleware::ReadOnly::API_VERSIONS.any? do |version|
+              request.path.include?("/api/v#{version}/geo/graphql")
+            end
+          end
+
           def sign_in_route?
             return unless request.post?
 

ee/lib/gitlab/geo/json_request.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Geo
+    class JsonRequest < BaseRequest
+      def headers
+        super.merge({ 'Content-Type' => 'application/json' })
+      end
+    end
+  end
+end

ee/spec/lib/ee/gitlab/auth/auth_finders_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe EE::Gitlab::Auth::AuthFinders do
+  include described_class
+  include ::EE::GeoHelpers
+
+  let(:current_request) { ActionDispatch::Request.new(env) }
+  let(:env) do
+    {
+      'rack.input' => ''
+    }
+  end
+
+  let_it_be(:user) { create(:user) }
+
+  describe '#find_user_from_geo_token' do
+    subject { find_user_from_geo_token }
+
+    let_it_be(:primary) { create(:geo_node, :primary) }
+
+    let(:path) { '/api/v4/geo/graphql' }
+    let(:authorization_header) do
+      ::Gitlab::Geo::JsonRequest
+        .new(scope: ::Gitlab::Geo::API_SCOPE, authenticating_user_id: user.id)
+        .headers['Authorization']
+    end
+
+    before do
+      stub_current_geo_node(primary)
+
+      env['SCRIPT_NAME'] = path
+      current_request.headers['Authorization'] = authorization_header
+    end
+
+    it { is_expected.to eq(user) }
+
+    context 'when the path is not Geo specific' do
+      let(:path) { '/api/v4/test' }
+
+      it { is_expected.to eq(nil) }
+    end
+
+    context 'when the Authorization header is invalid' do
+      let(:authorization_header) { 'invalid' }
+
+      it { is_expected.to eq(nil) }
+    end
+
+    context 'when the Authorization header is nil' do
+      let(:authorization_header) { '' }
+
+      it { is_expected.to eq(nil) }
+    end
+
+    context 'when the Authorization header is a Geo header' do
+      it 'does not authenticate when the token expired' do
+        travel_to(2.minutes.from_now) { expect { subject }.to raise_error(::Gitlab::Auth::UnauthorizedError) }
+      end
+
+      it 'does not authenticate when clocks are not in sync' do
+        travel_to(2.minutes.ago) { expect { subject }.to raise_error(::Gitlab::Auth::UnauthorizedError) }
+      end
+
+      it 'does not authenticate with invalid decryption key error' do
+        allow_next_instance_of(::Gitlab::Geo::JwtRequestDecoder) do |instance|
+          expect(instance).to receive(:decode_auth_header).and_raise(Gitlab::Geo::InvalidDecryptionKeyError)
+        end
+
+        expect { subject }.to raise_error(::Gitlab::Auth::UnauthorizedError)
+      end
+
+      context 'when the scope is not API' do
+        let(:authorization_header) do
+          ::Gitlab::Geo::JsonRequest
+            .new(scope: 'invalid', authenticating_user_id: user.id)
+            .headers['Authorization']
+        end
+
+        it 'does not authenticate' do
+          expect { subject }.to raise_error(::Gitlab::Auth::UnauthorizedError)
+        end
+      end
+
+      context 'when it does not contain a user id' do
+        let(:authorization_header) do
+          ::Gitlab::Geo::JsonRequest
+            .new(scope: ::Gitlab::Geo::API_SCOPE)
+            .headers['Authorization']
+        end
+
+        it 'raises an unauthorize error' do
+          expect { subject }.to raise_error(::Gitlab::Auth::UnauthorizedError)
+        end
+      end
+    end
+
+    context 'when the user does not exist' do
+      let(:user) { create(:user) }
+
+      it 'raises an unauthorized error' do
+        user.delete
+
+        expect { subject }.to raise_error(::Gitlab::Auth::UnauthorizedError)
+      end
+    end
+
+    context 'when the geo_token_user_authentication feature flag is disabled' do
+      before do
+        stub_feature_flags(geo_token_user_authentication: false)
+      end
+
+      it 'returns immediately' do
+        expect(::Gitlab::Geo::JwtRequestDecoder).not_to receive(:geo_auth_attempt?)
+        expect(::Gitlab::Geo::JwtRequestDecoder).not_to receive(:new)
+        expect(subject).to be_nil
+      end
+    end
+  end
+end

ee/spec/lib/ee/gitlab/auth/request_authenticator_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe EE::Gitlab::Auth::RequestAuthenticator do
+  let(:env) do
+    {
+      'rack.input' => ''
+    }
+  end
+
+  let(:request) { ActionDispatch::Request.new(env) }
+
+  subject { Gitlab::Auth::RequestAuthenticator.new(request) }
+
+  describe '#find_sessionless_user' do
+    let_it_be(:geo_token_user) { build(:user) }
+    let_it_be(:dependency_proxy_user) { build(:user) }
+
+    it 'returns geo_token user first' do
+      allow_next_instance_of(::Gitlab::Auth::RequestAuthenticator) do |instance|
+        allow(instance).to receive(:find_user_from_geo_token).and_return(geo_token_user)
+        allow(instance).to receive(:find_user_from_dependency_proxy_token).and_return(dependency_proxy_user)
+      end
+
+      expect(subject.find_sessionless_user(:api)).to eq geo_token_user
+    end
+
+    it 'returns nil if no user found' do
+      expect(subject.find_sessionless_user(:api)).to be_nil
+    end
+
+    it 'rescue Gitlab::Auth::AuthenticationError exceptions' do
+      allow_next_instance_of(::Gitlab::Auth::RequestAuthenticator) do |instance|
+        allow(instance).to receive(:find_user_from_geo_token).and_raise(Gitlab::Auth::UnauthorizedError)
+      end
+
+      expect(subject.find_sessionless_user(:api)).to be_nil
+    end
+  end
+end

ee/spec/lib/gitlab/geo/json_request_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Geo::JsonRequest, :geo do
+  include ::EE::GeoHelpers
+
+  let(:geo_node) { create(:geo_node) }
+  let(:request) { described_class.new }
+
+  before do
+    stub_current_geo_node(geo_node)
+  end
+
+  describe '#headers' do
+    it 'contains the JSON request headers' do
+      expect(request.headers).to include('Content-Type' => 'application/json')
+    end
+  end
+end

ee/spec/models/geo_node_spec.rb

@@ -577,6 +577,12 @@ RSpec.describe GeoNode, :request_store, :geo, type: :model do
     end
   end
 
+  describe '#graphql_url' do
+    it 'returns an api url to the graphql endpoint' do
+      expect(new_primary_node.graphql_url).to eq("https://localhost:3000/gitlab/api/#{api_version}/geo/graphql")
+    end
+  end
+
   describe '#snapshot_url' do
     let(:project) { create(:project) }
     let(:snapshot_url) { "https://localhost:3000/gitlab/api/#{api_version}/projects/#{project.id}/snapshot" }

ee/spec/requests/api/geo_spec.rb

@@ -747,4 +747,45 @@ RSpec.describe API::Geo do
       expect(response).to have_gitlab_http_status(:forbidden)
     end
   end
+
+  describe 'POST /geo/node_proxy/:id/graphql' do
+    let(:headers) { { 'Content-Type' => 'application/json' } }
+    let(:unexisting_node_id) { non_existing_record_id }
+
+    before do
+      stub_current_geo_node(primary_node)
+    end
+
+    it_behaves_like '404 response' do
+      let(:request) { post api("/geo/node_proxy/#{unexisting_node_id}/graphql", admin) }
+    end
+
+    it 'denies access if not admin' do
+      post api("/geo/node_proxy/#{secondary_node.id}/graphql", user)
+
+      expect(response).to have_gitlab_http_status(:forbidden)
+    end
+
+    it 'requests the graphql endpoint with the post body and returns the output' do
+      stub_request(:post, secondary_node.graphql_url)
+        .with(body: { input: 'test' })
+        .to_return(status: 200, body: { testResponse: 'result' }.to_json, headers: headers)
+
+      post api("/geo/node_proxy/#{secondary_node.id}/graphql", admin), params: { input: 'test' }.to_json, headers: headers
+
+      expect(response).to have_gitlab_http_status(:ok)
+      expect(json_response).to eq('testResponse' => 'result')
+    end
+
+    it 'returns empty output if remote fails' do
+      stub_request(:post, secondary_node.graphql_url)
+        .with(body: { input: 'test' })
+        .to_return(status: 500)
+
+      post api("/geo/node_proxy/#{secondary_node.id}/graphql", admin), params: { input: 'test' }.to_json, headers: headers
+
+      expect(response).to have_gitlab_http_status(:ok)
+      expect(json_response).to be_empty
+    end
+  end
 end

ee/spec/services/geo/graphql_request_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Geo::GraphqlRequestService, :geo do
+  include ::EE::GeoHelpers
+  include ApiHelpers
+
+  let_it_be(:primary)   { create(:geo_node, :primary) }
+  let_it_be(:secondary) { create(:geo_node) }
+  let_it_be(:user) { create(:user) }
+
+  before do
+    stub_current_geo_node(primary)
+  end
+
+  RSpec::Matchers.define :jwt_token_with_data do |data|
+    match do |actual_headers|
+      geo_jwt_decoder = ::Gitlab::Geo::JwtRequestDecoder.new(actual_headers['Authorization'])
+      result = geo_jwt_decoder.decode
+
+      expect(result).to eq(data)
+    end
+  end
+
+  subject { described_class.new(secondary, user) }
+
+  describe '#execute' do
+    context 'when the node is nil' do
+      subject { described_class.new(nil, user) }
+
+      it 'fails and not make a request' do
+        expect(Gitlab::HTTP).not_to receive(:perform_request)
+
+        expect(subject.execute({})).to be_falsey
+      end
+    end
+
+    context 'when the user is nil' do
+      subject { described_class.new(secondary, nil) }
+
+      it 'makes an unauthenticated request' do
+        expect(Gitlab::HTTP).to receive(:perform_request)
+                                  .with(
+                                    Net::HTTP::Post,
+                                    secondary.graphql_url,
+                                    hash_including(headers: jwt_token_with_data(
+                                      scope: ::Gitlab::Geo::API_SCOPE
+                                    )))
+                                  .and_return(double(success?: true, parsed_response: 'response'))
+
+        expect(subject.execute({})).to be('response')
+      end
+    end
+
+    it 'sends a request with the authenticating user id in the headers' do
+      expect(Gitlab::HTTP).to receive(:perform_request)
+                                .with(
+                                  Net::HTTP::Post,
+                                  secondary.graphql_url,
+                                  hash_including(headers: jwt_token_with_data(
+                                    scope: ::Gitlab::Geo::API_SCOPE,
+                                    authenticating_user_id: user.id
+                                  )))
+                                .and_return(double(success?: true, parsed_response: 'response'))
+
+      expect(subject.execute({})).to be('response')
+    end
+  end
+end

ee/spec/services/geo/node_status_request_service_spec.rb

@@ -35,9 +35,9 @@ RSpec.describe Geo::NodeStatusRequestService, :geo do
                                   Net::HTTP::Post,
                                   primary.status_url,
                                   hash_including(body: hash_not_including('id')))
-                                .and_return(double(success?: true))
+                                .and_return(double(success?: true, parsed_response: { 'message' => 'Test' }))
 
-      described_class.new(args).execute
+      expect(described_class.new(args).execute).to be_truthy
     end
 
     it 'sends geo_node_id in the request' do
@@ -53,9 +53,9 @@ RSpec.describe Geo::NodeStatusRequestService, :geo do
                                   Net::HTTP::Post,
                                   primary.status_url,
                                   hash_including(body: hash_including('geo_node_id' => secondary.id)))
-                                .and_return(double(success?: true))
+                                .and_return(double(success?: true, parsed_response: { 'message' => 'Test' }))
 
-      described_class.new(args).execute
+      expect(described_class.new(args).execute).to be_truthy
     end
 
     it 'sends all of the data in the status JSONB field in the request' do
@@ -69,9 +69,9 @@ RSpec.describe Geo::NodeStatusRequestService, :geo do
                                     body: hash_including(
                                       'status' => hash_including(
                                         *GeoNodeStatus::RESOURCE_STATUS_FIELDS))))
-                                .and_return(double(success?: true))
+                                .and_return(double(success?: true, parsed_response: { 'message' => 'Test' }))
 
-      described_class.new(args).execute
+      expect(described_class.new(args).execute).to be_truthy
     end
   end
 end

ee/spec/services/geo/replication_toggle_request_service_spec.rb

@@ -19,7 +19,8 @@ RSpec.describe Geo::ReplicationToggleRequestService, :geo do
 
   it 'expires the geo cache on success' do
     response = double(success?: true,
-                      code: 200 )
+                      code: 200,
+                      parsed_response: { 'message' => 'Test' } )
     allow(Gitlab::HTTP).to receive(:perform_request).and_return(response)
     expect(Gitlab::Geo).to receive(:expire_cache!)
 

ee/spec/support/shared_examples/lib/gitlab/middleware/maintenance_mode_gitlab_ee_instance_shared_examples.rb

@@ -176,6 +176,13 @@ RSpec.shared_examples 'write access for a read-only GitLab (EE) instance in main
         expect(response).to be_redirect
         expect(subject).to disallow_request
       end
+
+      it "allows Geo POST GraphQL requests" do
+        response = request.post("/api/#{API::API.version}/geo/graphql")
+
+        expect(response).not_to be_redirect
+        expect(subject).not_to disallow_request
+      end
     end
   end
 end

lib/gitlab/auth/request_authenticator.rb

@@ -108,3 +108,5 @@ module Gitlab
     end
   end
 end
+
+Gitlab::Auth::RequestAuthenticator.prepend_mod

spec/lib/gitlab/auth/request_authenticator_spec.rb

@@ -21,8 +21,10 @@ RSpec.describe Gitlab::Auth::RequestAuthenticator do
     let_it_be(:session_user) { build(:user) }
 
     it 'returns sessionless user first' do
-      allow_any_instance_of(described_class).to receive(:find_sessionless_user).and_return(sessionless_user)
-      allow_any_instance_of(described_class).to receive(:find_user_from_warden).and_return(session_user)
+      allow_next_instance_of(described_class) do |instance|
+        allow(instance).to receive(:find_sessionless_user).and_return(sessionless_user)
+        allow(instance).to receive(:find_user_from_warden).and_return(session_user)
+      end
 
       expect(subject.user([:api])).to eq sessionless_user
     end

workhorse/internal/upstream/routes.go

@@ -390,6 +390,7 @@ func configureRoutes(u *upstream) {
 		u.route("", "^/api/v4/geo_nodes", defaultUpstream),
 		u.route("", "^/api/v4/geo_replication", defaultUpstream),
 		u.route("", "^/api/v4/geo/proxy_git_ssh", defaultUpstream),
+		u.route("", "^/api/v4/geo/graphql", defaultUpstream),
 
 		// Internal API routes
 		u.route("", "^/api/v4/internal", defaultUpstream),