Generate Sitemap dynamically

In this commit we create a controller that will generate
dynamically the sitemap for the `gitlab-org` group.

The controller is restricted for .com.

config/feature_flags/development/gitlab_org_sitemap.yml

+---
+name: gitlab_org_sitemap
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/46661
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/276915
+milestone: '13.6'
+type: development
+group: group::editor
+default_enabled: false

config/routes.rb

@@ -275,6 +275,10 @@ Rails.application.routes.draw do
     draw :profile
   end
 
+  Gitlab.ee do
+    get '/sitemap' => 'sitemap#show', format: :xml
+  end
+
   root to: "root#index"
 
   get '*unmatched_route', to: 'application#route_not_found'

ee/app/controllers/sitemap_controller.rb

+# frozen_string_literal: true
+
+class SitemapController < ApplicationController
+  skip_before_action :authenticate_user!
+
+  feature_category :metrics
+
+  def show
+    return render_404 unless Gitlab.com?
+    return render_404 unless Feature.enabled?(:gitlab_org_sitemap)
+
+    respond_to do |format|
+      format.xml do
+        response = Sitemap::CreateService.new.execute
+
+        xml_data = if response.success?
+                     response.payload[:sitemap]
+                   else
+                     xml_error(response.message)
+                   end
+
+        render inline: xml_data
+      end
+    end
+  end
+
+  private
+
+  def xml_error(message)
+    xml_builder = Builder::XmlMarkup.new(indent: 2)
+    xml_builder.instruct!
+    xml_builder.error message
+  end
+end

ee/app/services/sitemap/create_service.rb

+# frozen_string_literal: true
+
+module Sitemap
+  class CreateService
+    def execute
+      result = Gitlab::Sitemaps::Generator.execute
+
+      if result.is_a?(String)
+        error_response(result)
+      else
+        success_response(result)
+      end
+    end
+
+    private
+
+    def success_response(file)
+      Gitlab::AppLogger.info("Sitemap generated successfully")
+
+      ServiceResponse.success(payload: { sitemap: file.render } )
+    end
+
+    def error_response(message)
+      Gitlab::AppLogger.error("Sitemap error creating sitemap: #{message}")
+
+      ServiceResponse.error(
+        message: message
+      )
+    end
+  end
+end

ee/changelogs/unreleased/fj-add-controller-for-sitemap-poc.yml

+---
+title: Generate dynamically sitemap through controller
+merge_request: 46661
+author:
+type: changed

ee/lib/gitlab/sitemaps/generator.rb

@@ -10,19 +10,22 @@ module Gitlab
 
         def execute
           unless Gitlab.com?
-            return "The sitemap can only be generated for Gitlab.com"
+            return 'The sitemap can only be generated for Gitlab.com'
           end
 
           file = Sitemaps::SitemapFile.new
 
-          if gitlab_org_group
-            file.add_elements(generic_urls)
-            file.add_elements(gitlab_org_group)
-            file.add_elements(gitlab_org_subgroups)
-            file.add_elements(gitlab_org_projects)
-            file.save
+          return "The group '#{GITLAB_ORG_NAMESPACE}' was not found" unless gitlab_org_group
+
+          file.add_elements(generic_urls)
+          file.add_elements(gitlab_org_group)
+          file.add_elements(gitlab_org_subgroups)
+          file.add_elements(gitlab_org_projects)
+
+          if file.empty?
+            'No urls found to generate the sitemap'
           else
-            "The group '#{GITLAB_ORG_NAMESPACE}' was not found"
+            file
           end
         end
 
@@ -37,7 +40,7 @@ module Gitlab
         end
 
         def gitlab_org_group
-          @gitlab_org_group ||= GroupFinder.new(nil).execute(path: 'gitlab-org', parent_id: nil, visibility_level: Gitlab::VisibilityLevel::PUBLIC)
+          @gitlab_org_group ||= GroupFinder.new(nil).execute(path: GITLAB_ORG_NAMESPACE, parent_id: nil, visibility_level: Gitlab::VisibilityLevel::PUBLIC)
         end
 
         def gitlab_org_subgroups

ee/lib/gitlab/sitemaps/sitemap_file.rb

@@ -20,17 +20,23 @@ module Gitlab
       end
 
       def save
-        return if urls.empty?
+        return if empty?
 
         File.write(SITEMAP_FILE_PATH, render)
       end
 
       def render
+        return if empty?
+
         fragment = File.read(File.expand_path("fragments/sitemap_file.xml.builder", __dir__))
 
         instance_eval fragment
       end
 
+      def empty?
+        urls.empty?
+      end
+
       private
 
       def xml_builder

ee/lib/gitlab/sitemaps/url_extractor.rb

 # frozen_string_literal: true
 
+# Generating the urls for the project and groups is the most
+# expensive part of the sitemap generation because we need
+# to call the Rails route helpers.
+#
+# We could hardcode them but if a route changes the sitemap
+# urls will be invalid.
 module Gitlab
   module Sitemaps
     class UrlExtractor

ee/spec/controllers/sitemap_controller_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe SitemapController do
+  describe '#show' do
+    subject { get :show, format: :xml }
+
+    before do
+      allow(Gitlab).to receive(:com?).and_return(dot_com)
+    end
+
+    context 'when not Gitlab.com?' do
+      let(:dot_com) { false }
+
+      it 'returns :not_found' do
+        subject
+
+        expect(response).to have_gitlab_http_status(:not_found)
+      end
+    end
+
+    context 'when Gitlab.com?' do
+      let(:dot_com) { true }
+
+      context 'with an authenticated user' do
+        let(:flag_value) { true }
+
+        before do
+          stub_feature_flags(gitlab_org_sitemap: flag_value)
+
+          allow(Sitemap::CreateService).to receive_message_chain(:new, :execute).and_return(result)
+
+          subject
+        end
+
+        shared_examples 'gitlab_org_sitemap flag is disabled' do
+          let(:flag_value) { false }
+
+          it 'returns :not_found' do
+            expect(response).to have_gitlab_http_status(:not_found)
+          end
+        end
+
+        context 'when the sitemap generation raises an error' do
+          let(:result) { ServiceResponse.error(message: 'foo') }
+
+          it 'returns an xml error' do
+            expect(response).to have_gitlab_http_status(:ok)
+            expect(response.body).to include('<error>foo</error>')
+          end
+
+          it_behaves_like 'gitlab_org_sitemap flag is disabled'
+        end
+
+        context 'when the sitemap was created suscessfully' do
+          let(:result) { ServiceResponse.success(payload: { sitemap: 'foo' }) }
+
+          it 'returns sitemap' do
+            expect(response).to have_gitlab_http_status(:ok)
+            expect(response.body).to eq('foo')
+          end
+
+          it_behaves_like 'gitlab_org_sitemap flag is disabled'
+        end
+      end
+    end
+  end
+end

ee/spec/lib/gitlab/sitemaps/generator_spec.rb

@@ -42,12 +42,7 @@ RSpec.describe Gitlab::Sitemaps::Generator do
         let_it_be(:internal_subgroup_internal_project) { create(:project, :internal, namespace: internal_subgroup) }
 
         it 'includes default explore routes and gitlab-org group routes' do
-          new_path = Rails.root.join('tmp/tests/sitemap.xml')
-          stub_const('Gitlab::Sitemaps::SitemapFile::SITEMAP_FILE_PATH', new_path)
-
-          subject
-
-          content = File.read(new_path)
+          content = subject.render
 
           expect(content).to include('/explore/projects')
           expect(content).to include('/explore/groups')
@@ -63,8 +58,6 @@ RSpec.describe Gitlab::Sitemaps::Generator do
           expect(content).not_to include(public_subgroup_internal_project.full_path)
           expect(content).not_to include(internal_subgroup_private_project.full_path)
           expect(content).not_to include(internal_subgroup_internal_project.full_path)
-
-          File.delete(new_path)
         end
       end
     end

ee/spec/lib/gitlab/sitemaps/sitemap_file_spec.rb

@@ -10,6 +10,12 @@ RSpec.describe Gitlab::Sitemaps::SitemapFile do
   end
 
   describe '#render' do
+    it 'returns if no elements has been provided' do
+      expect(File).not_to receive(:read)
+
+      described_class.new.save # rubocop: disable Rails/SaveBang
+    end
+
     it 'generates a valid sitemap file' do
       freeze_time do
         content = subject.render

ee/spec/services/sitemap/create_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Sitemap::CreateService do
+  describe '#execute' do
+    subject { described_class.new.execute}
+
+    it 'returns the successful service response with the sitemap content' do
+      sitemap_file = Gitlab::Sitemaps::SitemapFile.new
+
+      allow(sitemap_file).to receive(:render).and_return('foo')
+      allow(Gitlab::Sitemaps::Generator).to receive(:execute).and_return(sitemap_file)
+
+      expect(subject).to be_success
+      expect(subject.payload[:sitemap]).to eq 'foo'
+    end
+
+    context 'when the sitemap generator returns an error' do
+      it 'returns an error service response' do
+        allow(Gitlab).to receive(:com?).and_return(false)
+
+        expect(subject).to be_error
+        expect(subject.message).to eq 'The sitemap can only be generated for Gitlab.com'
+      end
+    end
+  end
+end