Render 404 to crawlers instead of redirecting

The login page is protected by Cloudflare's WAF and it is causing issues
with search engine crawlers because Cloudflare returns a 503 when doing
a browser check

app/controllers/application_controller.rb

@@ -121,7 +121,7 @@ class ApplicationController < ActionController::Base
   end
 
   def route_not_found
-    if current_user
+    if current_user || browser.bot.search_engine?
       not_found
     else
       store_location_for(:user, request.fullpath) unless request.xhr?

changelogs/unreleased/258993-render-404-to-bots.yml

+---
+title: Render 404 to search engine crawlers instead of redirecting to login
+merge_request: 45552
+author:
+type: changed

spec/controllers/application_controller_spec.rb

@@ -171,6 +171,8 @@ RSpec.describe ApplicationController do
 
   describe '#route_not_found' do
     controller(described_class) do
+      skip_before_action :authenticate_user!, only: :index
+
       def index
         route_not_found
       end
@@ -184,6 +186,14 @@ RSpec.describe ApplicationController do
       expect(response).to have_gitlab_http_status(:not_found)
     end
 
+    it 'renders 404 if client is a search engine crawler' do
+      request.env['HTTP_USER_AGENT'] = 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)'
+
+      get :index
+
+      expect(response).to have_gitlab_http_status(:not_found)
+    end
+
     it 'redirects to login page if not authenticated' do
       get :index