Allow in robots namespaces and projectswith .git on it

changelogs/unreleased/36837-fj-fix-bug-robots-git-suffix.yml

+---
+title: Fix bug with robots and .git suffix
+merge_request: 45866
+author:
+type: fixed

public/robots.txt

@@ -42,7 +42,7 @@ Disallow: /groups/*/group_members
 
 # Project details
 User-Agent: *
-Disallow: /*/*.git
+Disallow: /*/*.git$
 Disallow: /*/archive/
 Disallow: /*/repository/archive*
 Disallow: /*/activity

spec/requests/robots_txt_spec.rb

@@ -13,7 +13,8 @@ RSpec.describe 'Robots.txt Requests', :aggregate_failures do
 
   it 'allows the requests' do
     requests = [
-      '/users/sign_in'
+      '/users/sign_in',
+      '/namespace/subnamespace/design.gitlab.com'
     ]
 
     requests.each do |request|
@@ -60,7 +61,8 @@ RSpec.describe 'Robots.txt Requests', :aggregate_failures do
       '/foo/bar/protected_branches',
       '/foo/bar/uploads/foo',
       '/foo/bar/project_members',
-      '/foo/bar/settings'
+      '/foo/bar/settings',
+      '/namespace/subnamespace/design.gitlab.com/settings'
     ]
 
     requests.each do |request|