Merge branch '272986-fj-disallow-webide-route-in-robots' into 'master'

Resolve "Disallow WebIDE route in Robots.txt"

See merge request gitlab-org/gitlab!46117

changelogs/unreleased/272986-fj-disallow-webide-route-in-robots.yml

+---
+title: Disallow WebIDE route in robots.txt
+merge_request: 46117
+author:
+type: changed

public/robots.txt

@@ -23,6 +23,7 @@ Disallow: /users
 Disallow: /help
 Disallow: /s/
 Disallow: /-/profile
+Disallow: /-/ide/
 # Only specifically allow the Sign In page to avoid very ugly search results
 Allow: /users/sign_in
 

spec/requests/robots_txt_spec.rb

@@ -37,6 +37,7 @@ RSpec.describe 'Robots.txt Requests', :aggregate_failures do
       '/help',
       '/s/',
       '/-/profile',
+      '/-/ide/project',
       '/foo/bar/new',
       '/foo/bar/edit',
       '/foo/bar/raw',