Add auto_ssl_enabled to pages_domains api

Add more docs for pages domains certificate api
Fix autoremoving certificates if they have not been passed

changelogs/unreleased/30660-allow-to-enable-disable-auto-ssl-letsencrypt-support-via-api.yml

+---
+title: Require explicit null parameters to remove pages domain certificate and allow to use Let's Encrypt certificates through API
+merge_request:
+author:
+type: changed

doc/api/pages_domains.md

@@ -22,6 +22,7 @@ curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/ap
     "domain": "ssl.domain.example",
     "url": "https://ssl.domain.example",
     "project_id": 1337,
+    "auto_ssl_enabled": false,
     "certificate": {
       "expired": false,
       "expiration": "2020-04-12T14:32:00.000Z"
@@ -55,6 +56,7 @@ curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/ap
   {
     "domain": "ssl.domain.example",
     "url": "https://ssl.domain.example",
+    "auto_ssl_enabled": false,
     "certificate": {
       "subject": "/O=Example, Inc./OU=Example Origin CA/CN=Example Origin Certificate",
       "expired": false,
@@ -76,7 +78,7 @@ GET /projects/:id/pages/domains/:domain
 | Attribute | Type           | Required | Description                              |
 | --------- | -------------- | -------- | ---------------------------------------- |
 | `id`      | integer/string | yes      | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user |
-| `domain`  | string         | yes      | The domain                               |
+| `domain`  | string         | yes      | The custom domain indicated by the user  |
 
 ```bash
 curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/pages/domains/www.domain.example
@@ -97,6 +99,7 @@ curl --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/ap
 {
   "domain": "ssl.domain.example",
   "url": "https://ssl.domain.example",
+  "auto_ssl_enabled": false,
   "certificate": {
     "subject": "/O=Example, Inc./OU=Example Origin CA/CN=Example Origin Certificate",
     "expired": false,
@@ -114,12 +117,13 @@ Creates a new pages domain. The user must have permissions to create new pages d
 POST /projects/:id/pages/domains
 ```
 
-| Attribute     | Type           | Required | Description                              |
-| ------------- | -------------- | -------- | ---------------------------------------- |
-| `id`          | integer/string | yes      | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user |
-| `domain`      | string         | yes      | The domain                               |
-| `certificate` | file/string    | no       | The certificate in PEM format with intermediates following in most specific to least specific order.|
-| `key`         | file/string    | no       | The certificate key in PEM format.       |
+| Attribute          | Type           | Required | Description                              |
+| -------------------| -------------- | -------- | ---------------------------------------- |
+| `id`               | integer/string | yes      | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user |
+| `domain`           | string         | yes      | The custom domain indicated by the user  |
+| `auto_ssl_enabled` | boolean        | no       | Enables [automatic generation](../user/project/pages/custom_domains_ssl_tls_certification/lets_encrypt_integration.md) of SSL certificates issued by Let's Encrypt for custom domains. |
+| `certificate`      | file/string    | no       | The certificate in PEM format with intermediates following in most specific to least specific order.|
+| `key`              | file/string    | no       | The certificate key in PEM format.       |
 
 ```bash
 curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" --form "domain=ssl.domain.example" --form "certificate=@/path/to/cert.pem" --form "key=@/path/to/key.pem" https://gitlab.example.com/api/v4/projects/5/pages/domains
@@ -129,10 +133,15 @@ curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" --form "domain
 curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" --form "domain=ssl.domain.example" --form "certificate=$CERT_PEM" --form "key=$KEY_PEM" https://gitlab.example.com/api/v4/projects/5/pages/domains
 ```
 
+```bash
+curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" --form "domain=ssl.domain.example" --form "auto_ssl_enabled=true" https://gitlab.example.com/api/v4/projects/5/pages/domains
+```
+
 ```json
 {
   "domain": "ssl.domain.example",
   "url": "https://ssl.domain.example",
+  "auto_ssl_enabled": true,
   "certificate": {
     "subject": "/O=Example, Inc./OU=Example Origin CA/CN=Example Origin Certificate",
     "expired": false,
@@ -150,12 +159,15 @@ Updates an existing project pages domain. The user must have permissions to chan
 PUT /projects/:id/pages/domains/:domain
 ```
 
-| Attribute     | Type           | Required | Description                              |
-| ------------- | -------------- | -------- | ---------------------------------------- |
-| `id`          | integer/string | yes      | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user |
-| `domain`      | string         | yes      | The domain                               |
-| `certificate` | file/string    | no       | The certificate in PEM format with intermediates following in most specific to least specific order.|
-| `key`         | file/string    | no       | The certificate key in PEM format.       |
+| Attribute          | Type           | Required | Description                              |
+| ------------------ | -------------- | -------- | ---------------------------------------- |
+| `id`               | integer/string | yes      | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user |
+| `domain`           | string         | yes      | The custom domain indicated by the user  |
+| `auto_ssl_enabled` | boolean        | no       | Enables [automatic generation](../user/project/pages/custom_domains_ssl_tls_certification/lets_encrypt_integration.md) of SSL certificates issued by Let's Encrypt for custom domains. |
+| `certificate`      | file/string    | no       | The certificate in PEM format with intermediates following in most specific to least specific order.|
+| `key`              | file/string    | no       | The certificate key in PEM format.       |
+
+### Adding certificate
 
 ```bash
 curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" --form "certificate=@/path/to/cert.pem" --form "key=@/path/to/key.pem" https://gitlab.example.com/api/v4/projects/5/pages/domains/ssl.domain.example
@@ -169,6 +181,7 @@ curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" --form "certifi
 {
   "domain": "ssl.domain.example",
   "url": "https://ssl.domain.example",
+  "auto_ssl_enabled": false,
   "certificate": {
     "subject": "/O=Example, Inc./OU=Example Origin CA/CN=Example Origin Certificate",
     "expired": false,
@@ -178,6 +191,36 @@ curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" --form "certifi
 }
 ```
 
+### Enabling Let's Encrypt integration for Pages custom domains
+
+```bash
+curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" --form "auto_ssl_enabled=true" https://gitlab.example.com/api/v4/projects/5/pages/domains/ssl.domain.example
+```
+
+```json
+{
+  "domain": "ssl.domain.example",
+  "url": "https://ssl.domain.example",
+  "auto_ssl_enabled": true
+}
+```
+
+### Removing certificate
+
+To remove the SSL certificate attached to the Pages domain, run:
+
+```bash
+curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" --form "certificate=" --form "key=" https://gitlab.example.com/api/v4/projects/5/pages/domains/ssl.domain.example
+```
+
+```json
+{
+  "domain": "ssl.domain.example",
+  "url": "https://ssl.domain.example",
+  "auto_ssl_enabled": false
+}
+```
+
 ## Delete pages domain
 
 Deletes an existing project pages domain.
@@ -189,7 +232,7 @@ DELETE /projects/:id/pages/domains/:domain
 | Attribute | Type           | Required | Description                              |
 | --------- | -------------- | -------- | ---------------------------------------- |
 | `id`      | integer/string | yes      | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user |
-| `domain`  | string         | yes      | The domain                               |
+| `domain`  | string         | yes      | The custom domain indicated by the user  |
 
 ```bash
 curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" https://gitlab.example.com/api/v4/projects/5/pages/domains/ssl.domain.example

lib/api/entities.rb

@@ -1681,6 +1681,7 @@ module API
       expose :verified?, as: :verified
       expose :verification_code, as: :verification_code
       expose :enabled_until
+      expose :auto_ssl_enabled
 
       expose :certificate,
         as: :certificate_expiration,
@@ -1696,6 +1697,7 @@ module API
       expose :verified?, as: :verified
       expose :verification_code, as: :verification_code
       expose :enabled_until
+      expose :auto_ssl_enabled
 
       expose :certificate,
         if: ->(pages_domain, _) { pages_domain.certificate? },

lib/api/pages_domains.rb

@@ -92,8 +92,10 @@ module API
         requires :domain, type: String, desc: 'The domain'
         # rubocop:disable Scalability/FileUploads
         # TODO: remove rubocop disable - https://gitlab.com/gitlab-org/gitlab/issues/14960
-        optional :certificate, allow_blank: false, types: [File, String], desc: 'The certificate', as: :user_provided_certificate
-        optional :key, allow_blank: false, types: [File, String], desc: 'The key', as: :user_provided_key
+        optional :certificate, types: [File, String], desc: 'The certificate', as: :user_provided_certificate
+        optional :key, types: [File, String], desc: 'The key', as: :user_provided_key
+        optional :auto_ssl_enabled, allow_blank: false, type: Boolean, default: false,
+                 desc: "Enables automatic generation of SSL certificates issued by Let's Encrypt for custom domains."
         # rubocop:enable Scalability/FileUploads
         all_or_none_of :user_provided_certificate, :user_provided_key
       end
@@ -116,14 +118,16 @@ module API
         requires :domain, type: String, desc: 'The domain'
         # rubocop:disable Scalability/FileUploads
         # TODO: remove rubocop disable - https://gitlab.com/gitlab-org/gitlab/issues/14960
-        optional :certificate, allow_blank: false, types: [File, String], desc: 'The certificate', as: :user_provided_certificate
-        optional :key, allow_blank: false, types: [File, String], desc: 'The key', as: :user_provided_key
+        optional :certificate, types: [File, String], desc: 'The certificate', as: :user_provided_certificate
+        optional :key, types: [File, String], desc: 'The key', as: :user_provided_key
+        optional :auto_ssl_enabled, allow_blank: true, type: Boolean,
+                 desc: "Enables automatic generation of SSL certificates issued by Let's Encrypt for custom domains."
         # rubocop:enable Scalability/FileUploads
       end
       put ":id/pages/domains/:domain", requirements: PAGES_DOMAINS_ENDPOINT_REQUIREMENTS do
         authorize! :update_pages, user_project
 
-        pages_domain_params = declared(params, include_parent_namespaces: false)
+        pages_domain_params = declared(params, include_parent_namespaces: false, include_missing: false)
 
         # Remove empty private key if certificate is not empty.
         if pages_domain_params[:user_provided_certificate] && !pages_domain_params[:user_provided_key]

spec/fixtures/api/schemas/public_api/v4/pages_domain/basic.json

@@ -7,6 +7,7 @@
     "verified": { "type": "boolean" },
     "verification_code": { "type": ["string", "null"] },
     "enabled_until": { "type": ["date", "null"] },
+    "auto_ssl_enabled": { "type": "boolean" },
     "certificate_expiration": {
       "type": "object",
       "properties": {
@@ -17,6 +18,6 @@
       "additionalProperties": false
     }
   },
-  "required": ["domain", "url", "project_id", "verified", "verification_code", "enabled_until"],
+  "required": ["domain", "url", "project_id", "verified", "verification_code", "enabled_until", "auto_ssl_enabled"],
   "additionalProperties": false
 }

spec/fixtures/api/schemas/public_api/v4/pages_domain/detail.json

@@ -6,6 +6,7 @@
     "verified": { "type": "boolean" },
     "verification_code": { "type": ["string", "null"] },
     "enabled_until": { "type": ["date", "null"] },
+    "auto_ssl_enabled": { "type": "boolean" },
     "certificate": {
       "type": "object",
       "properties": {
@@ -18,6 +19,6 @@
       "additionalProperties": false
     }
   },
-  "required": ["domain", "url", "verified", "verification_code", "enabled_until"],
+  "required": ["domain", "url", "verified", "verification_code", "enabled_until", "auto_ssl_enabled"],
   "additionalProperties": false
 }

spec/requests/api/pages_domains_spec.rb

@@ -3,15 +3,20 @@
 require 'spec_helper'
 
 describe API::PagesDomains do
-  set(:project) { create(:project, path: 'my.project', pages_https_only: false) }
-  set(:user) { create(:user) }
-  set(:admin) { create(:admin) }
+  let_it_be(:project) { create(:project, path: 'my.project', pages_https_only: false) }
+  let_it_be(:user) { create(:user) }
+  let_it_be(:admin) { create(:admin) }
 
-  set(:pages_domain) { create(:pages_domain, :without_key, :without_certificate, domain: 'www.domain.test', project: project) }
-  set(:pages_domain_secure) { create(:pages_domain, domain: 'ssl.domain.test', project: project) }
-  set(:pages_domain_expired) { create(:pages_domain, :with_expired_certificate, domain: 'expired.domain.test', project: project) }
+  let_it_be(:pages_domain) { create(:pages_domain, :without_key, :without_certificate, domain: 'www.domain.test', project: project) }
+  let_it_be(:pages_domain_secure) { create(:pages_domain, domain: 'ssl.domain.test', project: project) }
+  let_it_be(:pages_domain_with_letsencrypt) { create(:pages_domain, :letsencrypt, domain: 'letsencrypt.domain.test', project: project) }
+  let_it_be(:pages_domain_expired) { create(:pages_domain, :with_expired_certificate, domain: 'expired.domain.test', project: project) }
 
   let(:pages_domain_params) { build(:pages_domain, :without_key, :without_certificate, domain: 'www.other-domain.test').slice(:domain) }
+  let(:pages_domain_with_letsencrypt_params) do
+    build(:pages_domain, :without_key, :without_certificate, domain: 'www.other-domain.test', auto_ssl_enabled: true)
+      .slice(:domain, :auto_ssl_enabled)
+  end
   let(:pages_domain_secure_params) { build(:pages_domain, domain: 'ssl.other-domain.test', project: project).slice(:domain, :certificate, :key) }
   let(:pages_domain_secure_key_missmatch_params) {build(:pages_domain, :with_trusted_chain, project: project).slice(:domain, :certificate, :key) }
   let(:pages_domain_secure_missing_chain_params) {build(:pages_domain, :with_missing_chain, project: project).slice(:certificate) }
@@ -22,6 +27,7 @@ describe API::PagesDomains do
   let(:route_secure_domain) { "/projects/#{project.id}/pages/domains/#{pages_domain_secure.domain}" }
   let(:route_expired_domain) { "/projects/#{project.id}/pages/domains/#{pages_domain_expired.domain}" }
   let(:route_vacant_domain) { "/projects/#{project.id}/pages/domains/www.vacant-domain.test" }
+  let(:route_letsencrypt_domain) { "/projects/#{project.id}/pages/domains/#{pages_domain_with_letsencrypt.domain}" }
 
   before do
     allow(Gitlab.config.pages).to receive(:enabled).and_return(true)
@@ -47,9 +53,10 @@ describe API::PagesDomains do
           expect(response).to match_response_schema('public_api/v4/pages_domain_basics')
           expect(response).to include_pagination_headers
           expect(json_response).to be_an Array
-          expect(json_response.size).to eq(3)
+          expect(json_response.size).to eq(4)
           expect(json_response.last).to have_key('domain')
           expect(json_response.last).to have_key('project_id')
+          expect(json_response.last).to have_key('auto_ssl_enabled')
           expect(json_response.last).to have_key('certificate_expiration')
           expect(json_response.last['certificate_expiration']['expired']).to be true
           expect(json_response.first).not_to have_key('certificate_expiration')
@@ -73,7 +80,7 @@ describe API::PagesDomains do
         expect(response).to match_response_schema('public_api/v4/pages_domains')
         expect(response).to include_pagination_headers
         expect(json_response).to be_an Array
-        expect(json_response.size).to eq(3)
+        expect(json_response.size).to eq(4)
         expect(json_response.map { |pages_domain| pages_domain['domain'] }).to include(pages_domain.domain)
         expect(json_response.last).to have_key('domain')
       end
@@ -166,6 +173,7 @@ describe API::PagesDomains do
         expect(json_response['url']).to eq(pages_domain_secure.url)
         expect(json_response['certificate']['subject']).to eq(pages_domain_secure.subject)
         expect(json_response['certificate']['expired']).to be false
+        expect(json_response['auto_ssl_enabled']).to be false
       end
 
       it 'returns pages domain with an expired certificate' do
@@ -175,6 +183,18 @@ describe API::PagesDomains do
         expect(response).to match_response_schema('public_api/v4/pages_domain/detail')
         expect(json_response['certificate']['expired']).to be true
       end
+
+      it 'returns pages domain with letsencrypt' do
+        get api(route_letsencrypt_domain, user)
+
+        expect(response).to have_gitlab_http_status(200)
+        expect(response).to match_response_schema('public_api/v4/pages_domain/detail')
+        expect(json_response['domain']).to eq(pages_domain_with_letsencrypt.domain)
+        expect(json_response['url']).to eq(pages_domain_with_letsencrypt.url)
+        expect(json_response['certificate']['subject']).to eq(pages_domain_with_letsencrypt.subject)
+        expect(json_response['certificate']['expired']).to be false
+        expect(json_response['auto_ssl_enabled']).to be true
+      end
     end
 
     context 'when domain is vacant' do
@@ -246,6 +266,7 @@ describe API::PagesDomains do
         expect(pages_domain.domain).to eq(params[:domain])
         expect(pages_domain.certificate).to be_nil
         expect(pages_domain.key).to be_nil
+        expect(pages_domain.auto_ssl_enabled).to be false
       end
 
       it 'creates a new secure pages domain' do
@@ -257,6 +278,29 @@ describe API::PagesDomains do
         expect(pages_domain.domain).to eq(params_secure[:domain])
         expect(pages_domain.certificate).to eq(params_secure[:certificate])
         expect(pages_domain.key).to eq(params_secure[:key])
+        expect(pages_domain.auto_ssl_enabled).to be false
+      end
+
+      it 'creates domain with letsencrypt enabled' do
+        post api(route, user), params: pages_domain_with_letsencrypt_params
+        pages_domain = PagesDomain.find_by(domain: json_response['domain'])
+
+        expect(response).to have_gitlab_http_status(201)
+        expect(response).to match_response_schema('public_api/v4/pages_domain/detail')
+        expect(pages_domain.domain).to eq(pages_domain_with_letsencrypt_params[:domain])
+        expect(pages_domain.auto_ssl_enabled).to be true
+      end
+
+      it 'creates domain with letsencrypt enabled and provided certificate' do
+        post api(route, user), params: params_secure.merge(auto_ssl_enabled: true)
+        pages_domain = PagesDomain.find_by(domain: json_response['domain'])
+
+        expect(response).to have_gitlab_http_status(201)
+        expect(response).to match_response_schema('public_api/v4/pages_domain/detail')
+        expect(pages_domain.domain).to eq(params_secure[:domain])
+        expect(pages_domain.certificate).to eq(params_secure[:certificate])
+        expect(pages_domain.key).to eq(params_secure[:key])
+        expect(pages_domain.auto_ssl_enabled).to be true
       end
 
       it 'fails to create pages domain without key' do
@@ -323,13 +367,14 @@ describe API::PagesDomains do
 
     shared_examples_for 'put pages domain' do
       it 'updates pages domain removing certificate' do
-        put api(route_secure_domain, user)
+        put api(route_secure_domain, user), params: { certificate: nil, key: nil }
         pages_domain_secure.reload
 
         expect(response).to have_gitlab_http_status(200)
         expect(response).to match_response_schema('public_api/v4/pages_domain/detail')
         expect(pages_domain_secure.certificate).to be_nil
         expect(pages_domain_secure.key).to be_nil
+        expect(pages_domain_secure.auto_ssl_enabled).to be false
       end
 
       it 'updates pages domain adding certificate' do
@@ -342,6 +387,37 @@ describe API::PagesDomains do
         expect(pages_domain.key).to eq(params_secure[:key])
       end
 
+      it 'updates pages domain adding certificate with letsencrypt' do
+        put api(route_domain, user), params: params_secure.merge(auto_ssl_enabled: true)
+        pages_domain.reload
+
+        expect(response).to have_gitlab_http_status(200)
+        expect(response).to match_response_schema('public_api/v4/pages_domain/detail')
+        expect(pages_domain.certificate).to eq(params_secure[:certificate])
+        expect(pages_domain.key).to eq(params_secure[:key])
+        expect(pages_domain.auto_ssl_enabled).to be true
+      end
+
+      it 'updates pages domain enabling letsencrypt' do
+        put api(route_domain, user), params: { auto_ssl_enabled: true }
+        pages_domain.reload
+
+        expect(response).to have_gitlab_http_status(200)
+        expect(response).to match_response_schema('public_api/v4/pages_domain/detail')
+        expect(pages_domain.auto_ssl_enabled).to be true
+      end
+
+      it 'updates pages domain disabling letsencrypt while preserving the certificate' do
+        put api(route_letsencrypt_domain, user), params: { auto_ssl_enabled: false }
+        pages_domain_with_letsencrypt.reload
+
+        expect(response).to have_gitlab_http_status(200)
+        expect(response).to match_response_schema('public_api/v4/pages_domain/detail')
+        expect(pages_domain_with_letsencrypt.auto_ssl_enabled).to be false
+        expect(pages_domain_with_letsencrypt.key).to be
+        expect(pages_domain_with_letsencrypt.certificate).to be
+      end
+
       it 'updates pages domain with expired certificate' do
         put api(route_expired_domain, user), params: params_secure
         pages_domain_expired.reload