Merge remote-tracking branch 'upstream/master' into ce-to-ee-2018-03-19

Gemfile

@@ -218,7 +218,7 @@ gem 'asana', '~> 0.6.0'
 gem 'ruby-fogbugz', '~> 0.2.1'
 
 # Kubernetes integration
-gem 'kubeclient', '~> 2.2.0'
+gem 'kubeclient', '~> 3.0'
 
 # d3
 gem 'd3_rails', '~> 3.5.0'

Gemfile.lock

@@ -182,7 +182,7 @@ GEM
     diff-lcs (1.3)
     diffy (3.1.0)
     docile (1.1.5)
-    domain_name (0.5.20161021)
+    domain_name (0.5.20170404)
       unf (>= 0.0.5, < 1.0.0)
     doorkeeper (4.2.6)
       railties (>= 4.2)
@@ -430,14 +430,14 @@ GEM
     html2text (0.2.0)
       nokogiri (~> 1.6)
     htmlentities (4.3.4)
-    http (0.9.8)
+    http (2.2.2)
       addressable (~> 2.3)
       http-cookie (~> 1.0)
       http-form_data (~> 1.0.1)
       http_parser.rb (~> 0.6.0)
     http-cookie (1.0.3)
       domain_name (~> 0.5)
-    http-form_data (1.0.1)
+    http-form_data (1.0.3)
     http_parser.rb (0.6.0)
     httparty (0.13.7)
       json (~> 1.8)
@@ -481,10 +481,11 @@ GEM
     kgio (2.10.0)
     knapsack (1.16.0)
       rake
-    kubeclient (2.2.0)
-      http (= 0.9.8)
-      recursive-open-struct (= 1.0.0)
-      rest-client
+      timecop (>= 0.1.0)
+    kubeclient (3.0.0)
+      http (~> 2.2.2)
+      recursive-open-struct (~> 1.0.4)
+      rest-client (~> 2.0)
     launchy (2.4.3)
       addressable (~> 2.3)
     letter_opener (1.4.1)
@@ -738,7 +739,7 @@ GEM
     re2 (1.1.1)
     recaptcha (3.0.0)
       json
-    recursive-open-struct (1.0.0)
+    recursive-open-struct (1.0.5)
     redcarpet (3.4.0)
     redis (3.3.5)
     redis-actionpack (5.0.2)
@@ -766,7 +767,7 @@ GEM
     request_store (1.3.1)
     responders (2.3.0)
       railties (>= 4.2.0, < 5.1)
-    rest-client (2.0.0)
+    rest-client (2.0.2)
       http-cookie (>= 1.0.2, < 2.0)
       mime-types (>= 1.16, < 4.0)
       netrc (~> 0.8)
@@ -967,7 +968,7 @@ GEM
       json (>= 1.8.0)
     unf (0.1.4)
       unf_ext
-    unf_ext (0.0.7.4)
+    unf_ext (0.0.7.5)
     unicode-display_width (1.3.0)
     unicorn (5.1.0)
       kgio (~> 2.6)
@@ -1126,7 +1127,7 @@ DEPENDENCIES
   jwt (~> 1.5.6)
   kaminari (~> 1.0)
   knapsack (~> 1.16)
-  kubeclient (~> 2.2.0)
+  kubeclient (~> 3.0)
   letter_opener_web (~> 1.3.0)
   license_finder (~> 3.1)
   licensee (~> 8.7.0)

app/models/clusters/platforms/kubernetes.rb

@@ -136,7 +136,7 @@ module Clusters
         kubeclient = build_kubeclient!
 
         kubeclient.get_pods(namespace: actual_namespace).as_json
-      rescue KubeException => err
+      rescue Kubeclient::HttpError => err
         raise err unless err.error_code == 404
 
         []

app/models/project_services/kubernetes_service.rb

@@ -199,7 +199,7 @@ class KubernetesService < DeploymentService
     kubeclient = build_kubeclient!
 
     kubeclient.get_pods(namespace: actual_namespace).as_json
-  rescue KubeException => err
+  rescue Kubeclient::HttpError => err
     raise err unless err.error_code == 404
 
     []

app/services/ci/fetch_kubernetes_token_service.rb

@@ -32,7 +32,7 @@ module Ci
       kubeclient = build_kubeclient!
 
       kubeclient.get_secrets.as_json
-    rescue KubeException => err
+    rescue Kubeclient::HttpError => err
       raise err unless err.error_code == 404
 
       []

app/services/clusters/applications/check_installation_progress_service.rb

@@ -12,7 +12,7 @@ module Clusters
         else
           check_timeout
         end
-      rescue KubeException => ke
+      rescue Kubeclient::HttpError => ke
         app.make_errored!("Kubernetes error: #{ke.message}") unless app.errored?
       end
 

app/services/clusters/applications/install_service.rb

@@ -10,7 +10,7 @@ module Clusters
 
           ClusterWaitForAppInstallationWorker.perform_in(
             ClusterWaitForAppInstallationWorker::INTERVAL, app.name, app.id)
-        rescue KubeException => ke
+        rescue Kubeclient::HttpError => ke
           app.make_errored!("Kubernetes error: #{ke.message}")
         rescue StandardError
           app.make_errored!("Can't start installation process")

changelogs/unreleased/update-gitlab-ci-yml-services-docs.yml

+---
+title: Update CI services documnetation
+merge_request: 17749
+author:
+type: other

doc/ci/docker/using_docker_images.md

@@ -58,7 +58,7 @@ your job and is linked to the Docker image that the `image` keyword defines.
 This allows you to access the service image during build time.
 
 The service image can run any application, but the most common use case is to
-run a database container, eg. `mysql`. It's easier and faster to use an
+run a database container, e.g., `mysql`. It's easier and faster to use an
 existing image and run it as an additional container than install `mysql` every
 time the project is built.
 
@@ -83,6 +83,67 @@ So, in order to access your database service you have to connect to the host
 named `mysql` instead of a socket or `localhost`. Read more in [accessing the
 services](#accessing-the-services).
 
+### How the health check of services works
+
+Services are designed to provide additional functionality which is **network accessible**.
+It may be a database like MySQL, or Redis, and even `docker:dind` which
+allows you to use Docker in Docker. It can be practically anything that is
+required for the CI/CD job to proceed and is accessed by network.
+
+To make sure this works, the Runner:
+
+1. checks which ports are exposed from the container by default
+1. starts a special container that waits for these ports to be accessible
+
+When the second stage of the check fails, either because there is no opened port in the
+service, or the service was not started properly before the timeout and the port is not
+responding, it prints the warning: `*** WARNING: Service XYZ probably didn't start properly`.
+
+In most cases it will affect the job, but there may be situations when the job
+will still succeed even if that warning was printed. For example:
+
+- The service was started a little after the warning was raised, and the job is
+  not using the linked service from the very beginning. In that case, when the
+  job needed to access the service, it may have been already there waiting for
+  connections.
+- The service container is not providing any networking service, but it's doing
+  something with the job's directory (all services have the job directory mounted
+  as a volume under `/builds`). In that case, the service will do its job, and
+  since the job is not trying to connect to it, it won't fail.
+
+### What services are not for
+
+As it was mentioned before, this feature is designed to provide **network accessible**
+services. A database is the simplest example of such a service.
+
+NOTE: **Note:**
+The services feature is not designed to, and will not add any software from the
+defined `services` image(s) to the job's container.
+
+For example, if you have the following `services` defined in your job, the `php`,
+`node` or `go` commands will **not** be available for your script, and thus
+the job will fail:
+
+```yaml
+job:
+  services:
+  - php:7
+  - node:latest
+  - golang:1.10
+  image: alpine:3.7
+  script:
+  - php -v
+  - node -v
+  - go version
+```
+
+If you need to have `php`, `node` and `go` available for your script, you should
+either:
+
+- choose an existing Docker image that contains all required tools, or
+- create your own Docker image, which will have all the required tools included
+  and use that in your job
+
 ### Accessing the services
 
 Let's say that you need a Wordpress instance to test some API integration with

doc/development/testing_guide/end_to_end_tests.md

@@ -22,7 +22,7 @@ You can find these nightly pipelines at [GitLab QA pipelines page][gitlab-qa-pip
 
 It is possible to run end-to-end tests (eventually being run within a
 [GitLab QA pipeline][gitlab-qa-pipelines]) for a merge request by triggering
-the `package-qa` manual action, that should be present in a merge request
+the `package-and-qa` manual action, that should be present in a merge request
 widget.
 
 Manual action that starts end-to-end tests is also available in merge requests

lib/gitlab/kubernetes/namespace.rb

@@ -10,7 +10,7 @@ module Gitlab
 
       def exists?
         @client.get_namespace(name)
-      rescue ::KubeException => ke
+      rescue ::Kubeclient::HttpError => ke
         raise ke unless ke.error_code == 404
 
         false

qa/README.md

@@ -26,7 +26,7 @@ and corresponding views / partials / selectors in CE / EE.
 
 Whenever `qa:selectors` job fails in your merge request, you are supposed to
 fix [page objects](qa/page/README.md). You should also trigger end-to-end tests
-using `package-qa` manual action, to test if everything works fine.
+using `package-and-qa` manual action, to test if everything works fine.
 
 ## How can I use it?
 

qa/qa/page/README.md

@@ -40,7 +40,7 @@ the time it would take to build packages and test everything.
 That is why when someone changes `t.text_field :login` to
 `t.text_field :username` in the _new session_ view we won't know about this
 change until our GitLab QA nightly pipeline fails, or until someone triggers
-`package-qa` action in their merge request.
+`package-and-qa` action in their merge request.
 
 Obviously such a change would break all tests. We call this problem a _fragile
 tests problem_.

spec/lib/gitlab/kubernetes/namespace_spec.rb

@@ -9,7 +9,7 @@ describe Gitlab::Kubernetes::Namespace do
 
   describe '#exists?' do
     context 'when namespace do not exits' do
-      let(:exception) { ::KubeException.new(404, "namespace #{name} not found", nil) }
+      let(:exception) { ::Kubeclient::HttpError.new(404, "namespace #{name} not found", nil) }
 
       it 'returns false' do
         expect(client).to receive(:get_namespace).with(name).once.and_raise(exception)

spec/models/clusters/platforms/kubernetes_spec.rb

@@ -254,7 +254,7 @@ describe Clusters::Platforms::Kubernetes, :use_clean_rails_memory_store_caching
         stub_kubeclient_deployments(status: 500)
       end
 
-      it { expect { subject }.to raise_error(KubeException) }
+      it { expect { subject }.to raise_error(Kubeclient::HttpError) }
     end
 
     context 'when kubernetes responds with 404s' do

spec/models/project_services/kubernetes_service_spec.rb

@@ -372,7 +372,7 @@ describe KubernetesService, :use_clean_rails_memory_store_caching do
         stub_kubeclient_deployments(status: 500)
       end
 
-      it { expect { subject }.to raise_error(KubeException) }
+      it { expect { subject }.to raise_error(Kubeclient::HttpError) }
     end
 
     context 'when kubernetes responds with 404s' do

spec/services/clusters/applications/install_service_spec.rb

@@ -34,7 +34,7 @@ describe Clusters::Applications::InstallService do
 
     context 'when k8s cluster communication fails' do
       before do
-        error = KubeException.new(500, 'system failure', nil)
+        error = Kubeclient::HttpError.new(500, 'system failure', nil)
         expect(helm_client).to receive(:install).with(install_command).and_raise(error)
       end