Merge branch 'ce-to-ee-2018-09-04' into 'master'

CE upstream - 2018-09-04 06:21 UTC

See merge request gitlab-org/gitlab-ee!7221

.flayignore

-*.erb
-lib/gitlab/sanitizers/svg/whitelist.rb
-lib/gitlab/diff/position_tracer.rb
-app/controllers/projects/approver_groups_controller.rb
-app/controllers/projects/approvers_controller.rb
-app/controllers/projects/protected_branches/merge_access_levels_controller.rb
-app/controllers/projects/protected_branches/push_access_levels_controller.rb
-app/controllers/projects/protected_tags/create_access_levels_controller.rb
-app/helpers/system_note_helper.rb
-app/policies/project_policy.rb
-app/models/concerns/relative_positioning.rb
-app/workers/stuck_merge_jobs_worker.rb
-lib/gitlab/redis/*.rb
-lib/gitlab/gitaly_client/operation_service.rb
-app/models/project_services/packagist_service.rb
-lib/gitlab/background_migration/normalize_ldap_extern_uids_range.rb
-lib/gitlab/background_migration/*
-app/models/project_services/kubernetes_service.rb
-lib/gitlab/workhorse.rb
-lib/gitlab/ci/trace/chunked_io.rb
-lib/gitlab/gitaly_client/ref_service.rb
-lib/gitlab/gitaly_client/commit_service.rb
-lib/gitlab/git/commit.rb
-lib/gitlab/git/tag.rb
-
-ee/db/**/*
-ee/app/serializers/ee/merge_request_widget_entity.rb
-ee/lib/api/epics.rb
-ee/lib/api/geo_nodes.rb
-ee/lib/ee/api/group_boards.rb
-ee/lib/ee/api/boards.rb
-ee/lib/ee/gitlab/ldap/sync/admin_users.rb
-ee/app/workers/geo/file_download_dispatch_worker/job_artifact_job_finder.rb
-ee/app/workers/geo/file_download_dispatch_worker/lfs_object_job_finder.rb
-ee/spec/**/*

Gemfile

@@ -375,7 +375,6 @@ group :development, :test do
   gem 'scss_lint', '~> 0.56.0', require: false
   gem 'haml_lint', '~> 0.26.0', require: false
   gem 'simplecov', '~> 0.14.0', require: false
-  gem 'flay', '~> 2.10.0', require: false
   gem 'bundler-audit', '~> 0.5.0', require: false
 
   gem 'benchmark-ips', '~> 2.3.0', require: false

Gemfile.lock

@@ -233,11 +233,6 @@ GEM
     fast_gettext (1.6.0)
     ffaker (2.4.0)
     ffi (1.9.18)
-    flay (2.10.0)
-      erubis (~> 2.7.0)
-      path_expander (~> 1.0)
-      ruby_parser (~> 3.0)
-      sexp_processor (~> 4.0)
     flipper (0.13.0)
     flipper-active_record (0.13.0)
       activerecord (>= 3.2, < 6)
@@ -617,7 +612,6 @@ GEM
     parser (2.5.1.0)
       ast (~> 2.4.0)
     parslet (1.8.2)
-    path_expander (1.0.2)
     peek (1.0.1)
       concurrent-ruby (>= 0.9.0)
       concurrent-ruby-ext (>= 0.9.0)
@@ -1058,7 +1052,6 @@ DEPENDENCIES
   faraday_middleware-aws-signers-v4
   fast_blank
   ffaker (~> 2.4)
-  flay (~> 2.10.0)
   flipper (~> 0.13.0)
   flipper-active_record (~> 0.13.0)
   flipper-active_support_cache_store (~> 0.13.0)

Gemfile.rails5.lock

@@ -236,11 +236,6 @@ GEM
     fast_gettext (1.6.0)
     ffaker (2.4.0)
     ffi (1.9.18)
-    flay (2.10.0)
-      erubis (~> 2.7.0)
-      path_expander (~> 1.0)
-      ruby_parser (~> 3.0)
-      sexp_processor (~> 4.0)
     flipper (0.13.0)
     flipper-active_record (0.13.0)
       activerecord (>= 3.2, < 6)
@@ -621,7 +616,6 @@ GEM
     parser (2.5.1.0)
       ast (~> 2.4.0)
     parslet (1.8.2)
-    path_expander (1.0.2)
     peek (1.0.1)
       concurrent-ruby (>= 0.9.0)
       concurrent-ruby-ext (>= 0.9.0)
@@ -1068,7 +1062,6 @@ DEPENDENCIES
   faraday_middleware-aws-signers-v4
   fast_blank
   ffaker (~> 2.4)
-  flay (~> 2.10.0)
   flipper (~> 0.13.0)
   flipper-active_record (~> 0.13.0)
   flipper-active_support_cache_store (~> 0.13.0)

doc/api/commits.md

@@ -464,7 +464,7 @@ Example response:
    },
    {
       "started_at" : null,
-      "name" : "flay",
+      "name" : "test",
       "allow_failure" : false,
       "status" : "pending",
       "created_at" : "2016-01-19T08:40:25.832Z",

doc/ci/caching/index.md

@@ -14,6 +14,64 @@ starting from GitLab 9.0.
 Make sure you read the [`cache` reference](../yaml/README.md#cache) to learn
 how it is defined in `.gitlab-ci.yml`.
 
+## Cache vs artifacts
+
+NOTE: **Note:**
+Be careful if you use cache and artifacts to store the same path in your jobs
+as **caches are restored before artifacts** and the content would be overwritten.
+
+Don't mix the caching with passing artifacts between stages. Caching is not
+designed to pass artifacts between stages. Cache is for runtime dependencies
+needed to compile the project:
+
+- `cache` - **Use for temporary storage for project dependencies.** Not useful
+  for keeping intermediate build results, like `jar` or `apk` files.
+  Cache was designed to be used to speed up invocations of subsequent runs of a
+  given job, by keeping things like dependencies (e.g., npm packages, Go vendor
+  packages, etc.) so they don't have to be re-fetched from the public internet.
+  While the cache can be abused to pass intermediate build results between stages,
+  there may be cases where artifacts are a better fit.
+- `artifacts` - **Use for stage results that will be passed between stages.**
+  Artifacts were designed to upload some compiled/generated bits of the build,
+  and they can be fetched by any number of concurrent Runners. They are
+  guaranteed to be available and are there to pass data between jobs. They are
+  also exposed to be downloaded from the UI. **Artifacts can only exist in
+  directories relative to the build directory** and specifying paths which don't
+  comply to this rule trigger an unintuitive and illogical error message (an
+  enhancement is discussed at
+  https://gitlab.com/gitlab-org/gitlab-ce/issues/15530). Artifacts need to be
+  uploaded to the GitLab instance (not only the GitLab runner) before the next
+  stage job(s) can start, so you need to evaluate carefully whether your
+  bandwidth allows you to profit from parallelization with stages and shared
+  artifacts before investing time in changes to the setup.
+
+It's sometimes confusing because the name artifact sounds like something that
+is only useful outside of the job, like for downloading a final image. But
+artifacts are also available in between stages within a pipeline. So if you
+build your application by downloading all the required modules, you might want
+to declare them as artifacts so that each subsequent stage can depend on them
+being there. There are some optimizations like declaring an
+[expiry time](../yaml/README.md#artifacts-expire_in) so you don't keep artifacts
+around too long, and using [dependencies](../yaml/README.md#dependencies) to
+control exactly where artifacts are passed around.
+
+In summary:
+
+- Caches are disabled if not defined globally or per job (using `cache:`)
+- Caches are available for all jobs in your `.gitlab-ci.yml` if enabled globally
+- Caches can be used by subsequent pipelines of that very same job (a script in
+  a stage) in which the cache was created (if not defined globally).
+- Caches are stored where the Runner is installed **and** uploaded to S3 if
+  [distributed cache is enabled](https://docs.gitlab.com/runner/configuration/autoscale.html#distributed-runners-caching)
+- Caches defined per job are only used either a) for the next pipeline of that job,
+  or b) if that same cache is also defined in a subsequent job of the same pipeline
+- Artifacts are disabled if not defined per job (using `artifacts:`)
+- Artifacts can only be enabled per job, not globally
+- Artifacts are created during a pipeline and can be used by the subsequent
+  jobs of that currently active pipeline
+- Artifacts are always uploaded to GitLab (known as coordinator)
+- Artifacts can have an expiration value for controlling disk usage (30 days by default).
+
 ## Good caching practices
 
 We have the cache from the perspective of the developers (who consume a cache
@@ -467,60 +525,3 @@ Behind the scenes, this works by increasing a counter in the database, and the
 value of that counter is used to create the key for the cache by appending an
 integer to it: `-1`, `-2`, etc. After a push, a new key is generated and the
 old cache is not valid anymore.
-
-## Cache vs artifacts
-
-NOTE: **Note:**
-Be careful if you use cache and artifacts to store the same path in your jobs
-as **caches are restored before artifacts** and the content would be overwritten.
-
-Don't mix the caching with passing artifacts between stages. Caching is not
-designed to pass artifacts between stages. Cache is for runtime dependencies
-needed to compile the project:
-
-- `cache` - **Use for temporary storage for project dependencies.** Not useful
-  for keeping intermediate build results, like `jar` or `apk` files.
-  Cache was designed to be used to speed up invocations of subsequent runs of a
-  given job, by keeping things like dependencies (e.g., npm packages, Go vendor
-  packages, etc.) so they don't have to be re-fetched from the public internet.
-  While the cache can be abused to pass intermediate build results between stages,
-  there may be cases where artifacts are a better fit.
-- `artifacts` - **Use for stage results that will be passed between stages.**
-  Artifacts were designed to upload some compiled/generated bits of the build,
-  and they can be fetched by any number of concurrent Runners. They are
-  guaranteed to be available and are there to pass data between jobs. They are
-  also exposed to be downloaded from the UI. **Artifacts can only exist in
-  directories relative to the build directory** and specifying paths which don't
-  comply to this rule trigger an unintuitive and unlogical error message (an
-  enhancement is discussed at
-  https://gitlab.com/gitlab-org/gitlab-ce/issues/15530). Artifacts need to be
-  uploaded to the GitLab instance (not only the GitLab runner) before the next
-  stage job(s) can start, so you need to evaluate carefully whether your
-  bandwidth allows you to profit from parallelization with stages and shared
-  artifacts before investing time in changes to the setup.
-
-It's sometimes confusing because the name artifact sounds like something that
-is only useful outside of the job, like for downloading a final image. But
-artifacts are also available in between stages within a pipeline. So if you
-build your application by downloading all the required modules, you might want
-to declare them as artifacts so that each subsequent stage can depend on them
-being there. There are some optimizations like declaring an
-[expiry time](../yaml/README.md#artifacts-expire_in) so you don't keep artifacts
-around too long, and using [dependencies](../yaml/README.md#dependencies) to
-control exactly where artifacts are passed around.
-
-So, to sum up:
-- Caches are disabled if not defined globally or per job (using `cache:`)
-- Caches are available for all jobs in your `.gitlab-ci.yml` if enabled globally
-- Caches can be used by subsequent pipelines of that very same job (a script in
-  a stage) in which the cache was created (if not defined globally).
-- Caches are stored where the Runner is installed **and** uploaded to S3 if
-  [distributed cache is enabled](https://docs.gitlab.com/runner/configuration/autoscale.html#distributed-runners-caching)
-- Caches defined per job are only used either a) for the next pipeline of that job,
-  or b) if that same cache is also defined in a subsequent job of the same pipeline
-- Artifacts are disabled if not defined per job (using `artifacts:`)
-- Artifacts can only be enabled per job, not globally
-- Artifacts are created during a pipeline and can be used by the subsequent
-  jobs of that currently active pipeline
-- Artifacts are always uploaded to GitLab (known as coordinator)
-- Artifacts can have an expiration value for controlling disk usage (30 days by default)

doc/install/installation.md

@@ -12,7 +12,7 @@ Since installations from source don't have Runit, Sidekiq can't be terminated an
 
 ## Select Version to Install
 
-Make sure you view [this installation guide](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/doc/install/installation.md) from the branch (version) of GitLab you would like to install (e.g., `11-2-stable`).
+Make sure you view [this installation guide](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/doc/install/installation.md) from the branch (version) of GitLab you would like to install (e.g., `11-3-stable`).
 You can select the branch in the version dropdown in the top left corner of GitLab (below the menu bar).
 
 If the highest number stable branch is unclear please check the [GitLab Blog](https://about.gitlab.com/blog/) for installation guide links by version.
@@ -300,9 +300,9 @@ sudo usermod -aG redis git
 ### Clone the Source
 
     # Clone GitLab repository
-    sudo -u git -H git clone https://gitlab.com/gitlab-org/gitlab-ce.git -b 11-2-stable gitlab
+    sudo -u git -H git clone https://gitlab.com/gitlab-org/gitlab-ce.git -b 11-3-stable gitlab
 
-**Note:** You can change `11-2-stable` to `master` if you want the *bleeding edge* version, but never install master on a production server!
+**Note:** You can change `11-3-stable` to `master` if you want the *bleeding edge* version, but never install master on a production server!
 
 ### Configure It
 
@@ -483,7 +483,7 @@ For more information about configuring Gitaly see
 
     sudo -u git -H bundle exec rake gitlab:setup RAILS_ENV=production
     # Type 'yes' to create the database tables.
-    
+
     # or you can skip the question by adding force=yes
     sudo -u git -H bundle exec rake gitlab:setup RAILS_ENV=production force=yes
 

doc/user/gitlab_com/index.md

@@ -74,22 +74,17 @@ or over the size limit, you can [reduce your repository size with Git](../projec
 ## Shared Runners
 
 Shared Runners on GitLab.com run in [autoscale mode] and powered by
-Google Cloud Platform and DigitalOcean. Autoscaling means reduced
+Google Cloud Platform. Autoscaling means reduced
 waiting times to spin up CI/CD jobs, and isolated VMs for each project,
 thus maximizing security.
 They're free to use for public open source projects and limited to 2000 CI
 minutes per month per group for private projects. Read about all
 [GitLab.com plans](https://about.gitlab.com/pricing/).
 
-In case of DigitalOcean based Runners, all your CI/CD jobs run on ephemeral
-instances with 2GB of RAM, CoreOS and the latest Docker Engine installed.
-Instances provide 2 vCPUs and 60GB of SSD disk space. The default region of the
-VMs is NYC1.
-
-In case of Google Cloud Platform based Runners, all your CI/CD jobs run on
-ephemeral instances with 3.75GB of RAM, CoreOS and the latest Docker Engine
+All your CI/CD jobs run on [n1-standard-1 instances](https://cloud.google.com/compute/docs/machine-types) with 3.75GB of RAM, CoreOS and the latest Docker Engine
 installed. Instances provide 1 vCPU and 25GB of HDD disk space. The default
 region of the VMs is US East1.
+Each instance is used only for one job, this ensures any sensitive data left on the system can't be accessed by other people their CI jobs.
 
 Jobs handled by the shared Runners on GitLab.com (`shared-runners-manager-X.gitlab.com`),
 **will be timed out after 3 hours**, regardless of the timeout configured in a

lib/tasks/flay.rake

-desc 'Code duplication analyze via flay'
-task :flay do
-  output = `bundle exec flay --mass 35 app/ lib/gitlab/ ee/ 2> #{File::NULL}`
-
-  if output.include?("Similar code found") || output.include?("IDENTICAL code found")
-    puts output
-    exit 1
-  end
-end

lib/tasks/lint.rake

@@ -34,7 +34,6 @@ unless Rails.env.production?
         config_lint
         lint:haml
         scss_lint
-        flay
         gettext:lint
         gettext:updated_check
         lint:static_verification