Merge branch 'ce-to-ee-2018-02-08' into 'master'

CE upstream - 2018-02-08 15:28 UTC

See merge request gitlab-org/gitlab-ee!4451

app/assets/javascripts/boards/components/new_list_dropdown.js

 /* eslint-disable func-names, no-new, space-before-function-paren, one-var,
    promise/catch-or-return */
+import axios from '~/lib/utils/axios_utils';
 import _ from 'underscore';
 import CreateLabelDropdown from '../../create_label';
 
@@ -28,9 +29,9 @@ gl.issueBoards.newListDropdownInit = () => {
 
     $this.glDropdown({
       data(term, callback) {
-        $.get($this.attr('data-list-labels-path'))
-          .then((resp) => {
-            callback(resp);
+        axios.get($this.attr('data-list-labels-path'))
+          .then(({ data }) => {
+            callback(data);
           });
       },
       renderRow (label) {

app/assets/javascripts/dispatcher.js

 /* eslint-disable func-names, space-before-function-paren, no-var, prefer-arrow-callback, wrap-iife, no-shadow, consistent-return, one-var, one-var-declaration-per-line, camelcase, default-case, no-new, quotes, no-duplicate-case, no-case-declarations, no-fallthrough, max-len */
-import MergeRequest from './merge_request';
 import Flash from './flash';
 import GfmAutoComplete from './gfm_auto_complete';
-import ZenMode from './zen_mode';
-import initNotes from './init_notes';
-import initIssuableSidebar from './init_issuable_sidebar';
 import { convertPermissionToBoolean } from './lib/utils/common_utils';
 import GlFieldErrors from './gl_field_errors';
 import Shortcuts from './shortcuts';
-import ShortcutsIssuable from './shortcuts_issuable';
-import Diff from './diff';
 import SearchAutocomplete from './search_autocomplete';
 
 // EE-only
 import UsersSelect from './users_select';
 import UserCallout from './user_callout';
+import ZenMode from './zen_mode';
 import initCompareAutocomplete from './compare_autocomplete';
 import initGeoInfoModal from 'ee/init_geo_info_modal'; // eslint-disable-line import/first
 import initGroupAnalytics from 'ee/init_group_analytics'; // eslint-disable-line import/first
@@ -301,17 +296,10 @@ var Dispatcher;
             .catch(fail);
           break;
         case 'projects:merge_requests:show':
-          new Diff();
-          new ZenMode();
-
-          initIssuableSidebar();
-          initNotes();
-
-          const mrShowNode = document.querySelector('.merge-request');
-          window.mergeRequest = new MergeRequest({
-            action: mrShowNode.dataset.mrAction,
-          });
-          shortcut_handler = new ShortcutsIssuable(true);
+          import('./pages/projects/merge_requests/show')
+            .then(callDefault)
+            .catch(fail);
+          shortcut_handler = true;
           break;
         case 'dashboard:activity':
           import('./pages/dashboard/activity')

app/assets/javascripts/gl_form.js

-/* global autosize */
-
+import autosize from 'autosize';
 import GfmAutoComplete from './gfm_auto_complete';
 import dropzoneInput from './dropzone_input';
 import textUtils from './lib/utils/text_markdown';

app/assets/javascripts/pages/projects/merge_requests/show/index.js

+import MergeRequest from '~/merge_request';
+import ZenMode from '~/zen_mode';
+import initNotes from '~/init_notes';
+import initIssuableSidebar from '~/init_issuable_sidebar';
+import ShortcutsIssuable from '~/shortcuts_issuable';
+import Diff from '~/diff';
+import { handleLocationHash } from '~/lib/utils/common_utils';
+
+export default () => {
+  new Diff(); // eslint-disable-line no-new
+  new ZenMode(); // eslint-disable-line no-new
+
+  initIssuableSidebar(); // eslint-disable-line no-new
+  initNotes(); // eslint-disable-line no-new
+
+  const mrShowNode = document.querySelector('.merge-request');
+
+  window.mergeRequest = new MergeRequest({
+    action: mrShowNode.dataset.mrAction,
+  });
+
+  new ShortcutsIssuable(true); // eslint-disable-line no-new
+  handleLocationHash();
+};

app/models/user.rb

@@ -569,7 +569,7 @@ class User < ActiveRecord::Base
     gpg_keys.each(&:update_invalid_gpg_signatures)
   end
 
-  # Returns the groups a user has access to
+  # Returns the groups a user has access to, either through a membership or a project authorization
   def authorized_groups
     union = Gitlab::SQL::Union
       .new([groups.select(:id), authorized_projects.select(:namespace_id)])
@@ -577,6 +577,11 @@ class User < ActiveRecord::Base
     Group.where("namespaces.id IN (#{union.to_sql})") # rubocop:disable GitlabSecurity/SqlInjection
   end
 
+  # Returns the groups a user is a member of, either directly or through a parent group
+  def membership_groups
+    Gitlab::GroupHierarchy.new(groups).base_and_descendants
+  end
+
   # Returns a relation of groups the user has access to, including their parent
   # and child groups (recursively).
   def all_expanded_groups

changelogs/unreleased/feature-oidc-groups-claim.yml

+---
+title: Add groups to OpenID Connect claims
+merge_request: 16929
+author: Hassan Zamani

config/initializers/doorkeeper_openid_connect.rb

@@ -31,6 +31,7 @@ Doorkeeper::OpenidConnect.configure do
       o.claim(:website)        { |user| user.full_website_url if user.website_url? }
       o.claim(:profile)        { |user| Gitlab::Routing.url_helpers.user_url user }
       o.claim(:picture)        { |user| user.avatar_url(only_path: false) }
+      o.claim(:groups)         { |user| user.membership_groups.map(&:full_path) }
     end
   end
 end

config/locales/doorkeeper.en.yml

@@ -68,7 +68,7 @@ en:
       read_user:
         Read-only access to the user's profile information, like username, public email and full name
       openid:
-        The ability to authenticate using GitLab, and read-only access to the user's profile information
+        The ability to authenticate using GitLab, and read-only access to the user's profile information and group memberships
       sudo:
         Access to the Sudo feature, to perform API actions as any user in the system (only available for admins)
     flash:

doc/development/i18n/externalization.md

@@ -126,6 +126,9 @@ strings and remove any strings that aren't used anymore. You should check this
 file in. Once the changes are on master, they will be picked up by
 [Crowdin](http://translate.gitlab.com) and be presented for translation.
 
+If there are merge conflicts in the `gitlab.pot` file, you can delete the file
+and regenerate it using the same command. Confirm that you are not deleting any strings accidentally by looking over the diff.
+
 The command also updates the translation files for each language: `locale/*/gitlab.po`
 These changes can be discarded, the languange files will be updated by Crowdin
 automatically.

doc/install/installation.md

@@ -299,9 +299,9 @@ sudo usermod -aG redis git
 ### Clone the Source
 
     # Clone GitLab repository
-    sudo -u git -H git clone https://gitlab.com/gitlab-org/gitlab-ce.git -b 10-4-stable gitlab
+    sudo -u git -H git clone https://gitlab.com/gitlab-org/gitlab-ce.git -b 10-5-stable gitlab
 
-**Note:** You can change `10-4-stable` to `master` if you want the *bleeding edge* version, but never install master on a production server!
+**Note:** You can change `10-5-stable` to `master` if you want the *bleeding edge* version, but never install master on a production server!
 
 ### Configure It
 

doc/integration/openid_connect_provider.md

@@ -39,6 +39,7 @@ Currently the following user information is shared with clients:
 | `website`        | `string`  | URL for the user's website
 | `profile`        | `string`  | URL for the user's GitLab profile
 | `picture`        | `string`  | URL for the user's GitLab avatar
+| `groups`         | `array`   | Names of the groups the user is a member of
 
 [OpenID Connect]: http://openid.net/connect/ "OpenID Connect website"
 [doorkeeper-openid_connect]: https://github.com/doorkeeper-gem/doorkeeper-openid_connect "Doorkeeper::OpenidConnect website"

doc/user/project/merge_requests/work_in_progress_merge_requests.md

@@ -7,7 +7,8 @@ have been marked a **Work In Progress**.
 ![Blocked Accept Button](img/wip_blocked_accept_button.png)
 
 To mark a merge request a Work In Progress, simply start its title with `[WIP]`
-or `WIP:`.
+or `WIP:`. As an alternative, you're also able to do it by sending a commit 
+with its title starting with `wip` or `WIP` to the merge request's source branch.
 
 ![Mark as WIP](img/wip_mark_as_wip.png)
 

doc/user/project/pages/getting_started_part_three.md

@@ -155,15 +155,40 @@ Certificates are NOT required to add to your custom
 (sub)domain on your GitLab Pages project, though they are
 highly recommendable.
 
-The importance of having any website securely served under HTTPS
-is explained on the introductory section of the blog post
-[Secure GitLab Pages with StartSSL](https://about.gitlab.com/2016/06/24/secure-gitlab-pages-with-startssl/#https-a-quick-overview).
+Let's start with an introduction to the importance of HTTPS.
+Alternatively, jump ahead to [adding certificates to your project](#adding-certificates-to-your-project).
 
-The reason why certificates are so important is that they encrypt
+#### Why should I care about HTTPS?
+
+This might be your first question. If our sites are hosted by GitLab Pages,
+they are static, hence we are not dealing with server-side scripts
+nor credit card transactions, then why do we need secure connections?
+
+Back in the 1990s, where HTTPS came out, [SSL](https://en.wikipedia.org/wiki/Transport_Layer_Security#SSL_1.0.2C_2.0_and_3.0) was considered a "special"
+security measure, necessary just for big companies, like banks and shoppings sites
+with financial transactions.
+Now we have a different picture. [According to Josh Aas](https://letsencrypt.org/2015/10/29/phishing-and-malware.html), Executive Director at [ISRG](https://en.wikipedia.org/wiki/Internet_Security_Research_Group):
+
+> _We’ve since come to realize that HTTPS is important for almost all websites. It’s important for any website that allows people to log in with a password, any website that [tracks its users](https://www.washingtonpost.com/news/the-switch/wp/2013/12/10/nsa-uses-google-cookies-to-pinpoint-targets-for-hacking/) in any way, any website that [doesn’t want its content altered](http://arstechnica.com/tech-policy/2014/09/why-comcasts-javascript-ad-injections-threaten-security-net-neutrality/), and for any site that offers content people might not want others to know they are consuming. We’ve also learned that any site not secured by HTTPS [can be used to attack other sites](http://krebsonsecurity.com/2015/04/dont-be-fodder-for-chinas-great-cannon/)._
+
+Therefore, the reason why certificates are so important is that they encrypt
 the connection between the **client** (you, me, your visitors)
 and the **server** (where you site lives), through a keychain of
 authentications and validations.
 
+How about taking Josh's advice and protecting our sites too? We will be
+well supported, and we'll contribute to a safer internet.
+
+#### Organizations supporting HTTPS
+
+There is a huge movement in favor of securing all the web. W3C fully
+[supports the cause](https://w3ctag.github.io/web-https/) and explains very well
+the reasons for that. Richard Barnes, a writer for Mozilla Security Blog,
+suggested that [Firefox would deprecate HTTP](https://blog.mozilla.org/security/2015/04/30/deprecating-non-secure-http/),
+and would no longer accept unsecured connections. Recently, Mozilla published a
+[communication](https://blog.mozilla.org/security/2016/03/29/march-2016-ca-communication/)
+reiterating the importance of HTTPS.
+
 ### Issuing Certificates
 
 GitLab Pages accepts [PEM](https://support.quovadisglobal.com/kb/a37/what-is-pem-format.aspx) certificates issued by

doc/user/project/pages/index.md

@@ -54,7 +54,6 @@ _Blog posts for securing GitLab Pages custom domains with SSL/TLS certificates:_
 
 - [CloudFlare](https://about.gitlab.com/2017/02/07/setting-up-gitlab-pages-with-cloudflare-certificates/)
 - [Let's Encrypt](https://about.gitlab.com/2016/04/11/tutorial-securing-your-gitlab-pages-with-tls-and-letsencrypt/) (outdated)
-- [StartSSL](https://about.gitlab.com/2016/06/24/secure-gitlab-pages-with-startssl/) (deprecated)
 
 ## Advanced use
 

spec/javascripts/gl_form_spec.js

-import Autosize from 'autosize';
+import autosize from 'autosize';
 import GLForm from '~/gl_form';
 import '~/lib/utils/text_utility';
 import '~/lib/utils/common_utils';
 
-window.autosize = Autosize;
-
 describe('GLForm', () => {
   describe('when instantiated', function () {
     beforeEach((done) => {
@@ -13,14 +11,12 @@ describe('GLForm', () => {
       spyOn($.prototype, 'off').and.returnValue(this.textarea);
       spyOn($.prototype, 'on').and.returnValue(this.textarea);
       spyOn($.prototype, 'css');
-      spyOn(window, 'autosize');
 
-      this.glForm = new GLForm(this.form);
+      this.glForm = new GLForm(this.form, false);
       setTimeout(() => {
         $.prototype.off.calls.reset();
         $.prototype.on.calls.reset();
         $.prototype.css.calls.reset();
-        window.autosize.calls.reset();
         done();
       });
     });
@@ -43,10 +39,6 @@ describe('GLForm', () => {
         expect($.prototype.on).toHaveBeenCalledWith('mouseup.autosize', jasmine.any(Function));
       });
 
-      it('should autosize the textarea', () => {
-        expect(window.autosize).toHaveBeenCalledWith(jasmine.any(Object));
-      });
-
       it('should set the resize css property to vertical', () => {
         expect($.prototype.css).toHaveBeenCalledWith('resize', 'vertical');
       });
@@ -74,7 +66,7 @@ describe('GLForm', () => {
           spyOn($.prototype, 'data');
           spyOn($.prototype, 'outerHeight').and.returnValue(200);
           spyOn(window, 'outerHeight').and.returnValue(400);
-          spyOn(window.autosize, 'destroy');
+          spyOn(autosize, 'destroy');
 
           this.glForm.destroyAutosize();
         });
@@ -88,7 +80,7 @@ describe('GLForm', () => {
         });
 
         it('should call autosize destroy', () => {
-          expect(window.autosize.destroy).toHaveBeenCalledWith(this.textarea);
+          expect(autosize.destroy).toHaveBeenCalledWith(this.textarea);
         });
 
         it('should set the data-height attribute', () => {
@@ -107,9 +99,9 @@ describe('GLForm', () => {
       it('should return undefined if the data-height equals the outerHeight', () => {
         spyOn($.prototype, 'outerHeight').and.returnValue(200);
         spyOn($.prototype, 'data').and.returnValue(200);
-        spyOn(window.autosize, 'destroy');
+        spyOn(autosize, 'destroy');
         expect(this.glForm.destroyAutosize()).toBeUndefined();
-        expect(window.autosize.destroy).not.toHaveBeenCalled();
+        expect(autosize.destroy).not.toHaveBeenCalled();
       });
     });
   });

spec/models/user_spec.rb

@@ -1635,14 +1635,37 @@ describe User do
   describe '#authorized_groups' do
     let!(:user) { create(:user) }
     let!(:private_group) { create(:group) }
+    let!(:child_group) { create(:group, parent: private_group) }
+
+    let!(:project_group) { create(:group) }
+    let!(:project) { create(:project, group: project_group) }
 
     before do
       private_group.add_user(user, Gitlab::Access::MASTER)
+      project.add_master(user)
     end
 
     subject { user.authorized_groups }
 
-    it { is_expected.to eq([private_group]) }
+    it { is_expected.to contain_exactly private_group, project_group }
+  end
+
+  describe '#membership_groups' do
+    let!(:user) { create(:user) }
+    let!(:parent_group) { create(:group) }
+    let!(:child_group) { create(:group, parent: parent_group) }
+
+    before do
+      parent_group.add_user(user, Gitlab::Access::MASTER)
+    end
+
+    subject { user.membership_groups }
+
+    if Group.supports_nested_groups?
+      it { is_expected.to contain_exactly parent_group, child_group }
+    else
+      it { is_expected.to contain_exactly parent_group }
+    end
   end
 
   describe '#authorized_projects', :delete do

spec/requests/openid_connect_spec.rb

@@ -65,10 +65,20 @@ describe 'OpenID Connect requests' do
         )
       end
 
-      let(:public_email) { build :email, email: 'public@example.com' }
-      let(:private_email) { build :email, email: 'private@example.com' }
+      let!(:public_email) { build :email, email: 'public@example.com' }
+      let!(:private_email) { build :email, email: 'private@example.com' }
 
-      it 'includes all user information' do
+      let!(:group1) { create :group, path: 'group1' }
+      let!(:group2) { create :group, path: 'group2' }
+      let!(:group3) { create :group, path: 'group3', parent: group2 }
+      let!(:group4) { create :group, path: 'group4', parent: group3 }
+
+      before do
+        group1.add_user(user, GroupMember::OWNER)
+        group3.add_user(user, Gitlab::Access::DEVELOPER)
+      end
+
+      it 'includes all user information and group memberships' do
         request_user_info
 
         expect(json_response).to eq({
@@ -79,7 +89,13 @@ describe 'OpenID Connect requests' do
           'email_verified' => true,
           'website'        => 'https://example.com',
           'profile'        => 'http://localhost/alice',
-          'picture'        => "http://localhost/uploads/-/system/user/avatar/#{user.id}/dk.png"
+          'picture'        => "http://localhost/uploads/-/system/user/avatar/#{user.id}/dk.png",
+          'groups'         =>
+            if Group.supports_nested_groups?
+              ['group1', 'group2/group3', 'group2/group3/group4']
+            else
+              ['group1', 'group2/group3']
+            end
         })
       end
     end

vendor/gitignore/Android.gitignore

@@ -54,3 +54,10 @@ google-services.json
 freeline.py
 freeline/
 freeline_project_description.json
+
+# fastlane
+fastlane/report.xml
+fastlane/Preview.html
+fastlane/screenshots
+fastlane/test_output
+fastlane/readme.md

vendor/gitignore/Dart.gitignore

 # See https://www.dartlang.org/tools/private-files.html
 
 # Files and directories created by pub
+.dart_tool/
 .packages
 .pub/
 build/

vendor/gitignore/Global/JetBrains.gitignore

-# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm
 # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
 
 # User-specific stuff:
@@ -9,7 +9,6 @@
 # Sensitive or high-churn files:
 .idea/**/dataSources/
 .idea/**/dataSources.ids
-.idea/**/dataSources.xml
 .idea/**/dataSources.local.xml
 .idea/**/sqlDataSources.xml
 .idea/**/dynamic.xml

vendor/gitignore/Python.gitignore

@@ -45,6 +45,7 @@ nosetests.xml
 coverage.xml
 *.cover
 .hypothesis/
+.pytest_cache/
 
 # Translations
 *.mo

vendor/gitignore/ROS.gitignore

@@ -13,6 +13,8 @@ msg/*Feedback.msg
 msg/*Goal.msg
 msg/*Result.msg
 msg/_*.py
+build_isolated/
+devel_isolated/
 
 # Generated by dynamic reconfigure
 *.cfgc

vendor/gitignore/TeX.gitignore

@@ -10,6 +10,7 @@
 *.fot
 *.cb
 *.cb2
+.*.lb
 
 ## Intermediate documents:
 *.dvi

vendor/gitignore/VisualStudio.gitignore

@@ -237,6 +237,7 @@ _UpgradeReport_Files/
 Backup*/
 UpgradeLog*.XML
 UpgradeLog*.htm
+ServiceFabricBackup/
 
 # SQL Server files
 *.mdf

vendor/gitlab-ci-yml/Auto-DevOps.gitlab-ci.yml

@@ -110,12 +110,15 @@ performance:
     kubernetes: active
 
 sast:
-  image: registry.gitlab.com/gitlab-org/gl-sast:latest
+  image: docker:latest
   variables:
-    POSTGRES_DB: "false"
+    DOCKER_DRIVER: overlay2
   allow_failure: true
+  services:
+    - docker:dind
   script:
-    - sast .
+    - setup_docker
+    - sast
   artifacts:
     paths: [gl-sast-report.json]
 
@@ -285,6 +288,12 @@ production:
   export TILLER_NAMESPACE=$KUBE_NAMESPACE
 
   function sast_container() {
+    if [[ -n "$CI_REGISTRY_USER" ]]; then
+      echo "Logging to GitLab Container Registry with CI credentials..."
+      docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" "$CI_REGISTRY"
+      echo ""
+    fi
+
     docker run -d --name db arminc/clair-db:latest
     docker run -p 6060:6060 --link db:postgres -d --name clair arminc/clair-local-scan:v2.0.1
     apk add -U wget ca-certificates
@@ -309,7 +318,12 @@ production:
   function sast() {
     case "$CI_SERVER_VERSION" in
       *-ee)
-        /app/bin/run "$@"
+        # Extract "MAJOR.MINOR" from CI_SERVER_VERSION and generate "MAJOR-MINOR-stable"
+        SAST_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')
+
+        docker run --volume "$PWD:/code" \
+                   --volume /var/run/docker.sock:/var/run/docker.sock \
+                   "registry.gitlab.com/gitlab-org/security-products/sast:$SAST_VERSION" /app/bin/run /code
         ;;
       *)
         echo "GitLab EE is required"
@@ -346,6 +360,12 @@ production:
       replicas="$new_replicas"
     fi
 
+    if [[ "$CI_PROJECT_VISIBILITY" != "public" ]]; then
+      secret_name='gitlab-registry'
+    else
+      secret_name=''
+    fi
+
     helm upgrade --install \
       --wait \
       --set service.enabled="$service_enabled" \
@@ -353,6 +373,7 @@ production:
       --set image.repository="$CI_APPLICATION_REPOSITORY" \
       --set image.tag="$CI_APPLICATION_TAG" \
       --set image.pullPolicy=IfNotPresent \
+      --set image.secrets[0].name="$secret_name" \
       --set application.track="$track" \
       --set application.database_url="$DATABASE_URL" \
       --set service.url="$CI_ENVIRONMENT_URL" \
@@ -482,6 +503,9 @@ production:
 
   function create_secret() {
     echo "Create secret..."
+    if [[ "$CI_PROJECT_VISIBILITY" == "public" ]]; then
+      return
+    fi
 
     kubectl create secret -n "$KUBE_NAMESPACE" \
       docker-registry gitlab-registry \