Merge branch '7-8-rc4' into '7-8-stable'

Changes for 7.8.rc4

See merge request !1564

CHANGELOG

 v 7.8.0 (unreleased)
+  - Fix access control and protection against XSS for note attachments and other uploads.
   - Replace highlight.js with rouge-fork rugments (Stefan Tatschner)
   - Make project search case insensitive (Hannes Rosenögger)
   - Include issue/mr participants in list of recipients for reassign/close/reopen emails
@@ -60,6 +61,8 @@ v 7.8.0 (unreleased)
   - Remove deprecated Group#owner_id from API
   - Show projects user contributed to on user page. Show stars near project on user page.
   - Improve database performance for GitLab
+  - Add Asana service (Jeremy Benoist)
+  - Improve project web hooks with extra data
 
 v 7.7.2
   - Update GitLab Shell to version 2.4.2 that fixes a bug when developers can push to protected branch

app/assets/stylesheets/highlight/dark.scss

@@ -12,6 +12,7 @@
     border-left: 1px solid #666;
   }
 
+  // highlight line via anchor
   pre.hll {
     background-color: #fff !important;
   }

app/assets/stylesheets/highlight/monokai.scss

@@ -12,6 +12,11 @@
     border-left: 1px solid #555;
   }
 
+  // highlight line via anchor
+  pre.hll {
+    background-color: #49483e !important;
+  }
+
   .hll { background-color: #49483e }
   .c { color: #75715e } /* Comment */
   .err { color: #960050; background-color: #1e0010 } /* Error */

app/assets/stylesheets/highlight/solarized_dark.scss

@@ -12,6 +12,11 @@
     border-left: 1px solid #113b46;
   }
 
+  // highlight line via anchor
+  pre.hll {
+    background-color: #073642 !important;
+  }
+
   /* Solarized Dark
 
   For use with Jekyll and Pygments

app/assets/stylesheets/highlight/solarized_light.scss

@@ -12,6 +12,11 @@
     border-left: 1px solid #c5d0d4;
   }
 
+  // highlight line via anchor
+  pre.hll {
+    background-color: #eee8d5 !important;
+  }
+
   /* Solarized Light
 
   For use with Jekyll and Pygments

app/assets/stylesheets/highlight/white.scss

@@ -12,6 +12,11 @@
     border-left: 1px solid #bbb;
   }
 
+  // highlight line via anchor
+  pre.hll {
+    background-color: #f8eec7 !important;
+  }
+
   .hll { background-color: #f8f8f8 }
   .c { color: #999988; font-style: italic; }
   .err { color: #a61717; background-color: #e3d2d2; }

app/controllers/import/gitorious_controller.rb

+class Import::GitoriousController < Import::BaseController
+
+  def new
+    redirect_to client.authorize_url(callback_import_gitorious_url)
+  end
+
+  def callback
+    session[:gitorious_repos] = params[:repos]
+    redirect_to status_import_gitorious_url
+  end
+
+  def status
+    @repos = client.repos
+
+    @already_added_projects = current_user.created_projects.where(import_type: "gitorious")
+    already_added_projects_names = @already_added_projects.pluck(:import_source)
+
+    @repos.to_a.reject! { |repo| already_added_projects_names.include? repo.full_name }
+  end
+
+  def jobs
+    jobs = current_user.created_projects.where(import_type: "gitorious").to_json(only: [:id, :import_status])
+    render json: jobs
+  end
+
+  def create
+    @repo_id = params[:repo_id]
+    repo = client.repo(@repo_id)
+    @target_namespace = params[:new_namespace].presence || repo.namespace
+    @project_name = repo.name
+
+    namespace = get_or_create_namespace || (render and return)
+
+    @project = Gitlab::GitoriousImport::ProjectCreator.new(repo, namespace, current_user).execute
+  end
+
+  private
+
+  def client
+    @client ||= Gitlab::GitoriousImport::Client.new(session[:gitorious_repos])
+  end
+
+end

app/controllers/projects/uploads_controller.rb

+class Projects::UploadsController < Projects::ApplicationController
+  layout "project"
+
+  before_filter :project
+
+  def show
+    path = File.join(project.path_with_namespace, params[:secret])
+    uploader = FileUploader.new('uploads', path)
+
+    uploader.retrieve_from_store!(params[:filename])
+
+    if uploader.file.exists?
+      # Right now, these are always images, so we can safely render them inline.
+      send_file uploader.file.path, disposition: 'inline'
+    else
+      not_found!
+    end
+  end
+end

app/controllers/uploads_controller.rb

+class UploadsController < ApplicationController
+  def show
+    model = params[:model].camelize.constantize.find(params[:id])
+    uploader = model.send(params[:mounted_as])
+
+    if uploader.file_storage?
+      if !model.respond_to?(:project) || can?(current_user, :read_project, model.project)
+        disposition = uploader.image? ? 'inline' : 'attachment'
+        send_file uploader.file.path, disposition: disposition
+      else
+        not_found!
+      end
+    else
+      redirect_to uploader.url
+    end
+  end
+end

app/views/import/gitorious/status.html.haml

+%h3.page-title
+  %i.fa.fa-gitorious
+  Import repositories from Gitorious.org
+
+%p.light
+  Select projects you want to import.
+%hr
+%p
+  = button_tag 'Import all projects', class: "btn btn-success js-import-all"
+
+%table.table.import-jobs
+  %thead
+    %tr
+      %th From Gitorious
+      %th To GitLab
+      %th Status
+  %tbody
+    - @already_added_projects.each do |project|
+      %tr{id: "project_#{project.id}", class: "#{project_status_css_class(project.import_status)}"}
+        %td= project.import_source
+        %td
+          %strong= link_to project.path_with_namespace, project
+        %td.job-status
+          - if project.import_status == 'finished'
+            %span.cgreen
+              %i.fa.fa-check
+              done
+          - else
+            = project.human_import_status_name
+
+    - @repos.each do |repo|
+      %tr{id: "repo_#{repo.id}"}
+        %td= repo.full_name
+        %td.import-target
+          = repo.full_name
+        %td.import-actions.job-status
+          = button_tag "Import", class: "btn js-add-to-import"
+
+:coffeescript
+  $ ->
+    new ImporterStatus("#{jobs_import_gitorious_path}", "#{import_gitorious_path}")

app/views/projects/new.html.haml

@@ -66,6 +66,13 @@
               Import projects from GitLab.com
             = render 'gitlab_import_modal'
 
+      .project-import.form-group
+        .col-sm-2
+        .col-sm-10
+          = link_to new_import_gitorious_path do
+            %i.fa.fa-heart
+            Import projects from Gitorious.org
+
       %hr.prepend-botton-10
 
       .form-group

config/initializers/static_files.rb

+begin
+  app = Rails.application
+
+  # The `ActionDispatch::Static` middleware intercepts requests for static files 
+  # by checking if they exist in the `/public` directory. 
+  # We're replacing it with our `Gitlab::Middleware::Static` that does the same,
+  # except ignoring `/uploads`, letting those go through to the GitLab Rails app.
+
+  app.config.middleware.swap(
+    ActionDispatch::Static, 
+    Gitlab::Middleware::Static, 
+    app.paths["public"].first, 
+    app.config.static_cache_control
+  )
+rescue
+  # If ActionDispatch::Static wasn't loaded onto the stack (like in production), 
+  # an exception is raised.
+end

config/routes.rb

@@ -67,9 +67,29 @@ Gitlab::Application.routes.draw do
       get :callback
       get :jobs
     end
+
+    resource :gitorious, only: [:create, :new], controller: :gitorious do
+      get :status
+      get :callback
+      get :jobs
+    end
   end
 
+  #
+  # Uploads
+  #
 
+  scope path: :uploads do
+    # Note attachments and User/Group/Project avatars
+    get ":model/:mounted_as/:id/:filename", 
+        to:           "uploads#show", 
+        constraints:  { model: /note|user|group|project/, mounted_as: /avatar|attachment/, filename: /.+/ }
+
+    # Project markdown uploads
+    get ":id/:secret/:filename", 
+        to:           "projects/uploads#show", 
+        constraints:  { id: /[a-zA-Z.0-9_\-]+\/[a-zA-Z.0-9_\-]+/, filename: /.+/ }
+  end
 
   #
   # Explore area

doc/api/projects.md

 # Projects
 
+
+### Project visibility level
+
+Project in GitLab has be either private, internal or public.
+You can determine it by `visibility_level` field in project. 
+
+Constants for project visibility levels are next:
+
+* Private. `visibility_level` is `0`. 
+  Project access must be granted explicitly for each user.
+
+* Internal. `visibility_level` is `10`.
+  The project can be cloned by any logged in user.
+ 
+* Public. `visibility_level` is `20`.
+  The project can be cloned without any authentication.
+
+
 ## List projects
 
 Get a list of projects accessible by the authenticated user.

doc/update/6.x-or-7.x-to-7.8.md

@@ -164,8 +164,6 @@ git diff 6-0-stable:config/gitlab.yml.example 7-8-stable:config/gitlab.yml.examp
 * Make `/home/git/gitlab/config/gitlab.yml` the same as https://gitlab.com/gitlab-org/gitlab-ce/blob/7-8-stable/config/gitlab.yml.example but with your settings.
 * Make `/home/git/gitlab/config/unicorn.rb` the same as https://gitlab.com/gitlab-org/gitlab-ce/blob/7-8-stable/config/unicorn.rb.example but with your settings.
 * Make `/home/git/gitlab-shell/config.yml` the same as https://gitlab.com/gitlab-org/gitlab-shell/blob/v2.4.3/config.yml.example but with your settings.
-* HTTP setups: Make `/etc/nginx/sites-available/gitlab` the same as https://gitlab.com/gitlab-org/gitlab-ce/blob/7-8-stable/lib/support/nginx/gitlab but with your settings.
-* HTTPS setups: Make `/etc/nginx/sites-available/gitlab-ssl` the same as https://gitlab.com/gitlab-org/gitlab-ce/blob/7-8-stablef/lib/support/nginx/gitlab-ssl but with your settings.
 * Copy rack attack middleware config
 
 ```bash
@@ -178,6 +176,12 @@ sudo -u git -H cp config/initializers/rack_attack.rb.example config/initializers
 sudo cp lib/support/logrotate/gitlab /etc/logrotate.d/gitlab
 ```
 
+### Change Nginx settings
+
+* HTTP setups: Make `/etc/nginx/sites-available/gitlab` the same as https://gitlab.com/gitlab-org/gitlab-ce/blob/7-8-stable/lib/support/nginx/gitlab but with your settings.
+* HTTPS setups: Make `/etc/nginx/sites-available/gitlab-ssl` the same as https://gitlab.com/gitlab-org/gitlab-ce/blob/7-8-stablef/lib/support/nginx/gitlab-ssl but with your settings.
+* A new `location /uploads/` section has been added that needs to have the same content as the existing `location @gitlab` section.
+
 ## 9. Start application
 
     sudo service gitlab start

doc/update/7.7-to-7.8.md

@@ -75,8 +75,9 @@ git diff origin/7-6-stable:config/gitlab.yml.example origin/7-8-stable:config/gi
 
 #### Change Nginx settings
 
-* HTTP setups: Make `/etc/nginx/sites-available/gitlab` the same as [`lib/support/nginx/gitlab`](/lib/support/nginx/gitlab) but with your settings
-* HTTPS setups: Make `/etc/nginx/sites-available/gitlab-ssl` the same as [`lib/support/nginx/gitlab-ssl`](/lib/support/nginx/gitlab-ssl) but with your setting
+* HTTP setups: Make `/etc/nginx/sites-available/gitlab` the same as [`lib/support/nginx/gitlab`](/lib/support/nginx/gitlab) but with your settings.
+* HTTPS setups: Make `/etc/nginx/sites-available/gitlab-ssl` the same as [`lib/support/nginx/gitlab-ssl`](/lib/support/nginx/gitlab-ssl) but with your settings.
+* A new `location /uploads/` section has been added that needs to have the same content as the existing `location @gitlab` section.
 
 #### Setup time zone (optional)
 

doc/web_hooks/web_hooks.md

@@ -24,16 +24,19 @@ Triggered when you push to the repository except when pushing tags.
   "project_id": 15,
   "repository": {
     "name": "Diaspora",
-    "url": "git@example.com:diaspora.git",
+    "url": "git@example.com:mike/diasporadiaspora.git",
     "description": "",
-    "homepage": "http://example.com/diaspora"
+    "homepage": "http://example.com/mike/diaspora", 
+    "git_http_url":"http://example.com/mike/diaspora.git",
+    "git_ssh_url":"git@example.com:mike/diaspora.git",
+    "visibility_level":0
   },
   "commits": [
     {
       "id": "b6568db1bc1dcd7f8b4d5a946b0b91f9dacd7327",
       "message": "Update Catalan translation to e38cb41.",
       "timestamp": "2011-12-12T14:27:31+02:00",
-      "url": "http://example.com/diaspora/commits/b6568db1bc1dcd7f8b4d5a946b0b91f9dacd7327",
+      "url": "http://example.com/mike/diaspora/commit/b6568db1bc1dcd7f8b4d5a946b0b91f9dacd7327",
       "author": {
         "name": "Jordi Mallach",
         "email": "jordi@softcatala.org"
@@ -43,7 +46,7 @@ Triggered when you push to the repository except when pushing tags.
       "id": "da1560886d4f094c3e6c9ef40349f7d38b5d27d7",
       "message": "fixed readme",
       "timestamp": "2012-01-03T23:36:29+02:00",
-      "url": "http://example.com/diaspora/commits/da1560886d4f094c3e6c9ef40349f7d38b5d27d7",
+      "url": "http://example.com/mike/diaspora/commit/da1560886d4f094c3e6c9ef40349f7d38b5d27d7",
       "author": {
         "name": "GitLab dev user",
         "email": "gitlabdev@dv6700.(none)"
@@ -72,8 +75,13 @@ Triggered when you create (or delete) tags to the repository.
     "name": "jsmith",
     "url": "ssh://git@example.com/jsmith/example.git",
     "description": "",
-    "homepage": "http://example.com/jsmith/example"
-  }
+    "homepage": "http://example.com/jsmith/example",
+    "git_http_url":"http://example.com/jsmith/example.git",
+    "git_ssh_url":"git@example.com:jsmith/example.git",
+    "visibility_level":0
+  },
+  "commits": [],
+  "total_commits_count": 0
 }
 ```
 

lib/gitlab/gitorious_import/client.rb

+module Gitlab
+  module GitoriousImport
+    GITORIOUS_HOST = "https://gitorious.org"
+
+    class Client
+      attr_reader :repo_list
+
+      def initialize(repo_list)
+        @repo_list = repo_list
+      end
+
+      def authorize_url(redirect_uri)
+        "#{GITORIOUS_HOST}/gitlab-import?callback_url=#{redirect_uri}"
+      end
+
+      def repos
+        @repos ||= repo_names.map { |full_name| Repository.new(full_name) }
+      end
+
+      def repo(id)
+        repos.find { |repo| repo.id == id }
+      end
+
+      private
+
+      def repo_names
+        repo_list.to_s.split(',').map(&:strip).reject(&:blank?)
+      end
+    end
+
+    Repository = Struct.new(:full_name) do
+      def id
+        Digest::SHA1.hexdigest(full_name)
+      end
+
+      def namespace
+        segments.first
+      end
+
+      def path
+        segments.last
+      end
+
+      def name
+        path.titleize
+      end
+
+      def description
+        ""
+      end
+
+      def import_url
+        "#{GITORIOUS_HOST}/#{full_name}.git"
+      end
+
+      private
+
+      def segments
+        full_name.split('/')
+      end
+    end
+  end
+end

lib/gitlab/gitorious_import/project_creator.rb

+module Gitlab
+  module GitoriousImport
+    class ProjectCreator
+      attr_reader :repo, :namespace, :current_user
+
+      def initialize(repo, namespace, current_user)
+        @repo = repo
+        @namespace = namespace
+        @current_user = current_user
+      end
+
+      def execute
+        @project = Project.new(
+          name: repo.name,
+          path: repo.path,
+          description: repo.description,
+          namespace: namespace,
+          creator: current_user,
+          visibility_level: Gitlab::VisibilityLevel::PUBLIC,
+          import_type: "gitorious",
+          import_source: repo.full_name,
+          import_url: repo.import_url
+        )
+
+        if @project.save!
+          @project.reload
+
+          if @project.import_failed?
+            @project.import_retry
+          else
+            @project.import_start
+          end
+        end
+
+        @project
+      end
+    end
+  end
+end

lib/gitlab/middleware/static.rb

+module Gitlab
+  module Middleware
+    class Static < ActionDispatch::Static
+      UPLOADS_REGEX = /\A\/uploads(\/|\z)/.freeze
+
+      def call(env)
+        return @app.call(env) if env['PATH_INFO'] =~ UPLOADS_REGEX
+
+        super
+      end
+    end
+  end
+end

lib/gitlab/push_data_builder.rb

@@ -41,6 +41,9 @@ module Gitlab
             url: project.url_to_repo,
             description: project.description,
             homepage: project.web_url,
+            git_http_url: project.http_url_to_repo,
+            git_ssh_url: project.ssh_url_to_repo,
+            visibility_level: project.visibility_level
           },
           commits: [],
           total_commits_count: commits_count

lib/support/nginx/gitlab

 ## GitLab
-## Contributors: randx, yin8086, sashkab, orkoden, axilleas, bbodenmiller
+## Contributors: randx, yin8086, sashkab, orkoden, axilleas, bbodenmiller, DouweM
 ##
 ## Lines starting with two hashes (##) are comments with information.
 ## Lines starting with one hash (#) are configuration parameters that can be uncommented.
@@ -56,6 +56,27 @@ server {
     try_files $uri $uri/index.html $uri.html @gitlab;
   }
 
+  ## We route uploads through GitLab to prevent XSS and enforce access control.
+  location /uploads/ {
+    ## If you use HTTPS make sure you disable gzip compression
+    ## to be safe against BREACH attack.
+    # gzip off;
+
+    ## https://github.com/gitlabhq/gitlabhq/issues/694
+    ## Some requests take more than 30 seconds.
+    proxy_read_timeout      300;
+    proxy_connect_timeout   300;
+    proxy_redirect          off;
+
+    proxy_set_header    Host                $http_host;
+    proxy_set_header    X-Real-IP           $remote_addr;
+    proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
+    proxy_set_header    X-Forwarded-Proto   $scheme;
+    proxy_set_header    X-Frame-Options     SAMEORIGIN;
+
+    proxy_pass http://gitlab;
+  }
+
   ## If a file, which is not found in the root folder is requested,
   ## then the proxy passes the request to the upsteam (gitlab unicorn).
   location @gitlab {

lib/support/nginx/gitlab-ssl

 ## GitLab
-## Contributors: randx, yin8086, sashkab, orkoden, axilleas, bbodenmiller
+## Contributors: randx, yin8086, sashkab, orkoden, axilleas, bbodenmiller, DouweM
 ##
 ## Modified from nginx http version
 ## Modified from http://blog.phusion.nl/2012/04/21/tutorial-setting-up-gitlab-on-debian-6/
@@ -101,6 +101,28 @@ server {
     try_files $uri $uri/index.html $uri.html @gitlab;
   }
 
+  ## We route uploads through GitLab to prevent XSS and enforce access control.
+  location /uploads/ {
+    ## If you use HTTPS make sure you disable gzip compression
+    ## to be safe against BREACH attack.
+    gzip off;
+
+    ## https://github.com/gitlabhq/gitlabhq/issues/694
+    ## Some requests take more than 30 seconds.
+    proxy_read_timeout      300;
+    proxy_connect_timeout   300;
+    proxy_redirect          off;
+
+    proxy_set_header    Host                $http_host;
+    proxy_set_header    X-Real-IP           $remote_addr;
+    proxy_set_header    X-Forwarded-Ssl     on;
+    proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
+    proxy_set_header    X-Forwarded-Proto   $scheme;
+    proxy_set_header    X-Frame-Options     SAMEORIGIN;
+
+    proxy_pass http://gitlab;
+  }
+
   ## If a file, which is not found in the root folder is requested,
   ## then the proxy passes the request to the upsteam (gitlab unicorn).
   location @gitlab {

spec/controllers/import/gitorious_controller_spec.rb

+require 'spec_helper'
+
+describe Import::GitoriousController do
+  let(:user) { create(:user) }
+
+  before do
+    sign_in(user)
+  end
+
+  describe "GET new" do
+    it "redirects to import endpoint on gitorious.org" do
+      get :new
+
+      expect(controller).to redirect_to("https://gitorious.org/gitlab-import?callback_url=http://test.host/import/gitorious/callback")
+    end
+  end
+
+  describe "GET callback" do
+    it "stores repo list in session" do
+      get :callback, repos: 'foo/bar,baz/qux'
+
+      expect(session[:gitorious_repos]).to eq('foo/bar,baz/qux')
+    end
+  end
+
+  describe "GET status" do
+    before do
+      @repo = OpenStruct.new(full_name: 'asd/vim')
+    end
+
+    it "assigns variables" do
+      @project = create(:project, import_type: 'gitorious', creator_id: user.id)
+      controller.stub_chain(:client, :repos).and_return([@repo])
+
+      get :status
+
+      expect(assigns(:already_added_projects)).to eq([@project])
+      expect(assigns(:repos)).to eq([@repo])
+    end
+
+    it "does not show already added project" do
+      @project = create(:project, import_type: 'gitorious', creator_id: user.id, import_source: 'asd/vim')
+      controller.stub_chain(:client, :repos).and_return([@repo])
+
+      get :status
+
+      expect(assigns(:already_added_projects)).to eq([@project])
+      expect(assigns(:repos)).to eq([])
+    end
+  end
+
+  describe "POST create" do
+    before do
+      @repo = Gitlab::GitoriousImport::Repository.new('asd/vim')
+    end
+
+    it "takes already existing namespace" do
+      namespace = create(:namespace, name: "asd", owner: user)
+      expect(Gitlab::GitoriousImport::ProjectCreator).
+        to receive(:new).with(@repo, namespace, user).
+        and_return(double(execute: true))
+      controller.stub_chain(:client, :repo).and_return(@repo)
+
+      post :create, format: :js
+    end
+  end
+end

spec/lib/gitlab/gitorious_import/project_creator.rb

+require 'spec_helper'
+
+describe Gitlab::GitoriousImport::ProjectCreator do
+  let(:user) { create(:user) }
+  let(:repo) { Gitlab::GitoriousImport::Repository.new('foo/bar-baz-qux') }
+  let(:namespace){ create(:namespace) }
+
+  it 'creates project' do
+    allow_any_instance_of(Project).to receive(:add_import_job)
+
+    project_creator = Gitlab::GitoriousImport::ProjectCreator.new(repo, namespace, user)
+    project_creator.execute
+    project = Project.last
+
+    expect(project.name).to eq("Bar Baz Qux")
+    expect(project.path).to eq("bar-baz-qux")
+    expect(project.namespace).to eq(namespace)
+    expect(project.visibility_level).to eq(Gitlab::VisibilityLevel::PUBLIC)
+    expect(project.import_type).to eq("gitorious")
+    expect(project.import_source).to eq("foo/bar-baz-qux")
+    expect(project.import_url).to eq("https://gitorious.org/foo/bar-baz-qux.git")
+  end
+end

spec/lib/gitlab/push_data_builder_spec.rb

@@ -13,6 +13,9 @@ describe 'Gitlab::PushDataBuilder' do
     it { expect(data[:after]).to eq('5937ac0a7beb003549fc5fd26fc247adbce4a52e') }
     it { expect(data[:ref]).to eq('refs/heads/master') }
     it { expect(data[:commits].size).to eq(3) }
+    it { expect(data[:repository][:git_http_url]).to eq(project.http_url_to_repo) }
+    it { expect(data[:repository][:git_ssh_url]).to eq(project.ssh_url_to_repo) }
+    it { expect(data[:repository][:visibility_level]).to eq(project.visibility_level) }
     it { expect(data[:total_commits_count]).to eq(3) }
   end