Merge branch '8-17-stable-pick-yarn-support' into '8-17-stable'

Pick Yarn support into 8.17

See merge request !9591

.gitlab-ci.yml

@@ -107,7 +107,10 @@ setup-test-env:
   <<: *dedicated-runner
   stage: prepare
   script:
-    - npm install
+    - node --version
+    - yarn --version
+    - yarn install --pure-lockfile
+    - yarn check # ensure that yarn.lock matches package.json
     - bundle exec rake gitlab:assets:compile
     - bundle exec ruby -Ispec -e 'require "spec_helper" ; TestEnv.init'
   artifacts:
@@ -302,7 +305,6 @@ karma:
   <<: *use-db
   <<: *dedicated-runner
   script:
-    - npm link istanbul
     - bundle exec rake karma
   artifacts:
     name: coverage-javascript
@@ -381,11 +383,9 @@ lint:javascript:
     paths:
       - node_modules/
   stage: test
-  image: "node:7.1"
-  before_script:
-    - npm install
+  before_script: []
   script:
-    - npm --silent run eslint
+    - yarn run eslint
 
 lint:javascript:report:
   <<: *dedicated-runner
@@ -393,12 +393,10 @@ lint:javascript:report:
     paths:
       - node_modules/
   stage: post-test
-  image: "node:7.1"
-  before_script:
-    - npm install
+  before_script: []
   script:
     - find app/ spec/ -name '*.js' -or -name '*.js.es6' -exec sed --in-place 's|/\* eslint-disable .*\*/||' {} \; # run report over all files
-    - npm --silent run eslint-report || true # ignore exit code
+    - yarn run eslint-report || true # ignore exit code
   artifacts:
     name: eslint-report
     expire_in: 31d

changelogs/unreleased/add-yarn-documentation.yml

+---
+title: add rake tasks to handle yarn dependencies and update documentation
+merge_request: 9316
+author:

changelogs/unreleased/replace-npm-with-yarn.yml

+---
+title: replace npm with yarn and add yarn.lock
+merge_request: 9055
+author:

doc/install/installation.md

@@ -155,15 +155,18 @@ page](https://golang.org/dl).
 ## 4. Node
 
 Since GitLab 8.17, GitLab requires the use of node >= v4.3.0 to compile
-javascript assets. In many distros the version provided by the official package
-repositories is out of date, so we'll need to install through the following
-commands:
+javascript assets, and yarn >= v0.17.0 to manage javascript dependencies.
+In many distros the versions provided by the official package  repositories
+are out of date, so we'll need to install through the following commands:
 
     # install node v7.x
     curl --location https://deb.nodesource.com/setup_7.x | bash -
     sudo apt-get install -y nodejs
 
-Visit the official website for [node](https://nodejs.org/en/download/package-manager/) if you have any trouble with this step.
+    # install yarn
+    curl --location https://yarnpkg.com/install.sh | bash -
+
+Visit the official websites for [node](https://nodejs.org/en/download/package-manager/) and [yarn](https://yarnpkg.com/en/docs/install/) if you have any trouble with these steps.
 
 ## 5. System Users
 
@@ -465,7 +468,7 @@ Check if GitLab and its environment are configured correctly:
 
 ### Compile Assets
 
-    sudo -u git -H npm install --production
+    sudo -u git -H yarn install --production --pure-lockfile
     sudo -u git -H bundle exec rake gitlab:assets:compile RAILS_ENV=production NODE_ENV=production
 
 ### Start Your GitLab Instance

lib/gitlab/upgrader.rb

@@ -60,15 +60,17 @@ module Gitlab
         "Get latest code" => %W(#{Gitlab.config.git.bin_path} fetch),
         "Switch to new version" => %W(#{Gitlab.config.git.bin_path} checkout v#{latest_version}),
         "Install gems" => %W(bundle),
-        "Install node modules" => %W(npm install --production),
         "Migrate DB" => %W(bundle exec rake db:migrate),
-        "Recompile assets" => %W(bundle exec rake gitlab:assets:clean gitlab:assets:compile),
+        "Recompile assets" => %W(bundle exec rake yarn:install gitlab:assets:clean gitlab:assets:compile),
         "Clear cache" => %W(bundle exec rake cache:clear)
       }
     end
 
     def env
-      { 'RAILS_ENV' => 'production' }
+      {
+        'RAILS_ENV' => 'production',
+        'NODE_ENV' => 'production'
+      }
     end
 
     def upgrade

lib/tasks/eslint.rake

 unless Rails.env.production?
   desc "GitLab | Run ESLint"
-  task :eslint do
-    system("npm", "run", "eslint")
+  task eslint: ['yarn:check'] do
+    unless system('yarn run eslint')
+      abort('rake eslint failed')
+    end
   end
 end
-

lib/tasks/gitlab/assets.rake

 namespace :gitlab do
   namespace :assets do
     desc 'GitLab | Assets | Compile all frontend assets'
-    task :compile do
-      Rake::Task['assets:precompile'].invoke
-      Rake::Task['webpack:compile'].invoke
-      Rake::Task['gitlab:assets:fix_urls'].invoke
-    end
+    task compile: [
+      'yarn:check',
+      'assets:precompile',
+      'webpack:compile',
+      'gitlab:assets:fix_urls'
+    ]
 
     desc 'GitLab | Assets | Clean up old compiled frontend assets'
-    task :clean do
-      Rake::Task['assets:clean'].invoke
-    end
+    task clean: ['assets:clean']
 
     desc 'GitLab | Assets | Remove all compiled frontend assets'
-    task :purge do
-      Rake::Task['assets:clobber'].invoke
-    end
+    task purge: ['assets:clobber']
+
+    desc 'GitLab | Assets | Uninstall frontend dependencies'
+    task purge_modules: ['yarn:clobber']
 
     desc 'GitLab | Assets | Fix all absolute url references in CSS'
     task :fix_urls do

lib/tasks/karma.rake

 unless Rails.env.production?
-  Rake::Task['karma'].clear if Rake::Task.task_defined?('karma')
-
   namespace :karma do
     desc 'GitLab | Karma | Generate fixtures for JavaScript tests'
     RSpec::Core::RakeTask.new(:fixtures) do |t|
@@ -10,16 +8,13 @@ unless Rails.env.production?
     end
 
     desc 'GitLab | Karma | Run JavaScript tests'
-    task :tests do
-      sh "npm run karma" do |ok, res|
+    task tests: ['yarn:check'] do
+      sh "yarn run karma" do |ok, res|
         abort('rake karma:tests failed') unless ok
       end
     end
   end
 
   desc 'GitLab | Karma | Shortcut for karma:fixtures and karma:tests'
-  task :karma do
-    Rake::Task['karma:fixtures'].invoke
-    Rake::Task['karma:tests'].invoke
-  end
+  task karma: ['karma:fixtures', 'karma:tests']
 end

lib/tasks/yarn.rake

+
+namespace :yarn do
+  desc 'Ensure Yarn is installed'
+  task :available do
+    unless system('yarn --version', out: File::NULL)
+      warn(
+        'Error: Yarn executable was not detected in the system.'.color(:red),
+        'Download Yarn at https://yarnpkg.com/en/docs/install'.color(:green)
+      )
+      abort
+    end
+  end
+
+  desc 'Ensure Node dependencies are installed'
+  task check: ['yarn:available'] do
+    unless system('yarn check --ignore-engines', out: File::NULL)
+      warn(
+        'Error: You have unmet dependencies. (`yarn check` command failed)'.color(:red),
+        'Run `yarn install` to install missing modules.'.color(:green)
+      )
+      abort
+    end
+  end
+
+  desc 'Install Node dependencies with Yarn'
+  task install: ['yarn:available'] do
+    unless system('yarn install --pure-lockfile --ignore-engines')
+      abort 'Error: Unable to install node modules.'.color(:red)
+    end
+  end
+
+  desc 'Remove Node dependencies'
+  task :clobber do
+    warn 'Purging ./node_modules directory'.color(:red)
+    FileUtils.rm_rf 'node_modules'
+  end
+end
+
+desc 'Install Node dependencies with Yarn'
+task yarn: ['yarn:install']

package.json

@@ -3,12 +3,12 @@
   "scripts": {
     "dev-server": "webpack-dev-server --config config/webpack.config.js",
     "eslint": "eslint --max-warnings 0 --ext .js,.js.es6 .",
-    "eslint-fix": "npm run eslint -- --fix",
-    "eslint-report": "npm run eslint -- --format html --output-file ./eslint-report.html",
+    "eslint-fix": "eslint --max-warnings 0 --ext .js,.js.es6 --fix .",
+    "eslint-report": "eslint --max-warnings 0 --ext .js,.js.es6 --format html --output-file ./eslint-report.html .",
     "karma": "karma start config/karma.config.js --single-run",
     "karma-start": "karma start config/karma.config.js",
     "webpack": "webpack --config config/webpack.config.js",
-    "webpack-prod": "NODE_ENV=production npm run webpack"
+    "webpack-prod": "NODE_ENV=production webpack --config config/webpack.config.js"
   },
   "dependencies": {
     "babel-core": "^6.22.1",