Merge branch 'usage-ping-port' into 'master'

Usage ping port

Closes #27750

See merge request !10481

app/assets/javascripts/dispatcher.js

@@ -367,6 +367,9 @@ const ShortcutsBlob = require('./shortcuts_blob');
         case 'admin':
           new Admin();
           switch (path[1]) {
+            case 'cohorts':
+              new gl.UsagePing();
+              break;
             case 'groups':
               new UsersSelect();
               break;

app/assets/javascripts/main.js

@@ -165,6 +165,7 @@ import './syntax_highlight';
 import './task_list';
 import './todos';
 import './tree';
+import './usage_ping';
 import './user';
 import './user_tabs';
 import './username_validator';

app/assets/javascripts/usage_ping.js

+function UsagePing() {
+  const usageDataUrl = $('.usage-data').data('endpoint');
+
+  $.ajax({
+    type: 'GET',
+    url: usageDataUrl,
+    dataType: 'html',
+    success(html) {
+      $('.usage-data').html(html);
+    },
+  });
+}
+
+window.gl = window.gl || {};
+window.gl.UsagePing = UsagePing;

app/controllers/admin/application_settings_controller.rb

@@ -17,6 +17,18 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController
     end
   end
 
+  def usage_data
+    respond_to do |format|
+      format.html do
+        usage_data = Gitlab::UsageData.data
+        usage_data_json = params[:pretty] ? JSON.pretty_generate(usage_data) : usage_data.to_json
+
+        render html: Gitlab::Highlight.highlight('payload.json', usage_data_json)
+      end
+      format.json { render json: Gitlab::UsageData.to_json }
+    end
+  end
+
   def reset_runners_token
     @application_setting.reset_runners_registration_token!
     flash[:notice] = 'New runners registration token has been generated!'
@@ -135,6 +147,7 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController
       :version_check_enabled,
       :terminal_max_session_time,
       :polling_interval_multiplier,
+      :usage_ping_enabled,
 
       disabled_oauth_sign_in_sources: [],
       import_sources: [],

app/controllers/admin/cohorts_controller.rb

+class Admin::CohortsController < Admin::ApplicationController
+  def index
+    if current_application_settings.usage_ping_enabled
+      cohorts_results = Rails.cache.fetch('cohorts', expires_in: 1.day) do
+        CohortsService.new.execute
+      end
+
+      @cohorts = CohortsSerializer.new.represent(cohorts_results)
+    end
+  end
+end

app/controllers/projects/git_http_controller.rb

@@ -5,6 +5,8 @@ class Projects::GitHttpController < Projects::GitHttpClientController
   # GET /foo/bar.git/info/refs?service=git-receive-pack (git push)
   def info_refs
     if upload_pack? && upload_pack_allowed?
+      log_user_activity
+
       render_ok
     elsif receive_pack? && receive_pack_allowed?
       render_ok
@@ -106,4 +108,8 @@ class Projects::GitHttpController < Projects::GitHttpClientController
   def access_klass
     @access_klass ||= wiki? ? Gitlab::GitAccessWiki : Gitlab::GitAccess
   end
+
+  def log_user_activity
+    Users::ActivityService.new(user, 'pull').execute
+  end
 end

app/controllers/sessions_controller.rb

@@ -35,6 +35,7 @@ class SessionsController < Devise::SessionsController
       # hide the signed-in notification
       flash[:notice] = nil
       log_audit_event(current_user, with: authentication_method)
+      log_user_activity(current_user)
     end
   end
 
@@ -123,6 +124,10 @@ class SessionsController < Devise::SessionsController
       for_authentication.security_event
   end
 
+  def log_user_activity(user)
+    Users::ActivityService.new(user, 'login').execute
+  end
+
   def load_recaptcha
     Gitlab::Recaptcha.load_configurations!
   end

app/models/application_setting.rb

@@ -238,7 +238,8 @@ class ApplicationSetting < ActiveRecord::Base
       terminal_max_session_time: 0,
       two_factor_grace_period: 48,
       user_default_external: false,
-      polling_interval_multiplier: 1
+      polling_interval_multiplier: 1,
+      usage_ping_enabled: true
     }
   end
 

app/serializers/cohort_activity_month_entity.rb

+class CohortActivityMonthEntity < Grape::Entity
+  include ActionView::Helpers::NumberHelper
+
+  expose :total do |cohort_activity_month|
+    number_with_delimiter(cohort_activity_month[:total])
+  end
+
+  expose :percentage do |cohort_activity_month|
+    number_to_percentage(cohort_activity_month[:percentage], precision: 0)
+  end
+end

app/serializers/cohort_entity.rb

+class CohortEntity < Grape::Entity
+  include ActionView::Helpers::NumberHelper
+
+  expose :registration_month do |cohort|
+    cohort[:registration_month].strftime('%b %Y')
+  end
+
+  expose :total do |cohort|
+    number_with_delimiter(cohort[:total])
+  end
+
+  expose :inactive do |cohort|
+    number_with_delimiter(cohort[:inactive])
+  end
+
+  expose :activity_months, using: CohortActivityMonthEntity
+end

app/serializers/cohorts_entity.rb

+class CohortsEntity < Grape::Entity
+  expose :months_included
+  expose :cohorts, using: CohortEntity
+end

app/serializers/cohorts_serializer.rb

+class CohortsSerializer < AnalyticsGenericSerializer
+  entity CohortsEntity
+end

app/services/cohorts_service.rb

+class CohortsService
+  MONTHS_INCLUDED = 12
+
+  def execute
+    {
+      months_included: MONTHS_INCLUDED,
+      cohorts: cohorts
+    }
+  end
+
+  # Get an array of hashes that looks like:
+  #
+  #     [
+  #       {
+  #         registration_month: Date.new(2017, 3),
+  #         activity_months: [3, 2, 1],
+  #         total: 3
+  #         inactive: 0
+  #      },
+  #      etc.
+  #
+  # The `months` array is always from oldest to newest, so it's always
+  # non-strictly decreasing from left to right.
+  def cohorts
+    months = Array.new(MONTHS_INCLUDED) { |i| i.months.ago.beginning_of_month.to_date }
+
+    Array.new(MONTHS_INCLUDED) do
+      registration_month = months.last
+      activity_months = running_totals(months, registration_month)
+
+      # Even if no users registered in this month, we always want to have a
+      # value to fill in the table.
+      inactive = counts_by_month[[registration_month, nil]].to_i
+
+      months.pop
+
+      {
+        registration_month: registration_month,
+        activity_months: activity_months,
+        total: activity_months.first[:total],
+        inactive: inactive
+      }
+    end
+  end
+
+  private
+
+  # Calculate a running sum of active users, so users active in later months
+  # count as active in this month, too. Start with the most recent month first,
+  # for calculating the running totals, and then reverse for displaying in the
+  # table.
+  #
+  # Each month has a total, and a percentage of the overall total, as keys.
+  def running_totals(all_months, registration_month)
+    month_totals =
+      all_months
+        .map { |activity_month| counts_by_month[[registration_month, activity_month]] }
+        .reduce([]) { |result, total| result << result.last.to_i + total.to_i }
+        .reverse
+
+    overall_total = month_totals.first
+
+    month_totals.map do |total|
+      { total: total, percentage: total.zero? ? 0 : 100 * total / overall_total }
+    end
+  end
+
+  # Get a hash that looks like:
+  #
+  #     {
+  #       [created_at_month, last_activity_on_month] => count,
+  #       [created_at_month, last_activity_on_month_2] => count_2,
+  #       # etc.
+  #     }
+  #
+  # created_at_month can never be nil, but last_activity_on_month can (when a
+  # user has never logged in, just been created). This covers the last
+  # MONTHS_INCLUDED months.
+  def counts_by_month
+    @counts_by_month ||=
+      begin
+        created_at_month = column_to_date('created_at')
+        last_activity_on_month = column_to_date('last_activity_on')
+
+        User
+          .where('created_at > ?', MONTHS_INCLUDED.months.ago.end_of_month)
+          .group(created_at_month, last_activity_on_month)
+          .reorder("#{created_at_month} ASC", "#{last_activity_on_month} ASC")
+          .count
+      end
+  end
+
+  def column_to_date(column)
+    if Gitlab::Database.postgresql?
+      "CAST(DATE_TRUNC('month', #{column}) AS date)"
+    else
+      "STR_TO_DATE(DATE_FORMAT(#{column}, '%Y-%m-01'), '%Y-%m-%d')"
+    end
+  end
+end

app/services/event_create_service.rb

@@ -72,6 +72,8 @@ class EventCreateService
 
   def push(project, current_user, push_data)
     create_event(project, current_user, Event::PUSHED, data: push_data)
+
+    Users::ActivityService.new(current_user, 'push').execute
   end
 
   private

app/services/users/activity_service.rb

+module Users
+  class ActivityService
+    def initialize(author, activity)
+      @author = author.respond_to?(:user) ? author.user : author
+      @activity = activity
+    end
+
+    def execute
+      return unless @author && @author.is_a?(User)
+
+      record_activity
+    end
+
+    private
+
+    def record_activity
+      Gitlab::UserActivities.record(@author.id)
+
+      Rails.logger.debug("Recorded activity: #{@activity} for User ID: #{@author.id} (username: #{@author.username}")
+    end
+  end
+end

app/views/admin/application_settings/_form.html.haml

@@ -477,7 +477,7 @@
           diagrams in Asciidoc documents using an external PlantUML service.
 
   %fieldset
-    %legend Usage statistics
+    %legend#usage-statistics Usage statistics
     .form-group
       .col-sm-offset-2.col-sm-10
         .checkbox
@@ -486,6 +486,19 @@
             Version check enabled
           .help-block
             Let GitLab inform you when an update is available.
+    .form-group
+      .col-sm-offset-2.col-sm-10
+        .checkbox
+          = f.label :usage_ping_enabled do
+            = f.check_box :usage_ping_enabled
+            Usage ping enabled
+            = link_to icon('question-circle'), help_page_path("user/admin_area/settings/usage_statistics", anchor: "usage-data")
+          .help-block
+            Every week GitLab will report license usage back to GitLab, Inc.
+            Disable this option if you do not want this to occur. To see the
+            JSON payload that will be sent, visit the
+            = succeed '.' do
+              = link_to "Cohorts page", admin_cohorts_path(anchor: 'usage-ping')
 
   %fieldset
     %legend Email

app/views/admin/cohorts/_cohorts_table.html.haml

+.bs-callout.clearfix
+  %p
+    User cohorts are shown for the last #{@cohorts[:months_included]}
+    months. Only users with activity are counted in the cohort total; inactive
+    users are counted separately.
+    = link_to icon('question-circle'), help_page_path('user/admin_area/user_cohorts', anchor: 'cohorts'), title: 'About this feature', target: '_blank'
+
+.table-holder
+  %table.table
+    %thead
+      %tr
+        %th Registration month
+        %th Inactive users
+        %th Cohort total
+        - @cohorts[:months_included].times do |i|
+          %th Month #{i}
+    %tbody
+      - @cohorts[:cohorts].each do |cohort|
+        %tr
+          %td= cohort[:registration_month]
+          %td= cohort[:inactive]
+          %td= cohort[:total]
+          - cohort[:activity_months].each do |activity_month|
+            %td
+              - next if cohort[:total] == '0'
+              = activity_month[:percentage]
+              %br
+              = activity_month[:total]

app/views/admin/cohorts/_usage_ping.html.haml

+%h2#usage-ping Usage ping
+
+.bs-callout.clearfix
+  %p
+    User cohorts are shown because the usage ping is enabled. The data sent with
+    this is shown below. To disable this, visit
+    = succeed '.' do
+      = link_to 'application settings', admin_application_settings_path(anchor: 'usage-statistics')
+
+%pre.usage-data.js-syntax-highlight.code.highlight{ data: { endpoint: usage_data_admin_application_settings_path(format: :html, pretty: true) } }

app/views/admin/cohorts/index.html.haml

+- @no_container = true
+= render "admin/dashboard/head"
+
+%div{ class: container_class }
+  - if @cohorts
+    = render 'cohorts_table'
+    = render 'usage_ping'
+  - else
+    .bs-callout.bs-callout-warning.clearfix
+      %p
+        User cohorts are only shown when the
+        = link_to 'usage ping', help_page_path('user/admin_area/usage_statistics'), target: '_blank'
+        is enabled. To enable it and see user cohorts,
+        visit
+        = succeed '.' do
+          = link_to 'application settings', admin_application_settings_path(anchor: 'usage-statistics')

app/views/admin/dashboard/_head.html.haml

@@ -27,3 +27,7 @@
           = link_to admin_runners_path, title: 'Runners' do
             %span
               Runners
+        = nav_link path: 'cohorts#index' do
+          = link_to admin_cohorts_path, title: 'Cohorts' do
+            %span
+              Cohorts

app/workers/gitlab_usage_ping_worker.rb

+class GitlabUsagePingWorker
+  LEASE_TIMEOUT = 86400
+
+  include Sidekiq::Worker
+  include CronjobQueue
+  include HTTParty
+
+  def perform
+    return unless current_application_settings.usage_ping_enabled
+
+    # Multiple Sidekiq workers could run this. We should only do this at most once a day.
+    return unless try_obtain_lease
+
+    begin
+      HTTParty.post(url,
+                    body: Gitlab::UsageData.to_json(force_refresh: true),
+                    headers: { 'Content-type' => 'application/json' }
+                   )
+    rescue HTTParty::Error => e
+      Rails.logger.info "Unable to contact GitLab, Inc.: #{e}"
+    end
+  end
+
+  def try_obtain_lease
+    Gitlab::ExclusiveLease.new('gitlab_usage_ping_worker:ping', timeout: LEASE_TIMEOUT).try_obtain
+  end
+
+  def url
+    'https://version.gitlab.com/usage_data'
+  end
+end

app/workers/schedule_update_user_activity_worker.rb

+class ScheduleUpdateUserActivityWorker
+  include Sidekiq::Worker
+  include CronjobQueue
+
+  def perform(batch_size = 500)
+    Gitlab::UserActivities.new.each_slice(batch_size) do |batch|
+      UpdateUserActivityWorker.perform_async(Hash[batch])
+    end
+  end
+end

app/workers/update_user_activity_worker.rb

+class UpdateUserActivityWorker
+  include Sidekiq::Worker
+  include DedicatedSidekiqQueue
+
+  def perform(pairs)
+    pairs = cast_data(pairs)
+    ids = pairs.keys
+    conditions = 'WHEN id = ? THEN ? ' * ids.length
+
+    User.where(id: ids).
+      update_all([
+        "last_activity_on = CASE #{conditions} ELSE last_activity_on END",
+        *pairs.to_a.flatten
+      ])
+
+    Gitlab::UserActivities.new.delete(*ids)
+  end
+
+  private
+
+  def cast_data(pairs)
+    pairs.each_with_object({}) do |(key, value), new_pairs|
+      new_pairs[key.to_i] = Time.at(value.to_i).to_s(:db)
+    end
+  end
+end

changelogs/unreleased/usage-ping-port.yml

+---
+title: Add usage ping to CE
+merge_request:
+author:

config/initializers/1_settings.rb

@@ -110,6 +110,14 @@ class Settings < Settingslogic
 
       URI.parse(url_without_path).host
     end
+
+    # Random cron time every Sunday to load balance usage pings
+    def cron_random_weekly_time
+      hour = rand(24)
+      minute = rand(60)
+
+      "#{minute} #{hour} * * 0"
+    end
   end
 end
 
@@ -204,8 +212,8 @@ Settings.gitlab['email_from'] ||= ENV['GITLAB_EMAIL_FROM'] || "gitlab@#{Settings
 Settings.gitlab['email_display_name'] ||= ENV['GITLAB_EMAIL_DISPLAY_NAME'] || 'GitLab'
 Settings.gitlab['email_reply_to'] ||= ENV['GITLAB_EMAIL_REPLY_TO'] || "noreply@#{Settings.gitlab.host}"
 Settings.gitlab['email_subject_suffix'] ||= ENV['GITLAB_EMAIL_SUBJECT_SUFFIX'] || ""
-Settings.gitlab['base_url']   ||= Settings.send(:build_base_gitlab_url)
-Settings.gitlab['url']        ||= Settings.send(:build_gitlab_url)
+Settings.gitlab['base_url']   ||= Settings.__send__(:build_base_gitlab_url)
+Settings.gitlab['url']        ||= Settings.__send__(:build_gitlab_url)
 Settings.gitlab['user']       ||= 'git'
 Settings.gitlab['user_home']  ||= begin
   Etc.getpwnam(Settings.gitlab['user']).dir
@@ -215,7 +223,7 @@ end
 Settings.gitlab['time_zone'] ||= nil
 Settings.gitlab['signup_enabled'] ||= true if Settings.gitlab['signup_enabled'].nil?
 Settings.gitlab['signin_enabled'] ||= true if Settings.gitlab['signin_enabled'].nil?
-Settings.gitlab['restricted_visibility_levels'] = Settings.send(:verify_constant_array, Gitlab::VisibilityLevel, Settings.gitlab['restricted_visibility_levels'], [])
+Settings.gitlab['restricted_visibility_levels'] = Settings.__send__(:verify_constant_array, Gitlab::VisibilityLevel, Settings.gitlab['restricted_visibility_levels'], [])
 Settings.gitlab['username_changing_enabled'] = true if Settings.gitlab['username_changing_enabled'].nil?
 Settings.gitlab['issue_closing_pattern'] = '((?:[Cc]los(?:e[sd]?|ing)|[Ff]ix(?:e[sd]|ing)?|[Rr]esolv(?:e[sd]?|ing))(:?) +(?:(?:issues? +)?%{issue_ref}(?:(?:, *| +and +)?)|([A-Z][A-Z0-9_]+-\d+))+)' if Settings.gitlab['issue_closing_pattern'].nil?
 Settings.gitlab['default_projects_features'] ||= {}
@@ -228,7 +236,7 @@ Settings.gitlab.default_projects_features['wiki']               = true if Settin
 Settings.gitlab.default_projects_features['snippets']           = true if Settings.gitlab.default_projects_features['snippets'].nil?
 Settings.gitlab.default_projects_features['builds']             = true if Settings.gitlab.default_projects_features['builds'].nil?
 Settings.gitlab.default_projects_features['container_registry'] = true if Settings.gitlab.default_projects_features['container_registry'].nil?
-Settings.gitlab.default_projects_features['visibility_level']   = Settings.send(:verify_constant, Gitlab::VisibilityLevel, Settings.gitlab.default_projects_features['visibility_level'], Gitlab::VisibilityLevel::PRIVATE)
+Settings.gitlab.default_projects_features['visibility_level']   = Settings.__send__(:verify_constant, Gitlab::VisibilityLevel, Settings.gitlab.default_projects_features['visibility_level'], Gitlab::VisibilityLevel::PRIVATE)
 Settings.gitlab['domain_whitelist'] ||= []
 Settings.gitlab['import_sources'] ||= %w[github bitbucket gitlab google_code fogbugz git gitlab_project gitea]
 Settings.gitlab['trusted_proxies'] ||= []
@@ -242,7 +250,7 @@ Settings.gitlab_ci['shared_runners_enabled'] = true if Settings.gitlab_ci['share
 Settings.gitlab_ci['all_broken_builds']     = true if Settings.gitlab_ci['all_broken_builds'].nil?
 Settings.gitlab_ci['add_pusher']            = false if Settings.gitlab_ci['add_pusher'].nil?
 Settings.gitlab_ci['builds_path']           = Settings.absolute(Settings.gitlab_ci['builds_path'] || "builds/")
-Settings.gitlab_ci['url']                 ||= Settings.send(:build_gitlab_ci_url)
+Settings.gitlab_ci['url']                 ||= Settings.__send__(:build_gitlab_ci_url)
 
 #
 # Reply by email
@@ -281,7 +289,7 @@ Settings.pages['https']           = false if Settings.pages['https'].nil?
 Settings.pages['host']            ||= "example.com"
 Settings.pages['port']            ||= Settings.pages.https ? 443 : 80
 Settings.pages['protocol']        ||= Settings.pages.https ? "https" : "http"
-Settings.pages['url']             ||= Settings.send(:build_pages_url)
+Settings.pages['url']             ||= Settings.__send__(:build_pages_url)
 Settings.pages['external_http']   ||= false unless Settings.pages['external_http'].present?
 Settings.pages['external_https']  ||= false unless Settings.pages['external_https'].present?
 
@@ -355,6 +363,14 @@ Settings.cron_jobs['remove_unreferenced_lfs_objects_worker']['job_class'] = 'Rem
 Settings.cron_jobs['stuck_import_jobs_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['stuck_import_jobs_worker']['cron'] ||= '15 * * * *'
 Settings.cron_jobs['stuck_import_jobs_worker']['job_class'] = 'StuckImportJobsWorker'
+Settings.cron_jobs['gitlab_usage_ping_worker'] ||= Settingslogic.new({})
+Settings.cron_jobs['gitlab_usage_ping_worker']['cron'] ||= Settings.__send__(:cron_random_weekly_time)
+Settings.cron_jobs['gitlab_usage_ping_worker']['job_class'] = 'GitlabUsagePingWorker'
+
+# Every day at 00:30
+Settings.cron_jobs['schedule_update_user_activity_worker'] ||= Settingslogic.new({})
+Settings.cron_jobs['schedule_update_user_activity_worker']['cron'] ||= '30 0 * * *'
+Settings.cron_jobs['schedule_update_user_activity_worker']['job_class'] = 'ScheduleUpdateUserActivityWorker'
 
 #
 # GitLab Shell
@@ -369,7 +385,7 @@ Settings.gitlab_shell['ssh_host']     ||= Settings.gitlab.ssh_host
 Settings.gitlab_shell['ssh_port']     ||= 22
 Settings.gitlab_shell['ssh_user']     ||= Settings.gitlab.user
 Settings.gitlab_shell['owner_group']  ||= Settings.gitlab.user
-Settings.gitlab_shell['ssh_path_prefix'] ||= Settings.send(:build_gitlab_shell_ssh_path_prefix)
+Settings.gitlab_shell['ssh_path_prefix'] ||= Settings.__send__(:build_gitlab_shell_ssh_path_prefix)
 
 #
 # Repositories

config/routes/admin.rb

@@ -91,6 +91,7 @@ namespace :admin do
 
   resource :application_settings, only: [:show, :update] do
     resources :services, only: [:index, :edit, :update]
+    get :usage_data
     put :reset_runners_token
     put :reset_health_check_token
     put :clear_repository_check_states
@@ -105,6 +106,8 @@ namespace :admin do
     end
   end
 
+  resources :cohorts, only: :index
+
   resources :builds, only: :index do
     collection do
       post :cancel_all

config/sidekiq_queues.yml

@@ -53,3 +53,4 @@
   - [default, 1]
   - [pages, 1]
   - [system_hook_push, 1]
+  - [update_user_activity, 1]

db/migrate/20160713222618_add_usage_ping_to_application_settings.rb

+class AddUsagePingToApplicationSettings < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    add_column :application_settings, :usage_ping_enabled, :boolean, default: true, null: false
+  end
+end

db/migrate/20161007073613_create_user_activities.rb

+class CreateUserActivities < ActiveRecord::Migration
+  DOWNTIME = false
+
+  # This migration is a no-op. It just exists to match EE.
+  def change
+  end
+end

db/migrate/20170307125949_add_last_activity_on_to_users.rb

+class AddLastActivityOnToUsers < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    add_column :users, :last_activity_on, :date
+  end
+end

db/migrate/20170328010804_add_uuid_to_application_settings.rb

+class AddUuidToApplicationSettings < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_column :application_settings, :uuid, :string
+    execute("UPDATE application_settings SET uuid = #{quote(SecureRandom.uuid)}")
+  end
+
+  def down
+    remove_column :application_settings, :uuid
+  end
+end

db/post_migrate/20161128170531_drop_user_activities_table.rb

+class DropUserActivitiesTable < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  # This migration is a no-op. It just exists to match EE.
+  def change
+  end
+end

db/post_migrate/20170324160416_migrate_user_activities_to_users_last_activity_on.rb

+class MigrateUserActivitiesToUsersLastActivityOn < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  disable_ddl_transaction!
+
+  DOWNTIME = false
+  USER_ACTIVITY_SET_KEY = 'user/activities'.freeze
+  ACTIVITIES_PER_PAGE = 100
+  TIME_WHEN_ACTIVITY_SET_WAS_INTRODUCED = Time.utc(2016, 12, 1)
+
+  def up
+    return if activities_count(TIME_WHEN_ACTIVITY_SET_WAS_INTRODUCED, Time.now).zero?
+
+    day = Time.at(activities(TIME_WHEN_ACTIVITY_SET_WAS_INTRODUCED, Time.now).first.second)
+
+    transaction do
+      while day <= Time.now.utc.tomorrow
+        persist_last_activity_on(day: day)
+        day = day.tomorrow
+      end
+    end
+  end
+
+  def down
+    # This ensures we don't lock all users for the duration of the migration.
+    update_column_in_batches(:users, :last_activity_on, nil) do |table, query|
+      query.where(table[:last_activity_on].not_eq(nil))
+    end
+  end
+
+  private
+
+  def persist_last_activity_on(day:, page: 1)
+    activities_count = activities_count(day.at_beginning_of_day, day.at_end_of_day)
+
+    return if activities_count.zero?
+
+    activities = activities(day.at_beginning_of_day, day.at_end_of_day, page: page)
+
+    update_sql =
+      Arel::UpdateManager.new(ActiveRecord::Base).
+        table(users_table).
+        set(users_table[:last_activity_on] => day.to_date).
+        where(users_table[:username].in(activities.map(&:first))).
+        to_sql
+
+    connection.exec_update(update_sql, self.class.name, [])
+
+    unless last_page?(page, activities_count)
+      persist_last_activity_on(day: day, page: page + 1)
+    end
+  end
+
+  def users_table
+    @users_table ||= Arel::Table.new(:users)
+  end
+
+  def activities(from, to, page: 1)
+    Gitlab::Redis.with do |redis|
+      redis.zrangebyscore(USER_ACTIVITY_SET_KEY, from.to_i, to.to_i,
+        with_scores: true,
+        limit: limit(page))
+    end
+  end
+
+  def activities_count(from, to)
+    Gitlab::Redis.with do |redis|
+      redis.zcount(USER_ACTIVITY_SET_KEY, from.to_i, to.to_i)
+    end
+  end
+
+  def limit(page)
+    [offset(page), ACTIVITIES_PER_PAGE]
+  end
+
+  def total_pages(count)
+    (count.to_f / ACTIVITIES_PER_PAGE).ceil
+  end
+
+  def last_page?(page, count)
+    page >= total_pages(count)
+  end
+
+  def offset(page)
+    (page - 1) * ACTIVITIES_PER_PAGE
+  end
+end

db/schema.rb

@@ -116,6 +116,8 @@ ActiveRecord::Schema.define(version: 20170418103908) do
     t.integer "unique_ips_limit_time_window"
     t.boolean "unique_ips_limit_enabled", default: false, null: false
     t.decimal "polling_interval_multiplier", default: 1.0, null: false
+    t.boolean "usage_ping_enabled", default: true, null: false
+    t.string "uuid"
   end
 
   create_table "audit_events", force: :cascade do |t|
@@ -1301,6 +1303,7 @@ ActiveRecord::Schema.define(version: 20170418103908) do
     t.string "organization"
     t.boolean "authorized_projects_populated"
     t.boolean "ghost"
+    t.date "last_activity_on"
     t.boolean "notified_of_own_activity"
     t.boolean "require_two_factor_authentication_from_group", default: false, null: false
     t.integer "two_factor_grace_period", default: 48, null: false

doc/README.md

@@ -69,6 +69,7 @@ All technical content published by GitLab lives in the documentation, including:
 - [Sidekiq Troubleshooting](administration/troubleshooting/sidekiq.md) Debug when Sidekiq appears hung and is not processing jobs.
 - [System hooks](system_hooks/system_hooks.md) Notifications when users, projects and keys are changed.
 - [Update](update/README.md) Update guides to upgrade your installation.
+- [User cohorts](user/admin_area/user_cohorts.md) View user activity over time.
 - [Web terminals](administration/integration/terminal.md) Provide terminal access to environments from within GitLab.
 - [Welcome message](customization/welcome_message.md) Add a custom welcome message to the sign-in page.
 

doc/api/users.md

@@ -72,6 +72,7 @@ GET /users
     "organization": "",
     "last_sign_in_at": "2012-06-01T11:41:01Z",
     "confirmed_at": "2012-05-23T09:05:22Z",
+    "last_activity_on": "2012-05-23",
     "color_scheme_id": 2,
     "projects_limit": 100,
     "current_sign_in_at": "2012-06-02T06:36:55Z",
@@ -104,6 +105,7 @@ GET /users
     "organization": "",
     "last_sign_in_at": null,
     "confirmed_at": "2012-05-30T16:53:06.148Z",
+    "last_activity_on": "2012-05-23",
     "color_scheme_id": 3,
     "projects_limit": 100,
     "current_sign_in_at": "2014-03-19T17:54:13Z",
@@ -196,6 +198,7 @@ Parameters:
   "organization": "",
   "last_sign_in_at": "2012-06-01T11:41:01Z",
   "confirmed_at": "2012-05-23T09:05:22Z",
+  "last_activity_on": "2012-05-23",
   "color_scheme_id": 2,
   "projects_limit": 100,
   "current_sign_in_at": "2012-06-02T06:36:55Z",
@@ -320,6 +323,7 @@ GET /user
   "organization": "",
   "last_sign_in_at": "2012-06-01T11:41:01Z",
   "confirmed_at": "2012-05-23T09:05:22Z",
+  "last_activity_on": "2012-05-23",
   "color_scheme_id": 2,
   "projects_limit": 100,
   "current_sign_in_at": "2012-06-02T06:36:55Z",
@@ -365,6 +369,7 @@ GET /user
   "organization": "",
   "last_sign_in_at": "2012-06-01T11:41:01Z",
   "confirmed_at": "2012-05-23T09:05:22Z",
+  "last_activity_on": "2012-05-23",
   "color_scheme_id": 2,
   "projects_limit": 100,
   "current_sign_in_at": "2012-06-02T06:36:55Z",
@@ -986,3 +991,55 @@ Parameters:
 | --------- | ---- | -------- | ----------- |
 | `user_id` | integer | yes | The ID of the user |
 | `impersonation_token_id` | integer | yes | The ID of the impersonation token |
+
+### Get user activities (admin only)
+
+>**Note:** This API endpoint is only available on 8.15 (EE) and 9.1 (CE) and above.
+
+Get the last activity date for all users, sorted from oldest to newest.
+
+The activities that update the timestamp are:
+
+  - Git HTTP/SSH activities (such as clone, push)
+  - User logging in into GitLab
+
+By default, it shows the activity for all users in the last 6 months, but this can be
+amended by using the `from` parameter.
+
+```
+GET /user/activities
+```
+
+Parameters:
+
+| Attribute | Type | Required | Description |
+| --------- | ---- | -------- | ----------- |
+| `from` | string | no | Date string in the format YEAR-MONTH-DAY, e.g. `2016-03-11`. Defaults to 6 months ago. |
+
+```bash
+curl --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" https://gitlab.example.com/api/v3/user/activities
+```
+
+Example response:
+
+```json
+[
+  {
+    "username": "user1",
+    "last_activity_on": "2015-12-14",
+    "last_activity_at": "2015-12-14"
+  },
+  {
+    "username": "user2",
+    "last_activity_on": "2015-12-15",
+    "last_activity_at": "2015-12-15"
+  },
+  {
+    "username": "user3",
+    "last_activity_on": "2015-12-16",
+    "last_activity_at": "2015-12-16"
+  }
+]
+```
+
+Please note that `last_activity_at` is deprecated, please use `last_activity_on`.

doc/user/admin_area/settings/usage_statistics.md

+# Usage statistics
+
+GitLab Inc. will periodically collect information about your instance in order
+to perform various actions.
+
+All statistics are opt-in and you can always disable them from the admin panel.
+
+## Version check
+
+GitLab can inform you when an update is available and the importance of it.
+
+No information other than the GitLab version and the instance's domain name
+are collected.
+
+In the **Overview** tab you can see if your GitLab version is up to date. There
+are three cases: 1) you are up to date (green), 2) there is an update available
+(yellow) and 3) your version is vulnerable and a security fix is released (red).
+
+In any case, you will see a message informing you of the state and the
+importance of the update.
+
+If enabled, the version status will also be shown in the help page (`/help`)
+for all signed in users.
+
+## Usage ping
+
+> [Introduced][ee-557] in GitLab Enterprise Edition 8.10. More statistics
+[were added][ee-735] in GitLab Enterprise Edition
+8.12. [Moved to GitLab Community Edition][ce-23361] in 9.1.
+
+GitLab Inc. can collect non-sensitive information about how GitLab users
+use their GitLab instance upon the activation of a ping feature
+located in the admin panel (`/admin/application_settings`).
+
+You can see the **exact** JSON payload that your instance sends to GitLab
+in the "Usage statistics" section of the admin panel.
+
+Nothing qualitative is collected. Only quantitative. That means no project
+names, author names, comment bodies, names of labels, etc.
+
+The usage ping is sent in order for GitLab Inc. to have a better understanding
+of how our users use our product, and to be more data-driven when creating or
+changing features.
+
+The total number of the following is sent back to GitLab Inc.:
+
+- Comments
+- Groups
+- Users
+- Projects
+- Issues
+- Labels
+- CI builds
+- Snippets
+- Milestones
+- Todos
+- Pushes
+- Merge requests
+- Environments
+- Triggers
+- Deploy keys
+- Pages
+- Project Services
+- Projects using the Prometheus service
+- Issue Boards
+- CI Runners
+- Deployments
+- Geo Nodes
+- LDAP Groups
+- LDAP Keys
+- LDAP Users
+- LFS objects
+- Protected branches
+- Releases
+- Remote mirrors
+- Uploads
+- Web hooks
+
+Also, we track if you've installed Mattermost with GitLab.
+For example: `"mattermost_enabled":true"`.
+
+More data will be added over time. The goal of this ping is to be as light as
+possible, so it won't have any performance impact on your installation when
+the calculation is made.
+
+### Deactivate the usage ping
+
+By default, usage ping is opt-out. If you want to deactivate this feature, go to
+the Settings page of your administration panel and uncheck the Usage ping
+checkbox.
+
+## Privacy policy
+
+GitLab Inc. does **not** collect any sensitive information, like project names
+or the content of the comments. GitLab Inc. does not disclose or otherwise make
+available any of the data collected on a customer specific basis.
+
+Read more about this in the [Privacy policy](https://about.gitlab.com/privacy).
+
+[ee-557]: https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/557
+[ee-735]: https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/735
+[ce-23361]: https://gitlab.com/gitlab-org/gitlab-ce/issues/23361

doc/user/admin_area/user_cohorts.md

+# Cohorts
+
+> **Notes:**
+- [Introduced][ce-23361] in GitLab 9.1.
+
+As a benefit of having the [usage ping active](settings/usage_statistics.md),
+GitLab lets you analyze the users' activities of your GitLab installation.
+Under `/admin/cohorts`, when the usage ping is active, GitLab will show the
+monthly cohorts of new users and their activities over time.
+
+How do we read the user cohorts table? Let's take an example with the following
+user cohorts.
+
+![User cohort example](img/cohorts.png)
+
+For the cohort of June 2016, 163 users have been created on this server. One
+month later, in July 2016, 155 users (or 95% of the June cohort) are still
+active. Two months later, 139 users (or 85%) are still active. 9 months later,
+we can see that only 6% of this cohort are still active.
+
+How do we measure the activity of users? GitLab considers a user active if:
+* the user signs in
+* the user has Git activity (whether push or pull).
+
+### Setup
+
+1. Activate the usage ping as defined [in the documentation](settings/usage_statistics.md)
+2. Go to `/admin/cohorts` to see the user cohorts of the server
+
+[ce-23361]: https://gitlab.com/gitlab-org/gitlab-ce/issues/23361

lib/api/entities.rb

@@ -18,6 +18,12 @@ module API
       expose :bio, :location, :skype, :linkedin, :twitter, :website_url, :organization
     end
 
+    class UserActivity < Grape::Entity
+      expose :username
+      expose :last_activity_on
+      expose :last_activity_on, as: :last_activity_at # Back-compat
+    end
+
     class Identity < Grape::Entity
       expose :provider, :extern_uid
     end
@@ -25,6 +31,7 @@ module API
     class UserPublic < User
       expose :last_sign_in_at
       expose :confirmed_at
+      expose :last_activity_on
       expose :email
       expose :color_scheme_id, :projects_limit, :current_sign_in_at
       expose :identities, using: Entities::Identity

lib/api/helpers/internal_helpers.rb

@@ -60,6 +60,12 @@ module API
       rescue JSON::ParserError
         {}
       end
+
+      def log_user_activity(actor)
+        commands = Gitlab::GitAccess::DOWNLOAD_COMMANDS
+
+        ::Users::ActivityService.new(actor, 'Git SSH').execute if commands.include?(params[:action])
+      end
     end
   end
 end

lib/api/internal.rb

@@ -40,6 +40,8 @@ module API
         response = { status: access_status.status, message: access_status.message }
 
         if access_status.status
+          log_user_activity(actor)
+
           # Return the repository full path so that gitlab-shell has it when
           # handling ssh commands
           response[:repository_path] =

lib/api/issues.rb

@@ -35,7 +35,7 @@ module API
         optional :assignee_id, type: Integer, desc: 'The ID of a user to assign issue'
         optional :milestone_id, type: Integer, desc: 'The ID of a milestone to assign issue'
         optional :labels, type: String, desc: 'Comma-separated list of label names'
-        optional :due_date, type: String, desc: 'Date time string in the format YEAR-MONTH-DAY'
+        optional :due_date, type: String, desc: 'Date string in the format YEAR-MONTH-DAY'
         optional :confidential, type: Boolean, desc: 'Boolean parameter if the issue should be confidential'
       end
 

lib/api/users.rb

@@ -534,6 +534,21 @@ module API
         email.destroy
         current_user.update_secondary_emails!
       end
+
+      desc 'Get a list of user activities'
+      params do
+        optional :from, type: DateTime, default: 6.months.ago, desc: 'Date string in the format YEAR-MONTH-DAY'
+        use :pagination
+      end
+      get "activities" do
+        authenticated_as_admin!
+
+        activities = User.
+          where(User.arel_table[:last_activity_on].gteq(params[:from])).
+          reorder(last_activity_on: :asc)
+
+        present paginate(activities), with: Entities::UserActivity
+      end
     end
   end
 end

lib/gitlab/usage_data.rb

+module Gitlab
+  class UsageData
+    include Gitlab::CurrentSettings
+
+    class << self
+      def data(force_refresh: false)
+        Rails.cache.fetch('usage_data', force: force_refresh, expires_in: 2.weeks) { uncached_data }
+      end
+
+      def uncached_data
+        license_usage_data.merge(system_usage_data)
+      end
+
+      def to_json(force_refresh: false)
+        data(force_refresh: force_refresh).to_json
+      end
+
+      def system_usage_data
+        {
+          counts: {
+            boards: Board.count,
+            ci_builds: ::Ci::Build.count,
+            ci_pipelines: ::Ci::Pipeline.count,
+            ci_runners: ::Ci::Runner.count,
+            ci_triggers: ::Ci::Trigger.count,
+            deploy_keys: DeployKey.count,
+            deployments: Deployment.count,
+            environments: Environment.count,
+            groups: Group.count,
+            issues: Issue.count,
+            keys: Key.count,
+            labels: Label.count,
+            lfs_objects: LfsObject.count,
+            merge_requests: MergeRequest.count,
+            milestones: Milestone.count,
+            notes: Note.count,
+            pages_domains: PagesDomain.count,
+            projects: Project.count,
+            projects_prometheus_active: PrometheusService.active.count,
+            protected_branches: ProtectedBranch.count,
+            releases: Release.count,
+            services: Service.where(active: true).count,
+            snippets: Snippet.count,
+            todos: Todo.count,
+            uploads: Upload.count,
+            web_hooks: WebHook.count
+          }
+        }
+      end
+
+      def license_usage_data
+        usage_data = {
+          uuid: current_application_settings.uuid,
+          version: Gitlab::VERSION,
+          active_user_count: User.active.count,
+          recorded_at: Time.now,
+          mattermost_enabled: Gitlab.config.mattermost.enabled,
+          edition: 'CE'
+        }
+
+        usage_data
+      end
+    end
+  end
+end

lib/gitlab/user_activities.rb

+module Gitlab
+  class UserActivities
+    include Enumerable
+
+    KEY = 'users:activities'.freeze
+    BATCH_SIZE = 500
+
+    def self.record(key, time = Time.now)
+      Gitlab::Redis.with do |redis|
+        redis.hset(KEY, key, time.to_i)
+      end
+    end
+
+    def delete(*keys)
+      Gitlab::Redis.with do |redis|
+        redis.hdel(KEY, keys)
+      end
+    end
+
+    def each
+      cursor = 0
+      loop do
+        cursor, pairs =
+          Gitlab::Redis.with do |redis|
+            redis.hscan(KEY, cursor, count: BATCH_SIZE)
+          end
+
+        Hash[pairs].each { |pair| yield pair }
+
+        break if cursor == '0'
+      end
+    end
+  end
+end

spec/controllers/admin/application_settings_controller_spec.rb

@@ -3,12 +3,49 @@ require 'spec_helper'
 describe Admin::ApplicationSettingsController do
   include StubENV
 
+  let(:group) { create(:group) }
+  let(:project) { create(:project, namespace: group) }
   let(:admin) { create(:admin) }
+  let(:user) { create(:user)}
 
   before do
     stub_env('IN_MEMORY_APPLICATION_SETTINGS', 'false')
   end
 
+  describe 'GET #usage_data with no access' do
+    before do
+      sign_in(user)
+    end
+
+    it 'returns 404' do
+      get :usage_data, format: :html
+
+      expect(response.status).to eq(404)
+    end
+  end
+
+  describe 'GET #usage_data' do
+    before do
+      sign_in(admin)
+    end
+
+    it 'returns HTML data' do
+      get :usage_data, format: :html
+
+      expect(response.body).to start_with('<span')
+      expect(response.status).to eq(200)
+    end
+
+    it 'returns JSON data' do
+      get :usage_data, format: :json
+
+      body = JSON.parse(response.body)
+      expect(body["version"]).to eq(Gitlab::VERSION)
+      expect(body).to include('counts')
+      expect(response.status).to eq(200)
+    end
+  end
+
   describe 'PUT #update' do
     before do
       sign_in(admin)

spec/controllers/sessions_controller_spec.rb

@@ -16,7 +16,9 @@ describe SessionsController do
         end
       end
 
-      context 'when using valid password' do
+      context 'when using valid password', :redis do
+        include UserActivitiesHelpers
+
         let(:user) { create(:user) }
 
         it 'authenticates user correctly' do
@@ -37,6 +39,12 @@ describe SessionsController do
             subject.sign_out user
           end
         end
+
+        it 'updates the user activity' do
+          expect do
+            post(:create, user: { login: user.username, password: user.password })
+          end.to change { user_activity(user) }
+        end
       end
     end
 

spec/lib/gitlab/usage_data_spec.rb

+require 'spec_helper'
+
+describe Gitlab::UsageData do
+  let!(:project) { create(:empty_project) }
+  let!(:project2) { create(:empty_project) }
+  let!(:board) { create(:board, project: project) }
+
+  describe '#data' do
+    subject { Gitlab::UsageData.data }
+
+    it "gathers usage data" do
+      expect(subject.keys).to match_array(%i(
+        active_user_count
+        counts
+        recorded_at
+        mattermost_enabled
+        edition
+        version
+        uuid
+      ))
+    end
+
+    it "gathers usage counts" do
+      count_data = subject[:counts]
+
+      expect(count_data[:boards]).to eq(1)
+      expect(count_data[:projects]).to eq(2)
+
+      expect(count_data.keys).to match_array(%i(
+        boards
+        ci_builds
+        ci_pipelines
+        ci_runners
+        ci_triggers
+        deploy_keys
+        deployments
+        environments
+        groups
+        issues
+        keys
+        labels
+        lfs_objects
+        merge_requests
+        milestones
+        notes
+        projects
+        projects_prometheus_active
+        pages_domains
+        protected_branches
+        releases
+        services
+        snippets
+        todos
+        uploads
+        web_hooks
+      ))
+    end
+  end
+
+  describe '#license_usage_data' do
+    subject { Gitlab::UsageData.license_usage_data }
+
+    it "gathers license data" do
+      expect(subject[:uuid]).to eq(current_application_settings.uuid)
+      expect(subject[:version]).to eq(Gitlab::VERSION)
+      expect(subject[:active_user_count]).to eq(User.active.count)
+      expect(subject[:recorded_at]).to be_a(Time)
+    end
+  end
+end

spec/lib/gitlab/user_activities_spec.rb

+require 'spec_helper'
+
+describe Gitlab::UserActivities, :redis, lib: true do
+  let(:now) { Time.now }
+
+  describe '.record' do
+    context 'with no time given' do
+      it 'uses Time.now and records an activity in Redis' do
+        Timecop.freeze do
+          now # eager-load now
+          described_class.record(42)
+        end
+
+        Gitlab::Redis.with do |redis|
+          expect(redis.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]])
+        end
+      end
+    end
+
+    context 'with a time given' do
+      it 'uses the given time and records an activity in Redis' do
+        described_class.record(42, now)
+
+        Gitlab::Redis.with do |redis|
+          expect(redis.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]])
+        end
+      end
+    end
+  end
+
+  describe '.delete' do
+    context 'with a single key' do
+      context 'and key exists' do
+        it 'removes the pair from Redis' do
+          described_class.record(42, now)
+
+          Gitlab::Redis.with do |redis|
+            expect(redis.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]])
+          end
+
+          subject.delete(42)
+
+          Gitlab::Redis.with do |redis|
+            expect(redis.hscan(described_class::KEY, 0)).to eq(['0', []])
+          end
+        end
+      end
+
+      context 'and key does not exist' do
+        it 'removes the pair from Redis' do
+          Gitlab::Redis.with do |redis|
+            expect(redis.hscan(described_class::KEY, 0)).to eq(['0', []])
+          end
+
+          subject.delete(42)
+
+          Gitlab::Redis.with do |redis|
+            expect(redis.hscan(described_class::KEY, 0)).to eq(['0', []])
+          end
+        end
+      end
+    end
+
+    context 'with multiple keys' do
+      context 'and all keys exist' do
+        it 'removes the pair from Redis' do
+          described_class.record(41, now)
+          described_class.record(42, now)
+
+          Gitlab::Redis.with do |redis|
+            expect(redis.hscan(described_class::KEY, 0)).to eq(['0', [['41', now.to_i.to_s], ['42', now.to_i.to_s]]])
+          end
+
+          subject.delete(41, 42)
+
+          Gitlab::Redis.with do |redis|
+            expect(redis.hscan(described_class::KEY, 0)).to eq(['0', []])
+          end
+        end
+      end
+
+      context 'and some keys does not exist' do
+        it 'removes the existing pair from Redis' do
+          described_class.record(42, now)
+
+          Gitlab::Redis.with do |redis|
+            expect(redis.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]])
+          end
+
+          subject.delete(41, 42)
+
+          Gitlab::Redis.with do |redis|
+            expect(redis.hscan(described_class::KEY, 0)).to eq(['0', []])
+          end
+        end
+      end
+    end
+  end
+
+  describe 'Enumerable' do
+    before do
+      described_class.record(40, now)
+      described_class.record(41, now)
+      described_class.record(42, now)
+    end
+
+    it 'allows to read the activities sequentially' do
+      expected = { '40' => now.to_i.to_s, '41' => now.to_i.to_s, '42' => now.to_i.to_s }
+
+      actual = described_class.new.each_with_object({}) do |(key, time), actual|
+        actual[key] = time
+      end
+
+      expect(actual).to eq(expected)
+    end
+
+    context 'with many records' do
+      before do
+        1_000.times { |i| described_class.record(i, now) }
+      end
+
+      it 'is possible to loop through all the records' do
+        expect(described_class.new.count).to eq(1_000)
+      end
+    end
+  end
+end

spec/migrations/migrate_user_activities_to_users_last_activity_on_spec.rb

+# encoding: utf-8
+
+require 'spec_helper'
+require Rails.root.join('db', 'post_migrate', '20170324160416_migrate_user_activities_to_users_last_activity_on.rb')
+
+describe MigrateUserActivitiesToUsersLastActivityOn, :redis do
+  let(:migration) { described_class.new }
+  let!(:user_active_1) { create(:user) }
+  let!(:user_active_2) { create(:user) }
+
+  def record_activity(user, time)
+    Gitlab::Redis.with do |redis|
+      redis.zadd(described_class::USER_ACTIVITY_SET_KEY, time.to_i, user.username)
+    end
+  end
+
+  around do |example|
+    Timecop.freeze { example.run }
+  end
+
+  before do
+    record_activity(user_active_1, described_class::TIME_WHEN_ACTIVITY_SET_WAS_INTRODUCED + 2.months)
+    record_activity(user_active_2, described_class::TIME_WHEN_ACTIVITY_SET_WAS_INTRODUCED + 3.months)
+    mute_stdout { migration.up }
+  end
+
+  describe '#up' do
+    it 'fills last_activity_on from the legacy Redis Sorted Set' do
+      expect(user_active_1.reload.last_activity_on).to eq((described_class::TIME_WHEN_ACTIVITY_SET_WAS_INTRODUCED + 2.months).to_date)
+      expect(user_active_2.reload.last_activity_on).to eq((described_class::TIME_WHEN_ACTIVITY_SET_WAS_INTRODUCED + 3.months).to_date)
+    end
+  end
+
+  describe '#down' do
+    it 'sets last_activity_on to NULL for all users' do
+      mute_stdout { migration.down }
+
+      expect(user_active_1.reload.last_activity_on).to be_nil
+      expect(user_active_2.reload.last_activity_on).to be_nil
+    end
+  end
+
+  def mute_stdout
+    orig_stdout = $stdout
+    $stdout = StringIO.new
+    yield
+    $stdout = orig_stdout
+  end
+end

spec/requests/api/internal_spec.rb

@@ -147,10 +147,15 @@ describe API::Internal, api: true  do
     end
   end
 
-  describe "POST /internal/allowed" do
+  describe "POST /internal/allowed", :redis do
     context "access granted" do
       before do
         project.team << [user, :developer]
+        Timecop.freeze
+      end
+
+      after do
+        Timecop.return
       end
 
       context 'with env passed as a JSON' do
@@ -176,6 +181,7 @@ describe API::Internal, api: true  do
           expect(response).to have_http_status(200)
           expect(json_response["status"]).to be_truthy
           expect(json_response["repository_path"]).to eq(project.wiki.repository.path_to_repo)
+          expect(user).not_to have_an_activity_record
         end
       end
 
@@ -186,6 +192,7 @@ describe API::Internal, api: true  do
           expect(response).to have_http_status(200)
           expect(json_response["status"]).to be_truthy
           expect(json_response["repository_path"]).to eq(project.wiki.repository.path_to_repo)
+          expect(user).to have_an_activity_record
         end
       end
 
@@ -196,6 +203,7 @@ describe API::Internal, api: true  do
           expect(response).to have_http_status(200)
           expect(json_response["status"]).to be_truthy
           expect(json_response["repository_path"]).to eq(project.repository.path_to_repo)
+          expect(user).to have_an_activity_record
         end
       end
 
@@ -206,6 +214,7 @@ describe API::Internal, api: true  do
           expect(response).to have_http_status(200)
           expect(json_response["status"]).to be_truthy
           expect(json_response["repository_path"]).to eq(project.repository.path_to_repo)
+          expect(user).not_to have_an_activity_record
         end
 
         context 'project as /namespace/project' do
@@ -241,6 +250,7 @@ describe API::Internal, api: true  do
 
           expect(response).to have_http_status(200)
           expect(json_response["status"]).to be_falsey
+          expect(user).not_to have_an_activity_record
         end
       end
 
@@ -250,6 +260,7 @@ describe API::Internal, api: true  do
 
           expect(response).to have_http_status(200)
           expect(json_response["status"]).to be_falsey
+          expect(user).not_to have_an_activity_record
         end
       end
     end
@@ -267,6 +278,7 @@ describe API::Internal, api: true  do
 
           expect(response).to have_http_status(200)
           expect(json_response["status"]).to be_falsey
+          expect(user).not_to have_an_activity_record
         end
       end
 
@@ -276,6 +288,7 @@ describe API::Internal, api: true  do
 
           expect(response).to have_http_status(200)
           expect(json_response["status"]).to be_falsey
+          expect(user).not_to have_an_activity_record
         end
       end
     end

spec/requests/api/users_spec.rb

 require 'spec_helper'
 
-describe API::Users, api: true  do
+describe API::Users, api: true do
   include ApiHelpers
 
-  let(:user)  { create(:user) }
+  let(:user) { create(:user) }
   let(:admin) { create(:admin) }
-  let(:key)   { create(:key, user: user) }
-  let(:email)   { create(:email, user: user) }
+  let(:key) { create(:key, user: user) }
+  let(:email) { create(:email, user: user) }
   let(:omniauth_user) { create(:omniauth_user) }
   let(:ldap_user) { create(:omniauth_user, provider: 'ldapmain') }
   let(:ldap_blocked_user) { create(:omniauth_user, provider: 'ldapmain', state: 'ldap_blocked') }
@@ -129,7 +129,7 @@ describe API::Users, api: true  do
   end
 
   describe "POST /users" do
-    before{ admin }
+    before { admin }
 
     it "creates user" do
       expect do
@@ -214,9 +214,9 @@ describe API::Users, api: true  do
 
     it "does not create user with invalid email" do
       post api('/users', admin),
-        email: 'invalid email',
-        password: 'password',
-        name: 'test'
+           email: 'invalid email',
+           password: 'password',
+           name: 'test'
       expect(response).to have_http_status(400)
     end
 
@@ -242,12 +242,12 @@ describe API::Users, api: true  do
 
     it 'returns 400 error if user does not validate' do
       post api('/users', admin),
-        password: 'pass',
-        email: 'test@example.com',
-        username: 'test!',
-        name: 'test',
-        bio: 'g' * 256,
-        projects_limit: -1
+           password: 'pass',
+           email: 'test@example.com',
+           username: 'test!',
+           name: 'test',
+           bio: 'g' * 256,
+           projects_limit: -1
       expect(response).to have_http_status(400)
       expect(json_response['message']['password']).
         to eq(['is too short (minimum is 8 characters)'])
@@ -267,19 +267,19 @@ describe API::Users, api: true  do
     context 'with existing user' do
       before do
         post api('/users', admin),
-          email: 'test@example.com',
-          password: 'password',
-          username: 'test',
-          name: 'foo'
+             email: 'test@example.com',
+             password: 'password',
+             username: 'test',
+             name: 'foo'
       end
 
       it 'returns 409 conflict error if user with same email exists' do
         expect do
           post api('/users', admin),
-            name: 'foo',
-            email: 'test@example.com',
-            password: 'password',
-            username: 'foo'
+               name: 'foo',
+               email: 'test@example.com',
+               password: 'password',
+               username: 'foo'
         end.to change { User.count }.by(0)
         expect(response).to have_http_status(409)
         expect(json_response['message']).to eq('Email has already been taken')
@@ -288,10 +288,10 @@ describe API::Users, api: true  do
       it 'returns 409 conflict error if same username exists' do
         expect do
           post api('/users', admin),
-            name: 'foo',
-            email: 'foo@example.com',
-            password: 'password',
-            username: 'test'
+               name: 'foo',
+               email: 'foo@example.com',
+               password: 'password',
+               username: 'test'
         end.to change { User.count }.by(0)
         expect(response).to have_http_status(409)
         expect(json_response['message']).to eq('Username has already been taken')
@@ -416,12 +416,12 @@ describe API::Users, api: true  do
 
     it 'returns 400 error if user does not validate' do
       put api("/users/#{user.id}", admin),
-        password: 'pass',
-        email: 'test@example.com',
-        username: 'test!',
-        name: 'test',
-        bio: 'g' * 256,
-        projects_limit: -1
+          password: 'pass',
+          email: 'test@example.com',
+          username: 'test!',
+          name: 'test',
+          bio: 'g' * 256,
+          projects_limit: -1
       expect(response).to have_http_status(400)
       expect(json_response['message']['password']).
         to eq(['is too short (minimum is 8 characters)'])
@@ -488,7 +488,7 @@ describe API::Users, api: true  do
       key_attrs = attributes_for :key
       expect do
         post api("/users/#{user.id}/keys", admin), key_attrs
-      end.to change{ user.keys.count }.by(1)
+      end.to change { user.keys.count }.by(1)
     end
 
     it "returns 400 for invalid ID" do
@@ -580,7 +580,7 @@ describe API::Users, api: true  do
       email_attrs = attributes_for :email
       expect do
         post api("/users/#{user.id}/emails", admin), email_attrs
-      end.to change{ user.emails.count }.by(1)
+      end.to change { user.emails.count }.by(1)
     end
 
     it "returns a 400 for invalid ID" do
@@ -842,7 +842,7 @@ describe API::Users, api: true  do
       key_attrs = attributes_for :key
       expect do
         post api("/user/keys", user), key_attrs
-      end.to change{ user.keys.count }.by(1)
+      end.to change { user.keys.count }.by(1)
       expect(response).to have_http_status(201)
     end
 
@@ -880,7 +880,7 @@ describe API::Users, api: true  do
         delete api("/user/keys/#{key.id}", user)
 
         expect(response).to have_http_status(204)
-      end.to change{user.keys.count}.by(-1)
+      end.to change { user.keys.count}.by(-1)
     end
 
     it "returns 404 if key ID not found" do
@@ -963,7 +963,7 @@ describe API::Users, api: true  do
       email_attrs = attributes_for :email
       expect do
         post api("/user/emails", user), email_attrs
-      end.to change{ user.emails.count }.by(1)
+      end.to change { user.emails.count }.by(1)
       expect(response).to have_http_status(201)
     end
 
@@ -989,7 +989,7 @@ describe API::Users, api: true  do
         delete api("/user/emails/#{email.id}", user)
 
         expect(response).to have_http_status(204)
-      end.to change{user.emails.count}.by(-1)
+      end.to change { user.emails.count}.by(-1)
     end
 
     it "returns 404 if email ID not found" do
@@ -1158,6 +1158,49 @@ describe API::Users, api: true  do
     end
   end
 
+  context "user activities", :redis do
+    let!(:old_active_user) { create(:user, last_activity_on: Time.utc(2000, 1, 1)) }
+    let!(:newly_active_user) { create(:user, last_activity_on: 2.days.ago.midday) }
+
+    context 'last activity as normal user' do
+      it 'has no permission' do
+        get api("/user/activities", user)
+
+        expect(response).to have_http_status(403)
+      end
+    end
+
+    context 'as admin' do
+      it 'returns the activities from the last 6 months' do
+        get api("/user/activities", admin)
+
+        expect(response).to include_pagination_headers
+        expect(json_response.size).to eq(1)
+
+        activity = json_response.last
+
+        expect(activity['username']).to eq(newly_active_user.username)
+        expect(activity['last_activity_on']).to eq(2.days.ago.to_date.to_s)
+        expect(activity['last_activity_at']).to eq(2.days.ago.to_date.to_s)
+      end
+
+      context 'passing a :from parameter' do
+        it 'returns the activities from the given date' do
+          get api("/user/activities?from=2000-1-1", admin)
+
+          expect(response).to include_pagination_headers
+          expect(json_response.size).to eq(2)
+
+          activity = json_response.first
+
+          expect(activity['username']).to eq(old_active_user.username)
+          expect(activity['last_activity_on']).to eq(Time.utc(2000, 1, 1).to_date.to_s)
+          expect(activity['last_activity_at']).to eq(Time.utc(2000, 1, 1).to_date.to_s)
+        end
+      end
+    end
+  end
+
   describe 'GET /users/:user_id/impersonation_tokens' do
     let!(:active_personal_access_token) { create(:personal_access_token, user: user) }
     let!(:revoked_personal_access_token) { create(:personal_access_token, :revoked, user: user) }

spec/requests/git_http_spec.rb

@@ -3,6 +3,7 @@ require "spec_helper"
 describe 'Git HTTP requests', lib: true do
   include GitHttpHelpers
   include WorkhorseHelpers
+  include UserActivitiesHelpers
 
   it "gives WWW-Authenticate hints" do
     clone_get('doesnt/exist.git')
@@ -255,6 +256,14 @@ describe 'Git HTTP requests', lib: true do
                     expect(response.content_type.to_s).to eq(Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE)
                   end
                 end
+
+                it 'updates the user last activity', :redis do
+                  expect(user_activity(user)).to be_nil
+
+                  download(path, env) do |response|
+                    expect(user_activity(user)).to be_present
+                  end
+                end
               end
 
               context "when an oauth token is provided" do

spec/services/cohorts_service_spec.rb

+require 'spec_helper'
+
+describe CohortsService do
+  describe '#execute' do
+    def month_start(months_ago)
+      months_ago.months.ago.beginning_of_month.to_date
+    end
+
+    # In the interests of speed and clarity, this example has minimal data.
+    it 'returns a list of user cohorts' do
+      6.times do |months_ago|
+        months_ago_time = (months_ago * 2).months.ago
+
+        create(:user, created_at: months_ago_time, last_activity_on: Time.now)
+        create(:user, created_at: months_ago_time, last_activity_on: months_ago_time)
+      end
+
+      create(:user) # this user is inactive and belongs to the current month
+
+      expected_cohorts = [
+        {
+          registration_month: month_start(11),
+          activity_months: Array.new(12) { { total: 0, percentage: 0 } },
+          total: 0,
+          inactive: 0
+        },
+        {
+          registration_month: month_start(10),
+          activity_months: [{ total: 2, percentage: 100 }] + Array.new(10) { { total: 1, percentage: 50 } },
+          total: 2,
+          inactive: 0
+        },
+        {
+          registration_month: month_start(9),
+          activity_months: Array.new(10) { { total: 0, percentage: 0 } },
+          total: 0,
+          inactive: 0
+        },
+        {
+          registration_month: month_start(8),
+          activity_months: [{ total: 2, percentage: 100 }] + Array.new(8) { { total: 1, percentage: 50 } },
+          total: 2,
+          inactive: 0
+        },
+        {
+          registration_month: month_start(7),
+          activity_months: Array.new(8) { { total: 0, percentage: 0 } },
+          total: 0,
+          inactive: 0
+        },
+        {
+          registration_month: month_start(6),
+          activity_months: [{ total: 2, percentage: 100 }] + Array.new(6) { { total: 1, percentage: 50 } },
+          total: 2,
+          inactive: 0
+        },
+        {
+          registration_month: month_start(5),
+          activity_months: Array.new(6) { { total: 0, percentage: 0 } },
+          total: 0,
+          inactive: 0
+        },
+        {
+          registration_month: month_start(4),
+          activity_months: [{ total: 2, percentage: 100 }] + Array.new(4) { { total: 1, percentage: 50 } },
+          total: 2,
+          inactive: 0
+        },
+        {
+          registration_month: month_start(3),
+          activity_months: Array.new(4) { { total: 0, percentage: 0 } },
+          total: 0,
+          inactive: 0
+        },
+        {
+          registration_month: month_start(2),
+          activity_months: [{ total: 2, percentage: 100 }] + Array.new(2) { { total: 1, percentage: 50 } },
+          total: 2,
+          inactive: 0
+        },
+        {
+          registration_month: month_start(1),
+          activity_months: Array.new(2) { { total: 0, percentage: 0 } },
+          total: 0,
+          inactive: 0
+        },
+        {
+          registration_month: month_start(0),
+          activity_months: [{ total: 2, percentage: 100 }],
+          total: 2,
+          inactive: 1
+        },
+      ]
+
+      expect(described_class.new.execute).to eq(months_included: 12,
+                                                cohorts: expected_cohorts)
+    end
+  end
+end

spec/services/event_create_service_spec.rb

 require 'spec_helper'
 
 describe EventCreateService, services: true do
+  include UserActivitiesHelpers
+
   let(:service) { EventCreateService.new }
 
   describe 'Issues' do
@@ -111,6 +113,19 @@ describe EventCreateService, services: true do
     end
   end
 
+  describe '#push', :redis do
+    let(:project) { create(:empty_project) }
+    let(:user) { create(:user) }
+
+    it 'creates a new event' do
+      expect { service.push(project, user, {}) }.to change { Event.count }
+    end
+
+    it 'updates user last activity' do
+      expect { service.push(project, user, {}) }.to change { user_activity(user) }
+    end
+  end
+
   describe 'Project' do
     let(:user) { create :user }
     let(:project) { create(:empty_project) }

spec/services/users/activity_service_spec.rb

+require 'spec_helper'
+
+describe Users::ActivityService, services: true do
+  include UserActivitiesHelpers
+
+  let(:user) { create(:user) }
+
+  subject(:service) { described_class.new(user, 'type') }
+
+  describe '#execute', :redis do
+    context 'when last activity is nil' do
+      before do
+        service.execute
+      end
+
+      it 'sets the last activity timestamp for the user' do
+        expect(last_hour_user_ids).to eq([user.id])
+      end
+
+      it 'updates the same user' do
+        service.execute
+
+        expect(last_hour_user_ids).to eq([user.id])
+      end
+
+      it 'updates the timestamp of an existing user' do
+        Timecop.freeze(Date.tomorrow) do
+          expect { service.execute }.to change { user_activity(user) }.to(Time.now.to_i.to_s)
+        end
+      end
+
+      describe 'other user' do
+        it 'updates other user' do
+          other_user = create(:user)
+          described_class.new(other_user, 'type').execute
+
+          expect(last_hour_user_ids).to match_array([user.id, other_user.id])
+        end
+      end
+    end
+  end
+
+  def last_hour_user_ids
+    Gitlab::UserActivities.new.
+      select { |k, v| v >= 1.hour.ago.to_i.to_s }.
+      map { |k, _| k.to_i }
+  end
+end

spec/support/matchers/user_activity_matchers.rb

+RSpec::Matchers.define :have_an_activity_record do |expected|
+  match do |user|
+    expect(Gitlab::UserActivities.new.find { |k, _| k == user.id.to_s }).to be_present
+  end
+end

spec/support/user_activities_helpers.rb

+module UserActivitiesHelpers
+  def user_activity(user)
+    Gitlab::UserActivities.new.
+      find { |k, _| k == user.id.to_s }&.
+      second
+  end
+end

spec/workers/gitlab_usage_ping_worker_spec.rb

+require 'spec_helper'
+
+describe GitlabUsagePingWorker do
+  subject { GitlabUsagePingWorker.new }
+
+  it "sends POST request" do
+    stub_application_setting(usage_ping_enabled: true)
+
+    stub_request(:post, "https://version.gitlab.com/usage_data").
+        to_return(status: 200, body: '', headers: {})
+    expect(Gitlab::UsageData).to receive(:to_json).with({ force_refresh: true }).and_call_original
+    expect(subject).to receive(:try_obtain_lease).and_return(true)
+
+    expect(subject.perform.response.code.to_i).to eq(200)
+  end
+
+  it "does not run if usage ping is disabled" do
+    stub_application_setting(usage_ping_enabled: false)
+
+    expect(subject).not_to receive(:try_obtain_lease)
+    expect(subject).not_to receive(:perform)
+  end
+end

spec/workers/schedule_update_user_activity_worker_spec.rb

+require 'spec_helper'
+
+describe ScheduleUpdateUserActivityWorker, :redis do
+  let(:now) { Time.now }
+
+  before do
+    Gitlab::UserActivities.record('1', now)
+    Gitlab::UserActivities.record('2', now)
+  end
+
+  it 'schedules UpdateUserActivityWorker once' do
+    expect(UpdateUserActivityWorker).to receive(:perform_async).with({ '1' => now.to_i.to_s, '2' => now.to_i.to_s })
+
+    subject.perform
+  end
+
+  context 'when specifying a batch size' do
+    it 'schedules UpdateUserActivityWorker twice' do
+      expect(UpdateUserActivityWorker).to receive(:perform_async).with({ '1' => now.to_i.to_s })
+      expect(UpdateUserActivityWorker).to receive(:perform_async).with({ '2' => now.to_i.to_s })
+
+      subject.perform(1)
+    end
+  end
+end

spec/workers/update_user_activity_worker_spec.rb

+require 'spec_helper'
+
+describe UpdateUserActivityWorker, :redis do
+  let(:user_active_2_days_ago) { create(:user, current_sign_in_at: 10.months.ago) }
+  let(:user_active_yesterday_1) { create(:user) }
+  let(:user_active_yesterday_2) { create(:user) }
+  let(:user_active_today) { create(:user) }
+  let(:data) do
+    {
+      user_active_2_days_ago.id.to_s => 2.days.ago.at_midday.to_i.to_s,
+      user_active_yesterday_1.id.to_s => 1.day.ago.at_midday.to_i.to_s,
+      user_active_yesterday_2.id.to_s => 1.day.ago.at_midday.to_i.to_s,
+      user_active_today.id.to_s => Time.now.to_i.to_s
+    }
+  end
+
+  it 'updates users.last_activity_on' do
+    subject.perform(data)
+
+    aggregate_failures do
+      expect(user_active_2_days_ago.reload.last_activity_on).to eq(2.days.ago.to_date)
+      expect(user_active_yesterday_1.reload.last_activity_on).to eq(1.day.ago.to_date)
+      expect(user_active_yesterday_2.reload.last_activity_on).to eq(1.day.ago.to_date)
+      expect(user_active_today.reload.reload.last_activity_on).to eq(Date.today)
+    end
+  end
+
+  it 'deletes the pairs from Redis' do
+    data.each { |id, time| Gitlab::UserActivities.record(id, time) }
+
+    subject.perform(data)
+
+    expect(Gitlab::UserActivities.new.to_a).to be_empty
+  end
+end