Merge branch '230465-fj-populate-namespace-statistics-with-wiki-size' into 'master'

Update existing namespace statistics wiki wiki_size

See merge request gitlab-org/gitlab!55487

db/post_migrate/20210302074524_backfill_namespace_statistics_with_wiki_size.rb

+# frozen_string_literal: true
+
+class BackfillNamespaceStatisticsWithWikiSize < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  DELAY_INTERVAL = 2.minutes.to_i
+  BATCH_SIZE = 500
+  MIGRATION = 'PopulateNamespaceStatistics'
+
+  disable_ddl_transaction!
+
+  def up
+    return unless Gitlab.ee?
+
+    groups = exec_query <<~SQL
+      SELECT group_wiki_repositories.group_id
+      FROM group_wiki_repositories
+    SQL
+
+    groups.rows.flatten.in_groups_of(BATCH_SIZE, false).each_with_index do |group_ids, index|
+      migrate_in(index * DELAY_INTERVAL, MIGRATION, [group_ids, [:wiki_size]])
+    end
+  end
+
+  def down
+    # No-op
+  end
+end

db/schema_migrations/20210302074524

+251c0d811eee00f78e62f4d1e7f2893b2137fbf8121cefc4283c3217677b5447
\ No newline at end of file

ee/changelogs/unreleased/230465-fj-populate-namespace-statistics-with-wiki-size.yml

+---
+title: Update existing namespace statistics with wiki_size
+merge_request: 55487
+author:
+type: added

ee/lib/ee/gitlab/background_migration/populate_namespace_statistics.rb

+# frozen_string_literal: true
+
+module EE
+  module Gitlab
+    module BackgroundMigration
+      # This class creates/updates those namespace statistics
+      # that haven't been created nor initialized.
+      # It also updates the related namespace statistics
+      module PopulateNamespaceStatistics
+        extend ::Gitlab::Utils::Override
+
+        override :perform
+        def perform(group_ids, statistics)
+          # Updating group statistics might involve calling Gitaly.
+          # For example, when calculating `wiki_size`, we will need
+          # to perform the request to check if the repo exists and
+          # also the repository size.
+          #
+          # The `allow_n_plus_1_calls` method is only intended for
+          # dev and test. It won't be raised in prod.
+          ::Gitlab::GitalyClient.allow_n_plus_1_calls do
+            ::Group.includes(:route, :namespace_statistics, group_wiki_repository: :shard).where(id: group_ids).each do |group|
+              upsert_namespace_statistics(group, statistics)
+            end
+          end
+        end
+
+        private
+
+        def upsert_namespace_statistics(group, statistics)
+          response = ::Groups::UpdateStatisticsService.new(group, statistics: statistics).execute
+
+          error_message("#{response.message} group: #{group.id}") if response.error?
+        end
+
+        def logger
+          @logger ||= ::Gitlab::BackgroundMigration::Logger.build
+        end
+
+        def error_message(message)
+          logger.error(message: "Namespace Statistics Migration: #{message}")
+        end
+      end
+    end
+  end
+end

ee/spec/lib/ee/gitlab/background_migration/populate_namespace_statistics_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::BackgroundMigration::PopulateNamespaceStatistics do
+  include AfterNextHelpers
+
+  let_it_be(:namespaces) { table(:namespaces) }
+  let_it_be(:namespace_statistics) { table(:namespace_statistics) }
+
+  let!(:group1) { namespaces.create!(id: 10, type: 'Group', name: 'group1', path: 'group1') }
+  let!(:group2) { namespaces.create!(id: 20, type: 'Group', name: 'group2', path: 'group2') }
+  let!(:group1_stats) { namespace_statistics.create!(id: 10, namespace_id: 10) }
+
+  let(:repo_size) { 123456 }
+  let(:expected_repo_size) { repo_size.megabytes }
+  let(:ids) { namespaces.pluck(:id) }
+  let(:migration) { described_class.new }
+  let(:statistics) { [] }
+
+  subject do
+    migration.perform(ids, statistics)
+  end
+
+  before do
+    allow_next(Repository).to receive(:size).and_return(repo_size)
+  end
+
+  context 'when group wikis are not enabled' do
+    it 'does not update wiki stats' do
+      subject
+
+      expect(namespace_statistics.where(wiki_size: 0).count).to eq 2
+    end
+  end
+
+  it 'creates/updates all namespace_statistics and update root storage statistics', :aggregate_failures do
+    stub_licensed_features(group_wikis: true)
+
+    expect(namespace_statistics.count).to eq 1
+
+    expect(Namespaces::ScheduleAggregationWorker).to receive(:perform_async).with(group1.id)
+    expect(Namespaces::ScheduleAggregationWorker).to receive(:perform_async).with(group2.id)
+
+    subject
+
+    expect(namespace_statistics.count).to eq 2
+
+    namespace_statistics.all.each do |stat|
+      expect(stat.wiki_size).to eq expected_repo_size
+      expect(stat.storage_size).to eq expected_repo_size
+    end
+  end
+
+  context 'when just a stat is passed' do
+    let(:statistics) { [:wiki_size] }
+
+    it 'calls the statistics update service with just that stat' do
+      expect(Groups::UpdateStatisticsService).to receive(:new).with(anything, statistics: [:wiki_size]).twice.and_call_original
+
+      subject
+    end
+  end
+end

ee/spec/migrations/backfill_namespace_statistics_with_wiki_size_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require Rails.root.join('db', 'post_migrate', '20210302074524_backfill_namespace_statistics_with_wiki_size.rb')
+
+RSpec.describe BackfillNamespaceStatisticsWithWikiSize do
+  let_it_be(:shards) { table(:shards) }
+  let_it_be(:shard) { shards.create!(id: 1, name: 'default') }
+  let_it_be(:groups) { table(:namespaces) }
+  let_it_be(:group1) { groups.create!(id: 10, name: 'test1', path: 'test1', type: 'Group') }
+  let_it_be(:group2) { groups.create!(id: 20, name: 'test2', path: 'test2', type: 'Group') }
+  let_it_be(:group3) { groups.create!(id: 30, name: 'test3', path: 'test3', type: 'Group') }
+  let_it_be(:group4) { groups.create!(id: 40, name: 'test4', path: 'test4', type: 'Group') }
+  let_it_be(:group_wiki_repository) { table(:group_wiki_repositories) }
+  let_it_be(:group1_repo) { group_wiki_repository.create!(shard_id: 1, group_id: 10, disk_path: 'foo1') }
+  let_it_be(:group2_repo) { group_wiki_repository.create!(shard_id: 1, group_id: 20, disk_path: 'foo2') }
+  let_it_be(:group3_repo) { group_wiki_repository.create!(shard_id: 1, group_id: 30, disk_path: 'foo3') }
+
+  describe '#up' do
+    it 'correctly schedules background migrations' do
+      stub_const("#{described_class}::BATCH_SIZE", 2)
+
+      Sidekiq::Testing.fake! do
+        freeze_time do
+          migrate!
+
+          aggregate_failures do
+            expect(described_class::MIGRATION)
+              .to be_scheduled_migration([10, 20], ['wiki_size'])
+
+            expect(described_class::MIGRATION)
+              .to be_scheduled_delayed_migration(2.minutes, [30], ['wiki_size'])
+
+            expect(BackgroundMigrationWorker.jobs.size).to eq(2)
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/background_migration/populate_namespace_statistics.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # This class creates/updates those namespace statistics
+    # that haven't been created nor initialized.
+    # It also updates the related namespace statistics
+    # This is only required in EE
+    class PopulateNamespaceStatistics
+      def perform(group_ids, statistics)
+      end
+    end
+  end
+end
+
+Gitlab::BackgroundMigration::PopulateNamespaceStatistics.prepend_if_ee('EE::Gitlab::BackgroundMigration::PopulateNamespaceStatistics')