Merge branch 'topics/public_projects_count' into 'master'

Topics: add `non_private_projects_count` counter

See merge request gitlab-org/gitlab!79140

app/models/project.rb

@@ -2779,6 +2779,12 @@ class Project < ApplicationRecord
   end
 
   def save_topics
+    topic_ids_before = self.topic_ids
+    update_topics
+    Projects::Topic.update_non_private_projects_counter(topic_ids_before, self.topic_ids, visibility_level_previously_was, visibility_level)
+  end
+
+  def update_topics
     return if @topic_list.nil?
 
     @topic_list = @topic_list.split(',') if @topic_list.instance_of?(String)

app/models/projects/topic.rb

@@ -25,6 +25,29 @@ module Projects
       def search(query)
         fuzzy_search(query, [:name])
       end
+
+      def update_non_private_projects_counter(ids_before, ids_after, project_visibility_level_before, project_visibility_level_after)
+        project_visibility_level_before ||= project_visibility_level_after
+
+        topics_to_decrement = []
+        topics_to_increment = []
+        topic_ids_removed = ids_before - ids_after
+        topic_ids_retained = ids_before & ids_after
+        topic_ids_added = ids_after - ids_before
+
+        if project_visibility_level_before > Gitlab::VisibilityLevel::PRIVATE
+          topics_to_decrement += topic_ids_removed
+          topics_to_decrement += topic_ids_retained if project_visibility_level_after == Gitlab::VisibilityLevel::PRIVATE
+        end
+
+        if project_visibility_level_after > Gitlab::VisibilityLevel::PRIVATE
+          topics_to_increment += topic_ids_added
+          topics_to_increment += topic_ids_retained if project_visibility_level_before == Gitlab::VisibilityLevel::PRIVATE
+        end
+
+        where(id: topics_to_increment).update_counters(non_private_projects_count: 1) unless topics_to_increment.empty?
+        where(id: topics_to_decrement).where('non_private_projects_count > 0').update_counters(non_private_projects_count: -1) unless topics_to_decrement.empty?
+      end
     end
   end
 end

db/migrate/20220125122228_add_topics_non_private_projects_count.rb

+# frozen_string_literal: true
+
+class AddTopicsNonPrivateProjectsCount < Gitlab::Database::Migration[1.0]
+  def up
+    add_column :topics, :non_private_projects_count, :bigint, null: false, default: 0
+  end
+
+  def down
+    remove_column :topics, :non_private_projects_count
+  end
+end

db/migrate/20220125122725_add_topics_non_private_projects_count_index.rb

+# frozen_string_literal: true
+
+class AddTopicsNonPrivateProjectsCountIndex < Gitlab::Database::Migration[1.0]
+  INDEX_NAME = 'index_topics_non_private_projects_count'
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :topics, [:non_private_projects_count, :id], order: { non_private_projects_count: :desc }, name: INDEX_NAME
+  end
+
+  def down
+    remove_concurrent_index_by_name :topics, INDEX_NAME
+  end
+end

db/post_migrate/20220125122640_schedule_populate_topics_non_private_projects_count.rb

+# frozen_string_literal: true
+
+class SchedulePopulateTopicsNonPrivateProjectsCount < Gitlab::Database::Migration[1.0]
+  MIGRATION = 'PopulateTopicsNonPrivateProjectsCount'
+  BATCH_SIZE = 10_000
+  DELAY_INTERVAL = 2.minutes
+
+  disable_ddl_transaction!
+
+  def up
+    queue_background_migration_jobs_by_range_at_intervals(
+      define_batchable_model('topics'),
+      MIGRATION,
+      DELAY_INTERVAL,
+      batch_size: BATCH_SIZE,
+      track_jobs: true
+    )
+  end
+
+  def down
+    # no-op
+  end
+end

db/schema_migrations/20220125122228

+26600e01d8b31a4308d0e23564e4d4c52488ec87ad7990a410b7cc0c031f12e7
\ No newline at end of file

db/schema_migrations/20220125122640

+51c7ab860b952281bd7f65d68e7a539a8eee57cac3bbdaf439ff5593f5b065ed
\ No newline at end of file

db/schema_migrations/20220125122725

+7740d1e71571576a709ae5bfd46f60ea3fb4be3f48cddec2cca53f148096cdd7
\ No newline at end of file

db/structure.sql

@@ -20272,6 +20272,7 @@ CREATE TABLE topics (
     avatar text,
     description text,
     total_projects_count bigint DEFAULT 0 NOT NULL,
+    non_private_projects_count bigint DEFAULT 0 NOT NULL,
     CONSTRAINT check_26753fb43a CHECK ((char_length(avatar) <= 255)),
     CONSTRAINT check_5d1a07c8c8 CHECK ((char_length(description) <= 1024)),
     CONSTRAINT check_7a90d4c757 CHECK ((char_length(name) <= 255))
@@ -27935,6 +27936,8 @@ CREATE UNIQUE INDEX index_token_with_ivs_on_hashed_plaintext_token ON token_with
 
 CREATE UNIQUE INDEX index_token_with_ivs_on_hashed_token ON token_with_ivs USING btree (hashed_token);
 
+CREATE INDEX index_topics_non_private_projects_count ON topics USING btree (non_private_projects_count DESC, id);
+
 CREATE UNIQUE INDEX index_topics_on_name ON topics USING btree (name);
 
 CREATE INDEX index_topics_on_name_trigram ON topics USING gin (name gin_trgm_ops);

lib/gitlab/background_migration/populate_topics_non_private_projects_count.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # The class to populates the non private projects counter of topics
+    class PopulateTopicsNonPrivateProjectsCount
+      SUB_BATCH_SIZE = 100
+
+      # Temporary AR model for topics
+      class Topic < ActiveRecord::Base
+        include EachBatch
+
+        self.table_name = 'topics'
+      end
+
+      def perform(start_id, stop_id)
+        Topic.where(id: start_id..stop_id).each_batch(of: SUB_BATCH_SIZE) do |batch|
+          ActiveRecord::Base.connection.execute(<<~SQL)
+            WITH batched_relation AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (#{batch.select(:id).limit(SUB_BATCH_SIZE).to_sql})
+            UPDATE topics
+            SET non_private_projects_count = (
+              SELECT COUNT(*) 
+              FROM project_topics 
+              INNER JOIN projects 
+              ON project_topics.project_id = projects.id 
+              WHERE project_topics.topic_id = batched_relation.id 
+              AND projects.visibility_level > 0
+            )
+            FROM batched_relation
+            WHERE topics.id = batched_relation.id
+          SQL
+        end
+      end
+    end
+  end
+end

spec/lib/gitlab/background_migration/populate_topics_non_private_projects_count_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::BackgroundMigration::PopulateTopicsNonPrivateProjectsCount, schema: 20220125122640 do
+  it 'correctly populates the non private projects counters' do
+    namespaces = table(:namespaces)
+    projects = table(:projects)
+    topics = table(:topics)
+    project_topics = table(:project_topics)
+
+    group = namespaces.create!(name: 'group', path: 'group')
+    project_public = projects.create!(namespace_id: group.id, visibility_level: Gitlab::VisibilityLevel::PUBLIC)
+    project_internal = projects.create!(namespace_id: group.id, visibility_level: Gitlab::VisibilityLevel::INTERNAL)
+    project_private = projects.create!(namespace_id: group.id, visibility_level: Gitlab::VisibilityLevel::PRIVATE)
+    topic_1 = topics.create!(name: 'Topic1')
+    topic_2 = topics.create!(name: 'Topic2')
+    topic_3 = topics.create!(name: 'Topic3')
+    topic_4 = topics.create!(name: 'Topic4')
+    topic_5 = topics.create!(name: 'Topic5')
+    topic_6 = topics.create!(name: 'Topic6')
+    topic_7 = topics.create!(name: 'Topic7')
+    topic_8 = topics.create!(name: 'Topic8')
+
+    project_topics.create!(topic_id: topic_1.id, project_id: project_public.id)
+    project_topics.create!(topic_id: topic_2.id, project_id: project_internal.id)
+    project_topics.create!(topic_id: topic_3.id, project_id: project_private.id)
+    project_topics.create!(topic_id: topic_4.id, project_id: project_public.id)
+    project_topics.create!(topic_id: topic_4.id, project_id: project_internal.id)
+    project_topics.create!(topic_id: topic_5.id, project_id: project_public.id)
+    project_topics.create!(topic_id: topic_5.id, project_id: project_private.id)
+    project_topics.create!(topic_id: topic_6.id, project_id: project_internal.id)
+    project_topics.create!(topic_id: topic_6.id, project_id: project_private.id)
+    project_topics.create!(topic_id: topic_7.id, project_id: project_public.id)
+    project_topics.create!(topic_id: topic_7.id, project_id: project_internal.id)
+    project_topics.create!(topic_id: topic_7.id, project_id: project_private.id)
+    project_topics.create!(topic_id: topic_8.id, project_id: project_public.id)
+
+    subject.perform(topic_1.id, topic_7.id)
+
+    expect(topic_1.reload.non_private_projects_count).to eq(1)
+    expect(topic_2.reload.non_private_projects_count).to eq(1)
+    expect(topic_3.reload.non_private_projects_count).to eq(0)
+    expect(topic_4.reload.non_private_projects_count).to eq(2)
+    expect(topic_5.reload.non_private_projects_count).to eq(1)
+    expect(topic_6.reload.non_private_projects_count).to eq(1)
+    expect(topic_7.reload.non_private_projects_count).to eq(2)
+    expect(topic_8.reload.non_private_projects_count).to eq(0)
+  end
+end

spec/models/project_spec.rb

@@ -7418,6 +7418,67 @@ RSpec.describe Project, factory_default: :keep do
         expect(project.reload.topics.map(&:name)).to eq(%w[topic1 topic2 topic3])
       end
     end
+
+    context 'public topics counter' do
+      let_it_be(:topic_1) { create(:topic, name: 't1') }
+      let_it_be(:topic_2) { create(:topic, name: 't2') }
+      let_it_be(:topic_3) { create(:topic, name: 't3') }
+
+      let(:private) { Gitlab::VisibilityLevel::PRIVATE }
+      let(:internal) { Gitlab::VisibilityLevel::INTERNAL }
+      let(:public) { Gitlab::VisibilityLevel::PUBLIC }
+
+      subject do
+        project_updates = {
+          visibility_level: new_visibility,
+          topic_list: new_topic_list
+        }.compact
+
+        project.update!(project_updates)
+      end
+
+      using RSpec::Parameterized::TableSyntax
+
+      # rubocop:disable Lint/BinaryOperatorWithIdenticalOperands
+      where(:initial_visibility, :new_visibility, :new_topic_list, :expected_count_changes) do
+        ref(:private)  | nil            | 't2, t3' | [0, 0, 0]
+        ref(:internal) | nil            | 't2, t3' | [-1, 0, 1]
+        ref(:public)   | nil            | 't2, t3' | [-1, 0, 1]
+        ref(:private)  | ref(:public)   | nil      | [1, 1, 0]
+        ref(:private)  | ref(:internal) | nil      | [1, 1, 0]
+        ref(:private)  | ref(:private)  | nil      | [0, 0, 0]
+        ref(:internal) | ref(:public)   | nil      | [0, 0, 0]
+        ref(:internal) | ref(:internal) | nil      | [0, 0, 0]
+        ref(:internal) | ref(:private)  | nil      | [-1, -1, 0]
+        ref(:public)   | ref(:public)   | nil      | [0, 0, 0]
+        ref(:public)   | ref(:internal) | nil      | [0, 0, 0]
+        ref(:public)   | ref(:private)  | nil      | [-1, -1, 0]
+        ref(:private)  | ref(:public)   | 't2, t3' | [0, 1, 1]
+        ref(:private)  | ref(:internal) | 't2, t3' | [0, 1, 1]
+        ref(:private)  | ref(:private)  | 't2, t3' | [0, 0, 0]
+        ref(:internal) | ref(:public)   | 't2, t3' | [-1, 0, 1]
+        ref(:internal) | ref(:internal) | 't2, t3' | [-1, 0, 1]
+        ref(:internal) | ref(:private)  | 't2, t3' | [-1, -1, 0]
+        ref(:public)   | ref(:public)   | 't2, t3' | [-1, 0, 1]
+        ref(:public)   | ref(:internal) | 't2, t3' | [-1, 0, 1]
+        ref(:public)   | ref(:private)  | 't2, t3' | [-1, -1, 0]
+      end
+      # rubocop:enable Lint/BinaryOperatorWithIdenticalOperands
+
+      with_them do
+        it 'increments or decrements counters of topics' do
+          project.reload.update!(
+            visibility_level: initial_visibility,
+            topic_list: [topic_1.name, topic_2.name]
+          )
+
+          expect { subject }
+            .to change { topic_1.reload.non_private_projects_count }.by(expected_count_changes[0])
+            .and change { topic_2.reload.non_private_projects_count }.by(expected_count_changes[1])
+            .and change { topic_3.reload.non_private_projects_count }.by(expected_count_changes[2])
+        end
+      end
+    end
   end
 
   shared_examples 'all_runners' do