Merge branch '219002-remove-ghost-column' into 'master'

Drop obsolete ghost column

Closes #219002

See merge request gitlab-org/gitlab!32957

app/models/user.rb

@@ -1818,7 +1818,7 @@ class User < ApplicationRecord
   def update_highest_role?
     return false unless persisted?
 
-    (previous_changes.keys & %w(state user_type ghost)).any?
+    (previous_changes.keys & %w(state user_type)).any?
   end
 
   def update_highest_role_attribute

changelogs/unreleased/219002-remove-ghost-column.yml

+---
+title: Remove obsolete users.ghost column
+merge_request: 32957
+author:
+type: other

config/pseudonymizer.yml

@@ -465,7 +465,6 @@ tables:
     - auditor
     - require_two_factor_authentication_from_group
     - two_factor_grace_period
-    - ghost
     - last_activity_on
     - notified_of_own_activity
     - user_type

db/migrate/20200525114553_rename_user_type_index.rb

+# frozen_string_literal: true
+
+class RenameUserTypeIndex < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :users, [:state, :user_type], name: 'index_users_on_state_and_user_type'
+    remove_concurrent_index_by_name :users, 'index_users_on_state_and_user_type_internal'
+  end
+
+  def down
+    add_concurrent_index :users, [:state, :user_type], where: 'ghost IS NOT TRUE', name: 'index_users_on_state_and_user_type_internal'
+    remove_concurrent_index_by_name :users, 'index_users_on_state_and_user_type'
+  end
+end

db/post_migrate/20200525121014_drop_users_ghost_column.rb

+# frozen_string_literal: true
+
+class DropUsersGhostColumn < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    remove_concurrent_index_by_name :users, 'index_users_on_ghost'
+
+    with_lock_retries do
+      remove_column :users, :ghost
+    end
+  end
+
+  def down
+    unless column_exists?(:users, :ghost)
+      with_lock_retries do
+        add_column :users, :ghost, :boolean # rubocop:disable Migration/AddColumnsToWideTables
+      end
+    end
+
+    execute 'UPDATE users set ghost = TRUE WHERE user_type = 5'
+
+    add_concurrent_index :users, :ghost
+  end
+end

db/structure.sql

@@ -6806,7 +6806,6 @@ CREATE TABLE public.users (
     auditor boolean DEFAULT false NOT NULL,
     require_two_factor_authentication_from_group boolean DEFAULT false NOT NULL,
     two_factor_grace_period integer DEFAULT 48 NOT NULL,
-    ghost boolean,
     last_activity_on date,
     notified_of_own_activity boolean,
     preferred_language character varying,
@@ -10826,8 +10825,6 @@ CREATE INDEX index_users_on_email_trigram ON public.users USING gin (email publi
 
 CREATE INDEX index_users_on_feed_token ON public.users USING btree (feed_token);
 
-CREATE INDEX index_users_on_ghost ON public.users USING btree (ghost);
-
 CREATE INDEX index_users_on_group_view ON public.users USING btree (group_view);
 
 CREATE INDEX index_users_on_incoming_email_token ON public.users USING btree (incoming_email_token);
@@ -10844,7 +10841,7 @@ CREATE UNIQUE INDEX index_users_on_reset_password_token ON public.users USING bt
 
 CREATE INDEX index_users_on_state ON public.users USING btree (state);
 
-CREATE INDEX index_users_on_state_and_user_type_internal ON public.users USING btree (state, user_type) WHERE (ghost IS NOT TRUE);
+CREATE INDEX index_users_on_state_and_user_type ON public.users USING btree (state, user_type);
 
 CREATE UNIQUE INDEX index_users_on_static_object_token ON public.users USING btree (static_object_token);
 
@@ -13925,5 +13922,7 @@ COPY "schema_migrations" (version) FROM STDIN;
 20200515155620
 20200519115908
 20200519171058
+20200525114553
+20200525121014
 \.
 

doc/development/telemetry/usage_ping.md

@@ -257,11 +257,11 @@ Your Rails console will return the generated SQL queries.
 Example:
 
 ```ruby
- pry(main)> Gitlab::UsageData.count(User.active)
-   (0.4ms)  SELECT "features"."key" FROM "features"
-   (0.7ms)  SELECT MIN("users"."id") FROM "users" WHERE ("users"."state" IN ('active')) AND (ghost IS NOT TRUE) AND ("users"."user_type" IS NULL OR "users"."user_type" NOT IN (2, 1, 3))
-   (0.6ms)  SELECT MAX("users"."id") FROM "users" WHERE ("users"."state" IN ('active')) AND (ghost IS NOT TRUE) AND ("users"."user_type" IS NULL OR "users"."user_type" NOT IN (2, 1, 3))
-   (0.5ms)  SELECT COUNT("users"."id") FROM "users" WHERE ("users"."state" IN ('active')) AND (ghost IS NOT TRUE) AND ("users"."user_type" IS NULL OR "users"."user_type" NOT IN (2, 1, 3)) AND "users"."id" BETWEEN 0 AND 99999
+pry(main)> Gitlab::UsageData.count(User.active)
+   (2.6ms)  SELECT "features"."key" FROM "features"
+   (15.3ms)  SELECT MIN("users"."id") FROM "users" WHERE ("users"."state" IN ('active')) AND ("users"."user_type" IS NULL OR "users"."user_type" IN (6, 4))
+   (2.4ms)  SELECT MAX("users"."id") FROM "users" WHERE ("users"."state" IN ('active')) AND ("users"."user_type" IS NULL OR "users"."user_type" IN (6, 4))
+   (1.9ms)  SELECT COUNT("users"."id") FROM "users" WHERE ("users"."state" IN ('active')) AND ("users"."user_type" IS NULL OR "users"."user_type" IN (6, 4)) AND "users"."id" BETWEEN 1 AND 100000
 ```
 
 ### 3. Optimize queries with #database-lab

doc/development/understanding_explain_plans.md

@@ -315,25 +315,30 @@ the `Indexes:` section:
 ```sql
 Indexes:
     "users_pkey" PRIMARY KEY, btree (id)
-    "users_confirmation_token_key" UNIQUE CONSTRAINT, btree (confirmation_token)
-    "users_email_key" UNIQUE CONSTRAINT, btree (email)
-    "users_reset_password_token_key" UNIQUE CONSTRAINT, btree (reset_password_token)
-    "index_on_users_lower_email" btree (lower(email::text))
-    "index_on_users_lower_username" btree (lower(username::text))
+    "index_users_on_confirmation_token" UNIQUE, btree (confirmation_token)
+    "index_users_on_email" UNIQUE, btree (email)
+    "index_users_on_reset_password_token" UNIQUE, btree (reset_password_token)
+    "index_users_on_static_object_token" UNIQUE, btree (static_object_token)
+    "index_users_on_unlock_token" UNIQUE, btree (unlock_token)
     "index_on_users_name_lower" btree (lower(name::text))
+    "index_users_on_accepted_term_id" btree (accepted_term_id)
     "index_users_on_admin" btree (admin)
     "index_users_on_created_at" btree (created_at)
     "index_users_on_email_trigram" gin (email gin_trgm_ops)
     "index_users_on_feed_token" btree (feed_token)
-    "index_users_on_ghost" btree (ghost)
+    "index_users_on_group_view" btree (group_view)
     "index_users_on_incoming_email_token" btree (incoming_email_token)
+    "index_users_on_managing_group_id" btree (managing_group_id)
     "index_users_on_name" btree (name)
     "index_users_on_name_trigram" gin (name gin_trgm_ops)
+    "index_users_on_public_email" btree (public_email) WHERE public_email::text <> ''::text
     "index_users_on_state" btree (state)
-    "index_users_on_state_and_internal_attrs" btree (state) WHERE ghost <> true AND support_bot <> true
-    "index_users_on_support_bot" btree (support_bot)
+    "index_users_on_state_and_user_type" btree (state, user_type)
+    "index_users_on_unconfirmed_email" btree (unconfirmed_email) WHERE unconfirmed_email IS NOT NULL
+    "index_users_on_user_type" btree (user_type)
     "index_users_on_username" btree (username)
     "index_users_on_username_trigram" gin (username gin_trgm_ops)
+    "tmp_idx_on_user_id_where_bio_is_filled" btree (id) WHERE COALESCE(bio, ''::character varying)::text IS DISTINCT FROM ''::text
 ```
 
 Here we can see there is no index on the `twitter` column, which means