Account for uploaded files in storage statistics

Up to now we have not been taking uploads into consideration for
computing total storage usage for repositories. This changes that.

app/models/namespace.rb

@@ -96,7 +96,8 @@ class Namespace < ApplicationRecord
         'COALESCE(SUM(ps.snippets_size), 0) AS snippets_size',
         'COALESCE(SUM(ps.lfs_objects_size), 0) AS lfs_objects_size',
         'COALESCE(SUM(ps.build_artifacts_size), 0) AS build_artifacts_size',
-        'COALESCE(SUM(ps.packages_size), 0) AS packages_size'
+        'COALESCE(SUM(ps.packages_size), 0) AS packages_size',
+        'COALESCE(SUM(ps.uploads_size), 0) AS uploads_size'
       )
   end
 

app/models/namespace/root_storage_statistics.rb

@@ -11,6 +11,7 @@ class Namespace::RootStorageStatistics < ApplicationRecord
     packages_size
     #{SNIPPETS_SIZE_STAT_NAME}
     pipeline_artifacts_size
+    uploads_size
   ).freeze
 
   self.primary_key = :namespace_id
@@ -50,7 +51,8 @@ class Namespace::RootStorageStatistics < ApplicationRecord
         'COALESCE(SUM(ps.build_artifacts_size), 0) AS build_artifacts_size',
         'COALESCE(SUM(ps.packages_size), 0) AS packages_size',
         "COALESCE(SUM(ps.snippets_size), 0) AS #{SNIPPETS_SIZE_STAT_NAME}",
-        'COALESCE(SUM(ps.pipeline_artifacts_size), 0) AS pipeline_artifacts_size'
+        'COALESCE(SUM(ps.pipeline_artifacts_size), 0) AS pipeline_artifacts_size',
+        'COALESCE(SUM(ps.uploads_size), 0) AS uploads_size'
       )
   end
 

app/models/project_statistics.rb

@@ -19,14 +19,14 @@ class ProjectStatistics < ApplicationRecord
 
   before_save :update_storage_size
 
-  COLUMNS_TO_REFRESH = [:repository_size, :wiki_size, :lfs_objects_size, :commit_count, :snippets_size].freeze
+  COLUMNS_TO_REFRESH = [:repository_size, :wiki_size, :lfs_objects_size, :commit_count, :snippets_size, :uploads_size].freeze
   INCREMENTABLE_COLUMNS = {
     build_artifacts_size: %i[storage_size],
     packages_size: %i[storage_size],
     pipeline_artifacts_size: %i[storage_size],
     snippets_size: %i[storage_size]
   }.freeze
-  NAMESPACE_RELATABLE_COLUMNS = [:repository_size, :wiki_size, :lfs_objects_size].freeze
+  NAMESPACE_RELATABLE_COLUMNS = [:repository_size, :wiki_size, :lfs_objects_size, :uploads_size].freeze
 
   scope :for_project_ids, ->(project_ids) { where(project_id: project_ids) }
 
@@ -72,6 +72,12 @@ class ProjectStatistics < ApplicationRecord
     self.lfs_objects_size = project.lfs_objects.sum(:size)
   end
 
+  def update_uploads_size
+    return uploads_size unless Feature.enabled?(:count_uploads_size_in_storage_stats, project)
+
+    self.uploads_size = project.uploads.sum(:size)
+  end
+
   # `wiki_size` and `snippets_size` have no default value in the database
   # and the column can be nil.
   # This means that, when the columns were added, all rows had nil
@@ -98,6 +104,10 @@ class ProjectStatistics < ApplicationRecord
     # might try to update project statistics before the `pipeline_artifacts_size` column has been created.
     storage_size += pipeline_artifacts_size if self.class.column_names.include?('pipeline_artifacts_size')
 
+    # The `uploads_size` column was added on 20201105021637 but db/post_migrate/20190527194900_schedule_calculate_wiki_sizes.rb
+    # might try to update project statistics before the `uploads_size` column has been created.
+    storage_size += uploads_size if self.class.column_names.include?('uploads_size')
+
     self.storage_size = storage_size
   end
 

changelogs/unreleased/241980-uploads-count-as-used-storage.yml

+---
+title: Account for uploads as part of used repository storage
+merge_request: 46941
+author:
+type: added

config/feature_flags/development/count_uploads_size_in_storage_stats.yml

+---
+name: count_uploads_size_in_storage_stats
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/46941
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/281950
+milestone: '13.6'
+type: development
+group: group::project management
+default_enabled: false

db/migrate/20201105021637_add_uploads_size_to_project_statistics.rb

+# frozen_string_literal: true
+
+class AddUploadsSizeToProjectStatistics < ActiveRecord::Migration[6.0]
+  DOWNTIME = false
+
+  def up
+    add_column :project_statistics, :uploads_size, :bigint, default: 0, null: false
+  end
+
+  def down
+    remove_column :project_statistics, :uploads_size
+  end
+end

db/migrate/20201105024127_add_uploads_size_to_namespace_root_storage_statistics.rb

+# frozen_string_literal: true
+
+class AddUploadsSizeToNamespaceRootStorageStatistics < ActiveRecord::Migration[6.0]
+  DOWNTIME = false
+
+  def up
+    add_column :namespace_root_storage_statistics, :uploads_size, :bigint, default: 0, null: false
+  end
+
+  def down
+    remove_column :namespace_root_storage_statistics, :uploads_size
+  end
+end

db/schema_migrations/20201105021637

+b04e37c8713a333afbff4c7034d9f3ffca4a4490d0b563b73d9e6bffd45f3f6a
\ No newline at end of file

db/schema_migrations/20201105024127

+afeff8d6bf5ca14f955f5d387e8d219e852e9617b80fe07ed2e1b57af19fd4fa
\ No newline at end of file

db/structure.sql

@@ -13876,7 +13876,8 @@ CREATE TABLE namespace_root_storage_statistics (
     storage_size bigint DEFAULT 0 NOT NULL,
     packages_size bigint DEFAULT 0 NOT NULL,
     snippets_size bigint DEFAULT 0 NOT NULL,
-    pipeline_artifacts_size bigint DEFAULT 0 NOT NULL
+    pipeline_artifacts_size bigint DEFAULT 0 NOT NULL,
+    uploads_size bigint DEFAULT 0 NOT NULL
 );
 
 CREATE TABLE namespace_settings (
@@ -15260,7 +15261,8 @@ CREATE TABLE project_statistics (
     packages_size bigint DEFAULT 0 NOT NULL,
     wiki_size bigint,
     snippets_size bigint,
-    pipeline_artifacts_size bigint DEFAULT 0 NOT NULL
+    pipeline_artifacts_size bigint DEFAULT 0 NOT NULL,
+    uploads_size bigint DEFAULT 0 NOT NULL
 );
 
 CREATE SEQUENCE project_statistics_id_seq

spec/factories/project_statistics.rb

@@ -23,6 +23,7 @@ FactoryBot.define do
         project_statistics.packages_size = evaluator.size_multiplier * 5
         project_statistics.snippets_size = evaluator.size_multiplier * 6
         project_statistics.pipeline_artifacts_size = evaluator.size_multiplier * 7
+        project_statistics.uploads_size = evaluator.size_multiplier * 8
       end
     end
   end

spec/models/namespace/root_storage_statistics_spec.rb

@@ -45,6 +45,7 @@ RSpec.describe Namespace::RootStorageStatistics, type: :model do
         total_storage_size = stat1.storage_size + stat2.storage_size
         total_snippets_size = stat1.snippets_size + stat2.snippets_size
         total_pipeline_artifacts_size = stat1.pipeline_artifacts_size + stat2.pipeline_artifacts_size
+        total_uploads_size = stat1.uploads_size + stat2.uploads_size
 
         expect(root_storage_statistics.repository_size).to eq(total_repository_size)
         expect(root_storage_statistics.wiki_size).to eq(total_wiki_size)
@@ -54,6 +55,7 @@ RSpec.describe Namespace::RootStorageStatistics, type: :model do
         expect(root_storage_statistics.storage_size).to eq(total_storage_size)
         expect(root_storage_statistics.snippets_size).to eq(total_snippets_size)
         expect(root_storage_statistics.pipeline_artifacts_size).to eq(total_pipeline_artifacts_size)
+        expect(root_storage_statistics.uploads_size).to eq(total_uploads_size)
       end
 
       it 'works when there are no projects' do

spec/models/project_statistics_spec.rb

@@ -34,7 +34,8 @@ RSpec.describe ProjectStatistics do
         lfs_objects_size: 2.exabytes,
         build_artifacts_size: 1.exabyte,
         snippets_size: 1.exabyte,
-        pipeline_artifacts_size: 1.exabyte - 1
+        pipeline_artifacts_size: 512.petabytes - 1,
+        uploads_size: 512.petabytes
       )
 
       statistics.reload
@@ -46,7 +47,8 @@ RSpec.describe ProjectStatistics do
       expect(statistics.build_artifacts_size).to eq(1.exabyte)
       expect(statistics.storage_size).to eq(8.exabytes - 1)
       expect(statistics.snippets_size).to eq(1.exabyte)
-      expect(statistics.pipeline_artifacts_size).to eq(1.exabyte - 1)
+      expect(statistics.pipeline_artifacts_size).to eq(512.petabytes - 1)
+      expect(statistics.uploads_size).to eq(512.petabytes)
     end
   end
 
@@ -57,6 +59,7 @@ RSpec.describe ProjectStatistics do
       statistics.lfs_objects_size = 3
       statistics.build_artifacts_size = 4
       statistics.snippets_size = 5
+      statistics.uploads_size = 3
 
       expect(statistics.total_repository_size).to eq 5
     end
@@ -98,6 +101,7 @@ RSpec.describe ProjectStatistics do
       allow(statistics).to receive(:update_lfs_objects_size)
       allow(statistics).to receive(:update_snippets_size)
       allow(statistics).to receive(:update_storage_size)
+      allow(statistics).to receive(:update_uploads_size)
     end
 
     context "without arguments" do
@@ -111,6 +115,7 @@ RSpec.describe ProjectStatistics do
         expect(statistics).to have_received(:update_wiki_size)
         expect(statistics).to have_received(:update_lfs_objects_size)
         expect(statistics).to have_received(:update_snippets_size)
+        expect(statistics).to have_received(:update_uploads_size)
       end
     end
 
@@ -125,6 +130,7 @@ RSpec.describe ProjectStatistics do
         expect(statistics).not_to have_received(:update_repository_size)
         expect(statistics).not_to have_received(:update_wiki_size)
         expect(statistics).not_to have_received(:update_snippets_size)
+        expect(statistics).not_to have_received(:update_uploads_size)
       end
     end
 
@@ -139,10 +145,12 @@ RSpec.describe ProjectStatistics do
         expect(statistics).to have_received(:update_repository_size)
         expect(statistics).to have_received(:update_wiki_size)
         expect(statistics).to have_received(:update_snippets_size)
+        expect(statistics).to have_received(:update_uploads_size)
         expect(statistics.repository_size).to eq(0)
         expect(statistics.commit_count).to eq(0)
         expect(statistics.wiki_size).to eq(0)
         expect(statistics.snippets_size).to eq(0)
+        expect(statistics.uploads_size).to eq(0)
       end
     end
 
@@ -163,10 +171,12 @@ RSpec.describe ProjectStatistics do
         expect(statistics).to have_received(:update_repository_size)
         expect(statistics).to have_received(:update_wiki_size)
         expect(statistics).to have_received(:update_snippets_size)
+        expect(statistics).to have_received(:update_uploads_size)
         expect(statistics.repository_size).to eq(0)
         expect(statistics.commit_count).to eq(0)
         expect(statistics.wiki_size).to eq(0)
         expect(statistics.snippets_size).to eq(0)
+        expect(statistics.uploads_size).to eq(0)
       end
     end
 
@@ -211,6 +221,7 @@ RSpec.describe ProjectStatistics do
         expect(statistics).not_to receive(:update_wiki_size)
         expect(statistics).not_to receive(:update_lfs_objects_size)
         expect(statistics).not_to receive(:update_snippets_size)
+        expect(statistics).not_to receive(:update_uploads_size)
         expect(statistics).not_to receive(:save!)
         expect(Namespaces::ScheduleAggregationWorker)
           .not_to receive(:perform_async)
@@ -295,6 +306,26 @@ RSpec.describe ProjectStatistics do
     end
   end
 
+  describe '#update_uploads_size' do
+    let!(:upload1) { create(:upload, model: project, size: 1.megabyte) }
+    let!(:upload2) { create(:upload, model: project, size: 2.megabytes) }
+
+    it 'stores the size of related uploaded files' do
+      expect(statistics.update_uploads_size).to eq(3.megabytes)
+    end
+
+    context 'with feature flag disabled' do
+      before do
+        statistics.update_columns(uploads_size: 0)
+        stub_feature_flags(count_uploads_size_in_storage_stats: false)
+      end
+
+      it 'does not store the size of related uploaded files' do
+        expect(statistics.update_uploads_size).to eq(0)
+      end
+    end
+  end
+
   describe '#update_storage_size' do
     it "sums all storage counters" do
       statistics.update!(
@@ -302,12 +333,13 @@ RSpec.describe ProjectStatistics do
         wiki_size: 4,
         lfs_objects_size: 3,
         snippets_size: 2,
-        pipeline_artifacts_size: 3
+        pipeline_artifacts_size: 3,
+        uploads_size: 5
       )
 
       statistics.reload
 
-      expect(statistics.storage_size).to eq 14
+      expect(statistics.storage_size).to eq 19
     end
 
     it 'works during wiki_size backfill' do