added specs and renamed stuff accordingly

app/views/admin/application_settings/_pseudonymizer.html.haml

+= form_for @application_setting, url: admin_application_settings_path, html: { class: 'fieldset-form' } do |f|
+  = form_errors(@application_setting)
+
+  %fieldset
+    .form-group.row
+      .offset-sm-2.col-sm-10
+        - is_enabled = @application_setting.pseudonymizer_enabled?
+        - is_available = @application_setting.pseudonymizer_available?
+        .form-check
+          = f.label :pseudonymizer_enabled do
+            = f.check_box :pseudonymizer_enabled, disabled: !is_available
+            Enable Pseudonymizer Cron Job
+          .form-text.text-muted
+            - if is_enabled
+              = pseudonymizer_description_text
+            - else
+              - if is_available
+                = pseudonymizer_disabled_description_text
+              - else
+                = pseudonymizer_unavailable_description_text
+
+  = f.submit 'Save changes', class: "btn btn-success"
+

app/views/admin/application_settings/show.html.haml

@@ -237,7 +237,7 @@
   .settings-content
     = render 'usage'
 
-- if License.feature_available?(:pseudonymizer)
+- if Gitlab::CurrentSettings.pseudonymizer_can_be_configured?
   %section.settings.as-usage.no-animate#js-elt-database-dump-settings{ class: ('expanded' if expanded) }
     .settings-header
       %h4

app/workers/pseudonymizer_worker.rb

@@ -5,16 +5,16 @@ class PseudonymizerWorker
   def perform
     return unless Gitlab::CurrentSettings.pseudonymizer_enabled?
 
-    options = Pseudonymity::Options.new(
+    options = Pseudonymizer::Options.new(
       config: YAML.load_file(Rails.root.join(Gitlab.config.pseudonymizer.manifest)),
       start_at: Time.now.utc
     )
 
-    table = Pseudonymity::Table.new(options)
-    table.tables_to_csv
+    dumper = Pseudonymizer::Dumper.new(options)
+    dumper.tables_to_csv
 
-    upload = Pseudonymity::UploadService.new(options)
-    upload.upload
-    upload.cleanup
+    uploader = Pseudonymizer::Uploader.new(options)
+    uploader.upload
+    uploader.cleanup
   end
 end

config/gitlab.yml.example

@@ -733,9 +733,9 @@ production: &base
   pseudonymizer:
     enabled: false
     # Tables manifest that specifies the fields to extract and pseudonymize.
-    manifest: config/pseudonymizer.yml
-    # remote_directory: 'gitlab-elt'
+    manifest: lib/pseudonymizer/manifest.yml
     upload:
+      # remote_directory: 'gitlab-elt'
       # Fog storage connection settings, see http://fog.io/storage/ .
       connection:
       #   provider: AWS
@@ -896,16 +896,16 @@ test:
     path: tmp/tests/backups
   pseudonymizer:
     enabled: false
-    manifest: config/pseudonymizer.test.yml
+    manifest: lib/pseudonymizer/manifest.yml
     upload:
       # The remote 'directory' to store the CSV files. For S3, this would be the bucket name.
       remote_directory: gitlab-elt.test
       # Fog storage connection settings, see http://fog.io/storage/
       connection:
-        provider: AWS
+        provider: AWS # Only AWS supported at the moment
+        aws_access_key_id: AWS_ACCESS_KEY_ID
+        aws_secret_access_key: AWS_SECRET_ACCESS_KEY
         region: us-east-1
-        aws_access_key_id: minio
-        aws_secret_access_key: gdk-minio
   gitlab_shell:
     path: tmp/tests/gitlab-shell/
     hooks_path: tmp/tests/gitlab-shell/hooks/

config/initializers/1_settings.rb

@@ -370,9 +370,9 @@ Settings.cron_jobs['gitlab_usage_ping_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['gitlab_usage_ping_worker']['cron'] ||= Settings.__send__(:cron_for_usage_ping)
 Settings.cron_jobs['gitlab_usage_ping_worker']['job_class'] = 'GitlabUsagePingWorker'
 
-Settings.cron_jobs['gitlab_elt_database_dump'] ||= Settingslogic.new({})
-Settings.cron_jobs['gitlab_elt_database_dump']['cron'] ||= '0 23 * * *';
-Settings.cron_jobs['gitlab_elt_database_dump']['job_class'] ||= 'GitlabEltDataDumpWorker';
+Settings.cron_jobs['pseudonymizer'] ||= Settingslogic.new({})
+Settings.cron_jobs['pseudonymizer']['cron'] ||= '0 23 * * *';
+Settings.cron_jobs['pseudonymizer']['job_class'] ||= 'PseudonymizerWorker';
 
 Settings.cron_jobs['schedule_update_user_activity_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['schedule_update_user_activity_worker']['cron'] ||= '30 0 * * *'

ee/app/controllers/ee/admin/application_settings_controller.rb

@@ -20,6 +20,10 @@ module EE
           attrs << :email_additional_text
         end
 
+        if License.feature_available?(:pseudonymizer)
+          attrs << :pseudonymizer_enabled
+        end
+
         attrs
       end
     end

ee/app/models/ee/application_setting.rb

@@ -101,7 +101,7 @@ module EE
           slack_app_id: nil,
           slack_app_secret: nil,
           slack_app_verification_token: nil,
-          pseudonymizer_enabled: Settings.pseudonymizer['enabled'],
+          pseudonymizer_enabled: Settings.pseudonymizer['enabled']
         )
       end
     end

lib/pseudonymity/table.rb → lib/pseudonymizer/dumper.rb

@@ -3,7 +3,7 @@ require 'digest'
 require 'csv'
 require 'yaml'
 
-module Pseudonymity
+module Pseudonymizer
   class Anon
     def initialize(fields)
       @anon_fields = fields
@@ -25,7 +25,7 @@ module Pseudonymity
     end
   end
 
-  class Table
+  class Dumper
     attr_accessor :config
 
     def initialize(options)

lib/pseudonymity/options.rb → lib/pseudonymizer/options.rb

-module Pseudonymity
+module Pseudonymizer
   class Options
     attr_reader :config
     attr_reader :start_at
@@ -15,5 +15,9 @@ module Pseudonymity
     def upload_dir
       File.join(self.start_at.iso8601)
     end
+
+    def object_store_credentials
+      config.upload.connection.to_hash.deep_symbolize_keys
+    end
   end
 end

lib/pseudonymity/upload_service.rb → lib/pseudonymizer/uploader.rb

-module Pseudonymity
-  class UploadService
+module Pseudonymizer
+  class Uploader
     RemoteStorageUnavailableError = Class.new(StandardError)
 
     def initialize(options, progress = nil)
       @progress = progress || $stdout
+      @config = options.config
       @output_dir = options.output_dir
       @upload_dir = options.upload_dir
+      @connection_params = options.object_store_credentials
     end
 
     def upload
@@ -16,18 +18,6 @@ module Pseudonymity
       end
     end
 
-    def upload_file(file, directory)
-      progress.print "\t#{file} ... "
-
-      if directory.files.create(key: File.join(@upload_dir, File.basename(file)),
-                                body: File.open(file),
-                                public: false)
-        progress.puts "done".color(:green)
-      else
-        progress.puts "uploading CSV to #{remote_directory} failed".color(:red)
-      end
-    end
-
     def cleanup
       progress.print "Deleting tmp directory #{@output_dir} ... "
       return unless File.exist?(@output_dir)
@@ -41,24 +31,33 @@ module Pseudonymity
 
     private
 
-    def config
-      Gitlab.config.pseudonymizer
+    attr_reader :progress
+
+    def upload_file(file, directory)
+      progress.print "\t#{file} ... "
+
+      if directory.files.create(key: File.join(@upload_dir, File.basename(file)),
+                                body: File.open(file),
+                                public: false)
+        progress.puts "done".color(:green)
+      else
+        progress.puts "uploading CSV to #{remote_directory} failed".color(:red)
+      end
     end
 
     def remote_directory
-      connection_settings = config.upload.connection
-      if connection_settings.blank?
+      if @connection_params.blank?
         progress.puts "Cannot upload files, make sure the `pseudonimizer.upload.connection` is set properly".color(:red)
-        raise RemoteStorageUnavailableError.new(connection_settings)
+        raise RemoteStorageUnavailableError.new(@config)
       end
 
-      connect_to_remote_directory(connection_settings)
+      connect_to_remote_directory
     end
 
-    def connect_to_remote_directory(connection_settings)
+    def connect_to_remote_directory
       # our settings use string keys, but Fog expects symbols
-      connection = ::Fog::Storage.new(connection_settings.symbolize_keys)
-      remote_dir = config.upload.remote_directory
+      connection = ::Fog::Storage.new(@connection_params)
+      remote_dir = @config.upload.remote_directory
 
       # We only attempt to create the directory for local backups. For AWS
       # and other cloud providers, we cannot guarantee the user will have

lib/tasks/gitlab/db.rake

@@ -78,17 +78,17 @@ namespace :gitlab do
 
       abort "Pseudonymizer disabled." unless Gitlab::CurrentSettings.pseudonymizer_enabled?
 
-      options = Pseudonymity::Options.new(
+      options = Pseudonymizer::Options.new(
         config: YAML.load_file(Rails.root.join(Gitlab.config.pseudonymizer.manifest)),
         start_at: Time.now.utc
       )
 
-      table = Pseudonymity::Table.new(options)
-      table.tables_to_csv
+      dumper = Pseudonymizer::Dumper.new(options)
+      dumper.tables_to_csv
 
-      upload = Pseudonymity::UploadService.new(options, progress)
-      upload.upload
-      upload.cleanup
+      uploader = Pseudonymizer::Uploader.new(options, progress)
+      uploader.upload
+      uploader.cleanup
     end
 
     def progress

spec/lib/pseudonymity_spec.rb → spec/lib/pseudonymizer/dumper_spec.rb

 require 'spec_helper'
 
-describe Gitlab::Pseudonymity do
+describe Pseudonymizer::Dumper do
   let!(:project) { create(:project) }
   let(:base_dir) { Dir.mktmpdir }
-  subject(:pseudo) { Pseudonymity::Table.new }
+  let(:options) { Pseudonymizer::Options.new() }
+  subject(:pseudo) { described_class.new(options) }
 
   after do
     FileUtils.rm_rf(base_dir)

spec/lib/pseudonymizer/uploader_spec.rb

+require 'spec_helper'
+
+describe Pseudonymizer::Uploader do
+  let(:base_dir) { Dir.mktmpdir }
+  let(:options) do
+    Pseudonymizer::Options.new(config: Gitlab.config.pseudonymizer,
+                               start_at: Time.now.utc)
+  end
+  let(:remote_directory) { subject.send(:remote_directory) }
+  subject { described_class.new(options) }
+
+  def mock_file(file_name)
+    FileUtils.touch(File.join(base_dir, file_name))
+  end
+
+  before do
+    allow(options).to receive(:output_dir).and_return(base_dir)
+    stub_object_storage_pseudonymizer(options: options)
+
+    10.times {|i| mock_file("file_#{i}.test")}
+    mock_file("schema.yml")
+    mock_file("file_list.json")
+  end
+
+  describe "#upload" do
+    it "upload all file in the directory" do
+      subject.upload
+
+      expect(remote_directory.files.count).to eq(12)
+    end
+  end
+
+  describe "#cleanup" do
+    it "cleans the directory" do
+      subject.cleanup
+
+      expect(Dir[File.join(base_dir, "*")].length).to eq(0)
+    end
+  end
+
+  after do
+    FileUtils.rm_rf(base_dir)
+  end
+end

spec/support/helpers/stub_object_storage.rb

@@ -15,9 +15,14 @@ module StubObjectStorage
 
     return unless enabled
 
+    stub_object_storage(connection_params: uploader.object_store_credentials,
+                        remote_directory: remote_directory)
+  end
+
+  def stub_object_storage(connection_params:, remote_directory:)
     Fog.mock!
 
-    ::Fog::Storage.new(uploader.object_store_credentials).tap do |connection|
+    ::Fog::Storage.new(connection_params).tap do |connection|
       begin
         connection.directories.create(key: remote_directory)
       rescue Excon::Error::Conflict
@@ -57,4 +62,9 @@ module StubObjectStorage
         </InitiateMultipartUploadResult>
       EOS
   end
+
+  def stub_object_storage_pseudonymizer(options:)
+    stub_object_storage(connection_params: options.object_store_credentials,
+                        remote_directory: options.config.upload.remote_directory)
+  end
 end