Merge branch '331979-make-it-easier-to-use-snowplow-micro-in-development-2' into 'master'

Make it easier to use Snowplow Micro in development

See merge request gitlab-org/gitlab!72838

app/views/layouts/_loading_hints.html.haml

@@ -11,5 +11,5 @@
     = preload_link_tag(path_to_stylesheet('application_utilities'), crossorigin: css_crossorigin)
     = preload_link_tag(path_to_stylesheet('application'), crossorigin: css_crossorigin)
   = preload_link_tag(path_to_stylesheet("highlight/themes/#{user_color_scheme}"), crossorigin: css_crossorigin)
-  - if Gitlab::Tracking.enabled? && Gitlab::CurrentSettings.snowplow_collector_hostname
-    %link{ rel: 'preconnect', href: Gitlab::CurrentSettings.snowplow_collector_hostname, crossorigin: '' }
+  - if Gitlab::Tracking.enabled? && Gitlab::Tracking.collector_hostname
+    %link{ rel: 'preconnect', href: Gitlab::Tracking.collector_hostname, crossorigin: '' }

doc/development/snowplow/implementation.md

@@ -424,7 +424,7 @@ records the same events as the full Snowplow pipeline. To query events, use the
 To install and run Snowplow Micro, complete these steps to modify the 
 [GitLab Development Kit (GDK)](https://gitlab.com/gitlab-org/gitlab-development-kit):
 
-1. Ensure Docker is installed and running.
+1. Ensure [Docker is installed](https://docs.docker.com/get-docker/) and running.
 
 1. To install Snowplow Micro, clone the settings in 
 [this project](https://gitlab.com/gitlab-org/snowplow-micro-configuration).
@@ -436,73 +436,18 @@ To install and run Snowplow Micro, complete these steps to modify the
    ./snowplow-micro.sh
    ```
 
-1. Use GDK to start the PostgreSQL terminal and connect 
-   to the `gitlabhq_development` database:
+1. Set the environment variable to tell the GDK to use Snowplow Micro in development. This overrides two `application_settings` options:
+   - `snowplow_enabled` setting will instead return `true` from `Gitlab::Tracking.enabled?`
+   - `snowplow_collector_hostname` setting will instead always return `localhost:9090` (or whatever is set for `SNOWPLOW_MICRO_URI`) from `Gitlab::Tracking.collector_hostname`.
 
    ```shell
-   gdk psql -d gitlabhq_development
+   export SNOWPLOW_MICRO_ENABLE=1
    ```
 
-1. Update your instance's settings to enable Snowplow events and 
-   point to the Snowplow Micro collector:
+   Optionally, you can set the URI for you Snowplow Micro instance as well (the default value is `http://localhost:9090`):
 
    ```shell
-   update application_settings set snowplow_collector_hostname='localhost:9090', snowplow_enabled=true, snowplow_cookie_domain='.gitlab.com';
-   ```
-
-1. Update `DEFAULT_SNOWPLOW_OPTIONS` in `app/assets/javascripts/tracking/constants.js` to remove `forceSecureTracker: true`:
-
-   ```diff
-   diff --git a/app/assets/javascripts/tracking/constants.js b/app/assets/javascripts/tracking/constants.js
-   index 598111e4086..eff38074d4c 100644
-   --- a/app/assets/javascripts/tracking/constants.js
-   +++ b/app/assets/javascripts/tracking/constants.js
-   @@ -7,7 +7,6 @@ export const DEFAULT_SNOWPLOW_OPTIONS = {
-      appId: '',
-      userFingerprint: false,
-      respectDoNotTrack: true,
-   -  forceSecureTracker: true,
-      eventMethod: 'post',
-      contexts: { webPage: true, performanceTiming: true },
-      formTracking: false,
-   ```
-
-1. Update `options` in `lib/gitlab/tracking.rb` to add `protocol` and `port`:
-
-   ```diff
-   diff --git a/lib/gitlab/tracking.rb b/lib/gitlab/tracking.rb
-   index 618e359211b..e9084623c43 100644
-   --- a/lib/gitlab/tracking.rb
-   +++ b/lib/gitlab/tracking.rb
-   @@ -41,7 +41,9 @@ def options(group)
-              cookie_domain: Gitlab::CurrentSettings.snowplow_cookie_domain,
-              app_id: Gitlab::CurrentSettings.snowplow_app_id,
-              form_tracking: additional_features,
-   -          link_click_tracking: additional_features
-   +          link_click_tracking: additional_features,
-   +          protocol: 'http',
-   +          port: 9090
-            }.transform_keys! { |key| key.to_s.camelize(:lower).to_sym }
-          end
-   ```
-
-1. Update `emitter` in `lib/gitlab/tracking/destinations/snowplow.rb` to change `protocol`:
-
-   ```diff
-   diff --git a/lib/gitlab/tracking/destinations/snowplow.rb b/lib/gitlab/tracking/destinations/snowplow.rb
-   index 4fa844de325..5dd9d0eacfb 100644
-   --- a/lib/gitlab/tracking/destinations/snowplow.rb
-   +++ b/lib/gitlab/tracking/destinations/snowplow.rb
-   @@ -40,7 +40,7 @@ def tracker
-            def emitter
-              SnowplowTracker::AsyncEmitter.new(
-                Gitlab::CurrentSettings.snowplow_collector_hostname,
-   -            protocol: 'https'
-   +            protocol: 'http'
-              )
-            end
-          end
-
+   export SNOWPLOW_MICRO_URI=https://127.0.0.1:8080
    ```
 
 1. Restart GDK:

lib/gitlab/tracking.rb

@@ -6,7 +6,7 @@ module Gitlab
 
     class << self
       def enabled?
-        Gitlab::CurrentSettings.snowplow_enabled?
+        snowplow_micro_enabled? || Gitlab::CurrentSettings.snowplow_enabled?
       end
 
       def event(category, action, label: nil, property: nil, value: nil, context: [], project: nil, user: nil, namespace: nil, **extra) # rubocop:disable Metrics/ParameterLists
@@ -18,21 +18,25 @@ module Gitlab
       end
 
       def options(group)
-        additional_features = Feature.enabled?(:additional_snowplow_tracking, group, type: :ops)
-        {
-          namespace: SNOWPLOW_NAMESPACE,
-          hostname: Gitlab::CurrentSettings.snowplow_collector_hostname,
-          cookie_domain: Gitlab::CurrentSettings.snowplow_cookie_domain,
-          app_id: Gitlab::CurrentSettings.snowplow_app_id,
-          form_tracking: additional_features,
-          link_click_tracking: additional_features
-        }.transform_keys! { |key| key.to_s.camelize(:lower).to_sym }
+        snowplow.options(group)
+      end
+
+      def collector_hostname
+        snowplow.hostname
       end
 
       private
 
       def snowplow
-        @snowplow ||= Gitlab::Tracking::Destinations::Snowplow.new
+        @snowplow ||= if snowplow_micro_enabled?
+                        Gitlab::Tracking::Destinations::SnowplowMicro.new
+                      else
+                        Gitlab::Tracking::Destinations::Snowplow.new
+                      end
+      end
+
+      def snowplow_micro_enabled?
+        Rails.env.development? && Gitlab::Utils.to_boolean(ENV['SNOWPLOW_MICRO_ENABLE'])
       end
     end
   end

lib/gitlab/tracking/destinations/snowplow.rb

@@ -16,25 +16,53 @@ module Gitlab
           increment_total_events_counter
         end
 
+        def options(group)
+          additional_features = Feature.enabled?(:additional_snowplow_tracking, group, type: :ops)
+          {
+            namespace: Gitlab::Tracking::SNOWPLOW_NAMESPACE,
+            hostname: hostname,
+            cookie_domain: cookie_domain,
+            app_id: app_id,
+            form_tracking: additional_features,
+            link_click_tracking: additional_features
+          }.transform_keys! { |key| key.to_s.camelize(:lower).to_sym }
+        end
+
+        def hostname
+          Gitlab::CurrentSettings.snowplow_collector_hostname
+        end
+
         private
 
         def enabled?
           Gitlab::Tracking.enabled?
         end
 
+        def app_id
+          Gitlab::CurrentSettings.snowplow_app_id
+        end
+
+        def protocol
+          'https'
+        end
+
+        def cookie_domain
+          Gitlab::CurrentSettings.snowplow_cookie_domain
+        end
+
         def tracker
           @tracker ||= SnowplowTracker::Tracker.new(
             emitter,
             SnowplowTracker::Subject.new,
             Gitlab::Tracking::SNOWPLOW_NAMESPACE,
-            Gitlab::CurrentSettings.snowplow_app_id
+            app_id
           )
         end
 
         def emitter
           SnowplowTracker::AsyncEmitter.new(
-            Gitlab::CurrentSettings.snowplow_collector_hostname,
-            protocol: 'https',
+            hostname,
+            protocol: protocol,
             on_success: method(:increment_successful_events_emissions),
             on_failure: method(:failure_callback)
           )
@@ -68,8 +96,6 @@ module Gitlab
         end
 
         def log_failures(failures)
-          hostname = Gitlab::CurrentSettings.snowplow_collector_hostname
-
           failures.each do |failure|
             Gitlab::AppLogger.error("#{failure["se_ca"]} #{failure["se_ac"]} failed to be reported to collector at #{hostname}")
           end

lib/gitlab/tracking/destinations/snowplow_micro.rb

+# frozen_string_literal: true
+#
+module Gitlab
+  module Tracking
+    module Destinations
+      class SnowplowMicro < Snowplow
+        include ::Gitlab::Utils::StrongMemoize
+        extend ::Gitlab::Utils::Override
+
+        DEFAULT_URI = 'http://localhost:9090'
+
+        override :options
+        def options(group)
+          super.update(
+            protocol: uri.scheme,
+            port: uri.port,
+            force_secure_tracker: false
+          )
+        end
+
+        override :hostname
+        def hostname
+          "#{uri.host}:#{uri.port}"
+        end
+
+        private
+
+        def uri
+          strong_memoize(:snowplow_uri) do
+            uri = URI(ENV['SNOWPLOW_MICRO_URI'] || DEFAULT_URI)
+            uri = URI("http://#{ENV['SNOWPLOW_MICRO_URI']}") unless %w[http https].include?(uri.scheme)
+            uri
+          end
+        end
+
+        override :cookie_domain
+        def cookie_domain
+          '.gitlab.com'
+        end
+
+        override :protocol
+        def protocol
+          uri.scheme
+        end
+      end
+    end
+  end
+end

spec/lib/gitlab/tracking/destinations/snowplow_micro_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Tracking::Destinations::SnowplowMicro do
+  include StubENV
+
+  before do
+    stub_application_setting(snowplow_enabled: true)
+    stub_env('SNOWPLOW_MICRO_ENABLE', '1')
+    allow(Rails.env).to receive(:development?).and_return(true)
+  end
+
+  describe '#hostname' do
+    context 'when SNOWPLOW_MICRO_URI is set' do
+      before do
+        stub_env('SNOWPLOW_MICRO_URI', 'http://gdk.test:9091')
+      end
+
+      it 'returns hostname URI part' do
+        expect(subject.hostname).to eq('gdk.test:9091')
+      end
+    end
+
+    context 'when SNOWPLOW_MICRO_URI is without protocol' do
+      before do
+        stub_env('SNOWPLOW_MICRO_URI', 'gdk.test:9091')
+      end
+
+      it 'returns hostname URI part' do
+        expect(subject.hostname).to eq('gdk.test:9091')
+      end
+    end
+
+    context 'when SNOWPLOW_MICRO_URI is hostname only' do
+      before do
+        stub_env('SNOWPLOW_MICRO_URI', 'uriwithoutport')
+      end
+
+      it 'returns hostname URI with default HTTP port' do
+        expect(subject.hostname).to eq('uriwithoutport:80')
+      end
+    end
+
+    context 'when SNOWPLOW_MICRO_URI is not set' do
+      it 'returns localhost hostname' do
+        expect(subject.hostname).to eq('localhost:9090')
+      end
+    end
+  end
+end

spec/lib/gitlab/tracking_spec.rb

@@ -2,6 +2,8 @@
 require 'spec_helper'
 
 RSpec.describe Gitlab::Tracking do
+  include StubENV
+
   before do
     stub_application_setting(snowplow_enabled: true)
     stub_application_setting(snowplow_collector_hostname: 'gitfoo.com')
@@ -12,6 +14,25 @@ RSpec.describe Gitlab::Tracking do
   end
 
   describe '.options' do
+    shared_examples 'delegates to destination' do |klass|
+      before do
+        allow_next_instance_of(klass) do |instance|
+          allow(instance).to receive(:options).and_call_original
+        end
+      end
+
+      it "delegates to #{klass} destination" do
+        expect_next_instance_of(klass) do |instance|
+          expect(instance).to receive(:options)
+        end
+
+        subject.options(nil)
+      end
+    end
+
+    context 'when destination is Snowplow' do
+      it_behaves_like 'delegates to destination', Gitlab::Tracking::Destinations::Snowplow
+
       it 'returns useful client options' do
         expected_fields = {
           namespace: 'gl',
@@ -24,6 +45,32 @@ RSpec.describe Gitlab::Tracking do
 
         expect(subject.options(nil)).to match(expected_fields)
       end
+    end
+
+    context 'when destination is SnowplowMicro' do
+      before do
+        stub_env('SNOWPLOW_MICRO_ENABLE', '1')
+        allow(Rails.env).to receive(:development?).and_return(true)
+      end
+
+      it_behaves_like 'delegates to destination', Gitlab::Tracking::Destinations::SnowplowMicro
+
+      it 'returns useful client options' do
+        expected_fields = {
+          namespace: 'gl',
+          hostname: 'localhost:9090',
+          cookieDomain: '.gitlab.com',
+          appId: '_abc123_',
+          protocol: 'http',
+          port: 9090,
+          force_secure_tracker: false,
+          formTracking: true,
+          linkClickTracking: true
+        }
+
+        expect(subject.options(nil)).to match(expected_fields)
+      end
+    end
 
     it 'when feature flag is disabled' do
       stub_feature_flags(additional_snowplow_tracking: false)
@@ -71,7 +118,23 @@ RSpec.describe Gitlab::Tracking do
       end
     end
 
+    context 'when destination is Snowplow' do
+      before do
+        stub_env('SNOWPLOW_MICRO_ENABLE', '0')
+        allow(Rails.env).to receive(:development?).and_return(true)
+      end
+
       it_behaves_like 'delegates to destination', Gitlab::Tracking::Destinations::Snowplow
+    end
+
+    context 'when destination is SnowplowMicro' do
+      before do
+        stub_env('SNOWPLOW_MICRO_ENABLE', '1')
+        allow(Rails.env).to receive(:development?).and_return(true)
+      end
+
+      it_behaves_like 'delegates to destination', Gitlab::Tracking::Destinations::SnowplowMicro
+    end
 
     it 'tracks errors' do
       expect(Gitlab::ErrorTracking).to receive(:track_and_raise_for_dev_exception).with(