Merge branch 'pedropombeiro/26345/6-add-expand_variable_collection' into 'master'

Add #expand_variable_collection to Variables::Collection

See merge request gitlab-org/gitlab!54507

lib/gitlab/ci/pipeline/seed/build.rb

@@ -154,8 +154,8 @@ module Gitlab
           end
 
           def variable_expansion_errors
-            sorted_collection = evaluate_context.variables.sorted_collection(@pipeline.project)
-            errors = sorted_collection.errors
+            expanded_collection = evaluate_context.variables.sort_and_expand_all(@pipeline.project)
+            errors = expanded_collection.errors
             ["#{name}: #{errors}"] if errors
           end
 

lib/gitlab/ci/variables/collection.rb

@@ -63,11 +63,46 @@ module Gitlab
           Collection.new(@variables.reject(&block))
         end
 
-        # Returns a sorted Collection object, and sets errors property in case of an error
-        def sorted_collection(project)
-          Sort.new(self, project).collection
+        def expand_value(value, keep_undefined: false)
+          value.gsub(ExpandVariables::VARIABLES_REGEXP) do
+            match = Regexp.last_match
+            result = @variables_by_key[match[1] || match[2]]&.value
+            result ||= match[0] if keep_undefined
+            result
           end
         end
+
+        def sort_and_expand_all(project, keep_undefined: false)
+          return self if Feature.disabled?(:variable_inside_variable, project)
+
+          sorted = Sort.new(self)
+          return self.class.new(self, sorted.errors) unless sorted.valid?
+
+          new_collection = self.class.new
+
+          sorted.tsort.each do |item|
+            unless item.depends_on
+              new_collection.append(item)
+              next
+            end
+
+            # expand variables as they are added
+            variable = item.to_runner_variable
+            variable[:value] = new_collection.expand_value(variable[:value], keep_undefined: keep_undefined)
+            new_collection.append(variable)
+          end
+
+          new_collection
+        end
+
+        def to_s
+          "#{@variables_by_key.keys}, @errors='#{@errors}'"
+        end
+
+        protected
+
+        attr_reader :variables
+      end
     end
   end
 end

lib/gitlab/ci/variables/collection/item.rb

@@ -7,20 +7,21 @@ module Gitlab
         class Item
           include Gitlab::Utils::StrongMemoize
 
-          attr_reader :raw
-
           def initialize(key:, value:, public: true, file: false, masked: false, raw: false)
             raise ArgumentError, "`#{key}` must be of type String or nil value, while it was: #{value.class}" unless
               value.is_a?(String) || value.nil?
 
-            @variable = { key: key, value: value, public: public, file: file, masked: masked }
-            @raw = raw
+            @variable = { key: key, value: value, public: public, file: file, masked: masked, raw: raw }
           end
 
           def value
             @variable.fetch(:value)
           end
 
+          def raw
+            @variable.fetch(:raw)
+          end
+
           def [](key)
             @variable.fetch(key)
           end
@@ -46,7 +47,7 @@ module Gitlab
           #
           def to_runner_variable
             @variable.reject do |hash_key, hash_value|
-              hash_key == :file && hash_value == false
+              (hash_key == :file || hash_key == :raw) && hash_value == false
             end
           end
 
@@ -62,6 +63,12 @@ module Gitlab
               raise ArgumentError, "Unknown `#{resource.class}` variable resource!"
             end
           end
+
+          def to_s
+            return to_runner_variable.to_s unless depends_on
+
+            "#{to_runner_variable}, depends_on=#{depends_on}"
+          end
         end
       end
     end

lib/gitlab/ci/variables/collection/sort.rb

@@ -8,12 +8,11 @@ module Gitlab
           include TSort
           include Gitlab::Utils::StrongMemoize
 
-          def initialize(collection, project)
+          def initialize(collection)
             raise(ArgumentError, "A Gitlab::Ci::Variables::Collection object was expected") unless
               collection.is_a?(Collection)
 
             @collection = collection
-            @project = project
           end
 
           def valid?
@@ -23,8 +22,6 @@ module Gitlab
           # errors sorts an array of variables, ignoring unknown variable references,
           # and returning an error string if a circular variable reference is found
           def errors
-            return if Feature.disabled?(:variable_inside_variable, @project)
-
             strong_memoize(:errors) do
               # Check for cyclic dependencies and build error message in that case
               cyclic_vars = each_strongly_connected_component.filter_map do |component|
@@ -35,16 +32,6 @@ module Gitlab
             end
           end
 
-          # collection sorts a collection of variables, ignoring unknown variable references.
-          # If a circular variable reference is found, a new collection with the original array and an error is returned
-          def collection
-            return @collection if Feature.disabled?(:variable_inside_variable, @project)
-
-            return Gitlab::Ci::Variables::Collection.new(@collection, errors) if errors
-
-            Gitlab::Ci::Variables::Collection.new(tsort)
-          end
-
           private
 
           def tsort_each_node(&block)

spec/lib/gitlab/ci/variables/collection/item_spec.rb

@@ -202,6 +202,26 @@ RSpec.describe Gitlab::Ci::Variables::Collection::Item do
       end
     end
 
+    context 'when variable is raw' do
+      it 'does not export raw value when it is false' do
+        runner_variable = described_class
+                            .new(key: 'VAR', value: 'value', raw: false)
+                            .to_runner_variable
+
+        expect(runner_variable)
+          .to eq(key: 'VAR', value: 'value', public: true, masked: false)
+      end
+
+      it 'exports raw value when it is true' do
+        runner_variable = described_class
+                            .new(key: 'VAR', value: 'value', raw: true)
+                            .to_runner_variable
+
+        expect(runner_variable)
+          .to eq(key: 'VAR', value: 'value', public: true, raw: true, masked: false)
+      end
+    end
+
     context 'when referencing a variable' do
       it '#depends_on contains names of dependencies' do
         runner_variable = described_class.new(key: 'CI_VAR', value: '${CI_VAR_2}-123-$CI_VAR_3')

spec/lib/gitlab/ci/variables/collection/sort_spec.rb

@@ -4,15 +4,8 @@ require 'spec_helper'
 
 RSpec.describe Gitlab::Ci::Variables::Collection::Sort do
   describe '#initialize with non-Collection value' do
-    let_it_be(:project_with_flag_disabled) { create(:project) }
-    let_it_be(:project_with_flag_enabled) { create(:project) }
-
-    before do
-      stub_feature_flags(variable_inside_variable: [project_with_flag_enabled])
-    end
-
     context 'when FF :variable_inside_variable is disabled' do
-      subject { Gitlab::Ci::Variables::Collection::Sort.new([], project_with_flag_disabled) }
+      subject { Gitlab::Ci::Variables::Collection::Sort.new([]) }
 
       it 'raises ArgumentError' do
         expect { subject }.to raise_error(ArgumentError, /Collection object was expected/)
@@ -20,7 +13,7 @@ RSpec.describe Gitlab::Ci::Variables::Collection::Sort do
     end
 
     context 'when FF :variable_inside_variable is enabled' do
-      subject { Gitlab::Ci::Variables::Collection::Sort.new([], project_with_flag_enabled) }
+      subject { Gitlab::Ci::Variables::Collection::Sort.new([]) }
 
       it 'raises ArgumentError' do
         expect { subject }.to raise_error(ArgumentError, /Collection object was expected/)
@@ -29,88 +22,6 @@ RSpec.describe Gitlab::Ci::Variables::Collection::Sort do
   end
 
   describe '#errors' do
-    context 'when FF :variable_inside_variable is disabled' do
-      let_it_be(:project_with_flag_disabled) { create(:project) }
-      let_it_be(:project_with_flag_enabled) { create(:project) }
-
-      before do
-        stub_feature_flags(variable_inside_variable: [project_with_flag_enabled])
-      end
-
-      context 'table tests' do
-        using RSpec::Parameterized::TableSyntax
-
-        where do
-          {
-            "empty array": {
-              variables: []
-            },
-            "simple expansions": {
-              variables: [
-                { key: 'variable', value: 'value' },
-                { key: 'variable2', value: 'result' },
-                { key: 'variable3', value: 'key$variable$variable2' }
-              ]
-            },
-            "complex expansion": {
-              variables: [
-                { key: 'variable', value: 'value' },
-                { key: 'variable2', value: 'key${variable}' }
-              ]
-            },
-            "complex expansions with missing variable for Windows": {
-              variables: [
-                { key: 'variable', value: 'value' },
-                { key: 'variable3', value: 'key%variable%%variable2%' }
-              ]
-            },
-            "out-of-order variable reference": {
-              variables: [
-                { key: 'variable2', value: 'key${variable}' },
-                { key: 'variable', value: 'value' }
-              ]
-            },
-            "array with cyclic dependency": {
-              variables: [
-                { key: 'variable', value: '$variable2' },
-                { key: 'variable2', value: '$variable3' },
-                { key: 'variable3', value: 'key$variable$variable2' }
-              ]
-            },
-            "array with raw variable": {
-              variables: [
-                { key: 'variable', value: '$variable2' },
-                { key: 'variable2', value: '$variable3' },
-                { key: 'variable3', value: 'key$variable$variable2', raw: true }
-              ]
-            }
-          }
-        end
-
-        with_them do
-          let(:collection) { Gitlab::Ci::Variables::Collection.new(variables) }
-
-          subject { Gitlab::Ci::Variables::Collection::Sort.new(collection, project_with_flag_disabled) }
-
-          it 'does not report error' do
-            expect(subject.errors).to eq(nil)
-          end
-
-          it 'valid? reports true' do
-            expect(subject.valid?).to eq(true)
-          end
-        end
-      end
-    end
-
-    context 'when FF :variable_inside_variable is enabled' do
-      let_it_be(:project_with_flag_disabled) { create(:project) }
-      let_it_be(:project_with_flag_enabled) { create(:project) }
-
-      before do
-        stub_feature_flags(variable_inside_variable: [project_with_flag_enabled])
-      end
-
     context 'table tests' do
       using RSpec::Parameterized::TableSyntax
 
@@ -118,7 +29,7 @@ RSpec.describe Gitlab::Ci::Variables::Collection::Sort do
         {
           "empty array": {
             variables: [],
-              validation_result: nil
+            expected_errors: nil
           },
           "simple expansions": {
             variables: [
@@ -126,7 +37,7 @@ RSpec.describe Gitlab::Ci::Variables::Collection::Sort do
               { key: 'variable2', value: 'result' },
               { key: 'variable3', value: 'key$variable$variable2' }
             ],
-              validation_result: nil
+            expected_errors: nil
           },
           "cyclic dependency": {
             variables: [
@@ -134,7 +45,7 @@ RSpec.describe Gitlab::Ci::Variables::Collection::Sort do
               { key: 'variable2', value: '$variable3' },
               { key: 'variable3', value: 'key$variable$variable2' }
             ],
-              validation_result: 'circular variable reference detected: ["variable", "variable2", "variable3"]'
+            expected_errors: 'circular variable reference detected: ["variable", "variable2", "variable3"]'
           },
           "array with raw variable": {
             variables: [
@@ -142,7 +53,15 @@ RSpec.describe Gitlab::Ci::Variables::Collection::Sort do
               { key: 'variable2', value: '$variable3' },
               { key: 'variable3', value: 'key$variable$variable2', raw: true }
             ],
-              validation_result: nil
+            expected_errors: nil
+          },
+          "variable containing escaped variable reference": {
+            variables: [
+              { key: 'variable_a', value: 'value' },
+              { key: 'variable_b', value: '$$variable_a' },
+              { key: 'variable_c', value: '$variable_b' }
+            ],
+            expected_errors: nil
           }
         }
       end
@@ -150,88 +69,24 @@ RSpec.describe Gitlab::Ci::Variables::Collection::Sort do
       with_them do
         let(:collection) { Gitlab::Ci::Variables::Collection.new(variables) }
 
-          subject { Gitlab::Ci::Variables::Collection::Sort.new(collection, project_with_flag_enabled) }
+        subject { Gitlab::Ci::Variables::Collection::Sort.new(collection) }
 
-          it 'errors matches expected validation result' do
-            expect(subject.errors).to eq(validation_result)
+        it 'errors matches expected errors' do
+          expect(subject.errors).to eq(expected_errors)
         end
 
-          it 'valid? matches expected validation result' do
-            expect(subject.valid?).to eq(validation_result.nil?)
-          end
-        end
-      end
-    end
-  end
-
-  describe '#collection' do
-    context 'when FF :variable_inside_variable is disabled' do
-      before do
-        stub_feature_flags(variable_inside_variable: false)
-      end
-
-      context 'table tests' do
-        using RSpec::Parameterized::TableSyntax
-
-        where do
-          {
-            "empty array": {
-              variables: []
-            },
-            "simple expansions": {
-              variables: [
-                { key: 'variable', value: 'value' },
-                { key: 'variable2', value: 'result' },
-                { key: 'variable3', value: 'key$variable$variable2' }
-              ]
-            },
-            "complex expansion": {
-              variables: [
-                { key: 'variable', value: 'value' },
-                { key: 'variable2', value: 'key${variable}' }
-              ]
-            },
-            "complex expansions with missing variable for Windows": {
-              variables: [
-                { key: 'variable', value: 'value' },
-                { key: 'variable3', value: 'key%variable%%variable2%' }
-              ]
-            },
-            "out-of-order variable reference": {
-              variables: [
-                { key: 'variable2', value: 'key${variable}' },
-                { key: 'variable', value: 'value' }
-              ]
-            },
-            "array with cyclic dependency": {
-              variables: [
-                { key: 'variable', value: '$variable2' },
-                { key: 'variable2', value: '$variable3' },
-                { key: 'variable3', value: 'key$variable$variable2' }
-              ]
-            }
-          }
+        it 'valid? matches expected errors' do
+          expect(subject.valid?).to eq(expected_errors.nil?)
         end
 
-        with_them do
-          let_it_be(:project) { create(:project) }
-          let(:collection) { Gitlab::Ci::Variables::Collection.new(variables) }
-
-          subject { Gitlab::Ci::Variables::Collection::Sort.new(collection, project).collection }
-
-          it 'does not expand variables' do
-            is_expected.to be(collection)
-          end
+        it 'does not raise' do
+          expect { subject }.not_to raise_error
         end
       end
     end
-
-    context 'when FF :variable_inside_variable is enabled' do
-      before do
-        stub_licensed_features(group_saml_group_sync: true)
-        stub_feature_flags(variable_inside_variable: true)
   end
 
+  describe '#tsort' do
     context 'table tests' do
       using RSpec::Parameterized::TableSyntax
 
@@ -279,14 +134,6 @@ RSpec.describe Gitlab::Ci::Variables::Collection::Sort do
             ],
             result: %w[variable variable3 variable4]
           },
-            "cyclic dependency causes original array to be returned": {
-              variables: [
-                { key: 'variable2', value: '$variable3' },
-                { key: 'variable3', value: 'key$variable$variable2' },
-                { key: 'variable', value: '$variable2' }
-              ],
-              result: %w[variable2 variable3 variable]
-            },
           "raw variable does not get resolved": {
             variables: [
               { key: 'variable', value: '$variable2' },
@@ -307,15 +154,31 @@ RSpec.describe Gitlab::Ci::Variables::Collection::Sort do
       end
 
       with_them do
-          let_it_be(:project) { create(:project) }
         let(:collection) { Gitlab::Ci::Variables::Collection.new(variables) }
 
-          subject { Gitlab::Ci::Variables::Collection::Sort.new(collection, project).collection }
+        subject { Gitlab::Ci::Variables::Collection::Sort.new(collection).tsort }
 
         it 'returns correctly sorted variables' do
-            expect(subject.map { |var| var[:key] }).to eq(result)
+          expect(subject.pluck(:key)).to eq(result)
+        end
       end
     end
+
+    context 'cyclic dependency' do
+      let(:variables) do
+        [
+          { key: 'variable2', value: '$variable3' },
+          { key: 'variable3', value: 'key$variable$variable2' },
+          { key: 'variable', value: '$variable2' }
+        ]
+      end
+
+      let(:collection) { Gitlab::Ci::Variables::Collection.new(variables) }
+
+      subject { Gitlab::Ci::Variables::Collection::Sort.new(collection).tsort }
+
+      it 'raises TSort::Cyclic' do
+        expect { subject }.to raise_error(TSort::Cyclic)
       end
     end
   end

spec/lib/gitlab/ci/variables/collection_spec.rb

@@ -166,68 +166,337 @@ RSpec.describe Gitlab::Ci::Variables::Collection do
     end
   end
 
-  describe '#sorted_collection' do
-    let!(:project) { create(:project) }
-
-    subject { collection.sorted_collection(project) }
-
-    context 'when FF :variable_inside_variable is disabled' do
-      before do
-        stub_feature_flags(variable_inside_variable: false)
-      end
-
+  describe '#reject' do
     let(:collection) do
       described_class.new
-          .append(key: 'A', value: 'test-$B')
-          .append(key: 'B', value: 'test-$C')
-          .append(key: 'C', value: 'test')
-      end
-
-      it { is_expected.to be(collection) }
+        .append(key: 'CI_JOB_NAME', value: 'test-1')
+        .append(key: 'CI_BUILD_ID', value: '1')
+        .append(key: 'TEST1', value: 'test-3')
     end
 
-    context 'when FF :variable_inside_variable is enabled' do
-      before do
-        stub_feature_flags(variable_inside_variable: [project])
-      end
+    subject { collection.reject { |var| var[:key] =~ /\ACI_(JOB|BUILD)/ } }
 
-      let(:collection) do
-        described_class.new
-          .append(key: 'A', value: 'test-$B')
-          .append(key: 'B', value: 'test-$C')
-          .append(key: 'C', value: 'test')
+    it 'returns a Collection instance' do
+      is_expected.to be_an_instance_of(described_class)
     end
 
-      it { is_expected.to be_a(Gitlab::Ci::Variables::Collection) }
-
-      it 'returns sorted collection' do
-        expect(subject.to_a).to eq(
-          [
-            { key: 'C', value: 'test', masked: false, public: true },
-            { key: 'B', value: 'test-$C', masked: false, public: true },
-            { key: 'A', value: 'test-$B', masked: false, public: true }
-          ])
-      end
+    it 'returns correctly filtered Collection' do
+      comp = collection.to_runner_variables.reject { |var| var[:key] =~ /\ACI_(JOB|BUILD)/ }
+      expect(subject.to_runner_variables).to eq(comp)
     end
   end
 
-  describe '#reject' do
+  describe '#expand_value' do
     let(:collection) do
-      described_class.new
+      Gitlab::Ci::Variables::Collection.new
                      .append(key: 'CI_JOB_NAME', value: 'test-1')
                      .append(key: 'CI_BUILD_ID', value: '1')
+                     .append(key: 'RAW_VAR', value: '$TEST1', raw: true)
                      .append(key: 'TEST1', value: 'test-3')
     end
 
-    subject { collection.reject { |var| var[:key] =~ /\ACI_(JOB|BUILD)/ } }
+    context 'table tests' do
+      using RSpec::Parameterized::TableSyntax
+
+      where do
+        {
+          "empty value": {
+            value: '',
+            result: '',
+            keep_undefined: false
+          },
+          "simple expansions": {
+            value: 'key$TEST1-$CI_BUILD_ID',
+            result: 'keytest-3-1',
+            keep_undefined: false
+          },
+          "complex expansion": {
+            value: 'key${TEST1}-${CI_JOB_NAME}',
+            result: 'keytest-3-test-1',
+            keep_undefined: false
+          },
+          "complex expansions with raw variable": {
+            value: 'key${RAW_VAR}-${CI_JOB_NAME}',
+            result: 'key$TEST1-test-1',
+            keep_undefined: false
+          },
+          "missing variable not keeping original": {
+            value: 'key${MISSING_VAR}-${CI_JOB_NAME}',
+            result: 'key-test-1',
+            keep_undefined: false
+          },
+          "missing variable keeping original": {
+            value: 'key${MISSING_VAR}-${CI_JOB_NAME}',
+            result: 'key${MISSING_VAR}-test-1',
+            keep_undefined: true
+          }
+        }
+      end
+
+      with_them do
+        subject { collection.expand_value(value, keep_undefined: keep_undefined) }
+
+        it 'matches expected expansion' do
+          is_expected.to eq(result)
+        end
+      end
+    end
+  end
+
+  describe '#sort_and_expand_all' do
+    context 'when FF :variable_inside_variable is disabled' do
+      let_it_be(:project_with_flag_disabled) { create(:project) }
+      let_it_be(:project_with_flag_enabled) { create(:project) }
 
-    it 'returns a Collection instance' do
-      is_expected.to be_an_instance_of(described_class)
+      before do
+        stub_feature_flags(variable_inside_variable: [project_with_flag_enabled])
+      end
+
+      context 'table tests' do
+        using RSpec::Parameterized::TableSyntax
+
+        where do
+          {
+            "empty array": {
+              variables: [],
+              keep_undefined: false
+            },
+            "simple expansions": {
+              variables: [
+                { key: 'variable', value: 'value' },
+                { key: 'variable2', value: 'result' },
+                { key: 'variable3', value: 'key$variable$variable2' }
+              ],
+              keep_undefined: false
+            },
+            "complex expansion": {
+              variables: [
+                { key: 'variable', value: 'value' },
+                { key: 'variable2', value: 'key${variable}' }
+              ],
+              keep_undefined: false
+            },
+            "out-of-order variable reference": {
+              variables: [
+                { key: 'variable2', value: 'key${variable}' },
+                { key: 'variable', value: 'value' }
+              ],
+              keep_undefined: false
+            },
+            "complex expansions with raw variable": {
+              variables: [
+                { key: 'variable3', value: 'key_${variable}_${variable2}' },
+                { key: 'variable', value: '$variable2', raw: true },
+                { key: 'variable2', value: 'value2' }
+              ],
+              keep_undefined: false
+            },
+            "array with cyclic dependency": {
+              variables: [
+                { key: 'variable', value: '$variable2' },
+                { key: 'variable2', value: '$variable3' },
+                { key: 'variable3', value: 'key$variable$variable2' }
+              ],
+              keep_undefined: true
+            }
+          }
+        end
+
+        with_them do
+          let(:collection) { Gitlab::Ci::Variables::Collection.new(variables, keep_undefined: keep_undefined) }
+
+          subject { collection.sort_and_expand_all(project_with_flag_disabled) }
+
+          it 'returns Collection' do
+            is_expected.to be_an_instance_of(Gitlab::Ci::Variables::Collection)
+          end
+
+          it 'does not expand variables' do
+            var_hash = variables.pluck(:key, :value).to_h
+            expect(subject.to_hash).to eq(var_hash)
+          end
+        end
+      end
     end
 
-    it 'returns correctly filtered Collection' do
-      comp = collection.to_runner_variables.reject { |var| var[:key] =~ /\ACI_(JOB|BUILD)/ }
-      expect(subject.to_runner_variables).to eq(comp)
+    context 'when FF :variable_inside_variable is enabled' do
+      let_it_be(:project_with_flag_disabled) { create(:project) }
+      let_it_be(:project_with_flag_enabled) { create(:project) }
+
+      before do
+        stub_feature_flags(variable_inside_variable: [project_with_flag_enabled])
+      end
+
+      context 'table tests' do
+        using RSpec::Parameterized::TableSyntax
+
+        where do
+          {
+            "empty array": {
+              variables: [],
+              keep_undefined: false,
+              result: []
+            },
+            "simple expansions": {
+              variables: [
+                { key: 'variable', value: 'value' },
+                { key: 'variable2', value: 'result' },
+                { key: 'variable3', value: 'key$variable$variable2' },
+                { key: 'variable4', value: 'key$variable$variable3' }
+              ],
+              keep_undefined: false,
+              result: [
+                { key: 'variable', value: 'value' },
+                { key: 'variable2', value: 'result' },
+                { key: 'variable3', value: 'keyvalueresult' },
+                { key: 'variable4', value: 'keyvaluekeyvalueresult' }
+              ]
+            },
+            "complex expansion": {
+              variables: [
+                { key: 'variable', value: 'value' },
+                { key: 'variable2', value: 'key${variable}' }
+              ],
+              keep_undefined: false,
+              result: [
+                { key: 'variable', value: 'value' },
+                { key: 'variable2', value: 'keyvalue' }
+              ]
+            },
+            "unused variables": {
+              variables: [
+                { key: 'variable', value: 'value' },
+                { key: 'variable2', value: 'result2' },
+                { key: 'variable3', value: 'result3' },
+                { key: 'variable4', value: 'key$variable$variable3' }
+              ],
+              keep_undefined: false,
+              result: [
+                { key: 'variable', value: 'value' },
+                { key: 'variable2', value: 'result2' },
+                { key: 'variable3', value: 'result3' },
+                { key: 'variable4', value: 'keyvalueresult3' }
+              ]
+            },
+            "complex expansions": {
+              variables: [
+                { key: 'variable', value: 'value' },
+                { key: 'variable2', value: 'result' },
+                { key: 'variable3', value: 'key${variable}${variable2}' }
+              ],
+              keep_undefined: false,
+              result: [
+                { key: 'variable', value: 'value' },
+                { key: 'variable2', value: 'result' },
+                { key: 'variable3', value: 'keyvalueresult' }
+              ]
+            },
+            "out-of-order expansion": {
+              variables: [
+                { key: 'variable3', value: 'key$variable2$variable' },
+                { key: 'variable', value: 'value' },
+                { key: 'variable2', value: 'result' }
+              ],
+              keep_undefined: false,
+              result: [
+                { key: 'variable2', value: 'result' },
+                { key: 'variable', value: 'value' },
+                { key: 'variable3', value: 'keyresultvalue' }
+              ]
+            },
+            "out-of-order complex expansion": {
+              variables: [
+                { key: 'variable', value: 'value' },
+                { key: 'variable2', value: 'result' },
+                { key: 'variable3', value: 'key${variable2}${variable}' }
+              ],
+              keep_undefined: false,
+              result: [
+                { key: 'variable', value: 'value' },
+                { key: 'variable2', value: 'result' },
+                { key: 'variable3', value: 'keyresultvalue' }
+              ]
+            },
+            "missing variable": {
+              variables: [
+                { key: 'variable2', value: 'key$variable' }
+              ],
+              keep_undefined: false,
+              result: [
+                { key: 'variable2', value: 'key' }
+              ]
+            },
+            "missing variable keeping original": {
+              variables: [
+                { key: 'variable2', value: 'key$variable' }
+              ],
+              keep_undefined: true,
+              result: [
+                { key: 'variable2', value: 'key$variable' }
+              ]
+            },
+            "complex expansions with missing variable keeping original": {
+              variables: [
+                { key: 'variable4', value: 'key${variable}${variable2}${variable3}' },
+                { key: 'variable', value: 'value' },
+                { key: 'variable3', value: 'value3' }
+              ],
+              keep_undefined: true,
+              result: [
+                { key: 'variable', value: 'value' },
+                { key: 'variable3', value: 'value3' },
+                { key: 'variable4', value: 'keyvalue${variable2}value3' }
+              ]
+            },
+            "complex expansions with raw variable": {
+              variables: [
+                { key: 'variable3', value: 'key_${variable}_${variable2}' },
+                { key: 'variable', value: '$variable2', raw: true },
+                { key: 'variable2', value: 'value2' }
+              ],
+              keep_undefined: false,
+              result: [
+                { key: 'variable', value: '$variable2', raw: true },
+                { key: 'variable2', value: 'value2' },
+                { key: 'variable3', value: 'key_$variable2_value2' }
+              ]
+            },
+            "cyclic dependency causes original array to be returned": {
+              variables: [
+                { key: 'variable', value: '$variable2' },
+                { key: 'variable2', value: '$variable3' },
+                { key: 'variable3', value: 'key$variable$variable2' }
+              ],
+              keep_undefined: false,
+              result: [
+                { key: 'variable', value: '$variable2' },
+                { key: 'variable2', value: '$variable3' },
+                { key: 'variable3', value: 'key$variable$variable2' }
+              ]
+            }
+          }
+        end
+
+        with_them do
+          let(:collection) { Gitlab::Ci::Variables::Collection.new(variables) }
+
+          subject { collection.sort_and_expand_all(project_with_flag_enabled, keep_undefined: keep_undefined) }
+
+          it 'returns Collection' do
+            is_expected.to be_an_instance_of(Gitlab::Ci::Variables::Collection)
+          end
+
+          it 'expands variables' do
+            var_hash = result.to_h { |env| [env.fetch(:key), env.fetch(:value)] }
+              .with_indifferent_access
+            expect(subject.to_hash).to eq(var_hash)
+          end
+
+          it 'preserves raw attribute' do
+            expect(subject.pluck(:key, :raw).to_h).to eq(collection.pluck(:key, :raw).to_h)
+          end
+        end
+      end
     end
   end
 end