Add `#expand_variable_collection` to `Collection`

Returns an enumerable of variables with fully-expanded values

lib/expand_variables.rb

@@ -16,6 +16,17 @@ module ExpandVariables
       end
     end
 
+    # expand_variables_collection expands a Gitlab::Ci::Variables::Collection, ignoring unknown variable references.
+    # If a circular variable reference is found, the original Collection is returned
+    def expand_variables_collection(variables, project)
+      return variables if Feature.disabled?(:variable_inside_variable, project)
+
+      sorted_variables = variables.sorted_collection(project)
+      return sorted_variables if sorted_variables.errors
+
+      expand_sorted_variables_collection(sorted_variables)
+    end
+
     def possible_var_reference?(value)
       return unless value
 
@@ -34,7 +45,11 @@ module ExpandVariables
     end
 
     def match_or_blank_value(variables, last_match)
-      variables[last_match[1] || last_match[2]]
+      ref_var_name = last_match[1] || last_match[2]
+      ref_var = variables[ref_var_name]
+      return ref_var if ref_var.is_a?(String) # if entry is a simple "key" => "value" hash
+
+      ref_var[:value] if ref_var
     end
 
     def match_or_original_value(variables, last_match)
@@ -48,15 +63,25 @@ module ExpandVariables
       # Convert Collection to variables
       variables = variables.to_hash if variables.is_a?(Gitlab::Ci::Variables::Collection)
 
-      # Convert hash array to variables
+      # Convert hash array to hash of variables
       if variables.is_a?(Array)
         variables = variables.reduce({}) do |hash, variable|
-          hash[variable[:key]] = variable[:value]
+          hash[variable[:key]] = variable
           hash
         end
       end
 
       variables
     end
+
+    def expand_sorted_variables_collection(sorted_variables)
+      expanded_vars = {}
+
+      sorted_variables.map do |item|
+        item = item.merge(value: expand_existing(item.value, expanded_vars)) if item.depends_on
+
+        expanded_vars.store(item[:key], item)
+      end
+    end
   end
 end

lib/gitlab/ci/variables/collection.rb

@@ -63,10 +63,21 @@ module Gitlab
           Collection.new(@variables.reject(&block))
         end
 
+        def ==(other)
+          return @variables == other if other.is_a?(Array)
+          return false unless other.class == self.class
+
+          @variables == other.variables
+        end
+
         # Returns a sorted Collection object, and sets errors property in case of an error
         def sorted_collection(project)
           Sort.new(self, project).collection
         end
+
+        protected
+
+        attr_reader :variables
       end
     end
   end

lib/gitlab/ci/variables/collection/item.rb

@@ -25,6 +25,10 @@ module Gitlab
             @variable.fetch(key)
           end
 
+          def merge(*other_hashes)
+            self.class.fabricate(@variable.merge(*other_hashes))
+          end
+
           def ==(other)
             to_runner_variable == self.class.fabricate(other).to_runner_variable
           end

spec/lib/expand_variables_spec.rb

@@ -268,4 +268,220 @@ RSpec.describe ExpandVariables do
       end
     end
   end
+
+  describe '#expand_variables_collection' do
+    context 'when FF :variable_inside_variable is disabled' do
+      let_it_be(:project_with_flag_disabled) { create(:project) }
+      let_it_be(:project_with_flag_enabled) { create(:project) }
+
+      before do
+        stub_feature_flags(variable_inside_variable: [project_with_flag_enabled])
+      end
+
+      context 'table tests' do
+        using RSpec::Parameterized::TableSyntax
+
+        where do
+          {
+            "empty array": {
+              variables: []
+            },
+            "simple expansions": {
+              variables: [
+                { key: 'variable', value: 'value' },
+                { key: 'variable2', value: 'result' },
+                { key: 'variable3', value: 'key$variable$variable2' }
+              ]
+            },
+            "complex expansion": {
+              variables: [
+                { key: 'variable', value: 'value' },
+                { key: 'variable2', value: 'key${variable}' }
+              ]
+            },
+            "out-of-order variable reference": {
+              variables: [
+                { key: 'variable2', value: 'key${variable}' },
+                { key: 'variable', value: 'value' }
+              ]
+            },
+            "complex expansions with raw variable": {
+              variables: [
+                { key: 'variable3', value: 'key_${variable}_${variable2}' },
+                { key: 'variable', value: '$variable2', raw: true },
+                { key: 'variable2', value: 'value2' }
+              ]
+            },
+            "array with cyclic dependency": {
+              variables: [
+                { key: 'variable', value: '$variable2' },
+                { key: 'variable2', value: '$variable3' },
+                { key: 'variable3', value: 'key$variable$variable2' }
+              ]
+            }
+          }
+        end
+
+        with_them do
+          subject { ExpandVariables.expand_variables_collection(variables, project_with_flag_disabled) }
+
+          it 'does not expand variables' do
+            is_expected.to eq(variables)
+          end
+        end
+      end
+    end
+
+    context 'when FF :variable_inside_variable is enabled' do
+      let_it_be(:project_with_flag_disabled) { create(:project) }
+      let_it_be(:project_with_flag_enabled) { create(:project) }
+
+      before do
+        stub_feature_flags(variable_inside_variable: [project_with_flag_enabled])
+      end
+
+      context 'table tests' do
+        using RSpec::Parameterized::TableSyntax
+
+        where do
+          {
+            "empty array": {
+              variables: [],
+              result: []
+            },
+            "simple expansions": {
+              variables: [
+                { key: 'variable', value: 'value' },
+                { key: 'variable2', value: 'result' },
+                { key: 'variable3', value: 'key$variable$variable2' },
+                { key: 'variable4', value: 'key$variable$variable3' }
+              ],
+              result: [
+                { key: 'variable', value: 'value' },
+                { key: 'variable2', value: 'result' },
+                { key: 'variable3', value: 'keyvalueresult' },
+                { key: 'variable4', value: 'keyvaluekeyvalueresult' }
+              ]
+            },
+            "complex expansion": {
+              variables: [
+                { key: 'variable', value: 'value' },
+                { key: 'variable2', value: 'key${variable}' }
+              ],
+              result: [
+                { key: 'variable', value: 'value' },
+                { key: 'variable2', value: 'keyvalue' }
+              ]
+            },
+            "unused variables": {
+              variables: [
+                { key: 'variable', value: 'value' },
+                { key: 'variable2', value: 'result2' },
+                { key: 'variable3', value: 'result3' },
+                { key: 'variable4', value: 'key$variable$variable3' }
+              ],
+              result: [
+                { key: 'variable', value: 'value' },
+                { key: 'variable2', value: 'result2' },
+                { key: 'variable3', value: 'result3' },
+                { key: 'variable4', value: 'keyvalueresult3' }
+              ]
+            },
+            "complex expansions": {
+              variables: [
+                { key: 'variable', value: 'value' },
+                { key: 'variable2', value: 'result' },
+                { key: 'variable3', value: 'key${variable}${variable2}' }
+              ],
+              result: [
+                { key: 'variable', value: 'value' },
+                { key: 'variable2', value: 'result' },
+                { key: 'variable3', value: 'keyvalueresult' }
+              ]
+            },
+            "out-of-order expansion": {
+              variables: [
+                { key: 'variable3', value: 'key$variable2$variable' },
+                { key: 'variable', value: 'value' },
+                { key: 'variable2', value: 'result' }
+              ],
+              result: [
+                { key: 'variable2', value: 'result' },
+                { key: 'variable', value: 'value' },
+                { key: 'variable3', value: 'keyresultvalue' }
+              ]
+            },
+            "out-of-order complex expansion": {
+              variables: [
+                { key: 'variable', value: 'value' },
+                { key: 'variable2', value: 'result' },
+                { key: 'variable3', value: 'key${variable2}${variable}' }
+              ],
+              result: [
+                { key: 'variable', value: 'value' },
+                { key: 'variable2', value: 'result' },
+                { key: 'variable3', value: 'keyresultvalue' }
+              ]
+            },
+            "missing variable": {
+              variables: [
+                { key: 'variable2', value: 'key$variable' }
+              ],
+              result: [
+                { key: 'variable2', value: 'key$variable' }
+              ]
+            },
+            "complex expansions with missing variable": {
+              variables: [
+                { key: 'variable4', value: 'key${variable}${variable2}${variable3}' },
+                { key: 'variable', value: 'value' },
+                { key: 'variable3', value: 'value3' }
+              ],
+              result: [
+                { key: 'variable', value: 'value' },
+                { key: 'variable3', value: 'value3' },
+                { key: 'variable4', value: 'keyvalue${variable2}value3' }
+              ]
+            },
+            "complex expansions with raw variable": {
+              variables: [
+                { key: 'variable3', value: 'key_${variable}_${variable2}' },
+                { key: 'variable', value: '$variable2', raw: true },
+                { key: 'variable2', value: 'value2' }
+              ],
+              result: [
+                { key: 'variable', value: '$variable2', raw: true },
+                { key: 'variable2', value: 'value2' },
+                { key: 'variable3', value: 'key_$variable2_value2' }
+              ]
+            },
+            "cyclic dependency causes original array to be returned": {
+              variables: [
+                { key: 'variable', value: '$variable2' },
+                { key: 'variable2', value: '$variable3' },
+                { key: 'variable3', value: 'key$variable$variable2' }
+              ],
+              result: [
+                { key: 'variable', value: '$variable2' },
+                { key: 'variable2', value: '$variable3' },
+                { key: 'variable3', value: 'key$variable$variable2' }
+              ]
+            }
+          }
+        end
+
+        with_them do
+          subject do
+            coll = Gitlab::Ci::Variables::Collection.new(variables)
+
+            ExpandVariables.expand_variables_collection(coll, project_with_flag_enabled)
+          end
+
+          it 'expands variables' do
+            is_expected.to eq(result)
+          end
+        end
+      end
+    end
+  end
 end

spec/lib/gitlab/ci/variables/collection/item_spec.rb

@@ -180,6 +180,16 @@ RSpec.describe Gitlab::Ci::Variables::Collection::Item do
     end
   end
 
+  describe '#merge' do
+    it 'behaves like hash merge' do
+      item = described_class.new(**variable)
+      subject = item.merge(value: 'another thing')
+
+      expect(subject).not_to eq item
+      expect(subject[:value]).to eq 'another thing'
+    end
+  end
+
   describe '#to_runner_variable' do
     context 'when variable is not a file-related' do
       it 'returns a runner-compatible hash representation' do

spec/lib/gitlab/ci/variables/collection_spec.rb

@@ -113,6 +113,67 @@ RSpec.describe Gitlab::Ci::Variables::Collection do
     end
   end
 
+  describe '#==' do
+    variable = { key: 'VAR', value: 'value', public: true, masked: false }
+
+    context 'on empty collection' do
+      collection = described_class.new([])
+
+      it 'returns false for an array containing variable hash' do
+        expect(collection == [variable]).to eq(false)
+      end
+
+      it 'returns false for an unexpected type' do
+        expect(collection == variable).to eq(false)
+      end
+
+      it 'returns true for an empty array' do
+        expect(collection == []).to eq(true)
+      end
+
+      it 'returns true for the same object' do
+        expect(collection).to eq(collection)
+      end
+
+      it 'returns true for a similar object' do
+        expect(collection ==  described_class.new([])).to eq(true)
+      end
+    end
+
+    context 'on collection with a variable' do
+      collection = described_class.new([variable])
+
+      it 'returns false for an array containing other variable' do
+        expect(collection == [{ key: 'VAR', value: 'different value' }]).to eq(false)
+      end
+
+      it 'returns false for an empty array' do
+        expect(collection == []).to eq(false)
+      end
+
+      it 'returns false for an unexpected type' do
+        expect(collection == variable).to eq(false)
+      end
+
+      it 'returns false for a Collection with a variable with different attribute value' do
+        other = described_class.new([{ key: 'VAR', value: 'value', public: false, masked: false }])
+        expect(collection == other).to eq(false)
+      end
+
+      it 'returns true for an array containing variable hash' do
+        expect(collection == [variable]).to eq(true)
+      end
+
+      it 'returns true for the same object' do
+        expect(collection).to eq(collection)
+      end
+
+      it 'returns true for a similar object' do
+        expect(collection ==  described_class.new([variable])).to eq(true)
+      end
+    end
+  end
+
   describe '#size' do
     it 'returns zero for empty collection' do
       collection = described_class.new([])