Add retry with backoff for component git fetch

README.md

@@ -987,6 +987,12 @@ generated files are placed
 `CFHIGHLANDER_AWS_RETRY_LIMIT` - defaults to 10. Number of retries for AWS SDK before giving up.
 AWS SDK uses exponential backoff to make the API calls
 
+`CFHIGHLANDER_COMPONENT_FETCH_RETRIES` - defaults to 3. Number of retries when fetching a
+component from git before giving up. Uses exponential backoff between attempts.
+
+`CFHIGHLANDER_COMPONENT_FETCH_RETRY_DELAY` - defaults to 1. Base delay in seconds used for the
+exponential backoff between component fetch retries (delay doubles with each retry).
+
 
 
 ## Testing components

lib/cfhighlander.factory.templatefinder.rb

@@ -73,7 +73,24 @@ def findTemplateGit(cache_path, component_name, component_version, git_url, bran
           puts "Trying to load #{component_name}/#{component_version} from #{git_url}##{branch} ... "
           clone_opts = { depth: 1 }
           clone_opts[:branch] = branch if not (branch.nil? or branch.empty?)
-          Git.clone git_url, cache_path, clone_opts
+
+          max_retries = parse_non_negative_int(ENV['CFHIGHLANDER_COMPONENT_FETCH_RETRIES'], 3)
+          retry_delay = parse_non_negative_int(ENV['CFHIGHLANDER_COMPONENT_FETCH_RETRY_DELAY'], 1)
+
+          attempt = 0
+          begin
+            Git.clone git_url, cache_path, clone_opts
+          rescue StandardError => clone_error
+            attempt += 1
+            if attempt <= max_retries
+              delay = retry_delay * (2**(attempt - 1))
+              STDERR.puts "Component fetch failed (attempt #{attempt}/#{max_retries}), retrying in #{delay}s: #{clone_error}"
+              sleep delay
+              retry
+            else
+              raise clone_error
+            end
+          end
           puts "\t .. cached in #{cache_path}\n"
           # return from cache once it's cloned
           return findTemplateGit(cache_path, component_name, component_version, git_url, branch)
@@ -247,6 +264,20 @@ def build_meta(template_name, template_version, template_location)
             template_location: template_location)
       end
 
+      private
+
+      # Parses a value (typically from an env var) as a non-negative integer,
+      # falling back to the given default when the value is nil, empty,
+      # non-numeric or negative.
+      def parse_non_negative_int(value, default)
+        return default if value.nil? || value.to_s.strip.empty?
+
+        parsed = Integer(value)
+        parsed >= 0 ? parsed : default
+      rescue ArgumentError, TypeError
+        default
+      end
+
     end
 
   end

spec/test_templatefinder_spec.rb

@@ -0,0 +1,155 @@
+require_relative '../lib/cfhighlander.factory.templatefinder'
+require 'fileutils'
+require 'git'
+require 'rspec'
+require 'tmpdir'
+
+RSpec.describe Cfhighlander::Factory::TemplateFinder, "#findTemplateGit" do
+
+  let(:finder) { Cfhighlander::Factory::TemplateFinder.new }
+
+  around(:each) do |example|
+    Dir.mktmpdir do |dir|
+      @cache_path = File.join(dir, 'component')
+      example.run
+    end
+  end
+
+  around(:each) do |example|
+    original_retries = ENV['CFHIGHLANDER_COMPONENT_FETCH_RETRIES']
+    original_retry_delay = ENV['CFHIGHLANDER_COMPONENT_FETCH_RETRY_DELAY']
+    begin
+      example.run
+    ensure
+      if original_retries.nil?
+        ENV.delete('CFHIGHLANDER_COMPONENT_FETCH_RETRIES')
+      else
+        ENV['CFHIGHLANDER_COMPONENT_FETCH_RETRIES'] = original_retries
+      end
+
+      if original_retry_delay.nil?
+        ENV.delete('CFHIGHLANDER_COMPONENT_FETCH_RETRY_DELAY')
+      else
+        ENV['CFHIGHLANDER_COMPONENT_FETCH_RETRY_DELAY'] = original_retry_delay
+      end
+    end
+  end
+
+  before(:each) do
+    # keep the spec instant regardless of retry/delay env overrides
+    allow(finder).to receive(:sleep)
+    ENV.delete('CFHIGHLANDER_COMPONENT_FETCH_RETRIES')
+    ENV.delete('CFHIGHLANDER_COMPONENT_FETCH_RETRY_DELAY')
+  end
+
+  it "retries a failing clone and succeeds once it stops failing" do
+    ENV['CFHIGHLANDER_COMPONENT_FETCH_RETRIES'] = '3'
+
+    attempts = 0
+    allow(Git).to receive(:clone) do |_url, path, _opts|
+      attempts += 1
+      raise Git::GitExecuteError, "simulated github flakiness" if attempts < 3
+
+      FileUtils.mkdir_p path
+      File.write(File.join(path, 'mycomponent.cfhighlander.rb'), '')
+    end
+
+    component_name, location = finder.findTemplateGit(
+      @cache_path, 'mycomponent', 'latest', 'https://github.com/theonestack/hl-component-mycomponent', 'master'
+    )
+
+    expect(attempts).to eq(3)
+    expect(component_name).to eq('mycomponent')
+    expect(location).to eq("#{@cache_path}/")
+    expect(finder).to have_received(:sleep).twice
+    expect(finder).to have_received(:sleep).with(1)
+    expect(finder).to have_received(:sleep).with(2)
+  end
+
+  it "gives up after exhausting retries and returns nil without raising" do
+    ENV['CFHIGHLANDER_COMPONENT_FETCH_RETRIES'] = '2'
+    allow(STDERR).to receive(:puts)
+
+    attempts = 0
+    allow(Git).to receive(:clone) do |*_args|
+      attempts += 1
+      raise Git::GitExecuteError, "simulated github outage"
+    end
+
+    result = finder.findTemplateGit(
+      @cache_path, 'mycomponent', 'latest', 'https://github.com/theonestack/hl-component-mycomponent', 'master'
+    )
+
+    expect(result).to be_nil
+    # initial attempt + 2 retries
+    expect(attempts).to eq(3)
+    expect(finder).to have_received(:sleep).twice
+    expect(STDERR).to have_received(:puts).with(a_string_matching(/Failed to resolve component mycomponent@latest/))
+  end
+
+  it "falls back to the default retry count when the env var is invalid" do
+    ENV['CFHIGHLANDER_COMPONENT_FETCH_RETRIES'] = 'not-a-number'
+    allow(STDERR).to receive(:puts)
+
+    attempts = 0
+    allow(Git).to receive(:clone) do |*_args|
+      attempts += 1
+      raise Git::GitExecuteError, "simulated github outage"
+    end
+
+    result = finder.findTemplateGit(
+      @cache_path, 'mycomponent', 'latest', 'https://github.com/theonestack/hl-component-mycomponent', 'master'
+    )
+
+    expect(result).to be_nil
+    # invalid value falls back to the documented default of 3 retries:
+    # initial attempt + 3 retries
+    expect(attempts).to eq(4)
+    expect(finder).to have_received(:sleep).exactly(3).times
+  end
+
+  it "falls back to the default retry delay when the env var is negative" do
+    ENV['CFHIGHLANDER_COMPONENT_FETCH_RETRIES'] = '1'
+    ENV['CFHIGHLANDER_COMPONENT_FETCH_RETRY_DELAY'] = '-5'
+
+    attempts = 0
+    allow(Git).to receive(:clone) do |_url, path, _opts|
+      attempts += 1
+      raise Git::GitExecuteError, "simulated github flakiness" if attempts < 2
+
+      FileUtils.mkdir_p path
+      File.write(File.join(path, 'mycomponent.cfhighlander.rb'), '')
+    end
+
+    finder.findTemplateGit(
+      @cache_path, 'mycomponent', 'latest', 'https://github.com/theonestack/hl-component-mycomponent', 'master'
+    )
+
+    # negative delay falls back to the documented default of 1 second
+    expect(finder).to have_received(:sleep).once.with(1)
+  end
+
+  it "honours a custom retry delay and applies exponential backoff to it" do
+    ENV['CFHIGHLANDER_COMPONENT_FETCH_RETRIES'] = '3'
+    ENV['CFHIGHLANDER_COMPONENT_FETCH_RETRY_DELAY'] = '2'
+
+    attempts = 0
+    allow(Git).to receive(:clone) do |_url, path, _opts|
+      attempts += 1
+      raise Git::GitExecuteError, "simulated github flakiness" if attempts < 3
+
+      FileUtils.mkdir_p path
+      File.write(File.join(path, 'mycomponent.cfhighlander.rb'), '')
+    end
+
+    finder.findTemplateGit(
+      @cache_path, 'mycomponent', 'latest', 'https://github.com/theonestack/hl-component-mycomponent', 'master'
+    )
+
+    # custom delay of 2 backs off exponentially: delay * 2**(attempt-1)
+    expect(finder).to have_received(:sleep).twice
+    expect(finder).to have_received(:sleep).with(2)
+    expect(finder).to have_received(:sleep).with(4)
+  end
+
+end