Implement timeout on each integration rspec test

Author: kpaulisse

.rubocop.yml

@@ -19,7 +19,7 @@ Style/HashSyntax:
 
 # We are setting absurdly high limits but should work to tighten these up.
 Metrics/AbcSize:
-  Max: 200
+  Max: 100
   Severity: warning
 
 Metrics/BlockLength:
@@ -34,7 +34,7 @@ Metrics/CyclomaticComplexity:
   Severity: warning
 
 Metrics/MethodLength:
-  Max: 200
+  Max: 100
   Severity: warning
 
 Metrics/PerceivedComplexity:

script/cibuild

@@ -106,12 +106,30 @@ if [ "$RSPEC_TEST" = "true" ]; then
     cat "$OUTPUT_FILE"
     echo ""
 
+    # To avoid travis getting hung if it gets confused, we'll run each of these
+    # scripts individually with a timeout. This will hopefully address the problem
+    # of hung builds.
     echo "Running rspec integration tests"
-    time bundle exec rake spec:integration
-    exitcode=$?
-    if [ "$exitcode" -ne 0 ]; then RSPEC_EXITCODE="$exitcode"; fi
-    cat "$OUTPUT_FILE"
-    echo ""
+    for file in "${DIR}"/spec/octocatalog-diff/integration/*_spec.rb; do
+      retry=1
+      exitcode=255
+      for try in 1 2 3 ; do
+        if [ $retry -eq 1 ]; then
+          retry=0
+          echo "$(basename "$file") try ${try}"
+          "$DIR/script/timeout" 120 bundle exec rspec "$file"
+          exitcode=$?
+          if [ $exitcode -eq 124 ] && [ $try -eq 3 ]; then
+            RSPEC_EXITCODE="255"
+          elif [ $exitcode -eq 124 ] && [ $try -lt 3 ]; then
+            retry=1
+          fi
+        fi
+      done
+
+      if [ "$exitcode" -ne 0 ]; then RSPEC_EXITCODE="$exitcode"; fi
+      cat "$OUTPUT_FILE"
+    done
   done
   export PATH="$SAVED_PATH"
   unset PUPPET_VERSION

script/timeout

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+
+require 'shellwords'
+require 'timeout'
+
+seconds = ARGV.shift
+raise "Usage: #{__FILE__} <seconds> <command>" unless seconds
+begin
+  Timeout::timeout(seconds.to_i) do
+    system ARGV.map { |i| Shellwords.escape(i) }.join(" ")
+    exit $?.exitstatus
+  end
+rescue Timeout::Error
+  STDERR.puts "Timed out after #{seconds} seconds"
+  exit 255
+end

spec/octocatalog-diff/integration/integration_helper.rb

@@ -8,7 +8,6 @@
 require 'shellwords'
 require 'stringio'
 require 'tempfile'
-require 'timeout'
 
 module OctocatalogDiff
   class Integration
@@ -125,16 +124,7 @@ def self.integration(options = {})
         $stdout = stdout_strio
 
         # Run the OctocatalogDiff::Cli.cli and validate output format.
-        result = nil
-        3.times do
-          begin
-            result = timeout(120, Timeout::Error) { OctocatalogDiff::Cli.cli(argv, logger, options) }
-          rescue Timeout::Error => e
-            warn "Warning: #{e.class} #{e.message}"
-          end
-          break if result
-        end
-
+        result = OctocatalogDiff::Cli.cli(argv, logger, options)
         if result.is_a?(Integer)
           result = OpenStruct.new(exitcode: result, exception: nil, diffs: [], to: nil, from: nil)
         elsif result.is_a?(Hash)

spec/octocatalog-diff/tests/spec_helper.rb

@@ -3,7 +3,6 @@
 require 'rspec'
 require 'rspec/retry'
 require 'tempfile'
-require 'timeout'
 
 # Enable SimpleCov coverage testing?
 if ENV['COVERAGE']