Merge pull request #545 from cben/v4.y-test_real_cluster_ssl_verify

[v4.y] Test VERIFY_PEER / VERIFY_NONE work against real cluster

.github/workflows/actions.yml

@@ -10,18 +10,23 @@ on:
     - '**'
 jobs:
   build:
-    runs-on: ${{ matrix.os }}
+    runs-on: ${{ matrix.os_and_command.os }}
     strategy:
       matrix:
         ruby: [ '2.5', '2.6', '2.7', '3.0', '3.1', 'ruby-head', 'truffleruby-head' ]
-        os: ['ubuntu-latest', 'macos-latest']
-        task: [test]
+        os_and_command:
+        - os: 'macos-latest'
+          command: 'env TESTOPTS="--verbose" bundle exec rake test'
+        - os: ubuntu-latest
+          # Sometimes minitest starts and then just hangs printing nothing.
+          # Github by default kills after 6hours(!). Hopefully SIGTERM may let it print some details?
+          command: 'timeout --signal=TERM 3m env TESTOPTS="--verbose" test/config/update_certs_k0s.rb'
         include:
         # run rubocop against lowest supported ruby
         - os: ubuntu-latest
           ruby: '2.5'
-          task: rubocop
-    name: ${{ matrix.os }} ${{ matrix.ruby }} rake ${{ matrix.task }}
+          command: 'bundle exec rake rubocop'
+    name: ${{ matrix.os_and_command.os }} ${{ matrix.ruby }} rake ${{ matrix.os_and_command.command }}
     steps:
     - uses: actions/checkout@v2
     # actions/setup-ruby did not support truffle or bundler caching
@@ -31,5 +36,6 @@ jobs:
         bundler-cache: false # disable running 'bundle install' and caching installed gems see https://github.com/httprb/http/issues/572
     - run: gem install rake bundler
     - run: bundle install
-    - run: bundle exec rake ${{ matrix.task }}
+    - run: ${{ matrix.os_and_command.command }}
+    timeout-minutes: 10
 

test/config/update_certs_k0s.rb

@@ -22,7 +22,7 @@ def sh!(*cmd)
 
 sh! "#{DOCKER} container inspect #{CONTAINER} --format='exists' ||
   #{DOCKER} run -d --name #{CONTAINER} --hostname k0s --privileged -v /var/lib/k0s -p 6443:6443 \
-  docker.io/k0sproject/k0s:v1.23.3-k0s.1"
+  ghcr.io/k0sproject/k0s/k0s:v1.23.3-k0s.1"
 
 # sh! "#{DOCKER} exec #{CONTAINER} kubectl config view --raw"
 # is another way to dump kubeconfig but succeeds with dummy output even before admin.conf exists;
@@ -36,6 +36,16 @@ def sh!(*cmd)
 sh! "#{DOCKER} exec #{CONTAINER} cat /var/lib/k0s/pki/admin.crt  > test/config/external-cert.pem"
 sh! "#{DOCKER} exec #{CONTAINER} cat /var/lib/k0s/pki/admin.key  > test/config/external-key.rsa"
 
-sh! 'bundle exec rake test'
+# Wait for apiserver to be up.  To speed startup, this only retries connection errors;
+# without `--fail-with-body` curl still returns 0 for well-formed 4xx or 5xx responses.
+sleep(1) until sh?(
+  'curl --cacert test/config/external-ca.pem ' \
+  '--key test/config/external-key.rsa ' \
+  '--cert test/config/external-cert.pem  https://127.0.0.1:6443/healthz'
+)
+
+sh! 'env KUBECLIENT_TEST_REAL_CLUSTER=true bundle exec rake test'
 
 sh! "#{DOCKER} rm -f #{CONTAINER}"
+
+puts 'If you run this only for tests, cleanup by running: git restore test/config/'

test/test_config.rb

@@ -193,10 +193,6 @@ def check_context(context, ssl: true)
     end
   end
 
-  def config_file(name)
-    File.join(File.dirname(__FILE__), 'config', name)
-  end
-
   def stub_exec(command_regexp, creds)
     st = Minitest::Mock.new
     st.expect(:success?, true)

test/test_helper.rb

@@ -6,13 +6,23 @@
 require 'json'
 require 'kubeclient'
 
-# Assumes test files will be in a subdirectory with the same name as the
-# file suffix.  e.g. a file named foo.json would be a "json" subdirectory.
-def open_test_file(name)
-  File.new(File.join(File.dirname(__FILE__), name.split('.').last, name))
-end
+MiniTest::Test.class_eval do
+  # Assumes test files will be in a subdirectory with the same name as the
+  # file suffix.  e.g. a file named foo.json would be a "json" subdirectory.
+  def open_test_file(name)
+    File.new(File.join(File.dirname(__FILE__), name.split('.').last, name))
+  end
+
+  # kubeconfig files deviate from above convention.
+  # They link to relaved certs etc. with various extensions, all in same dir.
+  def config_file(name)
+    File.join(File.dirname(__FILE__), 'config', name)
+  end
 
-def stub_core_api_list
-  stub_request(:get, %r{/api/v1$})
-    .to_return(body: open_test_file('core_api_resource_list.json'), status: 200)
+  def stub_core_api_list
+    stub_request(:get, %r{/api/v1$})
+      .to_return(body: open_test_file('core_api_resource_list.json'), status: 200)
+  end
 end
+
+WebMock.disable_net_connect!

test/test_real_cluster.rb

@@ -0,0 +1,82 @@
+require_relative 'test_helper'
+
+class KubeclientRealClusterTest < MiniTest::Test
+  # Tests here actually connect to a cluster!
+  # For simplicity, these tests use same config/*.kubeconfig files as test_config.rb,
+  # so are intended to run from config/update_certs_k0s.rb script.
+  def setup
+    if ENV['KUBECLIENT_TEST_REAL_CLUSTER'] == 'true'
+      WebMock.enable_net_connect!
+    else
+      skip('Requires real cluster, see test/config/update_certs_k0s.rb.')
+    end
+  end
+
+  def teardown
+    WebMock.disable_net_connect! # Don't allow any connections in other tests.
+  end
+
+  def test_real_cluster_verify_peer
+    config = Kubeclient::Config.read(config_file('external.kubeconfig'))
+    context = config.context
+    # localhost and 127.0.0.1 are among names on the certificate
+    client1 = Kubeclient::Client.new(
+      'https://127.0.0.1:6443', 'v1',
+      ssl_options: context.ssl_options.merge(verify_ssl: OpenSSL::SSL::VERIFY_PEER),
+      auth_options: context.auth_options
+    )
+    client1.discover
+    client1.get_nodes
+    exercise_watcher_with_timeout(client1.watch_nodes)
+    # 127.0.0.2 also means localhost but is not included in the certificate.
+    client2 = Kubeclient::Client.new(
+      'https://127.0.0.2:6443', 'v1',
+      ssl_options: context.ssl_options.merge(verify_ssl: OpenSSL::SSL::VERIFY_PEER),
+      auth_options: context.auth_options
+    )
+    # TODO: all OpenSSL exceptions should be wrapped with Kubeclient error.
+    assert_raises(Kubeclient::HttpError, OpenSSL::SSL::SSLError) do
+      client2.discover
+    end
+    # Since discovery fails, methods like .get_nodes, .watch_nodes would all fail
+    # on method_missing -> discover.  Call lower-level methods to test actual connection.
+    assert_raises(Kubeclient::HttpError, OpenSSL::SSL::SSLError) do
+      client2.get_entities('Node', 'nodes', {})
+    end
+    assert_raises(Kubeclient::HttpError, OpenSSL::SSL::SSLError) do
+      exercise_watcher_with_timeout(client2.watch_entities('nodes'))
+    end
+  end
+
+  def test_real_cluster_verify_none
+    config = Kubeclient::Config.read(config_file('external.kubeconfig'))
+    context = config.context
+    # localhost and 127.0.0.1 are among names on the certificate
+    client1 = Kubeclient::Client.new(
+      'https://127.0.0.1:6443', 'v1',
+      ssl_options: context.ssl_options.merge(verify_ssl: OpenSSL::SSL::VERIFY_NONE),
+      auth_options: context.auth_options
+    )
+    client1.get_nodes
+    # 127.0.0.2 also means localhost but is not included in the certificate.
+    client2 = Kubeclient::Client.new(
+      'https://127.0.0.2:6443', 'v1',
+      ssl_options: context.ssl_options.merge(verify_ssl: OpenSSL::SSL::VERIFY_NONE),
+      auth_options: context.auth_options
+    )
+    client2.get_nodes
+  end
+
+  private
+
+  def exercise_watcher_with_timeout(watcher)
+    thread = Thread.new do
+      sleep(1)
+      watcher.finish
+    end
+    watcher.each do |_notice|
+      break
+    end
+    thread.join
+  end
+end