From 94dde982fa7dd06748e8c38aa08df1aa64bdc4be Mon Sep 17 00:00:00 2001 From: Beni Cherniavsky-Paskin Date: Sun, 27 Feb 2022 01:12:35 +0200 Subject: [PATCH 1/8] update_certs_k0s.rb: get k0s from Github's registry https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-docker-registry --- test/config/update_certs_k0s.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/config/update_certs_k0s.rb b/test/config/update_certs_k0s.rb index 26140322..830d8f4e 100755 --- a/test/config/update_certs_k0s.rb +++ b/test/config/update_certs_k0s.rb @@ -22,7 +22,7 @@ def sh!(*cmd) sh! "#{DOCKER} container inspect #{CONTAINER} --format='exists' || #{DOCKER} run -d --name #{CONTAINER} --hostname k0s --privileged -v /var/lib/k0s -p 6443:6443 \ - docker.io/k0sproject/k0s:v1.23.3-k0s.1" + ghcr.io/k0sproject/k0s/k0s:v1.23.3-k0s.1" # sh! "#{DOCKER} exec #{CONTAINER} kubectl config view --raw" # is another way to dump kubeconfig but succeeds with dummy output even before admin.conf exists; From 963ffeb27c8095196e7c397096422f7f953df4f1 Mon Sep 17 00:00:00 2001 From: Beni Cherniavsky-Paskin Date: Fri, 25 Feb 2022 16:50:42 +0200 Subject: [PATCH 2/8] Test VERIFY_PEER / VERIFY_NONE work against real cluster Followup to https://github.com/ManageIQ/kubeclient/pull/540, tests that it really fixes https://github.com/abonas/kubeclient/issues/525. That bug only existed on master, not v4.y branch, but testing the behavior on both is good. --- test/config/update_certs_k0s.rb | 10 +++- test/test_config.rb | 4 -- test/test_helper.rb | 8 ++++ test/test_real_cluster.rb | 82 +++++++++++++++++++++++++++++++++ 4 files changed, 99 insertions(+), 5 deletions(-) create mode 100644 test/test_real_cluster.rb diff --git a/test/config/update_certs_k0s.rb b/test/config/update_certs_k0s.rb index 830d8f4e..537e6cb8 100755 --- a/test/config/update_certs_k0s.rb +++ b/test/config/update_certs_k0s.rb @@ -36,6 +36,14 @@ def sh!(*cmd) sh! "#{DOCKER} exec #{CONTAINER} cat /var/lib/k0s/pki/admin.crt > test/config/external-cert.pem" sh! "#{DOCKER} exec #{CONTAINER} cat /var/lib/k0s/pki/admin.key > test/config/external-key.rsa" -sh! 'bundle exec rake test' +# Wait for apiserver to be up. To speed startup, this only retries connection errors; +# without `--fail-with-body` curl still returns 0 for well-formed 4xx or 5xx responses. +sleep(1) until sh?('curl --cacert test/config/external-ca.pem ' \ + '--key test/config/external-key.rsa ' \ + '--cert test/config/external-cert.pem https://127.0.0.1:6443/healthz') + +sh! 'env KUBECLIENT_TEST_REAL_CLUSTER=true bundle exec rake test' sh! "#{DOCKER} rm -f #{CONTAINER}" + +puts 'If you run this only for tests, cleanup by running: git restore test/config/' diff --git a/test/test_config.rb b/test/test_config.rb index 6097e35e..98d2114f 100644 --- a/test/test_config.rb +++ b/test/test_config.rb @@ -193,10 +193,6 @@ def check_context(context, ssl: true) end end - def config_file(name) - File.join(File.dirname(__FILE__), 'config', name) - end - def stub_exec(command_regexp, creds) st = Minitest::Mock.new st.expect(:success?, true) diff --git a/test/test_helper.rb b/test/test_helper.rb index 56965813..08ddadb7 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -12,6 +12,14 @@ def open_test_file(name) File.new(File.join(File.dirname(__FILE__), name.split('.').last, name)) end +# kubeconfig files deviate from above convention. +# They link to relaved certs etc. with various extensions, all in same dir. +def config_file(name) + File.join(File.dirname(__FILE__), 'config', name) +end + +WebMock.disable_net_connect! + def stub_core_api_list stub_request(:get, %r{/api/v1$}) .to_return(body: open_test_file('core_api_resource_list.json'), status: 200) diff --git a/test/test_real_cluster.rb b/test/test_real_cluster.rb new file mode 100644 index 00000000..59c9deb0 --- /dev/null +++ b/test/test_real_cluster.rb @@ -0,0 +1,82 @@ +require_relative 'test_helper' + +class KubeclientRealClusterTest < MiniTest::Test + # Tests here actually connect to a cluster! + # For simplicity, these tests use same config/*.kubeconfig files as test_config.rb, + # so are intended to run from config/update_certs_k0s.rb script. + def setup + if ENV['KUBECLIENT_TEST_REAL_CLUSTER'] == 'true' + WebMock.enable_net_connect! + else + skip('Requires real cluster, see test/config/update_certs_k0s.rb.') + end + end + + def teardown + WebMock.disable_net_connect! # Don't allow any connections in other tests. + end + + def test_real_cluster_verify_peer + config = Kubeclient::Config.read(config_file('external.kubeconfig')) + context = config.context + # localhost and 127.0.0.1 are among names on the certificate + client1 = Kubeclient::Client.new( + 'https://127.0.0.1:6443', 'v1', + ssl_options: context.ssl_options.merge(verify_ssl: OpenSSL::SSL::VERIFY_PEER), + auth_options: context.auth_options + ) + client1.discover + client1.get_nodes + exercise_watcher_with_timeout(client1.watch_nodes) + # 127.0.0.2 also means localhost but is not included in the certificate. + client2 = Kubeclient::Client.new( + 'https://127.0.0.2:6443', 'v1', + ssl_options: context.ssl_options.merge(verify_ssl: OpenSSL::SSL::VERIFY_PEER), + auth_options: context.auth_options + ) + # TODO: all OpenSSL exceptions should be wrapped with Kubeclient error. + assert_raises(Kubeclient::HttpError, OpenSSL::SSL::SSLError) do + client2.discover + end + # Since discovery fails, methods like .get_nodes, .watch_nodes would all fail + # on method_missing -> discover. Call lower-level methods to test actual connection. + assert_raises(Kubeclient::HttpError, OpenSSL::SSL::SSLError) do + client2.get_entities('Node', 'nodes', {}) + end + assert_raises(Kubeclient::HttpError, OpenSSL::SSL::SSLError) do + exercise_watcher_with_timeout(client2.watch_entities('nodes')) + end + end + + def test_real_cluster_verify_none + config = Kubeclient::Config.read(config_file('external.kubeconfig')) + context = config.context + # localhost and 127.0.0.1 are among names on the certificate + client1 = Kubeclient::Client.new( + 'https://127.0.0.1:6443', 'v1', + ssl_options: context.ssl_options.merge(verify_ssl: OpenSSL::SSL::VERIFY_NONE), + auth_options: context.auth_options + ) + client1.get_nodes + # 127.0.0.2 also means localhost but is not included in the certificate. + client2 = Kubeclient::Client.new( + 'https://127.0.0.2:6443', 'v1', + ssl_options: context.ssl_options.merge(verify_ssl: OpenSSL::SSL::VERIFY_NONE), + auth_options: context.auth_options + ) + client2.get_nodes + end + + private + + def exercise_watcher_with_timeout(watcher) + thread = Thread.new do + sleep(1) + watcher.finish + end + watcher.each do |_notice| + break + end + thread.join + end +end From 390ab9ecf4792f07c76f6e37ee39ef98d16b237a Mon Sep 17 00:00:00 2001 From: Beni Cherniavsky-Paskin Date: Sun, 27 Feb 2022 00:58:40 +0200 Subject: [PATCH 3/8] tests: Create real cluster on github CI Yay, `docker run --priviledged` is allowed! k0s starts up really fast too. (only on linux, keeping regular `rake test` on macos because no `docker`.) --- .github/workflows/actions.yml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index 162e3833..1c3a5ef7 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -10,18 +10,21 @@ on: - '**' jobs: build: - runs-on: ${{ matrix.os }} + runs-on: ${{ matrix.os_and_command.os }} strategy: matrix: ruby: [ '2.5', '2.6', '2.7', '3.0', '3.1', 'ruby-head', 'truffleruby-head' ] - os: ['ubuntu-latest', 'macos-latest'] - task: [test] + os_and_command: + - os: 'macos-latest' + command: 'bundle exec rake test' + - os: ubuntu-latest + command: 'test/config/update_certs_k0s.rb' include: # run rubocop against lowest supported ruby - os: ubuntu-latest ruby: '2.5' - task: rubocop - name: ${{ matrix.os }} ${{ matrix.ruby }} rake ${{ matrix.task }} + command: 'bundle exec rake rubocop' + name: ${{ matrix.os_and_command.os }} ${{ matrix.ruby }} rake ${{ matrix.os_and_command.command }} steps: - uses: actions/checkout@v2 # actions/setup-ruby did not support truffle or bundler caching @@ -31,5 +34,5 @@ jobs: bundler-cache: false # disable running 'bundle install' and caching installed gems see https://github.com/httprb/http/issues/572 - run: gem install rake bundler - run: bundle install - - run: bundle exec rake ${{ matrix.task }} + - run: ${{ matrix.os_and_command.command }} From 367d432ef7e873e5fab233d5b0685c242440a996 Mon Sep 17 00:00:00 2001 From: Beni Cherniavsky-Paskin Date: Sun, 6 Mar 2022 00:08:25 +0200 Subject: [PATCH 4/8] CI: set a short timeout Sometimes minitest starts and then just hangs printing ...nothing? Run options: --seed 34017 # Running: Error: The operation was canceled. Github's default job timeout is a generous 6 hours! https://docs.github.com/en/actions/learn-github-actions/usage-limits-billing-and-administration#usage-limits But waiting is not productive and wasteful (and will burn our free minutes). Setting outer job timeout longer for `bundle install`. Hoping an inner SIGTERM received from while job is still running may allow the test to print some traceback. I have no idea where it's getting stuck. (macos has no `timeout` command, but most or all stuck runs were linux.) (I would suspect `exercise_watcher_with_timeout` but also saw same symptoms on another PR without k0s tests) --- .github/workflows/actions.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index 1c3a5ef7..8633144d 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -18,7 +18,9 @@ jobs: - os: 'macos-latest' command: 'bundle exec rake test' - os: ubuntu-latest - command: 'test/config/update_certs_k0s.rb' + # Sometimes minitest starts and then just hangs printing nothing. + # Github by default kills after 6hours(!). Hopefully SIGTERM may let it print some details? + command: 'timeout --signal=TERM 3m test/config/update_certs_k0s.rb' include: # run rubocop against lowest supported ruby - os: ubuntu-latest @@ -35,4 +37,5 @@ jobs: - run: gem install rake bundler - run: bundle install - run: ${{ matrix.os_and_command.command }} + timeout-minutes: 10 From 35160d983fc2433189964cdd6e1c861ae98734c5 Mon Sep 17 00:00:00 2001 From: Beni Cherniavsky-Paskin Date: Wed, 9 Mar 2022 14:14:41 +0200 Subject: [PATCH 5/8] CI: debug where tests timeout --- .github/workflows/actions.yml | 4 ++-- test/config/update_certs_k0s.rb | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index 8633144d..3b9a7877 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -16,11 +16,11 @@ jobs: ruby: [ '2.5', '2.6', '2.7', '3.0', '3.1', 'ruby-head', 'truffleruby-head' ] os_and_command: - os: 'macos-latest' - command: 'bundle exec rake test' + command: 'env TESTOPTS="--verbose" bundle exec rake --trace test' - os: ubuntu-latest # Sometimes minitest starts and then just hangs printing nothing. # Github by default kills after 6hours(!). Hopefully SIGTERM may let it print some details? - command: 'timeout --signal=TERM 3m test/config/update_certs_k0s.rb' + command: 'timeout --signal=TERM 3m env TESTOPTS="--verbose" test/config/update_certs_k0s.rb' include: # run rubocop against lowest supported ruby - os: ubuntu-latest diff --git a/test/config/update_certs_k0s.rb b/test/config/update_certs_k0s.rb index 537e6cb8..1e4d5c11 100755 --- a/test/config/update_certs_k0s.rb +++ b/test/config/update_certs_k0s.rb @@ -42,7 +42,7 @@ def sh!(*cmd) '--key test/config/external-key.rsa ' \ '--cert test/config/external-cert.pem https://127.0.0.1:6443/healthz') -sh! 'env KUBECLIENT_TEST_REAL_CLUSTER=true bundle exec rake test' +sh! 'env KUBECLIENT_TEST_REAL_CLUSTER=true bundle exec rake --trace test' sh! "#{DOCKER} rm -f #{CONTAINER}" From 022f4a2b10e5dcf883ed41cc409645f63d430f89 Mon Sep 17 00:00:00 2001 From: Beni Cherniavsky-Paskin Date: Thu, 10 Mar 2022 11:16:55 +0200 Subject: [PATCH 6/8] Update test/config/update_certs_k0s.rb Co-authored-by: Michael Grosser --- test/config/update_certs_k0s.rb | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/config/update_certs_k0s.rb b/test/config/update_certs_k0s.rb index 1e4d5c11..ceb43b4d 100755 --- a/test/config/update_certs_k0s.rb +++ b/test/config/update_certs_k0s.rb @@ -38,9 +38,11 @@ def sh!(*cmd) # Wait for apiserver to be up. To speed startup, this only retries connection errors; # without `--fail-with-body` curl still returns 0 for well-formed 4xx or 5xx responses. -sleep(1) until sh?('curl --cacert test/config/external-ca.pem ' \ - '--key test/config/external-key.rsa ' \ - '--cert test/config/external-cert.pem https://127.0.0.1:6443/healthz') +sleep(1) until sh?( + 'curl --cacert test/config/external-ca.pem ' \ + '--key test/config/external-key.rsa ' \ + '--cert test/config/external-cert.pem https://127.0.0.1:6443/healthz' +) sh! 'env KUBECLIENT_TEST_REAL_CLUSTER=true bundle exec rake --trace test' From b2152ef6a6f0ce61ba217a0c36d28e31d0920253 Mon Sep 17 00:00:00 2001 From: Beni Cherniavsky-Paskin Date: Thu, 10 Mar 2022 11:22:37 +0200 Subject: [PATCH 7/8] test_handler: Don't pollute global namespace --- test/test_helper.rb | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/test/test_helper.rb b/test/test_helper.rb index 08ddadb7..042a08a0 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -6,21 +6,23 @@ require 'json' require 'kubeclient' -# Assumes test files will be in a subdirectory with the same name as the -# file suffix. e.g. a file named foo.json would be a "json" subdirectory. -def open_test_file(name) - File.new(File.join(File.dirname(__FILE__), name.split('.').last, name)) -end +MiniTest::Test.class_eval do + # Assumes test files will be in a subdirectory with the same name as the + # file suffix. e.g. a file named foo.json would be a "json" subdirectory. + def open_test_file(name) + File.new(File.join(File.dirname(__FILE__), name.split('.').last, name)) + end + + # kubeconfig files deviate from above convention. + # They link to relaved certs etc. with various extensions, all in same dir. + def config_file(name) + File.join(File.dirname(__FILE__), 'config', name) + end -# kubeconfig files deviate from above convention. -# They link to relaved certs etc. with various extensions, all in same dir. -def config_file(name) - File.join(File.dirname(__FILE__), 'config', name) + def stub_core_api_list + stub_request(:get, %r{/api/v1$}) + .to_return(body: open_test_file('core_api_resource_list.json'), status: 200) + end end WebMock.disable_net_connect! - -def stub_core_api_list - stub_request(:get, %r{/api/v1$}) - .to_return(body: open_test_file('core_api_resource_list.json'), status: 200) -end From 6be9b1410289b1584c675e9de2e02f37b17619ee Mon Sep 17 00:00:00 2001 From: Beni Cherniavsky-Paskin Date: Thu, 10 Mar 2022 11:59:00 +0200 Subject: [PATCH 8/8] drop --trace shows stacktrace within Rake itself, not within test. --- .github/workflows/actions.yml | 2 +- test/config/update_certs_k0s.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index 3b9a7877..fd1de5f2 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -16,7 +16,7 @@ jobs: ruby: [ '2.5', '2.6', '2.7', '3.0', '3.1', 'ruby-head', 'truffleruby-head' ] os_and_command: - os: 'macos-latest' - command: 'env TESTOPTS="--verbose" bundle exec rake --trace test' + command: 'env TESTOPTS="--verbose" bundle exec rake test' - os: ubuntu-latest # Sometimes minitest starts and then just hangs printing nothing. # Github by default kills after 6hours(!). Hopefully SIGTERM may let it print some details? diff --git a/test/config/update_certs_k0s.rb b/test/config/update_certs_k0s.rb index ceb43b4d..84059bf4 100755 --- a/test/config/update_certs_k0s.rb +++ b/test/config/update_certs_k0s.rb @@ -44,7 +44,7 @@ def sh!(*cmd) '--cert test/config/external-cert.pem https://127.0.0.1:6443/healthz' ) -sh! 'env KUBECLIENT_TEST_REAL_CLUSTER=true bundle exec rake --trace test' +sh! 'env KUBECLIENT_TEST_REAL_CLUSTER=true bundle exec rake test' sh! "#{DOCKER} rm -f #{CONTAINER}"