diff --git a/.ci/run_tests.sh b/.ci/run_tests.sh index 1145ae22727e..7dae076de6c7 100755 --- a/.ci/run_tests.sh +++ b/.ci/run_tests.sh @@ -8,7 +8,7 @@ function red() { echo -e "\033[1;31m$*\033[0m" } -export BUSTED_ARGS="--no-k -o htest -v --exclude-tags=flaky,ipv6" +export BUSTED_ARGS="--no-k -o htest --repeat 200 -v --exclude-tags=flaky,ipv6" if [ "$KONG_TEST_DATABASE" == "postgres" ]; then export TEST_CMD="bin/busted $BUSTED_ARGS,off" @@ -29,111 +29,11 @@ else export TEST_CMD="bin/busted $BUSTED_ARGS,postgres,db" fi -if [[ "$KONG_TEST_COVERAGE" = true ]]; then - export TEST_CMD="$TEST_CMD --keep-going" -fi + if [ "$TEST_SUITE" == "integration" ]; then - if [[ "$TEST_SPLIT" == first* ]]; then - # GitHub Actions, run first batch of integration tests - eval "$TEST_CMD" $(ls -d spec/02-integration/* | sort | grep -v 05-proxy) - elif [[ "$TEST_SPLIT" == second* ]]; then - # GitHub Actions, run second batch of integration tests - # Note that the split here is chosen carefully to result - # in a similar run time between the two batches, and should - # be adjusted if imbalance become significant in the future - eval "$TEST_CMD" $(ls -d spec/02-integration/* | sort | grep 05-proxy) + eval "$TEST_CMD" spec/02-integration/06-invalidations/02-core_entities_invalidations_spec.lua - else - # Non GitHub Actions - eval "$TEST_CMD" spec/02-integration/ - fi fi -if [ "$TEST_SUITE" == "dbless" ]; then - eval "$TEST_CMD" spec/02-integration/02-cmd \ - spec/02-integration/05-proxy \ - spec/02-integration/04-admin_api/02-kong_routes_spec.lua \ - spec/02-integration/04-admin_api/15-off_spec.lua \ - spec/02-integration/08-status_api/01-core_routes_spec.lua \ - spec/02-integration/08-status_api/03-readiness_endpoint_spec.lua \ - spec/02-integration/11-dbless -fi -if [ "$TEST_SUITE" == "plugins" ]; then - set +ex - rm -f .failed - - if [[ "$TEST_SPLIT" == first* ]]; then - # GitHub Actions, run first batch of plugin tests - PLUGINS=$(ls -d spec/03-plugins/* | head -n22) - - elif [[ "$TEST_SPLIT" == second* ]]; then - # GitHub Actions, run second batch of plugin tests - # Note that the split here is chosen carefully to result - # in a similar run time between the two batches, and should - # be adjusted if imbalance become significant in the future - PLUGINS=$(ls -d spec/03-plugins/* | tail -n+23) - - else - # Non GitHub Actions - PLUGINS=$(ls -d spec/03-plugins/*) - fi - - for p in $PLUGINS; do - echo - cyan "--------------------------------------" - cyan $(basename $p) - cyan "--------------------------------------" - echo - - $TEST_CMD $p || echo "* $p" >> .failed - done - - if [[ "$TEST_SPLIT" == second* ]] || [[ "$TEST_SPLIT" != first* ]]; then - cat kong-*.rockspec | grep kong- | grep -v zipkin | grep -v sidecar | grep "~" | grep -v kong-prometheus-plugin | while read line ; do - REPOSITORY=`echo $line | sed "s/\"/ /g" | awk -F" " '{print $1}'` - VERSION=`luarocks show $REPOSITORY | grep $REPOSITORY | head -1 | awk -F" " '{print $2}' | cut -f1 -d"-"` - REPOSITORY=`echo $REPOSITORY | sed -e 's/kong-prometheus-plugin/kong-plugin-prometheus/g'` - REPOSITORY=`echo $REPOSITORY | sed -e 's/kong-proxy-cache-plugin/kong-plugin-proxy-cache/g'` - - echo - cyan "--------------------------------------" - cyan $REPOSITORY $VERSION - cyan "--------------------------------------" - echo - - git clone https://github.com/Kong/$REPOSITORY.git --branch $VERSION --single-branch /tmp/test-$REPOSITORY || \ - git clone https://github.com/Kong/$REPOSITORY.git --branch v$VERSION --single-branch /tmp/test-$REPOSITORY - sed -i 's/grpcbin:9000/localhost:15002/g' /tmp/test-$REPOSITORY/spec/*.lua - sed -i 's/grpcbin:9001/localhost:15003/g' /tmp/test-$REPOSITORY/spec/*.lua - cp -R /tmp/test-$REPOSITORY/spec/fixtures/* spec/fixtures/ || true - pushd /tmp/test-$REPOSITORY - luarocks make - popd - - $TEST_CMD /tmp/test-$REPOSITORY/spec/ || echo "* $REPOSITORY" >> .failed - - done - fi - - if [ -f .failed ]; then - echo - red "--------------------------------------" - red "Plugin tests failed:" - red "--------------------------------------" - cat .failed - exit 1 - else - exit 0 - fi -fi -if [ "$TEST_SUITE" == "pdk" ]; then - prove -I. -r t -fi -if [ "$TEST_SUITE" == "unit" ]; then - unset KONG_TEST_NGINX_USER KONG_PG_PASSWORD KONG_TEST_PG_PASSWORD - scripts/autodoc - bin/busted -v -o htest spec/01-unit - make lint -fi diff --git a/spec/02-integration/06-invalidations/02-core_entities_invalidations_spec.lua b/spec/02-integration/06-invalidations/02-core_entities_invalidations_spec.lua index 860b6b961edc..727188fae654 100644 --- a/spec/02-integration/06-invalidations/02-core_entities_invalidations_spec.lua +++ b/spec/02-integration/06-invalidations/02-core_entities_invalidations_spec.lua @@ -5,6 +5,50 @@ local ssl_fixtures = require "spec.fixtures.ssl" local POLL_INTERVAL = 0.1 +local function get_admin_2_plugins_iterator_id() + local admin_client_2 = helpers.http_client("127.0.0.1", 9001) + local res = assert(admin_client_2:send { + method = "GET", + path = "/cache/plugins_iterator:version", + }) + local body = assert.res_status(200, res) + local json = cjson.decode(body) + return json.message +end + +local function assert_admin_2_wait(request, res_status, res_not_message) + local ret + helpers.wait_until(function() + local admin_client_2 = helpers.http_client("127.0.0.1", 9001) + finally(function() + admin_client_2:close() + end) + + local res = admin_client_2:send(request) + if not res then + return false + end + if res.status ~= res_status then + return false + end + local body = res:read_body() + if not body then + return false + end + local json = cjson.decode(body) + if res_not_message then + if not json.message:match("^[%w-]+$") then + return false + end + if json.message == res_not_message then + return false + end + end + ret = json + return true + end, 10) + return ret +end local function assert_proxy_2_wait(request, res_status, res_headers) helpers.wait_until(function() @@ -651,9 +695,9 @@ for _, strategy in helpers.each_strategy() do assert.res_status(201, admin_res) helpers.wait_for_all_config_update({ - forced_admin_port = 8001, - forced_proxy_port = 8000, - }) + forced_admin_port = 8001, + forced_proxy_port = 8000, + }) -- no need to wait for workers propagation (lua-resty-events) -- because our test instance only has 1 worker @@ -697,9 +741,9 @@ for _, strategy in helpers.each_strategy() do assert.res_status(200, admin_res) helpers.wait_for_all_config_update({ - forced_admin_port = 8001, - forced_proxy_port = 8000, - }) + forced_admin_port = 8001, + forced_proxy_port = 8000, + }) -- no need to wait for workers propagation (lua-resty-events) -- because our test instance only has 1 worker @@ -734,9 +778,9 @@ for _, strategy in helpers.each_strategy() do assert.res_status(200, admin_res) helpers.wait_for_all_config_update({ - forced_admin_port = 8001, - forced_proxy_port = 8000, - }) + forced_admin_port = 8001, + forced_proxy_port = 8000, + }) local cert_1_old = get_cert(8443, "test.wildcard.com") assert.certificate(cert_1_old).has.cn("localhost") @@ -769,9 +813,9 @@ for _, strategy in helpers.each_strategy() do assert.res_status(200, admin_res) helpers.wait_for_all_config_update({ - forced_admin_port = 8001, - forced_proxy_port = 8000, - }) + forced_admin_port = 8001, + forced_proxy_port = 8000, + }) local cert_1_old = get_cert(8443, "test.wildcard_updated.com") assert.certificate(cert_1_old).has.cn("localhost") @@ -803,9 +847,9 @@ for _, strategy in helpers.each_strategy() do assert.res_status(204, admin_res) helpers.wait_for_all_config_update({ - forced_admin_port = 8001, - forced_proxy_port = 8000, - }) + forced_admin_port = 8001, + forced_proxy_port = 8000, + }) -- no need to wait for workers propagation (lua-resty-events) -- because our test instance only has 1 worker @@ -906,6 +950,7 @@ for _, strategy in helpers.each_strategy() do } }, 200, { ["Dummy-Plugin"] = ngx.null }) + local iterator_id_before = get_admin_2_plugins_iterator_id() -- create Plugin local admin_res_plugin = assert(admin_client_1:send { @@ -928,6 +973,12 @@ for _, strategy in helpers.each_strategy() do forced_proxy_port = 8000, }) + -- wait for new plugins iterator version on node 2 + assert_admin_2_wait({ + method = "GET", + path = "/cache/plugins_iterator:version", + }, 200, iterator_id_before) + -- no need to wait for workers propagation (lua-resty-events) -- because our test instance only has 1 worker @@ -951,6 +1002,7 @@ for _, strategy in helpers.each_strategy() do end) it("on update", function() + local iterator_id_before = get_admin_2_plugins_iterator_id() local admin_res_plugin = assert(admin_client_1:send { method = "PATCH", path = "/plugins/" .. service_plugin_id, @@ -970,6 +1022,12 @@ for _, strategy in helpers.each_strategy() do forced_proxy_port = 8000, }) + -- wait for new plugins iterator version on node 2 + assert_admin_2_wait({ + method = "GET", + path = "/cache/plugins_iterator:version", + }, 200, iterator_id_before) + -- no need to wait for workers propagation (lua-resty-events) -- because our test instance only has 1 worker @@ -993,6 +1051,7 @@ for _, strategy in helpers.each_strategy() do end) it("on delete", function() + local iterator_id_before = get_admin_2_plugins_iterator_id() local admin_res_plugin = assert(admin_client_1:send { method = "DELETE", path = "/plugins/" .. service_plugin_id, @@ -1004,6 +1063,12 @@ for _, strategy in helpers.each_strategy() do forced_proxy_port = 8000, }) + -- wait for new plugins iterator version on node 2 + assert_admin_2_wait({ + method = "GET", + path = "/cache/plugins_iterator:version", + }, 200, iterator_id_before) + -- no need to wait for workers propagation (lua-resty-events) -- because our test instance only has 1 worker @@ -1057,6 +1122,7 @@ for _, strategy in helpers.each_strategy() do assert.is_nil(res.headers["Dummy-Plugin"]) end + local iterator_id_before = get_admin_2_plugins_iterator_id() local admin_res_plugin = assert(admin_client_1:send { method = "POST", path = "/plugins", @@ -1076,6 +1142,12 @@ for _, strategy in helpers.each_strategy() do forced_proxy_port = 8000, }) + -- wait for new plugins iterator version on node 2 + assert_admin_2_wait({ + method = "GET", + path = "/cache/plugins_iterator:version", + }, 200, iterator_id_before) + -- no need to wait for workers propagation (lua-resty-events) -- because our test instance only has 1 worker @@ -1109,6 +1181,7 @@ for _, strategy in helpers.each_strategy() do assert.res_status(200, res_1) assert.equal("1", res_1.headers["Dummy-Plugin"]) + local iterator_id_before = get_admin_2_plugins_iterator_id() local admin_res = assert(admin_client_1:send { method = "DELETE", path = "/plugins/" .. global_dummy_plugin_id, @@ -1120,6 +1193,12 @@ for _, strategy in helpers.each_strategy() do forced_proxy_port = 8000, }) + -- wait for new plugins iterator version on node 2 + assert_admin_2_wait({ + method = "GET", + path = "/cache/plugins_iterator:version", + }, 200, iterator_id_before) + -- no need to wait for workers propagation (lua-resty-events) -- because our test instance only has 1 worker