-
Notifications
You must be signed in to change notification settings - Fork 441
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[SDK] test: Add e2e test for tune function. (#2399)
* fix(sdk): fix error field metrics_collector in tune function. Signed-off-by: Electronic-Waste <2690692950@qq.com> * test(sdk): Add e2e tests for tune function. Signed-off-by: Electronic-Waste <2690692950@qq.com> * test(sdk): add missing field parameters. Signed-off-by: Electronic-Waste <2690692950@qq.com> * refactor(test/sdk): add run-e2e-tune-api.py. Signed-off-by: Electronic-Waste <2690692950@qq.com> * test(sdk): delete tune testing code in run-e2e-experiment. Signed-off-by: Electronic-Waste <2690692950@qq.com> * test(sdk): add blank lines. Signed-off-by: Electronic-Waste <2690692950@qq.com> * test(sdk): add verbose and temporarily delete e2e-experiment test. Signed-off-by: Electronic-Waste <2690692950@qq.com> * test(sdk): add namespace_labels. Signed-off-by: Electronic-Waste <2690692950@qq.com> * test(sdk): add time.sleep(5). Signed-off-by: Electronic-Waste <2690692950@qq.com> * test(sdk): add error output. Signed-off-by: Electronic-Waste <2690692950@qq.com> * test(sdk): build random image for tune. Signed-off-by: Electronic-Waste <2690692950@qq.com> * test(sdk): delete extra debug log. Signed-off-by: Electronic-Waste <2690692950@qq.com> * refactor(test/sdk): create separate workflow for tune. Signed-off-by: Electronic-Waste <2690692950@qq.com> * test(sdk): change api to API. Signed-off-by: Electronic-Waste <2690692950@qq.com> * test(sdk): change the permission of scripts. Signed-off-by: Electronic-Waste <2690692950@qq.com> * test(sdk): delete exit code & comment image pulling. Signed-off-by: Electronic-Waste <2690692950@qq.com> * test(sdk): delete image pulling phase. Signed-off-by: Electronic-Waste <2690692950@qq.com> * test(sdk): refactor workflow file to use template. Signed-off-by: Electronic-Waste <2690692950@qq.com> * test(sdk): mark experiments and trial-images as not required. Signed-off-by: Electronic-Waste <2690692950@qq.com> * test(sdk): pass tune-api param to setup-minikube.sh. Signed-off-by: Electronic-Waste <2690692950@qq.com> * test(sdk): fix err in template-e2e-test. Signed-off-by: Electronic-Waste <2690692950@qq.com> * test(sdk): add debug logs. Signed-off-by: Electronic-Waste <2690692950@qq.com> * test(sdk): reorder params and delete logs. Signed-off-by: Electronic-Waste <2690692950@qq.com> --------- Signed-off-by: Electronic-Waste <2690692950@qq.com>
- Loading branch information
1 parent
51b246f
commit b6f7cfd
Showing
9 changed files
with
341 additions
and
149 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,34 @@ | ||
name: E2E Test with tune API | ||
|
||
on: | ||
pull_request: | ||
paths-ignore: | ||
- "pkg/ui/v1beta1/frontend/**" | ||
|
||
concurrency: | ||
group: ${{ github.workflow }}-${{ github.ref }} | ||
cancel-in-progress: true | ||
|
||
jobs: | ||
e2e: | ||
runs-on: ubuntu-22.04 | ||
timeout-minutes: 120 | ||
steps: | ||
- name: Checkout | ||
uses: actions/checkout@v4 | ||
|
||
- name: Setup Test Env | ||
uses: ./.github/workflows/template-setup-e2e-test | ||
with: | ||
kubernetes-version: ${{ matrix.kubernetes-version }} | ||
|
||
- name: Run e2e test with tune API | ||
uses: ./.github/workflows/template-e2e-test | ||
with: | ||
tune-api: true | ||
|
||
strategy: | ||
fail-fast: false | ||
matrix: | ||
# Detail: https://hub.docker.com/r/kindest/node | ||
kubernetes-version: ["v1.27.11", "v1.28.7", "v1.29.2"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,97 @@ | ||
import argparse | ||
import logging | ||
|
||
from kubeflow.katib import KatibClient | ||
from kubeflow.katib import search | ||
from kubernetes import client | ||
from verify import verify_experiment_results | ||
|
||
# Experiment timeout is 40 min. | ||
EXPERIMENT_TIMEOUT = 60 * 40 | ||
|
||
# The default logging config. | ||
logging.basicConfig(level=logging.INFO) | ||
|
||
|
||
def run_e2e_experiment_create_by_tune( | ||
katib_client: KatibClient, | ||
exp_name: str, | ||
exp_namespace: str, | ||
): | ||
# Create Katib Experiment and wait until it is finished. | ||
logging.debug("Creating Experiment: {}/{}".format(exp_namespace, exp_name)) | ||
|
||
# Use the test case from get-started tutorial. | ||
# https://www.kubeflow.org/docs/components/katib/getting-started/#getting-started-with-katib-python-sdk | ||
# [1] Create an objective function. | ||
def objective(parameters): | ||
import time | ||
time.sleep(5) | ||
result = 4 * int(parameters["a"]) - float(parameters["b"]) ** 2 | ||
print(f"result={result}") | ||
|
||
# [2] Create hyperparameter search space. | ||
parameters = { | ||
"a": search.int(min=10, max=20), | ||
"b": search.double(min=0.1, max=0.2) | ||
} | ||
|
||
# [3] Create Katib Experiment with 4 Trials and 2 CPUs per Trial. | ||
# And Wait until Experiment reaches Succeeded condition. | ||
katib_client.tune( | ||
name=exp_name, | ||
namespace=exp_namespace, | ||
objective=objective, | ||
parameters=parameters, | ||
objective_metric_name="result", | ||
max_trial_count=4, | ||
resources_per_trial={"cpu": "2"}, | ||
) | ||
experiment = katib_client.wait_for_experiment_condition( | ||
exp_name, exp_namespace, timeout=EXPERIMENT_TIMEOUT | ||
) | ||
|
||
# Verify the Experiment results. | ||
verify_experiment_results(katib_client, experiment, exp_name, exp_namespace) | ||
|
||
# Print the Experiment and Suggestion. | ||
logging.debug(katib_client.get_experiment(exp_name, exp_namespace)) | ||
logging.debug(katib_client.get_suggestion(exp_name, exp_namespace)) | ||
|
||
|
||
if __name__ == "__main__": | ||
parser = argparse.ArgumentParser() | ||
parser.add_argument( | ||
"--namespace", type=str, required=True, help="Namespace for the Katib E2E test", | ||
) | ||
parser.add_argument( | ||
"--verbose", action="store_true", help="Verbose output for the Katib E2E test", | ||
) | ||
args = parser.parse_args() | ||
|
||
if args.verbose: | ||
logging.getLogger().setLevel(logging.DEBUG) | ||
|
||
katib_client = KatibClient() | ||
|
||
namespace_labels = client.CoreV1Api().read_namespace(args.namespace).metadata.labels | ||
if 'katib.kubeflow.org/metrics-collector-injection' not in namespace_labels: | ||
namespace_labels['katib.kubeflow.org/metrics-collector-injection'] = 'enabled' | ||
client.CoreV1Api().patch_namespace(args.namespace, {'metadata': {'labels': namespace_labels}}) | ||
|
||
# Test with run_e2e_experiment_create_by_tune | ||
exp_name = "tune-example" | ||
exp_namespace = args.namespace | ||
try: | ||
run_e2e_experiment_create_by_tune(katib_client, exp_name, exp_namespace) | ||
logging.info("---------------------------------------------------------------") | ||
logging.info(f"E2E is succeeded for Experiment created by tune: {exp_namespace}/{exp_name}") | ||
except Exception as e: | ||
logging.info("---------------------------------------------------------------") | ||
logging.info(f"E2E is failed for Experiment created by tune: {exp_namespace}/{exp_name}") | ||
raise e | ||
finally: | ||
# Delete the Experiment. | ||
logging.info("---------------------------------------------------------------") | ||
logging.info("---------------------------------------------------------------") | ||
katib_client.delete_experiment(exp_name, exp_namespace) |
Oops, something went wrong.