From 8ca49965643127f45c18a248993d3474fa17ce50 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Tue, 29 Aug 2023 15:10:00 -0400 Subject: [PATCH] Integration Test improvements (#613) - Save Solr Operator logs for each test (filtered to only the applicable lines for that test) - Print the log directory on failure - Include the KUBERNETES_VERSION on failure-retry scripts. --- .gitignore | 3 ++ go.mod | 2 +- tests/e2e/suite_test.go | 99 ++++++++++++++++++++++++++++++++++-- tests/e2e/test_utils_test.go | 3 +- 4 files changed, 101 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index bf3ff906..54bb6039 100644 --- a/.gitignore +++ b/.gitignore @@ -50,6 +50,9 @@ helm/*/charts # Directory to test generated files generated-check +# Integration test outputs +tests/**/output + # Python for the release wizard venv __pycache__ diff --git a/go.mod b/go.mod index eef7232c..6193a4c0 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/robfig/cron/v3 v3.0.1 github.com/stretchr/testify v1.8.2 golang.org/x/net v0.14.0 + golang.org/x/text v0.12.0 helm.sh/helm/v3 v3.11.1 k8s.io/api v0.27.2 k8s.io/apimachinery v0.27.2 @@ -124,7 +125,6 @@ require ( golang.org/x/sync v0.3.0 // indirect golang.org/x/sys v0.11.0 // indirect golang.org/x/term v0.11.0 // indirect - golang.org/x/text v0.12.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.12.0 // indirect gomodules.xyz/jsonpatch/v2 v2.3.0 // indirect diff --git a/tests/e2e/suite_test.go b/tests/e2e/suite_test.go index 770965b2..40ebaed1 100644 --- a/tests/e2e/suite_test.go +++ b/tests/e2e/suite_test.go @@ -18,6 +18,8 @@ package e2e import ( + "bufio" + "bytes" "context" "fmt" solrv1beta1 "github.com/apache/solr-operator/api/v1beta1" @@ -25,14 +27,23 @@ import ( "github.com/go-logr/logr" "github.com/onsi/ginkgo/v2/types" zkApi "github.com/pravega/zookeeper-operator/api/v1beta1" + "golang.org/x/text/cases" + "golang.org/x/text/language" + "io" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" "math/rand" + "os" + "path/filepath" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/config" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" + "strings" "testing" "time" @@ -44,6 +55,7 @@ const ( // Available environment variables to customize tests operatorImageEnv = "OPERATOR_IMAGE" solrImageEnv = "SOLR_IMAGE" + kubeVersionEnv = "KUBERNETES_VERSION" backupDirHostPath = "/tmp/backup" @@ -64,6 +76,9 @@ var ( operatorImage = getEnvWithDefault(operatorImageEnv, defaultOperatorImage) solrImage = getEnvWithDefault(solrImageEnv, defaultSolrImage) + kubeVersion = getEnvWithDefault(kubeVersionEnv, defaultSolrImage) + + outputDir = "output" ) // Run e2e tests using the Ginkgo runner. @@ -81,6 +96,9 @@ var _ = SynchronizedBeforeSuite(func(ctx context.Context) { logger = zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true)) logf.SetLogger(logger) + Expect(os.RemoveAll(outputDir+"/")).To(Succeed(), "Could not delete existing output directory before tests start: %s", outputDir) + Expect(os.Mkdir(outputDir, os.ModeDir|os.ModePerm)).To(Succeed(), "Could not create directory for test output: %s", outputDir) + var err error k8sConfig, err = config.GetConfig() Expect(err).NotTo(HaveOccurred(), "Could not load in default kubernetes config") @@ -139,6 +157,7 @@ type RetryCommand struct { randomSeed int64 operatorImage string solrImage string + kubeVersion string } // ColorableString for ReportEntry to use @@ -149,18 +168,20 @@ func (rc RetryCommand) ColorableString() string { // non-colorable String() is used by go's string formatting support but ignored by ReportEntry func (rc RetryCommand) String() string { return fmt.Sprintf( - "make e2e-tests TEST_FILES=%q TEST_FILTER=%q TEST_SEED=%d TEST_PARALLELISM=%d %s=%q %s=%q", + "make e2e-tests TEST_FILES=%q TEST_FILTER=%q TEST_SEED=%d TEST_PARALLELISM=%d %s=%q %s=%q %s=%q", rc.report.FileName(), rc.report.FullText(), rc.randomSeed, rc.parallelism, solrImageEnv, rc.solrImage, operatorImageEnv, rc.operatorImage, + kubeVersionEnv, rc.kubeVersion, ) } type FailureInformation struct { - namespace string + namespace string + outputDirectory string } // ColorableString for ReportEntry to use @@ -171,18 +192,38 @@ func (fi FailureInformation) ColorableString() string { // non-colorable String() is used by go's string formatting support but ignored by ReportEntry func (fi FailureInformation) String() string { return fmt.Sprintf( - "Namespace: %s\n", + "Namespace: %s\nLogs Directory: %s\n", fi.namespace, + fi.outputDirectory, ) } var _ = ReportAfterEach(func(report SpecReport) { + testName := cases.Title(language.AmericanEnglish, cases.NoLower).String(report.FullText()) + testOutputDir := outputDir + "/" + strings.ReplaceAll(strings.ReplaceAll(testName, " ", "-"), " ", "") + // We count "ran" as "passed" or "failed" + if report.State.Is(types.SpecStatePassed | types.SpecStateFailureStates) { + Expect(os.Mkdir(testOutputDir, os.ModeDir|os.ModePerm)).To(Succeed(), "Could not create directory for test output: %s", testOutputDir) + testOutputDir += "/" + // Always save the logs of the Solr Operator for the test + writePodLogsToFile( + testOutputDir+"solr-operator.log", + getSolrOperatorPodName(solrOperatorReleaseNamespace), + solrOperatorReleaseNamespace, + report.StartTime, + fmt.Sprintf("%q: %q", "namespace", testNamespace()), + ) + } + if report.Failed() { ginkgoConfig, _ := GinkgoConfiguration() + testOutputDir, _ := filepath.Abs(testOutputDir) AddReportEntry( "Failure Information", + types.CodeLocation{}, FailureInformation{ - namespace: testNamespace(), + namespace: testNamespace(), + outputDirectory: testOutputDir, }, ) AddReportEntry( @@ -194,7 +235,57 @@ var _ = ReportAfterEach(func(report SpecReport) { randomSeed: GinkgoRandomSeed(), operatorImage: operatorImage, solrImage: solrImage, + kubeVersion: kubeVersion, }, ) } }) + +func getSolrOperatorPodName(namespace string) string { + labelSelector := labels.SelectorFromSet(map[string]string{"control-plane": "solr-operator"}) + listOps := &client.ListOptions{ + Namespace: namespace, + LabelSelector: labelSelector, + Limit: int64(1), + } + + foundPods := &corev1.PodList{} + Expect(k8sClient.List(context.TODO(), foundPods, listOps)).To(Succeed(), "Could not fetch Solr Operator pod") + Expect(foundPods).ToNot(BeNil(), "No Solr Operator pods could be found") + Expect(foundPods.Items).ToNot(BeEmpty(), "No Solr Operator pods could be found") + return foundPods.Items[0].Name +} + +func writePodLogsToFile(filename string, podName string, podNamespace string, startTimeRaw time.Time, filterLinesWithString string) { + logFile, err := os.Create(filename) + defer logFile.Close() + Expect(err).ToNot(HaveOccurred(), "Could not open file to save logs: %s", filename) + + startTime := metav1.NewTime(startTimeRaw) + podLogOpts := corev1.PodLogOptions{ + SinceTime: &startTime, + } + req := rawK8sClient.CoreV1().Pods(podNamespace).GetLogs(podName, &podLogOpts) + podLogs, logsErr := req.Stream(context.Background()) + defer podLogs.Close() + Expect(logsErr).ToNot(HaveOccurred(), "Could not open stream to fetch pod logs. namespace: %s, pod: %s", podNamespace, podName) + + var logReader io.Reader + logReader = podLogs + + if filterLinesWithString != "" { + filteredWriter := bytes.NewBufferString("") + scanner := bufio.NewScanner(podLogs) + for scanner.Scan() { + line := scanner.Text() + if strings.Contains(line, filterLinesWithString) { + io.WriteString(filteredWriter, line) + io.WriteString(filteredWriter, "\n") + } + } + logReader = filteredWriter + } + + _, err = io.Copy(logFile, logReader) + Expect(err).ToNot(HaveOccurred(), "Could not write podLogs to file: %s", filename) +} diff --git a/tests/e2e/test_utils_test.go b/tests/e2e/test_utils_test.go index 2330e99a..cc4a7249 100644 --- a/tests/e2e/test_utils_test.go +++ b/tests/e2e/test_utils_test.go @@ -21,6 +21,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" solrv1beta1 "github.com/apache/solr-operator/api/v1beta1" "github.com/apache/solr-operator/controllers/util" @@ -87,7 +88,7 @@ func runSolrOperator(ctx context.Context) *release.Release { histClient := action.NewHistory(actionConfig) histClient.Max = 1 var solrOperatorHelmRelease *release.Release - if _, err = histClient.Run(solrOperatorReleaseName); err == driver.ErrReleaseNotFound { + if _, err = histClient.Run(solrOperatorReleaseName); errors.Is(err, driver.ErrReleaseNotFound) { installClient := action.NewInstall(actionConfig) installClient.ReleaseName = solrOperatorReleaseName