Skip to content

Commit

Permalink
Merge pull request #59 from linode/bucket-webhook
Browse files Browse the repository at this point in the history
Bucket webhook
  • Loading branch information
nolancon authored Sep 26, 2023
2 parents 8cf46d8 + ec69532 commit 0ffb520
Show file tree
Hide file tree
Showing 17 changed files with 285 additions and 21 deletions.
5 changes: 4 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,8 @@ build.init: $(UP)
run: go.build
@$(INFO) Running Crossplane locally out-of-cluster . . .
@# To see other arguments that can be provided, run the command with --help instead
@# TODO: Webhooks are not enabled for local run.
@# A workaround for tls certs is required.
$(GO_OUT_DIR)/provider --zap-devel

# Spin up a Kind cluster and localstack.
Expand Down Expand Up @@ -140,7 +142,7 @@ load-package: $(KIND) build
@$(MAKE) local.xpkg.sync
@$(INFO) deploying provider package $(PROJECT_NAME)
@$(KIND) load docker-image $(BUILD_REGISTRY)/$(PROJECT_NAME)-$(ARCH) -n $(KIND_CLUSTER_NAME)
@echo '{"apiVersion":"pkg.crossplane.io/v1alpha1","kind":"ControllerConfig","metadata":{"name":"config"},"spec":{"args":["--zap-devel", "--kube-client-rate=100000", "--reconcile-timeout=2s", "--max-reconcile-rate=5000", "--reconcile-concurrency=100", "--poll=30m", "--sync=1h"],"image":"$(BUILD_REGISTRY)/$(PROJECT_NAME)-$(ARCH)"}}' | $(KUBECTL) apply -f -
@echo '{"apiVersion":"pkg.crossplane.io/v1alpha1","kind":"ControllerConfig","metadata":{"name":"config"},"spec":{"args":["--zap-devel","--enable-validation-webhooks", "--kube-client-rate=100000", "--reconcile-timeout=2s", "--max-reconcile-rate=5000", "--reconcile-concurrency=100", "--poll=30m", "--sync=1h"],"image":"$(BUILD_REGISTRY)/$(PROJECT_NAME)-$(ARCH)"}}' | $(KUBECTL) apply -f -
@echo '{"apiVersion":"pkg.crossplane.io/v1","kind":"Provider","metadata":{"name":"$(PROJECT_NAME)"},"spec":{"package":"$(PROJECT_NAME)-$(VERSION).gz","packagePullPolicy":"Never","controllerConfigRef":{"name":"config"}}}' | $(KUBECTL) apply -f -
@$(OK) deploying provider package $(PROJECT_NAME) $(VERSION)

Expand Down Expand Up @@ -170,6 +172,7 @@ dev-cluster: $(KUBECTL) cluster
@$(INFO) Installing CRDs and ProviderConfig
@$(KUBECTL) apply -k https://github.com/crossplane/crossplane//cluster?ref=master
@$(KUBECTL) apply -R -f package/crds
@# TODO: apply package/webhookconfigurations when webhooks can be enabled locally.
@$(KUBECTL) apply -R -f e2e/localstack/localstack-provider-cfg.yaml
@$(OK) Installing CRDs and ProviderConfig

Expand Down
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ spec:
package: xpkg.upbound.io/linode/provider-ceph:v0.0.16
EOF
```
See [WEBHOOKS.md](docs/WEBHOOKS.md) for instructions on how to enable webhooks.

## Contact
- Slack: Join our [#provider-ceph](https://crossplane.slack.com/archives/C05RKQRNDHA) slack channel.
6 changes: 5 additions & 1 deletion apis/generate.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,15 +19,19 @@ limitations under the License.
// NOTE: See the below link for details on what is happening here.
// https://github.com/golang/go/wiki/Modules#how-can-i-track-tool-dependencies-for-a-module

// Remove existing CRDs
// Remove existing manifests
//go:generate rm -rf ../package/crds
//go:generate rm -rf ../package/webhookconfigurations

// Generate deepcopy methodsets and CRD manifests
//go:generate go run -tags generate sigs.k8s.io/controller-tools/cmd/controller-gen object:headerFile=../hack/boilerplate.go.txt paths=./... crd:crdVersions=v1 output:artifacts:config=../package/crds

// Generate crossplane-runtime methodsets (resource.Claim, etc)
//go:generate go run -tags generate github.com/crossplane/crossplane-tools/cmd/angryjet generate-methodsets --header-file=../hack/boilerplate.go.txt ./...

// Generate webhook manifests
//go:generate go run -tags generate sigs.k8s.io/controller-tools/cmd/controller-gen webhook paths=../internal/controller/bucket output:artifacts:config=../package/webhookconfigurations

package apis

import (
Expand Down
6 changes: 2 additions & 4 deletions pkg/utils/utils.go → apis/provider-ceph/v1alpha1/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,16 +14,14 @@ See the License for the specific language governing permissions and
limitations under the License.
*/

package utils

import "github.com/linode/provider-ceph/apis/provider-ceph/v1alpha1"
package v1alpha1

const (
HealthCheckLabelKey = "provider-ceph.crossplane.io"
HealthCheckLabelVal = "health-check-bucket"
)

func IsHealthCheckBucket(bucket *v1alpha1.Bucket) bool {
func IsHealthCheckBucket(bucket *Bucket) bool {
if val, ok := bucket.GetLabels()[HealthCheckLabelKey]; ok {
if val == HealthCheckLabelVal {
return true
Expand Down
19 changes: 17 additions & 2 deletions cmd/provider/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ import (
"k8s.io/klog/v2"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/log/zap"
"sigs.k8s.io/controller-runtime/pkg/webhook"

xpv1 "github.com/crossplane/crossplane-runtime/apis/common/v1"
"github.com/crossplane/crossplane-runtime/pkg/controller"
Expand All @@ -42,9 +43,11 @@ import (
"github.com/crossplane/crossplane-runtime/pkg/resource"

"github.com/linode/provider-ceph/apis"
providercephv1alpha1 "github.com/linode/provider-ceph/apis/provider-ceph/v1alpha1"
"github.com/linode/provider-ceph/apis/v1alpha1"
"github.com/linode/provider-ceph/internal/backendstore"
ceph "github.com/linode/provider-ceph/internal/controller"
"github.com/linode/provider-ceph/internal/controller/bucket"
"github.com/linode/provider-ceph/internal/features"
)

Expand Down Expand Up @@ -73,6 +76,9 @@ func main() {
enableManagementPolicies = app.Flag("enable-management-policies", "Enable support for Management Policies.").Default("false").Envar("ENABLE_MANAGEMENT_POLICIES").Bool()

autoPauseBucket = app.Flag("auto-pause-bucket", "Enable auto pause of reconciliation of ready buckets").Default("false").Envar("AUTO_PAUSE_BUCKET").Bool()

webhookTLSCertDir = app.Flag("webhook-tls-cert-dir", "The directory of TLS certificate that will be used by the webhook server. There should be tls.crt and tls.key files.").Default("/").Envar("WEBHOOK_TLS_CERT_DIR").String()
enableValidationWebhooks = app.Flag("enable-validation-webhooks", "Enable support for Webhooks.").Default("false").Bool()
)

var zo zap.Options
Expand Down Expand Up @@ -150,15 +156,15 @@ func main() {
leaderRetryDuration := *leaderRenew / two

mgr, err := ctrl.NewManager(cfg, ctrl.Options{
SyncPeriod: syncInterval,

SyncPeriod: syncInterval,
LeaderElection: *leaderElection,
LeaderElectionID: "crossplane-leader-election-provider-ceph-ibyaiby",
LeaderElectionNamespace: *namespace,
LeaderElectionResourceLock: resourcelock.LeasesResourceLock,
RenewDeadline: leaderRenew,
LeaseDuration: &leaseDuration,
RetryPeriod: &leaderRetryDuration,
WebhookServer: webhook.NewServer(webhook.Options{CertDir: *webhookTLSCertDir}),
})
kingpin.FatalIfError(err, "Cannot create controller manager")
kingpin.FatalIfError(apis.AddToScheme(mgr.GetScheme()), "Cannot add Ceph APIs to scheme")
Expand Down Expand Up @@ -196,6 +202,15 @@ func main() {
}

backendStore := backendstore.NewBackendStore()

if *enableValidationWebhooks {
bucketValidator := bucket.NewBucketValidator(backendStore)
kingpin.FatalIfError(ctrl.NewWebhookManagedBy(mgr).
For(&providercephv1alpha1.Bucket{}).
WithValidator(bucketValidator).
Complete(), "Cannot setup bucket validating webhook")
}

kingpin.FatalIfError(ceph.Setup(mgr, o, backendStore, *autoPauseBucket, *pollInterval, *reconcileTimeout), "Cannot setup Ceph controllers")
kingpin.FatalIfError(mgr.Start(ctrl.SetupSignalHandler()), "Cannot start controller manager")
}
36 changes: 36 additions & 0 deletions docs/WEBHOOKS.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
# Webhooks

## Enable Webhooks
- Webhooks are enabled in Crossplane by default from `v1.13` onwards. For previous versions of Crossplane, include the flag `--set webhooks.enabled=true` when [installing Crossplane via Helm](https://docs.crossplane.io/v1.11/software/install/#install-the-crossplane-helm-chart).
- To enable webhooks in Provider Ceph, set the `--enable-webhooks` flag for the Provider Ceph controller. See example below using a controller configuration:

`Provider` with reference to a `ControllerConfig` (**Note:** package version is omitted):
```
apiVersion: pkg.crossplane.io/v1
kind: Provider
metadata:
name: provider-ceph
spec:
package: xpkg.upbound.io/linode/provider-ceph:vX.X.X
controllerConfigRef:
name: provider-ceph
```
`ControllerConfig` with arguments:
```
apiVersion: pkg.crossplane.io/v1alpha1
kind: ControllerConfig
metadata:
name: provider-ceph
spec:
args:
- "--enable-validation-webhooks"
```
**Note:** `ControllerConfig` has been deprecated, but remains in use until an alternative exists.

## Bucket Admission Controlling Webhook
Provider Ceph provides Dynamic Admission Control for Buckets.
Create and Update operations on Buckets are blocked by the bucket admission webhook when:
- The Bucket contains one or more providers (`bucket.spec.Providers`) that do not exist (i.e. a `ProviderConfig` of the same name does not exist in the k8s cluster).

Future Work (not yet implemented):
- Bucket Lifecycle Configurations cannot be validated against a backend.
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ require (
k8s.io/apimachinery v0.28.0
k8s.io/client-go v0.28.0
k8s.io/klog/v2 v2.100.1
k8s.io/utils v0.0.0-20230726121419-3b25d923346b
sigs.k8s.io/controller-runtime v0.15.1
sigs.k8s.io/controller-tools v0.13.0
)
Expand Down Expand Up @@ -118,7 +119,6 @@ require (
k8s.io/apiextensions-apiserver v0.28.0 // indirect
k8s.io/component-base v0.28.0 // indirect
k8s.io/kube-openapi v0.0.0-20230816210353-14e408962443 // indirect
k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.3.0 // indirect
sigs.k8s.io/yaml v1.3.0 // indirect
Expand Down
2 changes: 1 addition & 1 deletion internal/backendstore/backend.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package backendstore

//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -generate
//go:generate go run -mod=mod github.com/maxbrunsfeld/counterfeiter/v6 -generate

import (
"context"
Expand Down
94 changes: 94 additions & 0 deletions internal/controller/bucket/bucket_webhook.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
/*
Copyright 2023.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package bucket

import (
"context"
"fmt"

"github.com/crossplane/crossplane-runtime/pkg/webhook"
"github.com/linode/provider-ceph/apis/provider-ceph/v1alpha1"
"github.com/linode/provider-ceph/internal/backendstore"
"github.com/linode/provider-ceph/internal/utils"
"github.com/pkg/errors"
"k8s.io/apimachinery/pkg/runtime"
"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
)

type BucketValidator struct {
validator *webhook.Validator
backendStore *backendstore.BackendStore
}

func NewBucketValidator(b *backendstore.BackendStore) *BucketValidator {
bucketValidator := &BucketValidator{}
validator := webhook.NewValidator()

validator.CreationChain = append(validator.CreationChain, bucketValidator.ValidateCreate)
validator.UpdateChain = append(validator.UpdateChain, bucketValidator.ValidateUpdate)
validator.DeletionChain = append(validator.DeletionChain, bucketValidator.ValidateDelete)

bucketValidator.validator = validator
bucketValidator.backendStore = b

return bucketValidator
}

//+kubebuilder:webhook:path=/validate-provider-ceph-ceph-crossplane-io-v1alpha1-bucket,mutating=false,failurePolicy=fail,sideEffects=None,groups=provider-ceph.ceph.crossplane.io,resources=buckets,verbs=create;update,versions=v1alpha1,name=bucket.providerceph.crossplane.io,admissionReviewVersions=v1

func (b *BucketValidator) ValidateCreate(ctx context.Context, obj runtime.Object) (admission.Warnings, error) {
bucket, ok := obj.(*v1alpha1.Bucket)
if !ok {
return nil, errors.New(errNotBucket)
}

return nil, b.validateCreateOrUpdate(bucket)
}

func (b *BucketValidator) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) (admission.Warnings, error) {
bucket, ok := newObj.(*v1alpha1.Bucket)
if !ok {
return nil, errors.New(errNotBucket)
}

return nil, b.validateCreateOrUpdate(bucket)
}

func (b *BucketValidator) ValidateDelete(ctx context.Context, obj runtime.Object) (admission.Warnings, error) {
return nil, nil
}

func (b *BucketValidator) validateCreateOrUpdate(bucket *v1alpha1.Bucket) error {
// Ignore validation for health check buckets as they do not
// behave as 'normal' buckets. For example, health check buckets
// need to be updated after their owning ProviderConfig has been deleted.
// This is to remove a finalizer and enable garbage collection.
if v1alpha1.IsHealthCheckBucket(bucket) {
return nil
}

if len(bucket.Spec.Providers) == 0 {
return nil
}

missingProviders := utils.MissingStrings(bucket.Spec.Providers, b.backendStore.GetAllActiveBackendNames())
if len(missingProviders) != 0 {
return errors.New(fmt.Sprintf("providers %v listed in bucket.Spec.Providers cannot be found", missingProviders))
}

return nil
}
3 changes: 1 addition & 2 deletions internal/controller/bucket/create.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ import (
"github.com/linode/provider-ceph/apis/provider-ceph/v1alpha1"
apisv1alpha1 "github.com/linode/provider-ceph/apis/v1alpha1"
s3internal "github.com/linode/provider-ceph/internal/s3"
"github.com/linode/provider-ceph/pkg/utils"
)

//nolint:maintidx,gocognit,gocyclo,cyclop,nolintlint // Function requires numerous checks.
Expand Down Expand Up @@ -73,7 +72,7 @@ func (c *external) Create(ctx context.Context, mg resource.Managed) (managed.Ext
return managed.ExternalCreation{}, errors.Wrap(err, errGetPC)
}

if utils.IsHealthCheckBucket(bucket) && pc.Spec.DisableHealthCheck {
if v1alpha1.IsHealthCheckBucket(bucket) && pc.Spec.DisableHealthCheck {
c.log.Info("Health check is disabled on backend - health-check-bucket will not be created", "backend name", beName)

continue
Expand Down
3 changes: 1 addition & 2 deletions internal/controller/bucket/delete.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ import (

"github.com/linode/provider-ceph/apis/provider-ceph/v1alpha1"
s3internal "github.com/linode/provider-ceph/internal/s3"
"github.com/linode/provider-ceph/pkg/utils"
)

func (c *external) Delete(ctx context.Context, mg resource.Managed) error {
Expand All @@ -25,7 +24,7 @@ func (c *external) Delete(ctx context.Context, mg resource.Managed) error {
ctx, cancel := context.WithTimeout(ctx, c.operationTimeout)
defer cancel()

if utils.IsHealthCheckBucket(bucket) {
if v1alpha1.IsHealthCheckBucket(bucket) {
c.log.Info("Delete is NOOP for health check bucket as it is owned by, and garbage collected on deletion of its related providerconfig", "bucket", bucket.Name)

return nil
Expand Down
3 changes: 1 addition & 2 deletions internal/controller/bucket/observe.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ import (

"github.com/linode/provider-ceph/apis/provider-ceph/v1alpha1"
s3internal "github.com/linode/provider-ceph/internal/s3"
"github.com/linode/provider-ceph/pkg/utils"
)

//nolint:gocyclo,cyclop // Function requires numerous checks.
Expand Down Expand Up @@ -51,7 +50,7 @@ func (c *external) Observe(ctx context.Context, mg resource.Managed) (managed.Ex
break
}
}
if !available && !utils.IsHealthCheckBucket(bucket) {
if !available && !v1alpha1.IsHealthCheckBucket(bucket) {
return managed.ExternalObservation{
ResourceExists: true,
ResourceUpToDate: false,
Expand Down
5 changes: 2 additions & 3 deletions internal/controller/bucket/update.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ import (
apisv1alpha1 "github.com/linode/provider-ceph/apis/v1alpha1"
"github.com/linode/provider-ceph/internal/backendstore"
s3internal "github.com/linode/provider-ceph/internal/s3"
"github.com/linode/provider-ceph/pkg/utils"
)

//nolint:gocyclo,cyclop // Function requires numerous checks.
Expand All @@ -32,7 +31,7 @@ func (c *external) Update(ctx context.Context, mg resource.Managed) (managed.Ext
ctx, cancel := context.WithTimeout(ctx, c.operationTimeout)
defer cancel()

if utils.IsHealthCheckBucket(bucket) {
if v1alpha1.IsHealthCheckBucket(bucket) {
c.log.Info("Update is NOOP for health check bucket - updates performed by health-check-controller", "bucket", bucket.Name)

return managed.ExternalUpdate{}, nil
Expand Down Expand Up @@ -69,7 +68,7 @@ func (c *external) Update(ctx context.Context, mg resource.Managed) (managed.Ext
}
}

if !utils.IsHealthCheckBucket(bucket) &&
if !v1alpha1.IsHealthCheckBucket(bucket) &&
allBucketsReady &&
(bucket.Spec.AutoPause || c.autoPauseBucket) &&
bucket.Annotations[meta.AnnotationKeyReconciliationPaused] == "" {
Expand Down
3 changes: 1 addition & 2 deletions internal/controller/providerconfig/healthcheck_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,6 @@ import (
apisv1alpha1 "github.com/linode/provider-ceph/apis/v1alpha1"
"github.com/linode/provider-ceph/internal/backendstore"
s3internal "github.com/linode/provider-ceph/internal/s3"
"github.com/linode/provider-ceph/pkg/utils"
)

const (
Expand Down Expand Up @@ -184,7 +183,7 @@ func (r *HealthCheckReconciler) createHealthCheckBucket(ctx context.Context, pro
hcBucket.Spec.ProviderConfigReference = &commonv1.Reference{Name: providerConfig.Name}
// Add health-check label so that bucket controller knows to ignore Update/Delete calls.
bucketLabels := make(map[string]string)
bucketLabels[utils.HealthCheckLabelKey] = utils.HealthCheckLabelVal
bucketLabels[v1alpha1.HealthCheckLabelKey] = v1alpha1.HealthCheckLabelVal
hcBucket.SetLabels(bucketLabels)

return r.kubeClient.Create(ctx, hcBucket)
Expand Down
11 changes: 11 additions & 0 deletions internal/utils/utils.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
package utils

import "k8s.io/utils/strings/slices"

// MissingStrings returns a slice of all strings that exist
// in sliceA, but not in sliceB.
func MissingStrings(sliceA, sliceB []string) []string {
return slices.Filter(nil, sliceA, func(s string) bool {
return !slices.Contains(sliceB, s)
})
}
Loading

0 comments on commit 0ffb520

Please sign in to comment.