From 04e919504f316027bce0f6365ae4d2840ff9a45c Mon Sep 17 00:00:00 2001 From: Chris Schaefer Date: Wed, 23 Sep 2020 05:33:18 -0400 Subject: [PATCH] Consistent Kubernetes service object naming (#399) Resolves https://github.com/spring-cloud/spring-cloud-skipper/issues/953 --- .../AbstractKubernetesDeployer.java | 10 ++ .../spi/kubernetes/KubernetesAppDeployer.java | 142 ++++++++++++------ 2 files changed, 108 insertions(+), 44 deletions(-) diff --git a/src/main/java/org/springframework/cloud/deployer/spi/kubernetes/AbstractKubernetesDeployer.java b/src/main/java/org/springframework/cloud/deployer/spi/kubernetes/AbstractKubernetesDeployer.java index 1f554c79..786745af 100755 --- a/src/main/java/org/springframework/cloud/deployer/spi/kubernetes/AbstractKubernetesDeployer.java +++ b/src/main/java/org/springframework/cloud/deployer/spi/kubernetes/AbstractKubernetesDeployer.java @@ -66,6 +66,8 @@ public class AbstractKubernetesDeployer { protected static final String SPRING_APP_KEY = "spring-app-id"; protected static final String SPRING_MARKER_KEY = "role"; protected static final String SPRING_MARKER_VALUE = "spring-app"; + protected static final String APP_NAME_PROPERTY_KEY = AppDeployer.PREFIX + "appName"; + protected static final String APP_NAME_KEY = "spring-application-name"; private static final String SERVER_PORT_KEY = "server.port"; @@ -115,6 +117,14 @@ Map createIdMap(String appId, AppDeploymentRequest request) { map.put(SPRING_GROUP_KEY, groupId); } map.put(SPRING_DEPLOYMENT_KEY, appId); + + // un-versioned app name provided by skipper + String appName = request.getDeploymentProperties().get(APP_NAME_PROPERTY_KEY); + + if (StringUtils.hasText(appName)) { + map.put(APP_NAME_KEY, appName); + } + return map; } diff --git a/src/main/java/org/springframework/cloud/deployer/spi/kubernetes/KubernetesAppDeployer.java b/src/main/java/org/springframework/cloud/deployer/spi/kubernetes/KubernetesAppDeployer.java index a8064ad6..14a25d02 100755 --- a/src/main/java/org/springframework/cloud/deployer/spi/kubernetes/KubernetesAppDeployer.java +++ b/src/main/java/org/springframework/cloud/deployer/spi/kubernetes/KubernetesAppDeployer.java @@ -139,47 +139,13 @@ public void undeploy(String appId) { throw new IllegalStateException(String.format("App '%s' is not deployed", appId)); } - List apps = client.services().withLabel(SPRING_APP_KEY, appId).list().getItems(); - if (apps != null) { - for (Service app : apps) { - String appIdToDelete = app.getMetadata().getName(); - logger.debug(String.format("Deleting Resources for: %s", appIdToDelete)); - Service svc = client.services().withName(appIdToDelete).get(); - try { - if (svc != null && "LoadBalancer".equals(svc.getSpec().getType())) { - int tries = 0; - int maxWait = properties.getMinutesToWaitForLoadBalancer() * 6; // we check 6 times per minute - while (tries++ < maxWait) { - if (svc.getStatus() != null && svc.getStatus().getLoadBalancer() != null - && svc.getStatus().getLoadBalancer().getIngress() != null && svc.getStatus() - .getLoadBalancer().getIngress().isEmpty()) { - if (tries % 6 == 0) { - logger.warn("Waiting for LoadBalancer to complete before deleting it ..."); - } - logger.debug(String.format("Waiting for LoadBalancer, try %d", tries)); - try { - Thread.sleep(10000L); - } - catch (InterruptedException e) { - } - svc = client.services().withName(appIdToDelete).get(); - } - else { - break; - } - } - logger.debug(String.format("LoadBalancer Ingress: %s", - svc.getStatus().getLoadBalancer().getIngress().toString())); - } - - deleteAllObjects(appIdToDelete); - } - catch (RuntimeException e) { - logger.error(e.getMessage(), e); - throw e; - } - } + try { + deleteAllObjects(appId); + } + catch (RuntimeException e) { + logger.error(e.getMessage(), e); + throw e; } } @@ -325,9 +291,9 @@ protected void createStatefulSet(AppDeploymentRequest request) { String statefulSetInitContainerImageName = this.deploymentPropertiesResolver.getStatefulSetInitContainerImageName(kubernetesDeployerProperties); podSpec.getInitContainers().add(createStatefulSetInitContainer(statefulSetInitContainerImageName)); - + Map deploymentLabels= this.deploymentPropertiesResolver.getDeploymentLabels(request.getDeploymentProperties()); - + StatefulSetSpec spec = new StatefulSetSpecBuilder().withNewSelector().addToMatchLabels(idMap) .addToMatchLabels(SPRING_MARKER_KEY, SPRING_MARKER_VALUE).endSelector() .withVolumeClaimTemplates(persistentVolumeClaimBuilder.build()).withServiceName(appId) @@ -402,15 +368,69 @@ protected void createService(AppDeploymentRequest request) { servicePorts.addAll(addAdditionalServicePorts(additionalServicePorts)); } - spec.withSelector(idMap).addAllToPorts(servicePorts); + spec.addAllToPorts(servicePorts); Map annotations = this.deploymentPropertiesResolver.getServiceAnnotations(request.getDeploymentProperties()); - client.services().createNew().withNewMetadata().withName(appId) + String serviceName = getServiceName(request, appId); + + // if called from skipper, use unique selectors to maintain connectivity + // between service and pods that are being brought up/down + if (request.getDeploymentProperties().containsKey(APP_NAME_PROPERTY_KEY)) { + spec.withSelector(Collections.singletonMap(APP_NAME_KEY, + request.getDeploymentProperties().get(APP_NAME_PROPERTY_KEY))); + + String groupId = request.getDeploymentProperties().get(AppDeployer.GROUP_PROPERTY_KEY); + + if (groupId != null) { + spec.addToSelector(SPRING_GROUP_KEY, groupId); + } + } else { + spec.withSelector(idMap); + } + + client.services().createOrReplaceWithNew().withNewMetadata().withName(serviceName) .withLabels(idMap).withAnnotations(annotations).addToLabels(SPRING_MARKER_KEY, SPRING_MARKER_VALUE) .endMetadata().withSpec(spec.build()).done(); } + // logic to support using un-versioned service names when called from skipper + private String getServiceName(AppDeploymentRequest request, String appId) { + String appName = request.getDeploymentProperties().get(APP_NAME_PROPERTY_KEY); + + // if we have an un-versioned app name from skipper + if(StringUtils.hasText(appName)) { + String serviceName = formatServiceName(request, appName); + + // need to check if a versioned service exists to maintain backwards compat.. + // version number itself isn't checked on as it could be different if create or upgrade + // which we don't know at runtime.... + List services = client.services().withLabel(SPRING_DEPLOYMENT_KEY).list().getItems(); + + for (Service service : services) { + String serviceMetadataName = service.getMetadata().getName(); + + if(serviceMetadataName.startsWith(serviceName + "-v")) { + return appId; + } + } + + return serviceName; + } + + // no un-versioned app name provided, maybe not called from skipper, return whatever is in appId + return appId; + } + + private String formatServiceName(AppDeploymentRequest request, String appName) { + String groupId = request.getDeploymentProperties().get(AppDeployer.GROUP_PROPERTY_KEY); + + String serviceName = groupId == null ? String.format("%s", appName) + : String.format("%s-%s", groupId, appName); + + return serviceName.replace('.', '-').toLowerCase(); + } + private Set addAdditionalServicePorts(String additionalServicePorts) { Set ports = new HashSet<>(); @@ -463,6 +483,7 @@ private String setIndexProperty(String name) { private void deleteAllObjects(String appIdToDelete) { Map labels = Collections.singletonMap(SPRING_APP_KEY, appIdToDelete); + waitForLoadBalancerReady(labels); deleteService(labels); deleteDeployment(labels); deleteStatefulSet(labels); @@ -520,4 +541,37 @@ private void deletePvc(Map labels) { logger.debug(String.format("PVC deleted for: %s - %b", labels, pvcsDeleted)); } } + + private void waitForLoadBalancerReady(Map labels) { + List services = client.services().withLabels(labels).list().getItems(); + + if (!services.isEmpty()) { + Service svc = services.get(0); + if (svc != null && "LoadBalancer".equals(svc.getSpec().getType())) { + int tries = 0; + int maxWait = properties.getMinutesToWaitForLoadBalancer() * 6; // we check 6 times per minute + while (tries++ < maxWait) { + if (svc.getStatus() != null && svc.getStatus().getLoadBalancer() != null + && svc.getStatus().getLoadBalancer().getIngress() != null && svc.getStatus() + .getLoadBalancer().getIngress().isEmpty()) { + if (tries % 6 == 0) { + logger.warn("Waiting for LoadBalancer to complete before deleting it ..."); + } + logger.debug(String.format("Waiting for LoadBalancer, try %d", tries)); + try { + Thread.sleep(10000L); + } + catch (InterruptedException e) { + } + svc = client.services().withLabels(labels).list().getItems().get(0); + } + else { + break; + } + } + logger.debug(String.format("LoadBalancer Ingress: %s", + svc.getStatus().getLoadBalancer().getIngress().toString())); + } + } + } }