From 4745d78b76ff6b73d60e1c900cd477f1138bec02 Mon Sep 17 00:00:00 2001 From: neilavery Date: Wed, 18 Oct 2023 11:46:32 +0100 Subject: [PATCH] first pass --- README.MD | 17 +++ resources/README.MD | 66 ++++++++ resources/config/application.properties | 28 ++++ resources/config/feature.properties | 8 + resources/kafka-docker-compose.yml | 48 ++++++ .../keycloak-realm/microcks-realm-sample.json | 143 ++++++++++++++++++ .../microcks-docker-compose-async-addon.yml | 69 +++++++++ resources/microcks-docker-compose-devmode.yml | 61 ++++++++ resources/microcks-docker-compose.yml | 48 ++++++ resources/microcks-kafdrop-addon.yml | 13 ++ 10 files changed, 501 insertions(+) create mode 100644 resources/README.MD create mode 100644 resources/config/application.properties create mode 100644 resources/config/feature.properties create mode 100644 resources/kafka-docker-compose.yml create mode 100644 resources/keycloak-realm/microcks-realm-sample.json create mode 100644 resources/microcks-docker-compose-async-addon.yml create mode 100644 resources/microcks-docker-compose-devmode.yml create mode 100644 resources/microcks-docker-compose.yml create mode 100644 resources/microcks-kafdrop-addon.yml diff --git a/README.MD b/README.MD index 2fa8a72..5ef6594 100644 --- a/README.MD +++ b/README.MD @@ -10,3 +10,20 @@ This demo environment uses: - [Microcks (Data generation)](https://microcks.io/) - [SpecMesh](https://specmesh.io/) for Provisioning, Governance and publishing to specs and associated schemas (and meta-data) to DataHub + + +Resources: +- https://microcks.io/documentation/installing/docker-compose/ +- + + +1. Normal mode +``` +docker-compose -f microcks-docker-compose.yml -f microcks-docker-compose-async-addon.yml up -d +``` +2. DEV mode +``` +docker-compose -f microcks-docker-compose-devmode.yml up -d +``` + +Next Steps: see https://microcks.io/blog/async-features-with-docker-compose/ \ No newline at end of file diff --git a/resources/README.MD b/resources/README.MD new file mode 100644 index 0000000..7be1706 --- /dev/null +++ b/resources/README.MD @@ -0,0 +1,66 @@ +# Microcks - Getting started with Docker Compose +Docker Compose is a tool for easily test and run multi-container applications. Microcks offers a simple way to set up the minimal required containers to have a functional environment in your local computer. + +## Usage +To get started, make sure you have Docker installed on your system. + +[//]: # (In your terminal issue the following commands:) + +[//]: # (Clone this repository.) + +[//]: # () +[//]: # (git clone https://github.com/microcks/microcks.git) + +[//]: # (Change to the install folder) + +[//]: # () +[//]: # (cd microcks/install/docker-compose) + +## Spin up the containers + +`docker-compose up -d` + +This will start the required containers and setup a simple environment for your usage. + +Open a new browser tab and point to the http://localhost:8080 endpoint. This will redirect you to the Keycloak Single Sign On page for login. + +Use the following default credentials: + +```text +Username: admin +Password: microcks123 +``` + +You will be redirected to the main dashboard page. You can now start using Microcks. + + +## Enabling Asynchronous API features + +Support for Asynchronous API features of Microcks are not enabled by default into the docker-compose.yml file. If you feel your local machine has enough resources to afford it, you can enable them using a slightly different command line. + +In your terminal use the following command instead: + +```text +docker compose -f microcks-docker-compose.yml -f microcks-docker-compose-async-addon.yml up -d +``` + + +Docker compose is now launching additional containers, namely zookeeper, kafka and the microcks-async-minion. The above command should produce the following output: + + +# More experimental things + +If you’re feeling lucky regarding your machine, you can even add the Kafdrop utility to visualize and troubleshoot Kafka messages with this command: + +```text +docker compose -f docker-compose.yml -f docker-compose-async-addon.yml -f kafdrop-addon.yml up -d +``` + +## Un-authenticated mode +A “keycloakless” version of docker compose is available thanks to: + +```text +docker compose -f docker-compose-devmode.yml up -d +``` + +This configuration enabled Asynchronous API features in a very lightweight mode using Red Panda broker instead of full-blown Apache Kafka distribution. diff --git a/resources/config/application.properties b/resources/config/application.properties new file mode 100644 index 0000000..dcf2131 --- /dev/null +++ b/resources/config/application.properties @@ -0,0 +1,28 @@ +# Async mocking support. +async-api.enabled=true + +# Access to Microcks API server. +%docker-compose.io.github.microcks.minion.async.client.MicrocksAPIConnector/mp-rest/url=http://microcks:8080 + +# Access to Keycloak through docker network +%docker-compose.keycloak.auth.url=http://keycloak:8080 + +# Access to Kafka broker. +%docker-compose.kafka.bootstrap.servers=kafka:19092 + +# Do not save any consumer-offset on the broker as there's a re-sync on each minion startup. +%docker-compose.mp.messaging.incoming.microcks-services-updates.enable.auto.commit=false +%docker-compose.mp.messaging.incoming.microcks-services-updates.bootstrap.servers=kafka:19092 + +# Explicitly telling the minion the protocols we want to support +%docker-compose.minion.supported-bindings=KAFKA,WS + +# %docker-compose.minion.supported-bindings=KAFKA,WS,MQTT,AMQP,NATS + +# %docker-compose.mqtt.server=localhost:1883 +# %docker-compose.mqtt.username=microcks +# %docker-compose.mqtt.password=microcks + +# %docker-compose.amqp.server=localhost:5672 +# %docker-compose.amqp.username=microcks +# %docker-compose.amqp.password=microcks \ No newline at end of file diff --git a/resources/config/feature.properties b/resources/config/feature.properties new file mode 100644 index 0000000..b088c9c --- /dev/null +++ b/resources/config/feature.properties @@ -0,0 +1,8 @@ +features.feature.async-api.enabled=true +features.feature.async-api.frequencies=3,10,30 +features.feature.async-api.default-binding=KAFKA +features.feature.async-api.endpoint-KAFKA=localhost:9092 +features.feature.async-api.endpoint-MQTT=my-mqtt-broker.apps.try.microcks.io:1883 +features.feature.async-api.endpoint-AMQP=my-amqp-broker.apps.try.microcks.io:5672 +features.feature.async-api.endpoint-WS=localhost:8081 +features.feature.async-api.endpoint-NATS=localhost:4222 \ No newline at end of file diff --git a/resources/kafka-docker-compose.yml b/resources/kafka-docker-compose.yml new file mode 100644 index 0000000..df00f33 --- /dev/null +++ b/resources/kafka-docker-compose.yml @@ -0,0 +1,48 @@ +--- +version: '2' +services: + zookeeper-1: + image: confluentinc/cp-zookeeper:latest + environment: + ZOOKEEPER_CLIENT_PORT: 2181 + ZOOKEEPER_TICK_TIME: 2000 + ports: + - 22181:2181 + + zookeeper-2: + image: confluentinc/cp-zookeeper:latest + environment: + ZOOKEEPER_CLIENT_PORT: 2181 + ZOOKEEPER_TICK_TIME: 2000 + ports: + - 32181:2181 + + kafka-1: + image: confluentinc/cp-kafka:latest + depends_on: + - zookeeper-1 + - zookeeper-2 + + ports: + - 29092:29092 + environment: + KAFKA_BROKER_ID: 1 + KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-1:9092,PLAINTEXT_HOST://localhost:29092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + kafka-2: + image: confluentinc/cp-kafka:latest + depends_on: + - zookeeper-1 + - zookeeper-2 + ports: + - 39092:39092 + environment: + KAFKA_BROKER_ID: 2 + KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-2:9092,PLAINTEXT_HOST://localhost:39092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 \ No newline at end of file diff --git a/resources/keycloak-realm/microcks-realm-sample.json b/resources/keycloak-realm/microcks-realm-sample.json new file mode 100644 index 0000000..3646daf --- /dev/null +++ b/resources/keycloak-realm/microcks-realm-sample.json @@ -0,0 +1,143 @@ +{ + "id": "microcks", + "realm": "microcks", + "displayName": "Microcks", + "enabled": true, + "sslRequired": "none", + "registrationAllowed": false, + "users" : [ + { + "username" : "admin", + "enabled": true, + "credentials" : [ + { "type" : "password", + "value" : "microcks123" } + ], + "realmRoles": [], + "applicationRoles": { + "realm-management": [ "manage-users", "manage-clients" ], + "account": [ "manage-account" ], + "microcks-app": [ "user", "manager", "admin"] + } + } + ], + "roles": { + "realm": [], + "client": { + "microcks-app": [ + { + "name": "user", + "composite": false, + "clientRole": true, + "containerId": "microcks" + }, + { + "name": "admin", + "composite": false, + "clientRole": true, + "containerId": "microcks" + }, + { + "name": "manager", + "composite": false, + "clientRole": true, + "containerId": "microcks" + } + ] + } + }, + "groups": [ + { + "name": "microcks", + "path": "/microcks", + "attributes": {}, + "realmRoles": [], + "clientRoles": {}, + "subGroups": [ + { + "name": "manager", + "path": "/microcks/manager", + "attributes": {}, + "realmRoles": [], + "clientRoles": {}, + "subGroups": [] + } + ] + } + ], + "defaultRoles": [], + "requiredCredentials": [ "password" ], + "scopeMappings": [], + "clientScopeMappings": { + "microcks-app": [ + { + "client": "microcks-app-js", + "roles": [ + "manager", + "admin", + "user" + ] + } + ], + "realm-management": [ + { + "client": "microcks-app-js", + "roles": [ + "manage-users", + "manage-clients" + ] + } + ] + }, + "clients": [ + { + "clientId": "microcks-app-js", + "enabled": true, + "publicClient": true, + "redirectUris": [ + "http://localhost:8080/*" + ], + "webOrigins": [ + "+" + ], + "fullScopeAllowed": false, + "protocolMappers": [ + { + "name": "microcks-group-mapper", + "protocol": "openid-connect", + "protocolMapper": "oidc-group-membership-mapper", + "consentRequired": false, + "config": { + "full.path": "true", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "microcks-groups", + "userinfo.token.claim": "true" + } + } + ] + } + ], + "applications": [ + { + "name": "microcks-app", + "enabled": true, + "bearerOnly": true, + "defaultRoles": [ + "user" + ] + }, + { + "name": "microcks-serviceaccount", + "secret": "ab54d329-e435-41ae-a900-ec6b3fe15c54", + "enabled": true, + "bearerOnly": false, + "publicClient": false, + "standardFlowEnabled": false, + "directAccessGrantsEnabled": true, + "serviceAccountsEnabled": true, + "clientAuthenticatorType": "client-secret" + } + ], + "keycloakVersion": "10.0.1" +} \ No newline at end of file diff --git a/resources/microcks-docker-compose-async-addon.yml b/resources/microcks-docker-compose-async-addon.yml new file mode 100644 index 0000000..124cc78 --- /dev/null +++ b/resources/microcks-docker-compose-async-addon.yml @@ -0,0 +1,69 @@ +version: '2' + +services: + zookeeper: + image: quay.io/strimzi/kafka:0.30.0-kafka-3.1.0 + container_name: microcks-zookeeper + command: [ + "sh", "-c", "bin/zookeeper-server-start.sh config/zookeeper.properties" + ] + ports: + - "2181:2181" + environment: + LOG_DIR: /tmp/logs + + kafka: + depends_on: + - zookeeper + image: quay.io/strimzi/kafka:0.30.0-kafka-3.1.0 + container_name: microcks-kafka + command: [ + "sh", "-c", + "bin/kafka-server-start.sh config/server.properties --override listeners=$${KAFKA_LISTENERS} --override advertised.listeners=$${KAFKA_ADVERTISED_LISTENERS} --override listener.security.protocol.map=$${KAFKA_LISTENER_SECURITY_PROTOCOL_MAP} --override zookeeper.connect=$${KAFKA_ZOOKEEPER_CONNECT}" + ] + ports: + - "9092:9092" + - "19092:19092" + environment: + LOG_DIR: "/tmp/logs" + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:19092,EXTERNAL://localhost:9092 + KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:19092,EXTERNAL://0.0.0.0:9092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,EXTERNAL:PLAINTEXT + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + + app: + depends_on: + - mongo + - keycloak + - postman + image: quay.io/microcks/microcks:1.8.0 + container_name: microcks + volumes: + - "./config:/deployments/config" + ports: + - "8080:8080" + - "9090:9090" + environment: + - SPRING_PROFILES_ACTIVE=prod + - SPRING_DATA_MONGODB_URI=mongodb://mongo:27017 + - SPRING_DATA_MONGODB_DATABASE=microcks + - POSTMAN_RUNNER_URL=http://postman:3000 + - TEST_CALLBACK_URL=http://microcks:8080 + - SERVICES_UPDATE_INTERVAL=0 0 0/2 * * * + - KEYCLOAK_URL=http://keycloak:8080 + - KEYCLOAK_PUBLIC_URL=http://localhost:18080 + - ASYNC_MINION_URL=http://microcks-async-minion:8081 + - KAFKA_BOOTSTRAP_SERVER=kafka:19092 + + async-minion: + depends_on: + - app + ports: + - "8081:8081" + image: quay.io/microcks/microcks-async-minion:1.8.0 + container_name: microcks-async-minion + restart: on-failure + volumes: + - "./config:/deployments/config" + environment: + - QUARKUS_PROFILE=docker-compose \ No newline at end of file diff --git a/resources/microcks-docker-compose-devmode.yml b/resources/microcks-docker-compose-devmode.yml new file mode 100644 index 0000000..94c590a --- /dev/null +++ b/resources/microcks-docker-compose-devmode.yml @@ -0,0 +1,61 @@ +version: '2' + +services: + mongo: + image: mongo:3.6.23 + container_name: microcks-db + volumes: + - "~/tmp/microcks-data:/data/db" + + postman: + image: quay.io/microcks/microcks-postman-runtime:latest + container_name: microcks-postman-runtime + + kafka: + image: vectorized/redpanda:v22.2.2 + container_name: microcks-kafka + command: [ + "redpanda", "start", + "--overprovisioned --smp 1 --memory 1G --reserve-memory 0M --node-id 0 --check=false", + "--kafka-addr PLAINTEXT://0.0.0.0:19092,EXTERNAL://0.0.0.0:9092", + "--advertise-kafka-addr PLAINTEXT://kafka:19092,EXTERNAL://localhost:9092" + ] + ports: + - "9092:9092" + - "19092:19092" + + app: + depends_on: + - mongo + - postman + image: quay.io/microcks/microcks:1.8.0 + container_name: microcks + volumes: + - "./config:/deployments/config" + ports: + - "8080:8080" + - "9090:9090" + environment: + - SPRING_PROFILES_ACTIVE=prod + - SPRING_DATA_MONGODB_URI=mongodb://mongo:27017 + - SPRING_DATA_MONGODB_DATABASE=microcks + - POSTMAN_RUNNER_URL=http://postman:3000 + - TEST_CALLBACK_URL=http://microcks:8080 + - SERVICES_UPDATE_INTERVAL=0 0 0/2 * * * + - KEYCLOAK_ENABLED=false + #- MAX_UPLOAD_FILE_SIZE=3MB + - ASYNC_MINION_URL=http://microcks-async-minion:8081 + - KAFKA_BOOTSTRAP_SERVER=kafka:19092 + + async-minion: + depends_on: + - app + ports: + - "8081:8081" + image: quay.io/microcks/microcks-async-minion:1.8.0 + container_name: microcks-async-minion + restart: on-failure + volumes: + - "./config:/deployments/config" + environment: + - QUARKUS_PROFILE=docker-compose \ No newline at end of file diff --git a/resources/microcks-docker-compose.yml b/resources/microcks-docker-compose.yml new file mode 100644 index 0000000..bdd64f1 --- /dev/null +++ b/resources/microcks-docker-compose.yml @@ -0,0 +1,48 @@ +version: '2' + +services: + mongo: + image: mongo:3.6.23 + container_name: microcks-db + volumes: + - "~/tmp/microcks-data:/data/db" + + keycloak: + image: quay.io/keycloak/keycloak:22.0.2 + container_name: microcks-sso + ports: + - "18080:8080" + environment: + KEYCLOAK_ADMIN: "admin" + KEYCLOAK_ADMIN_PASSWORD: "admin" + KC_HOSTNAME_ADMIN_URL: "http://localhost:18080" + KC_HOSTNAME_URL: "http://localhost:18080" + volumes: + - "./keycloak-realm/microcks-realm-sample.json:/opt/keycloak/data/import/microcks-realm.json" + command: + - start-dev --import-realm + + postman: + image: quay.io/microcks/microcks-postman-runtime:latest + container_name: microcks-postman-runtime + + app: + depends_on: + - mongo + - keycloak + - postman + image: quay.io/microcks/microcks:1.8.0 + container_name: microcks + ports: + - "8080:8080" + - "9090:9090" + environment: + - SPRING_PROFILES_ACTIVE=prod + - SPRING_DATA_MONGODB_URI=mongodb://mongo:27017 + - SPRING_DATA_MONGODB_DATABASE=microcks + - POSTMAN_RUNNER_URL=http://postman:3000 + - TEST_CALLBACK_URL=http://microcks:8080 + - SERVICES_UPDATE_INTERVAL=0 0 0/2 * * * + - KEYCLOAK_URL=http://keycloak:8080 + - KEYCLOAK_PUBLIC_URL=http://localhost:18080 + #- MAX_UPLOAD_FILE_SIZE=3MB \ No newline at end of file diff --git a/resources/microcks-kafdrop-addon.yml b/resources/microcks-kafdrop-addon.yml new file mode 100644 index 0000000..6d1aefa --- /dev/null +++ b/resources/microcks-kafdrop-addon.yml @@ -0,0 +1,13 @@ +version: '2' + +services: + kafdrop: + image: obsidiandynamics/kafdrop + restart: "no" + ports: + - "9000:9000" + environment: + KAFKA_BROKERCONNECT: "kafka:19092" + JVM_OPTS: "-Xms16M -Xmx48M -Xss180K -XX:-TieredCompilation -XX:+UseStringDeduplication -noverify" + depends_on: + - "kafka" \ No newline at end of file