Validate Kafka Data publishing via Microcks

README.MD

@@ -10,3 +10,20 @@ This demo environment uses:
 - [Microcks (Data generation)](https://microcks.io/)
 - [SpecMesh](https://specmesh.io/) for Provisioning, Governance and publishing to specs and associated schemas (and meta-data) to DataHub
 
+
+
+Resources:
+- https://microcks.io/documentation/installing/docker-compose/
+- 
+
+
+1. Normal mode 
+```
+docker-compose -f microcks-docker-compose.yml -f microcks-docker-compose-async-addon.yml up -d
+```
+2. DEV mode
+```
+docker-compose -f microcks-docker-compose-devmode.yml up -d
+```
+
+Next Steps: see https://microcks.io/blog/async-features-with-docker-compose/

resources/README.MD

@@ -0,0 +1,66 @@
+# Microcks - Getting started with Docker Compose
+Docker Compose is a tool for easily test and run multi-container applications. Microcks offers a simple way to set up the minimal required containers to have a functional environment in your local computer.
+
+## Usage
+To get started, make sure you have Docker installed on your system.
+
+[//]: # (In your terminal issue the following commands:)
+
+[//]: # (Clone this repository.)
+
+[//]: # ()
+[//]: # (git clone https://github.com/microcks/microcks.git)
+
+[//]: # (Change to the install folder)
+
+[//]: # ()
+[//]: # (cd microcks/install/docker-compose)
+
+## Spin up the containers
+
+`docker-compose up -d`
+
+This will start the required containers and setup a simple environment for your usage.
+
+Open a new browser tab and point to the http://localhost:8080 endpoint. This will redirect you to the Keycloak Single Sign On page for login. 
+
+Use the following default credentials:
+
+```text
+Username: admin
+Password: microcks123
+```
+
+You will be redirected to the main dashboard page. You can now start using Microcks.
+
+
+## Enabling Asynchronous API features
+
+Support for Asynchronous API features of Microcks are not enabled by default into the docker-compose.yml file. If you feel your local machine has enough resources to afford it, you can enable them using a slightly different command line.
+
+In your terminal use the following command instead:
+
+```text
+docker compose -f microcks-docker-compose.yml -f microcks-docker-compose-async-addon.yml up -d
+```
+
+
+Docker compose is now launching additional containers, namely zookeeper, kafka and the microcks-async-minion. The above command should produce the following output:
+
+
+# More experimental things
+
+If you’re feeling lucky regarding your machine, you can even add the Kafdrop utility to visualize and troubleshoot Kafka messages with this command:
+
+```text
+docker compose -f docker-compose.yml -f docker-compose-async-addon.yml -f kafdrop-addon.yml up -d
+```
+
+## Un-authenticated mode
+A “keycloakless” version of docker compose is available thanks to:
+
+```text
+docker compose -f docker-compose-devmode.yml up -d
+```
+
+This configuration enabled Asynchronous API features in a very lightweight mode using Red Panda broker instead of full-blown Apache Kafka distribution.

resources/config/application.properties

@@ -0,0 +1,28 @@
+# Async mocking support.
+async-api.enabled=true
+
+# Access to Microcks API server.
+%docker-compose.io.github.microcks.minion.async.client.MicrocksAPIConnector/mp-rest/url=http://microcks:8080
+
+# Access to Keycloak through docker network
+%docker-compose.keycloak.auth.url=http://keycloak:8080
+
+# Access to Kafka broker.
+%docker-compose.kafka.bootstrap.servers=kafka:19092
+
+# Do not save any consumer-offset on the broker as there's a re-sync on each minion startup.
+%docker-compose.mp.messaging.incoming.microcks-services-updates.enable.auto.commit=false
+%docker-compose.mp.messaging.incoming.microcks-services-updates.bootstrap.servers=kafka:19092
+
+# Explicitly telling the minion the protocols we want to support
+%docker-compose.minion.supported-bindings=KAFKA,WS
+
+# %docker-compose.minion.supported-bindings=KAFKA,WS,MQTT,AMQP,NATS
+
+# %docker-compose.mqtt.server=localhost:1883
+# %docker-compose.mqtt.username=microcks
+# %docker-compose.mqtt.password=microcks
+
+# %docker-compose.amqp.server=localhost:5672
+# %docker-compose.amqp.username=microcks
+# %docker-compose.amqp.password=microcks

resources/config/feature.properties

@@ -0,0 +1,8 @@
+features.feature.async-api.enabled=true
+features.feature.async-api.frequencies=3,10,30
+features.feature.async-api.default-binding=KAFKA
+features.feature.async-api.endpoint-KAFKA=localhost:9092
+features.feature.async-api.endpoint-MQTT=my-mqtt-broker.apps.try.microcks.io:1883
+features.feature.async-api.endpoint-AMQP=my-amqp-broker.apps.try.microcks.io:5672
+features.feature.async-api.endpoint-WS=localhost:8081
+features.feature.async-api.endpoint-NATS=localhost:4222

resources/kafka-docker-compose.yml

@@ -0,0 +1,48 @@
+---
+version: '2'
+services:
+  zookeeper-1:
+    image: confluentinc/cp-zookeeper:latest
+    environment:
+      ZOOKEEPER_CLIENT_PORT: 2181
+      ZOOKEEPER_TICK_TIME: 2000
+    ports:
+      - 22181:2181
+
+  zookeeper-2:
+    image: confluentinc/cp-zookeeper:latest
+    environment:
+      ZOOKEEPER_CLIENT_PORT: 2181
+      ZOOKEEPER_TICK_TIME: 2000
+    ports:
+      - 32181:2181
+
+  kafka-1:
+    image: confluentinc/cp-kafka:latest
+    depends_on:
+      - zookeeper-1
+      - zookeeper-2
+
+    ports:
+      - 29092:29092
+    environment:
+      KAFKA_BROKER_ID: 1
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-1:9092,PLAINTEXT_HOST://localhost:29092
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+  kafka-2:
+    image: confluentinc/cp-kafka:latest
+    depends_on:
+      - zookeeper-1
+      - zookeeper-2
+    ports:
+      - 39092:39092
+    environment:
+      KAFKA_BROKER_ID: 2
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-2:9092,PLAINTEXT_HOST://localhost:39092
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1

resources/keycloak-realm/microcks-realm-sample.json

@@ -0,0 +1,143 @@
+{
+  "id": "microcks",
+  "realm": "microcks",
+  "displayName": "Microcks",
+  "enabled": true,
+  "sslRequired": "none",
+  "registrationAllowed": false,
+  "users" : [
+    {
+      "username" : "admin",
+      "enabled": true,
+      "credentials" : [
+        { "type" : "password",
+          "value" : "microcks123" }
+      ],
+      "realmRoles": [],
+      "applicationRoles": {
+        "realm-management": [ "manage-users", "manage-clients" ],
+        "account": [ "manage-account" ],
+        "microcks-app": [ "user", "manager", "admin"]
+      }
+    }
+  ],
+  "roles": {
+    "realm": [],
+    "client": {
+      "microcks-app": [
+        {
+          "name": "user",
+          "composite": false,
+          "clientRole": true,
+          "containerId": "microcks"
+        },
+        {
+          "name": "admin",
+          "composite": false,
+          "clientRole": true,
+          "containerId": "microcks"
+        },
+        {
+          "name": "manager",
+          "composite": false,
+          "clientRole": true,
+          "containerId": "microcks"
+        }
+      ]
+    }
+  },
+  "groups": [
+    {
+      "name": "microcks",
+      "path": "/microcks",
+      "attributes": {},
+      "realmRoles": [],
+      "clientRoles": {},
+      "subGroups": [
+        {
+          "name": "manager",
+          "path": "/microcks/manager",
+          "attributes": {},
+          "realmRoles": [],
+          "clientRoles": {},
+          "subGroups": []
+        }
+      ]
+    }
+  ],
+  "defaultRoles": [],
+  "requiredCredentials": [ "password" ],
+  "scopeMappings": [],
+  "clientScopeMappings": {
+    "microcks-app": [
+      {
+        "client": "microcks-app-js",
+        "roles": [
+          "manager",
+          "admin",
+          "user"
+        ]
+      }
+    ],
+    "realm-management": [
+      {
+        "client": "microcks-app-js",
+        "roles": [
+          "manage-users",
+          "manage-clients"
+        ]
+      }
+    ]
+  },
+  "clients": [
+    {
+      "clientId": "microcks-app-js",
+      "enabled": true,
+      "publicClient": true,
+      "redirectUris": [
+        "http://localhost:8080/*"
+      ],
+      "webOrigins": [
+        "+"
+      ],
+      "fullScopeAllowed": false,
+      "protocolMappers": [
+        {
+          "name": "microcks-group-mapper",
+          "protocol": "openid-connect",
+          "protocolMapper": "oidc-group-membership-mapper",
+          "consentRequired": false,
+          "config": {
+            "full.path": "true",
+            "id.token.claim": "true",
+            "access.token.claim": "true",
+            "claim.name": "microcks-groups",
+            "userinfo.token.claim": "true"
+          }
+        }
+      ]
+    }
+  ],
+  "applications": [
+    {
+      "name": "microcks-app",
+      "enabled": true,
+      "bearerOnly": true,
+      "defaultRoles": [
+        "user"
+      ]
+    },
+    {
+      "name": "microcks-serviceaccount",
+      "secret": "ab54d329-e435-41ae-a900-ec6b3fe15c54",
+      "enabled": true,
+      "bearerOnly": false,
+      "publicClient": false,
+      "standardFlowEnabled": false,
+      "directAccessGrantsEnabled": true,
+      "serviceAccountsEnabled": true,
+      "clientAuthenticatorType": "client-secret"
+    }
+  ],
+  "keycloakVersion": "10.0.1"
+}

resources/microcks-docker-compose-async-addon.yml

@@ -0,0 +1,69 @@
+version: '2'
+
+services:
+  zookeeper:
+    image: quay.io/strimzi/kafka:0.30.0-kafka-3.1.0
+    container_name: microcks-zookeeper
+    command: [
+      "sh", "-c", "bin/zookeeper-server-start.sh config/zookeeper.properties"
+    ]
+    ports:
+      - "2181:2181"
+    environment:
+      LOG_DIR: /tmp/logs
+
+  kafka:
+    depends_on:
+      - zookeeper
+    image: quay.io/strimzi/kafka:0.30.0-kafka-3.1.0
+    container_name: microcks-kafka
+    command: [
+      "sh", "-c",
+      "bin/kafka-server-start.sh config/server.properties --override listeners=$${KAFKA_LISTENERS} --override advertised.listeners=$${KAFKA_ADVERTISED_LISTENERS} --override listener.security.protocol.map=$${KAFKA_LISTENER_SECURITY_PROTOCOL_MAP} --override zookeeper.connect=$${KAFKA_ZOOKEEPER_CONNECT}"
+    ]
+    ports:
+      - "9092:9092"
+      - "19092:19092"
+    environment:
+      LOG_DIR: "/tmp/logs"
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:19092,EXTERNAL://localhost:9092
+      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:19092,EXTERNAL://0.0.0.0:9092
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,EXTERNAL:PLAINTEXT
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+
+  app:
+    depends_on:
+      - mongo
+      - keycloak
+      - postman
+    image: quay.io/microcks/microcks:1.8.0
+    container_name: microcks
+    volumes:
+      - "./config:/deployments/config"
+    ports:
+      - "8080:8080"
+      - "9090:9090"
+    environment:
+      - SPRING_PROFILES_ACTIVE=prod
+      - SPRING_DATA_MONGODB_URI=mongodb://mongo:27017
+      - SPRING_DATA_MONGODB_DATABASE=microcks
+      - POSTMAN_RUNNER_URL=http://postman:3000
+      - TEST_CALLBACK_URL=http://microcks:8080
+      - SERVICES_UPDATE_INTERVAL=0 0 0/2 * * *
+      - KEYCLOAK_URL=http://keycloak:8080
+      - KEYCLOAK_PUBLIC_URL=http://localhost:18080
+      - ASYNC_MINION_URL=http://microcks-async-minion:8081
+      - KAFKA_BOOTSTRAP_SERVER=kafka:19092
+
+  async-minion:
+    depends_on:
+      - app
+    ports:
+      - "8081:8081"
+    image: quay.io/microcks/microcks-async-minion:1.8.0
+    container_name: microcks-async-minion
+    restart: on-failure
+    volumes:
+      - "./config:/deployments/config"
+    environment:
+      - QUARKUS_PROFILE=docker-compose