diff --git a/.ci/helm.sh b/.ci/helm.sh index 56dde3d3d..bab77af7f 100644 --- a/.ci/helm.sh +++ b/.ci/helm.sh @@ -453,7 +453,7 @@ function ci::verify_elasticsearch_sink() { function ci::verify_mongodb_source() { timesleep=$1 - kubectl exec mongo-dbz-0 -c mongo -- mongo -u debezium -p dbz --authenticationDatabase admin localhost:27017/inventory --eval 'db.products.update({"_id":NumberLong(104)},{$set:{weight:1.25}})' + kubectl exec mongo-dbz-0 -c mongo -- mongosh -u debezium -p dbz --authenticationDatabase admin localhost:27017/inventory --eval 'db.products.update({"_id":NumberLong(104)},{$set:{weight:1.25}})' sleep "$timesleep" kubectl logs --tail=-1 -l compute.functionmesh.io/name=source-sample | grep "records sent" if [ $? -eq 0 ]; then diff --git a/.ci/tests/integration/cases/mongodb-source/manifests.yaml b/.ci/tests/integration/cases/mongodb-source/manifests.yaml index 954b936ca..188a1d1f2 100644 --- a/.ci/tests/integration/cases/mongodb-source/manifests.yaml +++ b/.ci/tests/integration/cases/mongodb-source/manifests.yaml @@ -23,7 +23,7 @@ spec: cpu: "0.1" memory: 1G sourceConfig: - mongodb.hosts: rs0/mongo-dbz-0.mongo.default.svc.cluster.local:27017,rs0/mongo-dbz-1.mongo.default.svc.cluster.local:27017,rs0/mongo-dbz-2.mongo.default.svc.cluster.local:27017 + mongodb.hosts: rs0/mongo-dbz-0.mongo.default.svc.cluster.local:27017 mongodb.name: dbserver1 mongodb.user: debezium mongodb.password: dbz diff --git a/.ci/tests/integration/cases/mongodb-source/mongodb-dbz.yaml b/.ci/tests/integration/cases/mongodb-source/mongodb-dbz.yaml index 60c64d6e4..7c1f32384 100644 --- a/.ci/tests/integration/cases/mongodb-source/mongodb-dbz.yaml +++ b/.ci/tests/integration/cases/mongodb-source/mongodb-dbz.yaml @@ -35,7 +35,7 @@ spec: matchLabels: role: mongo serviceName: "mongo" - replicas: 3 + replicas: 1 template: metadata: labels: @@ -44,7 +44,7 @@ spec: terminationGracePeriodSeconds: 10 containers: - name: mongo - image: debezium/example-mongodb:0.10 + image: debezium/example-mongodb:2.6 env: - name: MONGODB_USER value: "debezium" diff --git a/.ci/tests/integration/cases/mongodb-source/verify.sh b/.ci/tests/integration/cases/mongodb-source/verify.sh index 2d9c0b382..6065d8029 100644 --- a/.ci/tests/integration/cases/mongodb-source/verify.sh +++ b/.ci/tests/integration/cases/mongodb-source/verify.sh @@ -45,7 +45,7 @@ function install_mongodb_server() { # install mongodb server kubectl apply -f "${mongodb_file}" num=0 - while [[ ${num} -lt 3 ]]; do + while [[ ${num} -lt 1 ]]; do sleep 5 kubectl get pods num=$(kubectl get pods -l role=mongo | wc -l) diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml index f21bb3fb0..f017a098b 100644 --- a/.github/workflows/trivy.yml +++ b/.github/workflows/trivy.yml @@ -11,9 +11,17 @@ on: - 'LICENSE' - 'mesh-worker-service/README.md' - 'tools/README.md' + +permissions: + pull-requests: write + issues: write + jobs: build: name: Scan + permissions: + pull-requests: write + issues: write runs-on: ubuntu-latest steps: - name: Checkout @@ -59,4 +67,56 @@ jobs: with: image-ref: 'function-mesh-operator:latest' format: 'table' - exit-code: '1' \ No newline at end of file + exit-code: '1' + + - name: Build runner images + run: | + PULSAR_IMAGE_TAG=3.2.2.6 PULSAR_IMAGE=streamnative/sn-platform KIND_PUSH=false images/build.sh + + - name: Run Trivy vulnerability scanner for java + id: scan-java-runner + uses: aquasecurity/trivy-action@master + with: + image-ref: 'pulsar-functions-java-runner:latest' + format: 'table' + exit-code: '0' + + - name: Run Trivy vulnerability scanner for python + id: scan-python-runner + uses: aquasecurity/trivy-action@master + with: + image-ref: 'pulsar-functions-python-runner:latest' + format: 'table' + exit-code: '0' + + - name: Run Trivy vulnerability scanner for go + id: scan-go-runner + uses: aquasecurity/trivy-action@master + with: + image-ref: 'pulsar-functions-go-runner:latest' + format: 'table' + exit-code: '0' + + - name: Run Trivy vulnerability scanner for java with pulsarctl + id: scan-java-pulsarctl-runner + uses: aquasecurity/trivy-action@master + with: + image-ref: 'pulsar-functions-pulsarctl-java-runner:latest' + format: 'table' + exit-code: '0' + + - name: Run Trivy vulnerability scanner for python with pulsarctl + id: scan-python-pulsarctl-runner + uses: aquasecurity/trivy-action@master + with: + image-ref: 'pulsar-functions-pulsarctl-python-runner:latest' + format: 'table' + exit-code: '0' + + - name: Run Trivy vulnerability scanner for go with pulsarctl + id: scan-go-pulsarctl-runner + uses: aquasecurity/trivy-action@master + with: + image-ref: 'pulsar-functions-pulsarctl-go-runner:latest' + format: 'table' + exit-code: '0' diff --git a/images/pulsar-functions-base-runner/Dockerfile b/images/pulsar-functions-base-runner/Dockerfile index 6b0c2a767..155f4b476 100644 --- a/images/pulsar-functions-base-runner/Dockerfile +++ b/images/pulsar-functions-base-runner/Dockerfile @@ -1,13 +1,13 @@ ARG PULSAR_IMAGE ARG PULSAR_IMAGE_TAG FROM ${PULSAR_IMAGE}:${PULSAR_IMAGE_TAG} as pulsar -FROM ubuntu:20.04 as functions-runner +FROM alpine:3.19 as functions-runner ENV GID=10001 ENV UID=10000 ENV USER=pulsar -RUN groupadd -g $GID pulsar -RUN adduser -u $UID --gid $GID --disabled-login --disabled-password --gecos '' $USER +RUN addgroup -g $GID pulsar +RUN adduser -u $UID -G pulsar -D -g '' $USER RUN mkdir -p /pulsar/bin/ \ && mkdir -p /pulsar/lib/ \ @@ -28,38 +28,36 @@ RUN echo "VERSION_TAG=${VERSION_TAG}" && \ VERSION_PATCH=$(echo $VERSION_TAG | cut -d. -f3) && \ if [ $VERSION_MAJOR -eq 2 ] && [ $VERSION_MINOR -eq 7 ]; then \ echo "Pulsar version is 2.7, use java 1.8" && \ - export JRE_PACKAGE_NAME=openjdk-8-jre-headless; \ + export JRE_PACKAGE_NAME=openjdk8; \ elif [ $VERSION_MAJOR -eq 2 ] && [ $VERSION_MINOR -eq 8 ]; then \ echo "Pulsar version is 2.8, use java 1.8" && \ - export JRE_PACKAGE_NAME=openjdk-8-jre-headless; \ + export JRE_PACKAGE_NAME=openjdk8; \ elif [ $VERSION_MAJOR -eq 2 ] && [ $VERSION_MINOR -eq 9 ]; then \ echo "Pulsar version is 2.9, use java 11" && \ - export JRE_PACKAGE_NAME=openjdk-11-jre-headless; \ + export JRE_PACKAGE_NAME=openjdk11; \ elif [ $VERSION_MAJOR -eq 2 ] && [ $VERSION_MINOR -eq 10 ]; then \ echo "Pulsar version is 2.10, use java 11" && \ - export JRE_PACKAGE_NAME=openjdk-11-jre-headless; \ + export JRE_PACKAGE_NAME=openjdk11; \ elif [ $VERSION_MAJOR -eq 2 ] && [ $VERSION_MINOR -eq 11 ]; then \ echo "Pulsar version is 2.11, use java 17" && \ - export JRE_PACKAGE_NAME=openjdk-17-jre-headless; \ + export JRE_PACKAGE_NAME=openjdk11; \ else \ echo "Pulsar version is not in the list, use java 17 instead" && \ - export JRE_PACKAGE_NAME=openjdk-17-jre-headless; \ + export JRE_PACKAGE_NAME=openjdk17; \ fi && \ - apt-get update \ - && apt-get -y dist-upgrade \ - && apt-get -y install $JRE_PACKAGE_NAME \ - && apt-get -y --purge autoremove \ - && apt-get autoclean \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* + apk update && apk add --no-cache $JRE_PACKAGE_NAME bash COPY --from=pulsar --chown=$UID:$GID /pulsar/conf /pulsar/conf COPY --from=pulsar --chown=$UID:$GID /pulsar/bin /pulsar/bin COPY --from=pulsar --chown=$UID:$GID /pulsar/lib /pulsar/lib +# remove the vertx jar since it's not need ans has a cve +RUN rm -rf /pulsar/lib/io.vertx-vertx-core-*.jar || true + # remove presto dependencies because they are not needed RUN rm -rf /pulsar/lib/presto || true RUN rm -rf /pulsar/conf/presto || true +RUN rm -rf /pulsar/bin/pulsar-detector || true ENV PULSAR_ROOT_LOGGER=INFO,CONSOLE ENV java.io.tmpdir=/pulsar/tmp/ diff --git a/images/pulsar-functions-base-runner/pulsarctl.Dockerfile b/images/pulsar-functions-base-runner/pulsarctl.Dockerfile index 3fc4e2c69..123cba403 100644 --- a/images/pulsar-functions-base-runner/pulsarctl.Dockerfile +++ b/images/pulsar-functions-base-runner/pulsarctl.Dockerfile @@ -1,13 +1,13 @@ ARG PULSAR_IMAGE ARG PULSAR_IMAGE_TAG FROM ${PULSAR_IMAGE}:${PULSAR_IMAGE_TAG} as pulsar -FROM ubuntu:20.04 as functions-runner +FROM alpine:3.19 as functions-runner ENV GID=10001 ENV UID=10000 ENV USER=pulsar -RUN groupadd -g $GID pulsar -RUN adduser -u $UID --gid $GID --disabled-login --disabled-password --gecos '' $USER +RUN addgroup -g $GID pulsar +RUN adduser -u $UID -G pulsar -D -g '' $USER RUN mkdir -p /pulsar/bin/ \ && mkdir -p /pulsar/lib/ \ @@ -19,13 +19,7 @@ RUN mkdir -p /pulsar/bin/ \ && mkdir -p /pulsar/examples/ \ && chown -R $UID:$GID /pulsar \ && chmod -R g=u /pulsar \ - && apt-get update \ - && apt-get -y dist-upgrade \ - && apt-get -y install wget \ - && apt-get -y --purge autoremove \ - && apt-get autoclean \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* \ + && apk update && apk add --no-cache wget bash \ && wget https://github.com/streamnative/pulsarctl/releases/latest/download/pulsarctl-amd64-linux.tar.gz -P /pulsar/bin/ \ && tar -xzf /pulsar/bin/pulsarctl-amd64-linux.tar.gz -C /pulsar/bin/ \ && rm -rf /pulsar/bin/pulsarctl-amd64-linux.tar.gz \ diff --git a/images/pulsar-functions-java-runner/pulsarctl.Dockerfile b/images/pulsar-functions-java-runner/pulsarctl.Dockerfile index f16a5dcb1..289694869 100644 --- a/images/pulsar-functions-java-runner/pulsarctl.Dockerfile +++ b/images/pulsar-functions-java-runner/pulsarctl.Dockerfile @@ -12,37 +12,33 @@ RUN echo "VERSION_TAG=${VERSION_TAG}" && \ VERSION_PATCH=$(echo $VERSION_TAG | cut -d. -f3) && \ if [ $VERSION_MAJOR -eq 2 ] && [ $VERSION_MINOR -eq 7 ]; then \ echo "Pulsar version is 2.7, use java 1.8" && \ - export JRE_PACKAGE_NAME=openjdk-8-jre-headless; \ + export JRE_PACKAGE_NAME=openjdk8; \ elif [ $VERSION_MAJOR -eq 2 ] && [ $VERSION_MINOR -eq 8 ]; then \ echo "Pulsar version is 2.8, use java 1.8" && \ - export JRE_PACKAGE_NAME=openjdk-8-jre-headless; \ + export JRE_PACKAGE_NAME=openjdk8; \ elif [ $VERSION_MAJOR -eq 2 ] && [ $VERSION_MINOR -eq 9 ]; then \ echo "Pulsar version is 2.9, use java 11" && \ - export JRE_PACKAGE_NAME=openjdk-11-jre-headless; \ + export JRE_PACKAGE_NAME=openjdk11; \ elif [ $VERSION_MAJOR -eq 2 ] && [ $VERSION_MINOR -eq 10 ]; then \ echo "Pulsar version is 2.10, use java 11" && \ - export JRE_PACKAGE_NAME=openjdk-11-jre-headless; \ + export JRE_PACKAGE_NAME=openjdk11; \ elif [ $VERSION_MAJOR -eq 2 ] && [ $VERSION_MINOR -eq 11 ]; then \ echo "Pulsar version is 2.11, use java 17" && \ - export JRE_PACKAGE_NAME=openjdk-17-jre-headless; \ + export JRE_PACKAGE_NAME=openjdk17; \ else \ echo "Pulsar version is not in the list, use java 17 instead" && \ - export JRE_PACKAGE_NAME=openjdk-17-jre-headless; \ + export JRE_PACKAGE_NAME=openjdk17; \ fi && \ - apt-get update \ - && apt-get -y dist-upgrade \ - && apt-get -y install $JRE_PACKAGE_NAME \ - && apt-get -y --purge autoremove \ - && apt-get autoclean \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* + apk update && apk add --no-cache $JRE_PACKAGE_NAME COPY --from=pulsar --chown=$UID:$GID /pulsar/conf /pulsar/conf -COPY --from=pulsar --chown=$UID:$GID /pulsar/bin /pulsar/bin COPY --from=pulsar --chown=$UID:$GID /pulsar/lib /pulsar/lib COPY --from=pulsar --chown=$UID:$GID /pulsar/instances/java-instance.jar /pulsar/instances/java-instance.jar COPY --from=pulsar --chown=$UID:$GID /pulsar/instances/deps /pulsar/instances/deps +# remove the vertx jar since it's not need ans has a cve +RUN rm -rf /pulsar/lib/io.vertx-vertx-core-*.jar || true + # remove presto dependencies because they are not needed RUN rm -rf /pulsar/lib/presto || true RUN rm -rf /pulsar/conf/presto || true diff --git a/images/pulsar-functions-python-runner/Dockerfile b/images/pulsar-functions-python-runner/Dockerfile index b40dd5e9f..13cb1e10a 100644 --- a/images/pulsar-functions-python-runner/Dockerfile +++ b/images/pulsar-functions-python-runner/Dockerfile @@ -11,6 +11,8 @@ COPY --from=pulsar --chown=$UID:$GID /pulsar/instances/deps /pulsar/instances/de # Below is a hacky way to copy /pulsar/pulsar-client if exist in pulsar image COPY --from=pulsar --chown=$UID:$GID /pulsar/README /pulsar/pulsar-clien* /pulsar/pulsar-client/ +ENV PULSAR_CLIENT_PYTHON_VERSION=3.5.0 + # Pulsar 2.8.0 removes /pulsar/cpp-client from docker image # But it required with Pulsar 2.7.X and below # to make this Dockerfile compalicate with different Pulsar versions @@ -19,27 +21,19 @@ COPY --from=pulsar --chown=$UID:$GID /pulsar/README /pulsar/cpp-clien* /tmp/puls RUN if [ -d "/tmp/pulsar/cpp-client" ]; then mv /tmp/pulsar/cpp-client /pulsar/cpp-client || true ; fi # Install some utilities -RUN apt-get update \ - && DEBIAN_FRONTEND=noninteractive apt-get install -y python3 python3-dev python3-setuptools python3-yaml python3-kazoo \ - libreadline-gplv2-dev libncursesw5-dev libssl-dev libsqlite3-dev tk-dev libgdbm-dev libc6-dev libbz2-dev \ - curl ca-certificates\ - && apt-get clean autoclean && apt-get autoremove --yes && rm -rf /var/lib/apt/lists/* +RUN apk update \ + && apk add --no-cache python3 python3-dev tk-dev curl ca-certificates\ + && mv /usr/lib/python3.11/EXTERNALLY-MANAGED /usr/lib/python3.11/EXTERNALLY-MANAGED.old RUN mkdir -p /etc/pki/tls/certs && cp /etc/ssl/certs/ca-certificates.crt /etc/pki/tls/certs/ca-bundle.crt RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py RUN python3 get-pip.py -RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10 - -RUN if [ -d "/pulsar/cpp-client" ]; then apt-get update \ - && apt install -y /pulsar/cpp-client/*.deb \ - && apt-get clean autoclean && apt-get autoremove --yes && rm -rf /var/lib/apt/lists/* ; fi - WORKDIR /pulsar RUN if [ -f "/pulsar/bin/install-pulsar-client-37.sh" ]; then /pulsar/bin/install-pulsar-client-37.sh || true ; fi -RUN if [ -f "/pulsar/bin/install-pulsar-client.sh" ]; then /pulsar/bin/install-pulsar-client.sh || pip3 install 'pulsar-client[all]==3.1.0' ; fi +RUN if [ -f "/pulsar/bin/install-pulsar-client.sh" ]; then /pulsar/bin/install-pulsar-client.sh || pip3 install 'pulsar-client[all]==3.5.0' ; fi # this dir is duplicate with the installed pulsar-client pip package, and maybe not compatible with the `_pulsar`(the .so library package) RUN rm -rf /pulsar/instances/python-instance/pulsar/ \ @@ -49,6 +43,6 @@ RUN rm -rf /pulsar/instances/python-instance/pulsar/ \ USER $USER # a temp solution from https://github.com/apache/pulsar/pull/15846 to fix python protobuf version error -RUN pip3 install protobuf==3.20.1 --user +RUN pip3 install protobuf==3.20.2 --user # to make the python runner could print json logs RUN pip3 install python-json-logger --user diff --git a/images/pulsar-functions-python-runner/pulsarctl.Dockerfile b/images/pulsar-functions-python-runner/pulsarctl.Dockerfile index 5e39acd80..649541264 100644 --- a/images/pulsar-functions-python-runner/pulsarctl.Dockerfile +++ b/images/pulsar-functions-python-runner/pulsarctl.Dockerfile @@ -11,28 +11,23 @@ COPY --from=pulsar --chown=$UID:$GID /pulsar/instances/python-instance /pulsar/i # Below is a hacky way to copy /pulsar/pulsar-client if exist in pulsar image COPY --from=pulsar --chown=$UID:$GID /pulsar/README /pulsar/pulsar-clien* /pulsar/pulsar-client/ +ENV PULSAR_CLIENT_PYTHON_VERSION=3.5.0 + # Pulsar 2.8.0 removes /pulsar/cpp-client from docker image # But it required with Pulsar 2.7.X and below # to make this Dockerfile compalicate with different Pulsar versions # Below is a hacky way to copy /pulsar/cpp-client if exist in pulsar image COPY --from=pulsar --chown=$UID:$GID /pulsar/README /pulsar/cpp-clien* /pulsar/cpp-client/ -RUN apt-get update \ - && DEBIAN_FRONTEND=noninteractive apt-get install -y python3 python3-dev python3-setuptools python3-yaml python3-kazoo \ - libreadline-gplv2-dev libncursesw5-dev libssl-dev libsqlite3-dev tk-dev libgdbm-dev libc6-dev libbz2-dev \ - curl ca-certificates\ - && apt-get clean autoclean && apt-get autoremove --yes && rm -rf /var/lib/apt/lists/* \ +RUN apk update \ + && apk add --no-cache python3 python3-dev tk-dev curl ca-certificates\ && mkdir -p /etc/pki/tls/certs && cp /etc/ssl/certs/ca-certificates.crt /etc/pki/tls/certs/ca-bundle.crt \ - && update-alternatives --install /usr/bin/python python /usr/bin/python3 10 \ && curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py \ + && mv /usr/lib/python3.11/EXTERNALLY-MANAGED /usr/lib/python3.11/EXTERNALLY-MANAGED.old \ && python3 get-pip.py && pip3 install --upgrade pip -RUN if [ -d "/pulsar/cpp-client" ]; then apt-get update \ - && apt install -y /pulsar/cpp-client/*.deb || true \ - && apt-get clean autoclean && apt-get autoremove --yes && rm -rf /var/lib/apt/lists/* ; fi - -RUN if [ -f "/pulsar/bin/install-pulsar-client-37.sh" ]; then /pulsar/bin/install-pulsar-client-37.sh || pip3 install 'pulsar-client[all]==3.1.0' ; fi -RUN if [ -f "/pulsar/bin/install-pulsar-client.sh" ]; then /pulsar/bin/install-pulsar-client.sh || pip3 install 'pulsar-client[all]==3.1.0' ; fi +RUN if [ -f "/pulsar/bin/install-pulsar-client-37.sh" ]; then /pulsar/bin/install-pulsar-client-37.sh || pip3 install 'pulsar-client[all]==3.5.0' ; fi +RUN if [ -f "/pulsar/bin/install-pulsar-client.sh" ]; then /pulsar/bin/install-pulsar-client.sh || pip3 install 'pulsar-client[all]==3.5.0' ; fi # this dir is duplicate with the installed pulsar-client pip package, and maybe not compatible with the `_pulsar`(the .so library package) RUN rm -rf /pulsar/instances/python-instance/pulsar/ \ @@ -43,6 +38,6 @@ WORKDIR /pulsar USER $USER # a temp solution from https://github.com/apache/pulsar/pull/15846 to fix python protobuf version error -RUN pip3 install protobuf==3.20.1 --user +RUN pip3 install protobuf==3.20.2 --user # to make the python runner could print json logs -RUN pip3 install python-json-logger --user \ No newline at end of file +RUN pip3 install python-json-logger --user