diff --git a/.github/workflows/api-test.yml b/.github/workflows/api-test.yml index e9923fd59c7c..6c6c2a85f610 100644 --- a/.github/workflows/api-test.yml +++ b/.github/workflows/api-test.yml @@ -83,7 +83,7 @@ jobs: -Dmaven.test.skip=true \ -Dmaven.javadoc.skip=true \ -Dspotless.skip=true \ - -Pdocker,release -Ddocker.tag=ci + -Pdocker,staging -Ddocker.tag=ci - name: Export Docker Images run: | docker save apache/dolphinscheduler-standalone-server:ci -o /tmp/standalone-image.tar \ @@ -91,7 +91,7 @@ jobs: - uses: actions/upload-artifact@v4 name: Upload Docker Images with: - name: standalone-image + name: standalone-image-api-test path: /tmp/standalone-image.tar retention-days: 1 api-test: @@ -140,7 +140,7 @@ jobs: - uses: actions/download-artifact@v4 name: Download Docker Images with: - name: standalone-image + name: standalone-image-api-test path: /tmp - name: Load Docker Images run: | diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index 268c313fe7ca..389dcf4f323a 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -89,7 +89,7 @@ jobs: - name: Build and Package on ${{ matrix.java }} run: | ./mvnw -B clean install \ - -Prelease \ + -Pstaging \ -Dmaven.test.skip=true \ -Dspotless.skip=true - name: Check dependency license diff --git a/.github/workflows/e2e-k8s.yml b/.github/workflows/e2e-k8s.yml index e5ad17ff86b0..d78d785773a1 100644 --- a/.github/workflows/e2e-k8s.yml +++ b/.github/workflows/e2e-k8s.yml @@ -73,7 +73,7 @@ jobs: -Dmaven.test.skip=true \ -Dmaven.javadoc.skip=true \ -Dspotless.skip=true \ - -Pdocker,release -Ddocker.tag=ci + -Pdocker,staging -Ddocker.tag=ci - name: Create k8s Kind Cluster run: | # install kubectl diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 8dc83efdc9f8..c88899a4dc69 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -85,7 +85,7 @@ jobs: -Dmaven.test.skip=true \ -Dmaven.javadoc.skip=true \ -Dspotless.skip=true \ - -Pdocker,release -Ddocker.tag=ci + -Pdocker,staging -Ddocker.tag=ci - name: Export Docker Images run: | docker save apache/dolphinscheduler-standalone-server:ci -o /tmp/standalone-image.tar \ @@ -93,7 +93,7 @@ jobs: - uses: actions/upload-artifact@v4 name: Upload Docker Images with: - name: standalone-image + name: standalone-image-e2e path: /tmp/standalone-image.tar retention-days: 1 e2e: @@ -168,7 +168,7 @@ jobs: - uses: actions/download-artifact@v4 name: Download Docker Images with: - name: standalone-image + name: standalone-image-e2e path: /tmp - name: Load Docker Images run: | diff --git a/.github/workflows/publish-nexus.yaml b/.github/workflows/publish-nexus.yaml new file mode 100644 index 000000000000..e430bb036f2d --- /dev/null +++ b/.github/workflows/publish-nexus.yaml @@ -0,0 +1,75 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: Publish Snapshot + +on: + release: + types: + - released + +jobs: + publish-snapshot: + if: github.repository == 'apache/dolphinscheduler' + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + submodules: true + - name: Maximize runner space + uses: ./.github/actions/maximize-build-space + with: + root-reserve-mb: 30720 + temp-reserve-mb: 10240 + remove-dotnet: 'true' + remove-android: 'true' + remove-haskell: 'true' + remove-codeql: 'true' + remove-docker-images: 'true' + - uses: actions/checkout@v4 + - name: Cache local Maven repository + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-backend + restore-keys: ${{ runner.os }}-maven- + - name: Setup JDK 8 + uses: actions/setup-java@v4 + with: + distribution: 'adopt' + java-version: 8 + - name: Set environment variables + run: | + if [[ ${{ github.event_name }} == "schedule" ]]; then + echo "SETTINGS_PATH=./.github/workflows/publish-nexus/snapshot-asf-settings.xml" >> $SETTINGS_PATH + elif [[ ${{ github.event_name }} == "release" ]]; then + echo "SETTINGS_PATH=./.github/workflows/publish-nexus/release-asf-settings.xml" >> $SETTINGS_PATH + else + echo "unknown event name: ${{ github.event_name }}" + exit 2 + fi + - name: Publish snapshot with dev branch + env: + ASF_USERNAME: ${{ secrets.NEXUS_USER }} + ASF_PASSWORD: ${{ secrets.NEXUS_PW }} + run: | + ./mvnw clean deploy \ + -s ${{ env.SETTINGS_PATH }} \ + -Dmaven.test.skip=true \ + -Dspotless.skip=true \ + -Pstaging diff --git a/.github/workflows/publish-nexus/release-asf-settings.xml b/.github/workflows/publish-nexus/release-asf-settings.xml new file mode 100644 index 000000000000..b6388f7e695a --- /dev/null +++ b/.github/workflows/publish-nexus/release-asf-settings.xml @@ -0,0 +1,34 @@ + + + + + + + apache.snapshots.https + ${env.ASF_USERNAME} + ${env.ASF_PASSWORD} + + + apache.releases.https + ${env.ASF_USERNAME} + ${env.ASF_PASSWORD} + + + diff --git a/.github/workflows/publish-nexus/snapshot-asf-settings.xml b/.github/workflows/publish-nexus/snapshot-asf-settings.xml new file mode 100644 index 000000000000..6529e98f0670 --- /dev/null +++ b/.github/workflows/publish-nexus/snapshot-asf-settings.xml @@ -0,0 +1,29 @@ + + + + + + + apache.snapshots.https + ${env.ASF_USERNAME} + ${env.ASF_PASSWORD} + + + diff --git a/.licenserc.yaml b/.licenserc.yaml index 1f0b43e502bb..6697c2cb73d8 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -48,6 +48,7 @@ header: - 'org.mockito.plugins.MockMaker' - tools/dependencies/known-dependencies.txt - '**/banner.txt' + - '**/*-banner.txt' - '.terraform.lock.hcl' - deploy/kubernetes/dolphinscheduler/README.md.gotmpl - .idea/vcs.xml diff --git a/config/plugins_config b/config/plugins_config new file mode 100644 index 000000000000..6fac612b01c0 --- /dev/null +++ b/config/plugins_config @@ -0,0 +1,110 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This mapping is used to resolve the Jar package name without version (or call artifactId) +# +# corresponding to the module in the user Config, helping Dolphinscheduler to load the correct Jar package. +# Don't modify the delimiter " -- ", just select the plugin you need + +--alert-plugins-- +dolphinscheduler-alert-aliyunVoice +dolphinscheduler-alert-dingtalk +dolphinscheduler-alert-email +dolphinscheduler-alert-feishu +dolphinscheduler-alert-http +dolphinscheduler-alert-pagerduty +dolphinscheduler-alert-prometheus +dolphinscheduler-alert-script +dolphinscheduler-alert-slack +dolphinscheduler-alert-telegram +dolphinscheduler-alert-webexteams +dolphinscheduler-alert-wechat +--end-- + +--datasource-plugins-- +dolphinscheduler-datasource-aliyunserverlessspark +dolphinscheduler-datasource-athena +dolphinscheduler-datasource-azure-sql +dolphinscheduler-datasource-clickhouse +dolphinscheduler-datasource-dameng +dolphinscheduler-datasource-databend +dolphinscheduler-datasource-db2 +dolphinscheduler-datasource-doris +dolphinscheduler-datasource-hana +dolphinscheduler-datasource-hive +dolphinscheduler-datasource-k8s +dolphinscheduler-datasource-kyuubi +dolphinscheduler-datasource-mysql +dolphinscheduler-datasource-oceanbase +dolphinscheduler-datasource-oracle +dolphinscheduler-datasource-postgresql +dolphinscheduler-datasource-presto +dolphinscheduler-datasource-redshift +dolphinscheduler-datasource-sagemaker +dolphinscheduler-datasource-snowflake +dolphinscheduler-datasource-spark +dolphinscheduler-datasource-sqlserver +dolphinscheduler-datasource-ssh +dolphinscheduler-datasource-starrocks +dolphinscheduler-datasource-trino +dolphinscheduler-datasource-vertica +dolphinscheduler-datasource-zeppelin +--end-- + +--storage-plugins-- +dolphinscheduler-storage-abs +dolphinscheduler-storage-gcs +dolphinscheduler-storage-hdfs +dolphinscheduler-storage-obs +dolphinscheduler-storage-oss +dolphinscheduler-storage-s3 +--end-- + +--task-plugins-- +dolphinscheduler-task-aliyunserverlessspark +dolphinscheduler-task-chunjun +dolphinscheduler-task-datafactory +dolphinscheduler-task-dataquality +dolphinscheduler-task-datasync +dolphinscheduler-task-datax +dolphinscheduler-task-dinky +dolphinscheduler-task-dms +dolphinscheduler-task-dvc +dolphinscheduler-task-emr +dolphinscheduler-task-flink +dolphinscheduler-task-flink-stream +dolphinscheduler-task-hivecli +dolphinscheduler-task-http +dolphinscheduler-task-java +dolphinscheduler-task-jupyter +dolphinscheduler-task-k8s +dolphinscheduler-task-kubeflow +dolphinscheduler-task-linkis +dolphinscheduler-task-mlflow +dolphinscheduler-task-mr +dolphinscheduler-task-openmldb +dolphinscheduler-task-procedure +dolphinscheduler-task-python +dolphinscheduler-task-pytorch +dolphinscheduler-task-remoteshell +dolphinscheduler-task-sagemaker +dolphinscheduler-task-seatunnel +dolphinscheduler-task-shell +dolphinscheduler-task-spark +dolphinscheduler-task-sql +dolphinscheduler-task-sqoop +dolphinscheduler-task-zeppelin +--end-- diff --git a/docs/docs/en/guide/installation/pseudo-cluster.md b/docs/docs/en/guide/installation/pseudo-cluster.md index 6561759a9f8b..851b0501a1c7 100644 --- a/docs/docs/en/guide/installation/pseudo-cluster.md +++ b/docs/docs/en/guide/installation/pseudo-cluster.md @@ -16,10 +16,27 @@ Pseudo-cluster deployment of DolphinScheduler requires external software support - `pstree` for macOS - `psmisc` for Fedora/Red/Hat/CentOS/Ubuntu/Debian -> **_Note:_** DolphinScheduler itself does not depend on Hadoop, Hive, Spark, but if you need to run tasks that depend on them, you need to have the corresponding environment support. +## Download Plugins Dependencies + +Starting from version 3.3.0, the binary package no longer provides plugin dependencies, and users need to download them by themselves. The plugin dependency package download address: [Plugin Dependency Package](https://repo.maven.apache.org/maven2/org/apache/dolphinscheduler) +You can also execute the following command to install plugin dependencies: + +```shell +bash ./bin/install-plugins.sh 3.3.0 +``` + +Usually, you do not need all connector plugins, you can specify the plugins you need by configuring `conf/plugins_config`. For example, if you only need the `dolphinscheduler-task-shell` plugin, you can modify the configuration file as follows: + +``` +--task-plugins-- +dolphinscheduler-task-shell +--end-- +``` ## DolphinScheduler Startup Environment +> **_Note:_** DolphinScheduler itself does not depend on Hadoop, Hive, Spark, but if you need to run tasks that depend on them, you need to have the corresponding environment support. + ### Configure User Exemption and Permissions Create a deployment user, and make sure to configure `sudo` without password. Here make an example to create user `dolphinscheduler`: diff --git a/docs/docs/en/guide/installation/standalone.md b/docs/docs/en/guide/installation/standalone.md index bf47c3b9a59e..646c1d48b0ab 100644 --- a/docs/docs/en/guide/installation/standalone.md +++ b/docs/docs/en/guide/installation/standalone.md @@ -14,6 +14,10 @@ If you want to deploy DolphinScheduler in production, we recommend you follow [c - JDK：download [JDK][jdk] (1.8 or 11), install and configure environment variable `JAVA_HOME` and append `bin` dir (included in `JAVA_HOME`) to `PATH` variable. You can skip this step if it already exists in your environment. - Binary package: download the DolphinScheduler binary package at [download page](https://dolphinscheduler.apache.org/en-us/download/). +## Download Plugin Dependencies + +Please refer to the [Download Plugin Dependencies](../installation/pseudo-cluster.md) in pseudo-cluster deployment. + ### Configure User Exemption and Permissions Create a deployment user, and make sure to configure `sudo` without password. Here make an example to create user `dolphinscheduler`: diff --git a/docs/docs/en/guide/upgrade/incompatible.md b/docs/docs/en/guide/upgrade/incompatible.md index 62852e24630e..c0549badd208 100644 --- a/docs/docs/en/guide/upgrade/incompatible.md +++ b/docs/docs/en/guide/upgrade/incompatible.md @@ -28,8 +28,8 @@ This document records the incompatible updates between each version. You need to ## 3.3.0 -* Remove the `udf-manage` function from the `resource center` ([#16209]) -* Remove the `Pigeon` from the `Task Plugin` ([#16218]) -* Uniformly name `process` in code as `workflow` ([#16515]) -* Deprecated upgrade code of 1.x and 2.x in 3.3.0-release ([#16543]) +* Remove the `udf-manage` function from the `resource center` ([#16209])(https://github.com/apache/dolphinscheduler/pull/16209) +* Remove the `Pigeon` from the `Task Plugin` ([#16218])(https://github.com/apache/dolphinscheduler/pull/16218) +* Uniformly name `process` in code as `workflow` ([#16515])(https://github.com/apache/dolphinscheduler/pull/16515) +* Deprecated upgrade code of 1.x and 2.x in 3.3.0-release ([#16543])(https://github.com/apache/dolphinscheduler/pull/16543) diff --git a/docs/docs/en/guide/upgrade/upgrade.md b/docs/docs/en/guide/upgrade/upgrade.md index 074b1ca442ed..dfb3016a07d6 100644 --- a/docs/docs/en/guide/upgrade/upgrade.md +++ b/docs/docs/en/guide/upgrade/upgrade.md @@ -72,7 +72,8 @@ Execution result: #### Upgrade version restriction -After version 3.3.X and later, we only support upgrading from 3.0.0. For versions lower than this, please download the historical version and upgrade to 3.0.0. +- After version 3.3.X and later, we only support upgrading from 3.0.0. For versions lower than this, please download the historical version and upgrade to 3.0.0. +- After version 3.3.X and later, binary packages no longer provide plugins dependencies by default, so when you use them for the first time, you need to download and install them yourself. For more information, please refer to [Pseudo-Cluster](../installation/pseudo-cluster.md). #### Precautions after the upgrade diff --git a/docs/docs/zh/guide/installation/pseudo-cluster.md b/docs/docs/zh/guide/installation/pseudo-cluster.md index 5c910ee759e5..0449951d4d97 100644 --- a/docs/docs/zh/guide/installation/pseudo-cluster.md +++ b/docs/docs/zh/guide/installation/pseudo-cluster.md @@ -16,10 +16,27 @@ - macOS 安装`pstree` - Fedora/Red/Hat/CentOS/Ubuntu/Debian 安装`psmisc` -> **_注意:_** DolphinScheduler 本身不依赖 Hadoop、Hive、Spark，但如果你运行的任务需要依赖他们，就需要有对应的环境支持 +## 下载插件依赖 + +从 3.3.0 版本开始，二进制包不再提供插件依赖，需要用户自行下载。插件依赖包下载地址：[插件依赖包](https://repo.maven.apache.org/maven2/org/apache/dolphinscheduler) +你也可以执行以下命令来安装插件依赖: + +```shell +bash ./bin/install-plugins.sh 3.3.0 +``` + +通常你并不需要所有的连接器插件，可以通过配置 `conf/plugins_config` 来指定你所需要的插件，例如，你只需要 `dolphinscheduler-task-shell` 插件，那么您可以修改配置文件如下： + +``` +--task-plugins-- +dolphinscheduler-task-shell +--end-- +``` ## 准备 DolphinScheduler 启动环境 +> **_注意:_** DolphinScheduler 本身不依赖 Hadoop、Hive、Spark，但如果你运行的任务需要依赖他们，就需要有对应的环境支持 + ### 配置用户免密及权限 创建部署用户，并且一定要配置 `sudo` 免密。以创建 dolphinscheduler 用户为例 diff --git a/docs/docs/zh/guide/installation/standalone.md b/docs/docs/zh/guide/installation/standalone.md index d669db472757..bc280d9ed746 100644 --- a/docs/docs/zh/guide/installation/standalone.md +++ b/docs/docs/zh/guide/installation/standalone.md @@ -12,6 +12,10 @@ Standalone 仅适用于 DolphinScheduler 的快速体验. - JDK：下载[JDK][jdk] (1.8 or 11)，安装并配置 `JAVA_HOME` 环境变量，并将其下的 `bin` 目录追加到 `PATH` 环境变量中。如果你的环境中已存在，可以跳过这步。 - 二进制包：在[下载页面](https://dolphinscheduler.apache.org/en-us/download/)下载 DolphinScheduler 二进制包 +## 下载插件依赖 + +请参考伪集群部署的[下载插件依赖](../installation/pseudo-cluster.md#下载插件依赖) + ## 配置用户免密及权限 创建部署用户，并且一定要配置 `sudo` 免密。以创建 dolphinscheduler 用户为例 diff --git a/docs/docs/zh/guide/upgrade/incompatible.md b/docs/docs/zh/guide/upgrade/incompatible.md index 9dd689f0ae8d..098992294dbf 100644 --- a/docs/docs/zh/guide/upgrade/incompatible.md +++ b/docs/docs/zh/guide/upgrade/incompatible.md @@ -26,8 +26,8 @@ ## 3.3.0 -* 从 `资源中心` 中移除了 `udf-manage` 功能 ([#16209]) -* 从 `任务插件` 中移除了 `Pigeon` 类型 ([#16218]) -* 统一代码中的 `process` 为 `workflow` ([#16515]) -* 在 3.3.0-release 中废弃了从 1.x 至 2.x 的升级代码 ([#16543]) +* 从 `资源中心` 中移除了 `udf-manage` 功能 ([#16209])(https://github.com/apache/dolphinscheduler/pull/16209) +* 从 `任务插件` 中移除了 `Pigeon` 类型 ([#16218])(https://github.com/apache/dolphinscheduler/pull/16218) +* 统一代码中的 `process` 为 `workflow` ([#16515])(https://github.com/apache/dolphinscheduler/pull/16515) +* 在 3.3.0-release 中废弃了从 1.x 至 2.x 的升级代码 ([#16543])(https://github.com/apache/dolphinscheduler/pull/16543) diff --git a/docs/docs/zh/guide/upgrade/upgrade.md b/docs/docs/zh/guide/upgrade/upgrade.md index c2cec6783af3..19ccad73cab2 100644 --- a/docs/docs/zh/guide/upgrade/upgrade.md +++ b/docs/docs/zh/guide/upgrade/upgrade.md @@ -73,7 +73,8 @@ jar 包 并添加到 `./tools/libs` 目录下，设置以下环境变量 #### 升级版本限制 -在 3.3.X 以及之后的版本，我们仅支持从 3.0.0 开始进行升级，低于此版本的请下载历史版本升级至 3.0.0。 +- 在 3.3.X 以及之后的版本，我们仅支持从 3.0.0 开始进行升级，低于此版本的请下载历史版本升级至 3.0.0。 +- 在 3.3.X 以及之后的版本，二进制包不再默认提供插件依赖，因此第一次使用时，需要自行下载安装。具体请参考请参照[伪集群部署(Pseudo-Cluster)](../installation/pseudo-cluster.md) #### 升级后的注意事项 diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/bin/start.sh b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/bin/start.sh index 56a493d9b7ed..5d34faee252c 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/bin/start.sh +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/bin/start.sh @@ -58,7 +58,9 @@ alert-plugins ) for plugin in ${PLUGINS_PATH[@]}; do - CP=$CP:"$DOLPHINSCHEDULER_HOME/plugins/$plugin/*" + if [ -d "$DOLPHINSCHEDULER_HOME/plugins/$plugin" ]; then + CP=$CP:"$DOLPHINSCHEDULER_HOME/plugins/$plugin/*" + fi done diff --git a/dolphinscheduler-api/src/main/bin/start.sh b/dolphinscheduler-api/src/main/bin/start.sh index eb60455c5350..e5d2a5b620eb 100644 --- a/dolphinscheduler-api/src/main/bin/start.sh +++ b/dolphinscheduler-api/src/main/bin/start.sh @@ -60,7 +60,9 @@ task-plugins ) for plugin in ${PLUGINS_PATH[@]}; do - CP=$CP:"$DOLPHINSCHEDULER_HOME/plugins/$plugin/*" + if [ -d "$DOLPHINSCHEDULER_HOME/plugins/$plugin" ]; then + CP=$CP:"$DOLPHINSCHEDULER_HOME/plugins/$plugin/*" + fi done $JAVA_HOME/bin/java $JAVA_OPTS \ diff --git a/dolphinscheduler-dist/pom.xml b/dolphinscheduler-dist/pom.xml index 046b5335183f..c9b5814456da 100644 --- a/dolphinscheduler-dist/pom.xml +++ b/dolphinscheduler-dist/pom.xml @@ -27,6 +27,10 @@ dolphinscheduler-dist ${project.artifactId} + + ${build.assembly.skip} + + @@ -73,71 +77,70 @@ apache-dolphinscheduler-${project.version} - + + + maven-assembly-plugin + + + dolphinscheduler-bin + + single + + package - - - release - - - - maven-assembly-plugin - - - dolphinscheduler-bin - - single - - package + + + src/main/assembly/dolphinscheduler-bin.xml + + true + + - - - src/main/assembly/dolphinscheduler-bin.xml - - true - - + + src + + single + + package + + + src/main/assembly/dolphinscheduler-src.xml + + true + + - - src - - single - - package - - - src/main/assembly/dolphinscheduler-src.xml - - true - - + + + + org.codehaus.mojo + exec-maven-plugin + ${exec-maven-plugin.version} + + ${build.assembly.skip} + + + + assembly-plugins + + exec + + package + + bash + ${project.basedir} + + src/main/assembly/assembly-plugins.sh + ${build.plugins.skip} + + + + + + + - - - - org.codehaus.mojo - exec-maven-plugin - ${exec-maven-plugin.version} - - - assembly-plugins - - exec - - package - - bash - ${project.basedir} - - src/main/assembly/assembly-plugins.sh - - - - - - - - + docker diff --git a/dolphinscheduler-dist/src/main/assembly/assembly-plugins.sh b/dolphinscheduler-dist/src/main/assembly/assembly-plugins.sh index 36cb430df2bb..80f7dd686038 100755 --- a/dolphinscheduler-dist/src/main/assembly/assembly-plugins.sh +++ b/dolphinscheduler-dist/src/main/assembly/assembly-plugins.sh @@ -17,6 +17,8 @@ # set -xeo pipefail +PLUGINS_ASSEMBLY_SKIP=$1 + DIST_DIR="$(pwd)/target" BIN_TAR_FILE="$DIST_DIR/apache-dolphinscheduler-*-bin.tar.gz" if [ ! -f $BIN_TAR_FILE ]; then @@ -36,12 +38,16 @@ storage-plugins task-plugins ) -for plugin_path in ${PLUGINS_PATH[@]} -do - cd $BIN_DIR/plugins/$plugin_path - find ./* -name "*.jar" | xargs -I {} mv {} ./ - ls -d */ | xargs -I {} rm -rf {} -done +if [ $PLUGINS_ASSEMBLY_SKIP == "true" ]; then + rm -rf $BIN_DIR/plugins/* +else + for plugin_path in ${PLUGINS_PATH[@]} + do + cd $BIN_DIR/plugins/$plugin_path + find ./* -name "*.jar" | xargs -I {} mv {} ./ + ls -d */ | xargs -I {} rm -rf {} + done +fi # move *-server/libs/*.jar to libs/ and create symbolic link in *-server/libs/ MODULES_PATH=( diff --git a/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-bin.xml b/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-bin.xml index 72018cfb9ef8..1dba3816300d 100644 --- a/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-bin.xml +++ b/dolphinscheduler-dist/src/main/assembly/dolphinscheduler-bin.xml @@ -107,6 +107,21 @@ 0755 + + ${basedir}/../config + conf + + + + ${basedir}/.././ + + mvnw + mvnw.cmd + + 0755 + . + + ${basedir}/release-docs . diff --git a/dolphinscheduler-dist/src/main/docker/alert-server.dockerfile b/dolphinscheduler-dist/src/main/docker/alert-server.dockerfile index cc8d0646c4dc..26e39c544759 100644 --- a/dolphinscheduler-dist/src/main/docker/alert-server.dockerfile +++ b/dolphinscheduler-dist/src/main/docker/alert-server.dockerfile @@ -27,7 +27,11 @@ RUN apt update ; \ WORKDIR $DOLPHINSCHEDULER_HOME -ADD ./target/apache-dolphinscheduler-*-bin $DOLPHINSCHEDULER_HOME +COPY ./target/apache-dolphinscheduler-*-bin.tar.gz $DOLPHINSCHEDULER_HOME +RUN tar -zxvf apache-dolphinscheduler-*-bin.tar.gz ; \ + rm -rf apache-dolphinscheduler-*-bin.tar.gz ; \ + mv apache-dolphinscheduler-*-bin/* . ; \ + rm -rf apache-dolphinscheduler-*-bin EXPOSE 12345 25333 diff --git a/dolphinscheduler-dist/src/main/docker/api-server.dockerfile b/dolphinscheduler-dist/src/main/docker/api-server.dockerfile index a0201810d2e0..941a46e79a74 100644 --- a/dolphinscheduler-dist/src/main/docker/api-server.dockerfile +++ b/dolphinscheduler-dist/src/main/docker/api-server.dockerfile @@ -27,7 +27,11 @@ RUN apt update ; \ WORKDIR $DOLPHINSCHEDULER_HOME -ADD ./target/apache-dolphinscheduler-*-bin $DOLPHINSCHEDULER_HOME +COPY ./target/apache-dolphinscheduler-*-bin.tar.gz $DOLPHINSCHEDULER_HOME +RUN tar -zxvf apache-dolphinscheduler-*-bin.tar.gz ; \ + rm -rf apache-dolphinscheduler-*-bin.tar.gz ; \ + mv apache-dolphinscheduler-*-bin/* . ; \ + rm -rf apache-dolphinscheduler-*-bin EXPOSE 12345 25333 diff --git a/dolphinscheduler-dist/src/main/docker/master-server.dockerfile b/dolphinscheduler-dist/src/main/docker/master-server.dockerfile index fbcc8bf62112..a89b6348b034 100644 --- a/dolphinscheduler-dist/src/main/docker/master-server.dockerfile +++ b/dolphinscheduler-dist/src/main/docker/master-server.dockerfile @@ -27,7 +27,11 @@ RUN apt update ; \ WORKDIR $DOLPHINSCHEDULER_HOME -ADD ./target/apache-dolphinscheduler-*-bin $DOLPHINSCHEDULER_HOME +COPY ./target/apache-dolphinscheduler-*-bin.tar.gz $DOLPHINSCHEDULER_HOME +RUN tar -zxvf apache-dolphinscheduler-*-bin.tar.gz ; \ + rm -rf apache-dolphinscheduler-*-bin.tar.gz ; \ + mv apache-dolphinscheduler-*-bin/* . ; \ + rm -rf apache-dolphinscheduler-*-bin EXPOSE 12345 25333 diff --git a/dolphinscheduler-dist/src/main/docker/standalone-server.dockerfile b/dolphinscheduler-dist/src/main/docker/standalone-server.dockerfile index b0cad0632a31..0abd547ea210 100644 --- a/dolphinscheduler-dist/src/main/docker/standalone-server.dockerfile +++ b/dolphinscheduler-dist/src/main/docker/standalone-server.dockerfile @@ -27,7 +27,11 @@ RUN apt update ; \ WORKDIR $DOLPHINSCHEDULER_HOME -ADD ./target/apache-dolphinscheduler-*-bin $DOLPHINSCHEDULER_HOME +COPY ./target/apache-dolphinscheduler-*-bin.tar.gz $DOLPHINSCHEDULER_HOME +RUN tar -zxvf apache-dolphinscheduler-*-bin.tar.gz ; \ + rm -rf apache-dolphinscheduler-*-bin.tar.gz ; \ + mv apache-dolphinscheduler-*-bin/* . ; \ + rm -rf apache-dolphinscheduler-*-bin EXPOSE 12345 25333 diff --git a/dolphinscheduler-dist/src/main/docker/tools.dockerfile b/dolphinscheduler-dist/src/main/docker/tools.dockerfile index c795a2b5074a..c50d91bc92c4 100644 --- a/dolphinscheduler-dist/src/main/docker/tools.dockerfile +++ b/dolphinscheduler-dist/src/main/docker/tools.dockerfile @@ -28,6 +28,10 @@ RUN apt update ; \ WORKDIR $DOLPHINSCHEDULER_HOME # see doc: https://dolphinscheduler.apache.org/en-us/docs/dev/user_doc/guide/upgrade.html -ADD ./target/apache-dolphinscheduler-*-bin $DOLPHINSCHEDULER_HOME +COPY ./target/apache-dolphinscheduler-*-bin.tar.gz $DOLPHINSCHEDULER_HOME +RUN tar -zxvf apache-dolphinscheduler-*-bin.tar.gz ; \ + rm -rf apache-dolphinscheduler-*-bin.tar.gz ; \ + mv apache-dolphinscheduler-*-bin/* . ; \ + rm -rf apache-dolphinscheduler-*-bin ENTRYPOINT [ "/bin/bash" ] diff --git a/dolphinscheduler-dist/src/main/docker/worker-server.dockerfile b/dolphinscheduler-dist/src/main/docker/worker-server.dockerfile index c8ce0b61c98c..4f4dfdb7cc55 100644 --- a/dolphinscheduler-dist/src/main/docker/worker-server.dockerfile +++ b/dolphinscheduler-dist/src/main/docker/worker-server.dockerfile @@ -27,7 +27,11 @@ RUN apt update ; \ WORKDIR $DOLPHINSCHEDULER_HOME -ADD ./target/apache-dolphinscheduler-*-bin $DOLPHINSCHEDULER_HOME +COPY ./target/apache-dolphinscheduler-*-bin.tar.gz $DOLPHINSCHEDULER_HOME +RUN tar -zxvf apache-dolphinscheduler-*-bin.tar.gz ; \ + rm -rf apache-dolphinscheduler-*-bin.tar.gz ; \ + mv apache-dolphinscheduler-*-bin/* . ; \ + rm -rf apache-dolphinscheduler-*-bin EXPOSE 12345 25333 diff --git a/dolphinscheduler-master/src/main/bin/start.sh b/dolphinscheduler-master/src/main/bin/start.sh index 74b7ba1112e1..de933f464ac9 100644 --- a/dolphinscheduler-master/src/main/bin/start.sh +++ b/dolphinscheduler-master/src/main/bin/start.sh @@ -60,7 +60,9 @@ task-plugins ) for plugin in ${PLUGINS_PATH[@]}; do - CP=$CP:"$DOLPHINSCHEDULER_HOME/plugins/$plugin/*" + if [ -d "$DOLPHINSCHEDULER_HOME/plugins/$plugin" ]; then + CP=$CP:"$DOLPHINSCHEDULER_HOME/plugins/$plugin/*" + fi done $JAVA_HOME/bin/java $JAVA_OPTS \ diff --git a/dolphinscheduler-standalone-server/src/main/bin/start.sh b/dolphinscheduler-standalone-server/src/main/bin/start.sh index 39596fc69c9a..47b73afdf0f1 100755 --- a/dolphinscheduler-standalone-server/src/main/bin/start.sh +++ b/dolphinscheduler-standalone-server/src/main/bin/start.sh @@ -65,7 +65,9 @@ task-plugins ) for plugin in ${PLUGINS_PATH[@]}; do - CP=$CP:"$DOLPHINSCHEDULER_HOME/plugins/$plugin/*" + if [ -d "$DOLPHINSCHEDULER_HOME/plugins/$plugin" ]; then + CP=$CP:"$DOLPHINSCHEDULER_HOME/plugins/$plugin/*" + fi done for jar in $(find $STANDALONE_HOME/libs/* -name "*.jar"); do diff --git a/dolphinscheduler-standalone-server/src/main/resources/application.yaml b/dolphinscheduler-standalone-server/src/main/resources/application.yaml index 3304d8ad5f0a..97b9b6d22d0c 100644 --- a/dolphinscheduler-standalone-server/src/main/resources/application.yaml +++ b/dolphinscheduler-standalone-server/src/main/resources/application.yaml @@ -23,6 +23,7 @@ spring: date-format: "yyyy-MM-dd HH:mm:ss" banner: charset: UTF-8 + location: classpath:standalone-banner.txt sql: init: schema-locations: classpath:sql/dolphinscheduler_h2.sql diff --git a/dolphinscheduler-standalone-server/src/main/resources/banner.txt b/dolphinscheduler-standalone-server/src/main/resources/standalone-banner.txt similarity index 100% rename from dolphinscheduler-standalone-server/src/main/resources/banner.txt rename to dolphinscheduler-standalone-server/src/main/resources/standalone-banner.txt diff --git a/dolphinscheduler-ui/pom.xml b/dolphinscheduler-ui/pom.xml index 2e61184ef503..117af4ed9fcf 100644 --- a/dolphinscheduler-ui/pom.xml +++ b/dolphinscheduler-ui/pom.xml @@ -35,54 +35,49 @@ 1.12.1 - - - release - - - - com.github.eirslett - frontend-maven-plugin - ${frontend-maven-plugin.version} + + + + com.github.eirslett + frontend-maven-plugin + ${frontend-maven-plugin.version} + + ${build.ui.skip} + false + + + + install node and pnpm + + install-node-and-pnpm + - ${build.ui.skip} - false + ${node.version} + ${pnpm.version} - - - install node and pnpm - - install-node-and-pnpm - - - ${node.version} - ${pnpm.version} - - - - pnpm install - - pnpm - - generate-resources - - install - - - - pnpm run build:prod - - pnpm - - - run build:prod - - - - - + + + pnpm install + + pnpm + + generate-resources + + install + + + + pnpm run build:prod + + pnpm + + + run build:prod + + + + + - - - + diff --git a/dolphinscheduler-worker/src/main/bin/start.sh b/dolphinscheduler-worker/src/main/bin/start.sh index 0a1b9dd83260..4930338c5a50 100644 --- a/dolphinscheduler-worker/src/main/bin/start.sh +++ b/dolphinscheduler-worker/src/main/bin/start.sh @@ -60,7 +60,9 @@ task-plugins ) for plugin in ${PLUGINS_PATH[@]}; do - CP=$CP:"$DOLPHINSCHEDULER_HOME/plugins/$plugin/*" + if [ -d "$DOLPHINSCHEDULER_HOME/plugins/$plugin" ]; then + CP=$CP:"$DOLPHINSCHEDULER_HOME/plugins/$plugin/*" + fi done $JAVA_HOME/bin/java $JAVA_OPTS \ diff --git a/pom.xml b/pom.xml index f4e553e17be4..2dd7aba16be3 100755 --- a/pom.xml +++ b/pom.xml @@ -95,7 +95,9 @@ true true true - false + true + false + true false false @@ -777,7 +779,36 @@ HEAD + + + apache.releases.https + Apache Release Distribution Repository + https://repository.apache.org/service/local/staging/deploy/maven2 + + + apache.snapshots.https + Apache Development Snapshot Repository + https://repository.apache.org/content/repositories/snapshots + + + + + release + + false + true + false + + + + staging + + false + false + false + + docker diff --git a/script/install-plugins.sh b/script/install-plugins.sh new file mode 100755 index 000000000000..67bb05d16dc0 --- /dev/null +++ b/script/install-plugins.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -eo pipefail + +# This script is used to download the plugins required during the running process. +# All are downloaded by default. You can also choose what you need. +# You only need to configure the plug-in name in config/plugin_config. + +# get ds home +DOLPHINSCHEDULER_HOME=$(cd $(dirname $0);cd ../;pwd) + +# plugins default version is 3.3.0, you can also choose a custom version. eg: 3.3.0: bash install-plugins.sh 3.3.0 +version=3.3.0 + +if [ -n "$1" ]; then + if [ "$1" == "dev" ]; then + version="dev-SNAPSHOT" + else + version="$1" + fi +fi + +echo "Install Dolphinscheduler plugins, usage version is ${version}" + +# create the plugins directory +if [ ! -d ${DOLPHINSCHEDULER_HOME}/plugins ]; then + mkdir -p ${DOLPHINSCHEDULER_HOME}/plugins +fi + +plugin_dir="" +while read line; do + if [ -z "$line" ]; then + continue + fi + + start_char=$(echo "$line" | cut -c 1) + + if [ "$start_char" == "-" ]; then + plugin_dir=$(echo ${line//--/}) + if [ "$plugin_dir" != "end" ]; then + mkdir -p ${DOLPHINSCHEDULER_HOME}/plugins/${plugin_dir} + fi + fi + + if [ "$start_char" != "-" ] && [ "$start_char" != "#" ]; then + echo "installing plugin: " $line + ${DOLPHINSCHEDULER_HOME}/mvnw dependency:get -DgroupId=org.apache.dolphinscheduler -DartifactId=${line} -Dversion=${version} -Ddest=${DOLPHINSCHEDULER_HOME}/plugins/${plugin_dir} + fi + +done < ${DOLPHINSCHEDULER_HOME}/conf/plugins_config +