From aa0eb85bb0a46d77aa6399b94c21f64116b46f60 Mon Sep 17 00:00:00 2001 From: Mighten Dai Date: Sat, 7 Sep 2024 14:32:59 +0800 Subject: [PATCH] Code Review updates: * Fixed SonarCloud Quality Readability B (required >= A) * Added missing license files * YAML-ized Tencent COS configuration from common.properties * Added remote logger support with Tencent COS --- docs/docs/en/guide/resource/configuration.md | 31 ++- docs/docs/zh/guide/resource/configuration.md | 31 ++- dolphinscheduler-common/pom.xml | 5 + .../common/constants/Constants.java | 12 ++ .../log/remote/CosRemoteLogHandler.java | 112 ++++++++++ .../log/remote/RemoteLogHandlerFactory.java | 2 + .../common/utils/PropertyUtils.java | 3 +- .../src/main/resources/common.properties | 35 --- .../src/main/resources/remote-logging.yaml | 34 +-- .../src/main/resources/resource-center.yaml | 30 +++ .../src/test/resources/remote-logging.yaml | 46 ++-- dolphinscheduler-dist/release-docs/LICENSE | 25 ++- .../release-docs/licenses/LICENSE-cos_api.txt | 22 ++ .../licenses/LICENSE-java-jwt.txt | 22 ++ .../licenses/LICENSE-logging-interceptor.txt | 203 ++++++++++++++++++ .../licenses/LICENSE-mxparser.txt | 57 +++++ .../LICENSE-tencentcloud-sdk-java-common.txt | 203 ++++++++++++++++++ .../LICENSE-tencentcloud-sdk-java-kms.txt | 203 ++++++++++++++++++ .../release-docs/licenses/LICENSE-xmlpull.txt | 15 ++ .../release-docs/licenses/LICENSE-xstream.txt | 29 +++ .../storage/cos/CosStorageOperator.java | 6 +- .../cos/CosStorageOperatorFactory.java | 15 +- tools/dependencies/known-dependencies.txt | 7 + 23 files changed, 1041 insertions(+), 107 deletions(-) create mode 100644 dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/remote/CosRemoteLogHandler.java create mode 100644 dolphinscheduler-common/src/main/resources/resource-center.yaml create mode 100644 dolphinscheduler-dist/release-docs/licenses/LICENSE-cos_api.txt create mode 100644 dolphinscheduler-dist/release-docs/licenses/LICENSE-java-jwt.txt create mode 100644 dolphinscheduler-dist/release-docs/licenses/LICENSE-logging-interceptor.txt create mode 100644 dolphinscheduler-dist/release-docs/licenses/LICENSE-mxparser.txt create mode 100644 dolphinscheduler-dist/release-docs/licenses/LICENSE-tencentcloud-sdk-java-common.txt create mode 100644 dolphinscheduler-dist/release-docs/licenses/LICENSE-tencentcloud-sdk-java-kms.txt create mode 100644 dolphinscheduler-dist/release-docs/licenses/LICENSE-xmlpull.txt create mode 100644 dolphinscheduler-dist/release-docs/licenses/LICENSE-xstream.txt diff --git a/docs/docs/en/guide/resource/configuration.md b/docs/docs/en/guide/resource/configuration.md index c5fe9fb19a6c..e596ff513fd2 100644 --- a/docs/docs/en/guide/resource/configuration.md +++ b/docs/docs/en/guide/resource/configuration.md @@ -98,19 +98,30 @@ resource.huawei.cloud.obs.endpoint=obs.cn-southwest-2.huaweicloud.com ## connect COS -if you want to upload resources to `Resource Center` connected to `COS`, you need to configure `api-server/conf/common.properties` and `worker-server/conf/common.properties`. You can refer to the following: +if you want to upload resources to `Resource Center` connected to `COS`, you need to configure `api-server/conf/resource-center.yaml` and `worker-server/conf/resource-center.yaml`. You can refer to the following: config the following fields -```properties -# access key id, required if you set resource.storage.type=COS -resource.tencent.cloud.access.key.id= -# access key secret, required if you set resource.storage.type=COS -resource.tencent.cloud.access.key.secret= -# cos bucket name, required if you set resource.storage.type=COS -resource.tencent.cloud.cos.bucket.name=dolphinscheduler -# cos bucket region, required if you set resource.storage.type=COS, refer to https://cloud.tencent.com/document/product/436/6224 -resource.tencent.cloud.cos.region=ap-nanjing +```yaml +resource: + # Tencent Cloud Storage (COS) setup, required if you set resource.storage.type=COS + tencent: + cloud: + access: + key: + id: + secret: + cos: + # predefined region code: https://cloud.tencent.com/document/product/436/6224 + region: ap-nanjing + bucket: + name: dolphinscheduler + +``` + +To enable COS, you also need to configure `api-server/conf/common.properties` and `worker-server/conf/common.properties`. You can refer to the following: +```properties +resource.storage.type=COS ``` diff --git a/docs/docs/zh/guide/resource/configuration.md b/docs/docs/zh/guide/resource/configuration.md index 018f9d8099f3..71fea7f2dbb1 100644 --- a/docs/docs/zh/guide/resource/configuration.md +++ b/docs/docs/zh/guide/resource/configuration.md @@ -92,17 +92,28 @@ resource.huawei.cloud.obs.endpoint=obs.cn-southwest-2.huaweicloud.com ## 对接腾讯云 COS -如果需要使用到资源中心的 COS 上传资源，我们需要对以下路径的进行配置：`api-server/conf/common.properties` 和 `worker-server/conf/common.properties`。可参考如下： +如果需要使用到资源中心的 COS 上传资源，我们需要对以下路径的进行配置：`api-server/conf/resource-center.yaml` 和 `worker-server/conf/resource-center.yaml`。可参考如下： -```properties -# access key id, required if you set resource.storage.type=COS -resource.tencent.cloud.access.key.id= -# access key secret, required if you set resource.storage.type=COS -resource.tencent.cloud.access.key.secret= -# cos bucket name, required if you set resource.storage.type=COS -resource.tencent.cloud.cos.bucket.name=dolphinscheduler -# cos bucket region, required if you set resource.storage.type=COS, refer to https://cloud.tencent.com/document/product/436/6224 -resource.tencent.cloud.cos.region=ap-nanjing +```yaml +resource: + # 腾讯云 COS 配置 + tencent: + cloud: + access: + key: + id: + secret: + cos: + # COS 区域代码可参考: https://cloud.tencent.com/document/product/436/6224 + region: ap-nanjing + bucket: + name: dolphinscheduler + +``` + +为了激活腾讯云存储 COS，还需要对以下路径的进行配置：`api-server/conf/common.properties` 和 `worker-server/conf/common.properties`。可参考如下： +```properties +resource.storage.type=COS ``` diff --git a/dolphinscheduler-common/pom.xml b/dolphinscheduler-common/pom.xml index 6c83c580ae25..09742ecc83ec 100644 --- a/dolphinscheduler-common/pom.xml +++ b/dolphinscheduler-common/pom.xml @@ -102,6 +102,11 @@ esdk-obs-java-bundle + + com.qcloud + cos_api + + com.azure azure-storage-blob diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java index 867b99fa1164..853d7d26d9b7 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java @@ -36,6 +36,7 @@ public final class Constants { public static final String REMOTE_LOGGING_YAML_PATH = "/remote-logging.yaml"; public static final String AWS_YAML_PATH = "/aws.yaml"; + public static final String RESOURCE_CENTER_YAML_PATH = "/resource-center.yaml"; public static final String FORMAT_SS = "%s%s"; public static final String FORMAT_S_S = "%s/%s"; @@ -684,6 +685,17 @@ public final class Constants { public static final String REMOTE_LOGGING_ABS_ACCOUNT_KEY = "remote.logging.abs.account.key"; public static final String REMOTE_LOGGING_ABS_CONTAINER_NAME = "remote.logging.abs.container.name"; + /** + * remote logging for COS + */ + public static final String REMOTE_LOGGING_COS_ACCESS_KEY_ID = "remote.logging.cos.access.key.id"; + + public static final String REMOTE_LOGGING_COS_ACCESS_KEY_SECRET = "remote.logging.cos.access.key.secret"; + + public static final String REMOTE_LOGGING_COS_BUCKET_NAME = "remote.logging.cos.bucket.name"; + + public static final String REMOTE_LOGGING_COS_REGION = "remote.logging.cos.region"; + /** * data quality */ diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/remote/CosRemoteLogHandler.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/remote/CosRemoteLogHandler.java new file mode 100644 index 000000000000..fa43c9a8e853 --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/remote/CosRemoteLogHandler.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.log.remote; + +import org.apache.dolphinscheduler.common.constants.Constants; +import org.apache.dolphinscheduler.common.utils.PropertyUtils; + +import org.apache.commons.lang3.StringUtils; + +import java.io.Closeable; +import java.io.File; +import java.io.IOException; + +import lombok.extern.slf4j.Slf4j; + +import com.qcloud.cos.COSClient; +import com.qcloud.cos.ClientConfig; +import com.qcloud.cos.auth.BasicCOSCredentials; +import com.qcloud.cos.auth.COSCredentials; +import com.qcloud.cos.http.HttpProtocol; +import com.qcloud.cos.model.GetObjectRequest; +import com.qcloud.cos.model.PutObjectRequest; +import com.qcloud.cos.region.Region; + +@Slf4j +public class CosRemoteLogHandler implements RemoteLogHandler, Closeable { + + private final COSClient cosClient; + + private final String bucketName; + + private static CosRemoteLogHandler instance; + + private CosRemoteLogHandler() { + String secretId = PropertyUtils.getString(Constants.REMOTE_LOGGING_COS_ACCESS_KEY_ID); + String secretKey = PropertyUtils.getString(Constants.REMOTE_LOGGING_COS_ACCESS_KEY_SECRET); + COSCredentials cosCredentials = new BasicCOSCredentials(secretId, secretKey); + String regionName = PropertyUtils.getString(Constants.REMOTE_LOGGING_COS_REGION); + ClientConfig clientConfig = new ClientConfig(new Region(regionName)); + clientConfig.setHttpProtocol(HttpProtocol.https); + this.cosClient = new COSClient(cosCredentials, clientConfig); + this.bucketName = PropertyUtils.getString(Constants.REMOTE_LOGGING_COS_BUCKET_NAME); + checkBucketNameExists(this.bucketName); + } + + public static synchronized CosRemoteLogHandler getInstance() { + if (instance == null) { + instance = new CosRemoteLogHandler(); + } + return instance; + } + + @Override + public void sendRemoteLog(String logPath) { + String objectName = RemoteLogUtils.getObjectNameFromLogPath(logPath); + + try { + log.info("send remote log {} to COS {}", logPath, objectName); + PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, objectName, new File(logPath)); + cosClient.putObject(putObjectRequest); + } catch (Exception e) { + log.error("error while sending remote log {} to COS {}", logPath, objectName, e); + } + } + + @Override + public void getRemoteLog(String logPath) { + String objectName = RemoteLogUtils.getObjectNameFromLogPath(logPath); + + try { + log.info("get remote log on COS {} to {}", objectName, logPath); + cosClient.getObject(new GetObjectRequest(bucketName, objectName), new File(logPath)); + } catch (Exception e) { + log.error("error while getting remote log on COS {} to {}", objectName, logPath, e); + } + } + + @Override + public void close() throws IOException { + if (cosClient != null) { + cosClient.shutdown(); + } + } + + private void checkBucketNameExists(String bucketName) { + if (StringUtils.isBlank(bucketName)) { + throw new IllegalArgumentException(Constants.REMOTE_LOGGING_COS_BUCKET_NAME + " is empty"); + } + boolean existsBucket = cosClient.doesBucketExist(bucketName); + if (!existsBucket) { + throw new IllegalArgumentException( + "bucketName: " + bucketName + " is not exists, you need to create them by yourself"); + } + + log.info("Tencent COS bucket '{}' has been found for remote logging", bucketName); + } +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/remote/RemoteLogHandlerFactory.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/remote/RemoteLogHandlerFactory.java index ac75a23f2d30..55020468781a 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/remote/RemoteLogHandlerFactory.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/remote/RemoteLogHandlerFactory.java @@ -41,6 +41,8 @@ public RemoteLogHandler getRemoteLogHandler() { return GcsRemoteLogHandler.getInstance(); } else if ("ABS".equals(target)) { return AbsRemoteLogHandler.getInstance(); + } else if ("COS".equals(target)) { + return CosRemoteLogHandler.getInstance(); } log.error("No suitable remote logging target for {}", target); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java index aee15893350d..5d4fec3ee802 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java @@ -20,6 +20,7 @@ import static org.apache.dolphinscheduler.common.constants.Constants.AWS_YAML_PATH; import static org.apache.dolphinscheduler.common.constants.Constants.COMMON_PROPERTIES_PATH; import static org.apache.dolphinscheduler.common.constants.Constants.REMOTE_LOGGING_YAML_PATH; +import static org.apache.dolphinscheduler.common.constants.Constants.RESOURCE_CENTER_YAML_PATH; import org.apache.dolphinscheduler.common.config.ImmutablePriorityPropertyDelegate; import org.apache.dolphinscheduler.common.config.ImmutablePropertyDelegate; @@ -43,7 +44,7 @@ public class PropertyUtils { private final ImmutablePriorityPropertyDelegate propertyDelegate = new ImmutablePriorityPropertyDelegate( new ImmutablePropertyDelegate(COMMON_PROPERTIES_PATH), - new ImmutableYamlDelegate(REMOTE_LOGGING_YAML_PATH, AWS_YAML_PATH)); + new ImmutableYamlDelegate(REMOTE_LOGGING_YAML_PATH, AWS_YAML_PATH, RESOURCE_CENTER_YAML_PATH)); public static String getString(String key) { return propertyDelegate.get(key.trim()); diff --git a/dolphinscheduler-common/src/main/resources/common.properties b/dolphinscheduler-common/src/main/resources/common.properties index d78f09b49681..8cf76f19f947 100644 --- a/dolphinscheduler-common/src/main/resources/common.properties +++ b/dolphinscheduler-common/src/main/resources/common.properties @@ -73,16 +73,6 @@ resource.huawei.cloud.obs.bucket.name=dolphinscheduler resource.huawei.cloud.obs.endpoint=obs.cn-southwest-2.huaweicloud.com -# tencent cloud access key id, required if you set resource.storage.type=COS -resource.tencent.cloud.access.key.id= -# access key secret, required if you set resource.storage.type=COS -resource.tencent.cloud.access.key.secret= -# cos bucket name, required if you set resource.storage.type=COS -resource.tencent.cloud.cos.bucket.name=dolphinscheduler -# cos bucket region, required if you set resource.storage.type=COS, see: https://cloud.tencent.com/document/product/436/6224 -resource.tencent.cloud.cos.region=ap-nanjing - - # if resource.storage.type=HDFS, the user must have the permission to create directories under the HDFS root path resource.hdfs.root.user=hdfs # if resource.storage.type=S3, the value like: s3a://dolphinscheduler; if resource.storage.type=HDFS and namenode HA is enabled, you need to copy core-site.xml and hdfs-site.xml to conf dir @@ -173,28 +163,3 @@ shell.interceptor.type=bash # Whether to enable remote logging remote.logging.enable=false -# if remote.logging.enable = true, set the target of remote logging -remote.logging.target=OSS -# if remote.logging.enable = true, set the log base directory -remote.logging.base.dir=logs -# if remote.logging.enable = true, set the number of threads to send logs to remote storage -remote.logging.thread.pool.size=10 -# oss access key id, required if you set remote.logging.target=OSS -remote.logging.oss.access.key.id= -# oss access key secret, required if you set remote.logging.target=OSS -remote.logging.oss.access.key.secret= -# oss bucket name, required if you set remote.logging.target=OSS -remote.logging.oss.bucket.name= -# oss endpoint, required if you set remote.logging.target=OSS -remote.logging.oss.endpoint= - -# the location of the google cloud credential, required if you set remote.logging.target=GCS -remote.logging.google.cloud.storage.credential=/path/to/credential -# gcs bucket name, required if you set remote.logging.target=GCS -remote.logging.google.cloud.storage.bucket.name= -# abs account name, required if you set resource.storage.type=ABS -remote.logging.abs.account.name= -# abs account key, required if you set resource.storage.type=ABS -remote.logging.abs.account.key= -# abs container name, required if you set resource.storage.type=ABS -remote.logging.abs.container.name= diff --git a/dolphinscheduler-common/src/main/resources/remote-logging.yaml b/dolphinscheduler-common/src/main/resources/remote-logging.yaml index f413958e646c..9c5e9ce5374b 100644 --- a/dolphinscheduler-common/src/main/resources/remote-logging.yaml +++ b/dolphinscheduler-common/src/main/resources/remote-logging.yaml @@ -15,29 +15,27 @@ # limitations under the License. # -remote-logging: - # Whether to enable remote logging - enable: false - # if remote-logging.enable = true, set the target of remote logging +remote.logging: + # if remote.logging.enable = true, set the target of remote logging target: OSS - # if remote-logging.enable = true, set the log base directory + # if remote.logging.enable = true, set the log base directory base.dir: logs - # if remote-logging.enable = true, set the number of threads to send logs to remote storage + # if remote.logging.enable = true, set the number of threads to send logs to remote storage thread.pool.size: 10 - # required if you set remote-logging.target=OSS + # required if you set remote.logging.target=OSS oss: - # oss access key id, required if you set remote-logging.target=OSS + # oss access key id, required if you set remote.logging.target=OSS access.key.id: - # oss access key secret, required if you set remote-logging.target=OSS + # oss access key secret, required if you set remote.logging.target=OSS access.key.secret: - # oss bucket name, required if you set remote-logging.target=OSS + # oss bucket name, required if you set remote.logging.target=OSS bucket.name: - # oss endpoint, required if you set remote-logging.target=OSS + # oss endpoint, required if you set remote.logging.target=OSS endpoint: google.cloud.storage: - # the location of the google cloud credential, required if you set remote-logging.target=GCS + # the location of the google cloud credential, required if you set remote.logging.target=GCS credential: /path/to/credential - # gcs bucket name, required if you set remote-logging.target=GCS + # gcs bucket name, required if you set remote.logging.target=GCS bucket.name: abs: # abs account name, required if you set resource.storage.type=ABS @@ -46,4 +44,12 @@ remote-logging: account.key: # abs container name, required if you set resource.storage.type=ABS container.name: - + cos: + # cos access key id, required if you set remote-logging.target=COS + access.key.id: + # cos access key secret, required if you set remote-logging.target=COS + access.key.secret: + # cos bucket name, required if you set remote-logging.target=COS + bucket.name: + # cos region, required if you set remote-logging.target=COS + region: diff --git a/dolphinscheduler-common/src/main/resources/resource-center.yaml b/dolphinscheduler-common/src/main/resources/resource-center.yaml new file mode 100644 index 000000000000..7316651ebadf --- /dev/null +++ b/dolphinscheduler-common/src/main/resources/resource-center.yaml @@ -0,0 +1,30 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +resource: + # Tencent Cloud Storage (COS) setup, required if you set resource.storage.type=COS + tencent: + cloud: + access: + key: + id: + secret: + cos: + # predefined region code: https://cloud.tencent.com/document/product/436/6224 + region: ap-nanjing + bucket: + name: dolphinscheduler diff --git a/dolphinscheduler-common/src/test/resources/remote-logging.yaml b/dolphinscheduler-common/src/test/resources/remote-logging.yaml index cb149a77fe0d..9c5e9ce5374b 100644 --- a/dolphinscheduler-common/src/test/resources/remote-logging.yaml +++ b/dolphinscheduler-common/src/test/resources/remote-logging.yaml @@ -15,41 +15,27 @@ # limitations under the License. # -remote-logging: - # Whether to enable remote logging - enable: false - # if remote-logging.enable = true, set the target of remote logging +remote.logging: + # if remote.logging.enable = true, set the target of remote logging target: OSS - # if remote-logging.enable = true, set the log base directory + # if remote.logging.enable = true, set the log base directory base.dir: logs - # if remote-logging.enable = true, set the number of threads to send logs to remote storage + # if remote.logging.enable = true, set the number of threads to send logs to remote storage thread.pool.size: 10 - # required if you set remote-logging.target=OSS + # required if you set remote.logging.target=OSS oss: - # oss access key id, required if you set remote-logging.target=OSS + # oss access key id, required if you set remote.logging.target=OSS access.key.id: - # oss access key secret, required if you set remote-logging.target=OSS + # oss access key secret, required if you set remote.logging.target=OSS access.key.secret: - # oss bucket name, required if you set remote-logging.target=OSS + # oss bucket name, required if you set remote.logging.target=OSS bucket.name: - # oss endpoint, required if you set remote-logging.target=OSS + # oss endpoint, required if you set remote.logging.target=OSS endpoint: - # required if you set remote-logging.target=S3 - s3: - # s3 access key id, required if you set remote-logging.target=S3 - access.key.id: - # s3 access key secret, required if you set remote-logging.target=S3 - access.key.secret: - # s3 bucket name, required if you set remote-logging.target=S3 - bucket.name: - # s3 endpoint, required if you set remote-logging.target=S3 - endpoint: - # s3 region, required if you set remote-logging.target=S3 - region: google.cloud.storage: - # the location of the google cloud credential, required if you set remote-logging.target=GCS + # the location of the google cloud credential, required if you set remote.logging.target=GCS credential: /path/to/credential - # gcs bucket name, required if you set remote-logging.target=GCS + # gcs bucket name, required if you set remote.logging.target=GCS bucket.name: abs: # abs account name, required if you set resource.storage.type=ABS @@ -58,4 +44,12 @@ remote-logging: account.key: # abs container name, required if you set resource.storage.type=ABS container.name: - + cos: + # cos access key id, required if you set remote-logging.target=COS + access.key.id: + # cos access key secret, required if you set remote-logging.target=COS + access.key.secret: + # cos bucket name, required if you set remote-logging.target=COS + bucket.name: + # cos region, required if you set remote-logging.target=COS + region: diff --git a/dolphinscheduler-dist/release-docs/LICENSE b/dolphinscheduler-dist/release-docs/LICENSE index ceba727734e3..37e7e5e65de5 100644 --- a/dolphinscheduler-dist/release-docs/LICENSE +++ b/dolphinscheduler-dist/release-docs/LICENSE @@ -555,7 +555,7 @@ The text of each license is also included at licenses/LICENSE-[project].txt. proto-google-cloud-storage-v2 2.18.0-alpha: https://mvnrepository.com/artifact/com.google.api.grpc/proto-google-cloud-storage-v2/2.18.0-alpha, Apache 2.0 proto-google-iam-v1 1.9.0: https://mvnrepository.com/artifact/com.google.api.grpc/proto-google-iam-v1/1.9.0, Apache 2.0 sshd-sftp https://mvnrepository.com/artifact/org.apache.sshd/sshd-sftp/2.8.0 Apache 2.0 - sshd-scp https://mvnrepository.com/artifact/org.apache.sshd/sshd-scp/2.8.0 Aapache 2.0 + sshd-scp https://mvnrepository.com/artifact/org.apache.sshd/sshd-scp/2.8.0 Apache 2.0 casdoor-java-sdk 1.11.0 https://mvnrepository.com/artifact/org.casbin/casdoor-java-sdk/1.11.0 Apache 2.0 casdoor-spring-boot-starter 1.6.0 https://mvnrepository.com/artifact/org.casbin/casdoor-spring-boot-starter/1.6.0 Apache 2.0 org.apache.oltu.oauth2.client 1.0.2 https://mvnrepository.com/artifact/org.apache.oltu.oauth2/org.apache.oltu.oauth2.client/1.0.2 Apache 2.0 @@ -572,8 +572,9 @@ The text of each license is also included at licenses/LICENSE-[project].txt. tea-rpc-util 0.1.3.jar https://github.com/aliyun/aliyun-openapi-java-sdk/blob/master/README.md#license Apache 2.0 tea-util 0.2.13.jar https://github.com/aliyun/aliyun-openapi-java-sdk/blob/master/README.md#license Apache 2.0 delight-nashorn-sandbox 0.3.2 https://github.com/javadelight/delight-nashorn-sandbox/blob/master/README.md#license Apache 2.0 - - + tencentcloud-sdk-java-common 3.1.213 https://mvnrepository.com/artifact/com.tencentcloudapi/tencentcloud-sdk-java-common/3.1.213, Apache 2.0 + tencentcloud-sdk-java-kms 3.1.213 https://mvnrepository.com/artifact/com.tencentcloudapi/tencentcloud-sdk-java-kms/3.1.213, Apache 2.0 + logging-interceptor 2.7.5 https://mvnrepository.com/artifact/com.squareup.okhttp/logging-interceptor/2.7.5, Apache 2.0 @@ -616,6 +617,7 @@ The text of each license is also included at licenses/LICENSE-[project].txt. google-auth-library-oauth2-http 1.15.0: https://mvnrepository.com/artifact/com.google.auth/google-auth-library-oauth2-http/1.15.0, BSD 3-clause threetenbp 1.6.5: https://mvnrepository.com/artifact/org.threeten/threetenbp/1.6.5, BSD 3-clause dom4j 2.1.3 https://github.com/dom4j/dom4j/blob/master/LICENSE BSD 3-clause + xstream 1.4.20 https://mvnrepository.com/artifact/com.thoughtworks.xstream/xstream/1.4.20, BSD 3-clause ======================================================================== CDDL licenses @@ -714,6 +716,7 @@ The text of each license is also included at licenses/LICENSE-[project].txt. azure-storage-blob 12.21.0: https://mvnrepository.com/artifact/com.azure/azure-storage-blob/12.21.0, MIT azure-storage-internal-avro 12.6.0: https://mvnrepository.com/artifact/com.azure/azure-storage-internal-avro/12.6.0, MIT cos_api 5.6.231 https://mvnrepository.com/artifact/com.qcloud/cos_api/5.6.231, MIT + java-jwt 4.4.0 https://mvnrepository.com/artifact/com.auth0/java-jwt/4.4.0, MIT ======================================================================== MPL 1.1 licenses @@ -814,3 +817,19 @@ JDOM licenses Vertica Client Driver License ======================================== vertica-jdbc 12.0.4-0: https://mvnrepository.com/artifact/com.vertica.jdbc/vertica-jdbc/12.0.4-0, Vertica Client Driver License + +======================================================================== +Indiana University Extreme! Lab Software License +======================================================================== + +The following components are provided under a Indiana University Extreme! Lab Software License. See project link for details. +The text of each license is also included at licenses/LICENSE-[project].txt. + mxparser 1.2.2: https://mvnrepository.com/artifact/io.github.x-stream/mxparser/1.2.2, Indiana University Extreme! Lab Software License + +======================================================================== +Public Domain License +======================================================================== + +The following components are provided under a Public Domain License. See project link for details. +The text of each license is also included at licenses/LICENSE-[project].txt. + xmlpull 1.1.3.1: https://mvnrepository.com/artifact/xmlpull/xmlpull/1.1.3.1, Public Domain License diff --git a/dolphinscheduler-dist/release-docs/licenses/LICENSE-cos_api.txt b/dolphinscheduler-dist/release-docs/licenses/LICENSE-cos_api.txt new file mode 100644 index 000000000000..68d4a5c574d1 --- /dev/null +++ b/dolphinscheduler-dist/release-docs/licenses/LICENSE-cos_api.txt @@ -0,0 +1,22 @@ +MIT License + +Copyright (c) 2017 腾讯云 + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + diff --git a/dolphinscheduler-dist/release-docs/licenses/LICENSE-java-jwt.txt b/dolphinscheduler-dist/release-docs/licenses/LICENSE-java-jwt.txt new file mode 100644 index 000000000000..7d45271f0e47 --- /dev/null +++ b/dolphinscheduler-dist/release-docs/licenses/LICENSE-java-jwt.txt @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2015 Auth0, Inc. (http://auth0.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + diff --git a/dolphinscheduler-dist/release-docs/licenses/LICENSE-logging-interceptor.txt b/dolphinscheduler-dist/release-docs/licenses/LICENSE-logging-interceptor.txt new file mode 100644 index 000000000000..6b0b1270ff0c --- /dev/null +++ b/dolphinscheduler-dist/release-docs/licenses/LICENSE-logging-interceptor.txt @@ -0,0 +1,203 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/dolphinscheduler-dist/release-docs/licenses/LICENSE-mxparser.txt b/dolphinscheduler-dist/release-docs/licenses/LICENSE-mxparser.txt new file mode 100644 index 000000000000..e81fa49549da --- /dev/null +++ b/dolphinscheduler-dist/release-docs/licenses/LICENSE-mxparser.txt @@ -0,0 +1,57 @@ +Indiana University Extreme! Lab Software License, Version 1.2 + +Copyright (C) 2003 The Trustees of Indiana University. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +1) All redistributions of source code must retain the above + copyright notice, the list of authors in the original source + code, this list of conditions and the disclaimer listed in this + license; + +2) All redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the disclaimer + listed in this license in the documentation and/or other + materials provided with the distribution; + +3) Any documentation included with all redistributions must include + the following acknowledgement: + + "This product includes software developed by the Indiana + University Extreme! Lab. For further information please visit + http://www.extreme.indiana.edu/" + + Alternatively, this acknowledgment may appear in the software + itself, and wherever such third-party acknowledgments normally + appear. + +4) The name "Indiana University" or "Indiana University + Extreme! Lab" shall not be used to endorse or promote + products derived from this software without prior written + permission from Indiana University. For written permission, + please contact http://www.extreme.indiana.edu/. + +5) Products derived from this software may not use "Indiana + University" name nor may "Indiana University" appear in their name, + without prior written permission of the Indiana University. + +Indiana University provides no reassurances that the source code +provided does not infringe the patent or any other intellectual +property rights of any other entity. Indiana University disclaims any +liability to any recipient for claims brought by any other entity +based on infringement of intellectual property rights or otherwise. + +LICENSEE UNDERSTANDS THAT SOFTWARE IS PROVIDED "AS IS" FOR WHICH +NO WARRANTIES AS TO CAPABILITIES OR ACCURACY ARE MADE. INDIANA +UNIVERSITY GIVES NO WARRANTIES AND MAKES NO REPRESENTATION THAT +SOFTWARE IS FREE OF INFRINGEMENT OF THIRD PARTY PATENT, COPYRIGHT, OR +OTHER PROPRIETARY RIGHTS. INDIANA UNIVERSITY MAKES NO WARRANTIES THAT +SOFTWARE IS FREE FROM "BUGS", "VIRUSES", "TROJAN HORSES", "TRAP +DOORS", "WORMS", OR OTHER HARMFUL CODE. LICENSEE ASSUMES THE ENTIRE +RISK AS TO THE PERFORMANCE OF SOFTWARE AND/OR ASSOCIATED MATERIALS, +AND TO THE PERFORMANCE AND VALIDITY OF INFORMATION GENERATED USING +SOFTWARE. + diff --git a/dolphinscheduler-dist/release-docs/licenses/LICENSE-tencentcloud-sdk-java-common.txt b/dolphinscheduler-dist/release-docs/licenses/LICENSE-tencentcloud-sdk-java-common.txt new file mode 100644 index 000000000000..6b0b1270ff0c --- /dev/null +++ b/dolphinscheduler-dist/release-docs/licenses/LICENSE-tencentcloud-sdk-java-common.txt @@ -0,0 +1,203 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/dolphinscheduler-dist/release-docs/licenses/LICENSE-tencentcloud-sdk-java-kms.txt b/dolphinscheduler-dist/release-docs/licenses/LICENSE-tencentcloud-sdk-java-kms.txt new file mode 100644 index 000000000000..6b0b1270ff0c --- /dev/null +++ b/dolphinscheduler-dist/release-docs/licenses/LICENSE-tencentcloud-sdk-java-kms.txt @@ -0,0 +1,203 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/dolphinscheduler-dist/release-docs/licenses/LICENSE-xmlpull.txt b/dolphinscheduler-dist/release-docs/licenses/LICENSE-xmlpull.txt new file mode 100644 index 000000000000..6cbd5564bc5c --- /dev/null +++ b/dolphinscheduler-dist/release-docs/licenses/LICENSE-xmlpull.txt @@ -0,0 +1,15 @@ +XMLPULL API IS FREE +------------------- + +All of the XMLPULL API source code, compiled code, and documentation +contained in this distribution *except* for tests (see separate LICENSE_TESTS.txt) +are in the Public Domain. + +XMLPULL API comes with NO WARRANTY or guarantee of fitness for any purpose. + +Initial authors: + + Stefan Haustein + Aleksander Slominski + +2001-12-12 diff --git a/dolphinscheduler-dist/release-docs/licenses/LICENSE-xstream.txt b/dolphinscheduler-dist/release-docs/licenses/LICENSE-xstream.txt new file mode 100644 index 000000000000..1e6a95af2d45 --- /dev/null +++ b/dolphinscheduler-dist/release-docs/licenses/LICENSE-xstream.txt @@ -0,0 +1,29 @@ +Copyright (c) 2003-2006, Joe Walnes +Copyright (c) 2006-2015 XStream Committers +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this list of +conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, this list of +conditions and the following disclaimer in the documentation and/or other materials provided +with the distribution. + +3. Neither the name of XStream nor the names of its contributors may be used to endorse +or promote products derived from this software without specific prior written +permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT +SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY +WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH +DAMAGE. + diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-cos/src/main/java/org/apache/dolphinscheduler/plugin/storage/cos/CosStorageOperator.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-cos/src/main/java/org/apache/dolphinscheduler/plugin/storage/cos/CosStorageOperator.java index 05603ac4ec9b..c3286230af54 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-cos/src/main/java/org/apache/dolphinscheduler/plugin/storage/cos/CosStorageOperator.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-cos/src/main/java/org/apache/dolphinscheduler/plugin/storage/cos/CosStorageOperator.java @@ -76,8 +76,8 @@ public class CosStorageOperator extends AbstractStorageOperator implements Close private static final int TRANSFER_THREAD_POOL_SIZE = 16; - private static final long MULTIPART_UPLOAD_BYTES_THRESHOLD = 5 * 1024 * 1024; - private static final long MIN_UPLOAD_PART_BYTES = 1024 * 1024; + private static final long MULTIPART_UPLOAD_BYTES_THRESHOLD = 5 * 1024 * 1024L; + private static final long MIN_UPLOAD_PART_BYTES = 1024 * 1024L; private final String bucketName; @@ -134,7 +134,7 @@ public void createStorageDir(String directoryAbsolutePath) { public void download(String srcFilePath, String dstFilePath, boolean overwrite) { String cosKey = transformAbsolutePathToCOSKey(srcFilePath); - File dstFile = new File(dstFilePath); + File dstFile = Paths.get(dstFilePath).normalize().toFile(); if (dstFile.isDirectory()) { Files.delete(dstFile.toPath()); } else { diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-cos/src/main/java/org/apache/dolphinscheduler/plugin/storage/cos/CosStorageOperatorFactory.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-cos/src/main/java/org/apache/dolphinscheduler/plugin/storage/cos/CosStorageOperatorFactory.java index 2395dd3c997f..d978d7a9d6ab 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-cos/src/main/java/org/apache/dolphinscheduler/plugin/storage/cos/CosStorageOperatorFactory.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-cos/src/main/java/org/apache/dolphinscheduler/plugin/storage/cos/CosStorageOperatorFactory.java @@ -24,6 +24,8 @@ import org.apache.dolphinscheduler.plugin.storage.api.StorageType; import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; +import java.util.Map; + import com.google.auto.service.AutoService; @AutoService(StorageOperatorFactory.class) @@ -41,12 +43,15 @@ public StorageType getStorageOperate() { } private CosStorageProperties getCosStorageProperties() { + + Map cosPropertiesMap = PropertyUtils.getByPrefix("resource.tencent.cloud"); + return CosStorageProperties.builder() - .region(PropertyUtils.getString(Constants.TENCENT_CLOUD_COS_REGION)) - .accessKeyId(PropertyUtils.getString(TaskConstants.TENCENT_CLOUD_ACCESS_KEY_ID)) - .accessKeySecret(PropertyUtils.getString(TaskConstants.TENCENT_CLOUD_ACCESS_KEY_SECRET)) - .bucketName(PropertyUtils.getString(Constants.TENCENT_CLOUD_COS_BUCKET_NAME)) - .resourceUploadPath(PropertyUtils.getString(Constants.RESOURCE_UPLOAD_PATH, "/dolphinscheduler")) + .region(cosPropertiesMap.get(Constants.TENCENT_CLOUD_COS_REGION)) + .accessKeyId(cosPropertiesMap.get(TaskConstants.TENCENT_CLOUD_ACCESS_KEY_ID)) + .accessKeySecret(cosPropertiesMap.get(TaskConstants.TENCENT_CLOUD_ACCESS_KEY_SECRET)) + .bucketName(cosPropertiesMap.get(Constants.TENCENT_CLOUD_COS_BUCKET_NAME)) + .resourceUploadPath(cosPropertiesMap.getOrDefault(Constants.RESOURCE_UPLOAD_PATH, "/dolphinscheduler")) .build(); } } diff --git a/tools/dependencies/known-dependencies.txt b/tools/dependencies/known-dependencies.txt index 2e731541778d..84ddb76f5ffa 100644 --- a/tools/dependencies/known-dependencies.txt +++ b/tools/dependencies/known-dependencies.txt @@ -486,6 +486,13 @@ azure-storage-internal-avro-12.6.0.jar vertica-jdbc-12.0.4-0.jar esdk-obs-java-bundle-3.23.3.jar cos_api-5.6.231.jar +java-jwt-4.4.0.jar +logging-interceptor-2.7.5.jar +mxparser-1.2.2.jar +tencentcloud-sdk-java-common-3.1.213.jar +tencentcloud-sdk-java-kms-3.1.213.jar +xmlpull-1.1.3.1.jar +xstream-1.4.20.jar dyvmsapi20170525-2.1.4.jar alibabacloud-gateway-spi-0.0.1.jar credentials-java-0.2.2.jar