From 1f876d397b1875b5546041c9beaaf05d3a3e518f Mon Sep 17 00:00:00 2001 From: Yubi Lee Date: Sat, 16 Sep 2023 18:50:35 +0900 Subject: [PATCH] RANGER-4418: Upgrade hadoop version and use shaded hadoop client artifacts --- agents-audit/pom.xml | 69 ++++----- agents-common/pom.xml | 63 +++----- .../ranger/plugin/util/RangerCache.java | 4 +- agents-cred/pom.xml | 50 ++---- build_ranger_using_docker.sh | 15 +- credentialbuilder/pom.xml | 70 ++------- distro/src/main/assembly/admin-web.xml | 15 +- distro/src/main/assembly/hbase-agent.xml | 4 +- distro/src/main/assembly/hdfs-agent.xml | 4 +- distro/src/main/assembly/hive-agent.xml | 5 +- distro/src/main/assembly/kms.xml | 20 ++- distro/src/main/assembly/knox-agent.xml | 8 +- distro/src/main/assembly/plugin-atlas.xml | 4 +- .../main/assembly/plugin-elasticsearch.xml | 10 +- distro/src/main/assembly/plugin-kafka.xml | 10 +- distro/src/main/assembly/plugin-kms.xml | 7 +- distro/src/main/assembly/plugin-kylin.xml | 4 +- distro/src/main/assembly/plugin-ozone.xml | 4 +- distro/src/main/assembly/plugin-presto.xml | 10 +- .../main/assembly/plugin-schema-registry.xml | 28 ++-- distro/src/main/assembly/plugin-solr.xml | 4 +- distro/src/main/assembly/plugin-sqoop.xml | 4 +- distro/src/main/assembly/plugin-trino.xml | 10 +- distro/src/main/assembly/plugin-yarn.xml | 4 +- distro/src/main/assembly/ranger-tools.xml | 6 +- distro/src/main/assembly/sample-client.xml | 4 +- distro/src/main/assembly/storm-agent.xml | 10 +- distro/src/main/assembly/tagsync.xml | 4 +- distro/src/main/assembly/usersync.xml | 5 +- embeddedwebserver/pom.xml | 5 + hbase-agent/pom.xml | 104 +++---------- hdfs-agent/pom.xml | 29 ++-- hive-agent/pom.xml | 22 +-- kms/pom.xml | 89 ++--------- .../kms/server/KMSAuthenticationFilter.java | 16 +- .../key/kms/server/KMSExceptionsProvider.java | 6 +- .../crypto/key/kms/server/KMSJMXServlet.java | 4 +- knox-agent/pom.xml | 29 ++-- plugin-kafka/pom.xml | 25 +-- .../client/TestNiFiRegistryClient.java | 14 +- plugin-ozone/pom.xml | 25 +-- plugin-schema-registry/pom.xml | 88 ++--------- plugin-yarn/pom.xml | 14 +- ranger-authn/pom.xml | 41 +++-- ranger-common-ha/pom.xml | 29 +--- .../src/main/assembly/plugin-sampleapp.xml | 4 +- .../src/main/assembly/sample-client.xml | 4 +- ranger-examples/plugin-sampleapp/pom.xml | 41 +---- ranger-hdfs-plugin-shim/pom.xml | 24 +++ ranger-metrics/pom.xml | 25 +-- .../metrics/sink/RangerMetricsJsonSink.java | 2 +- .../sink/RangerMetricsPrometheusSink.java | 2 +- .../source/RangerMetricsContainerSource.java | 2 +- ranger-ozone-plugin-shim/pom.xml | 20 +++ ranger-presto-plugin-shim/pom.xml | 25 +-- ranger-storm-plugin-shim/pom.xml | 20 +++ ranger-util/pom.xml | 7 +- ranger-yarn-plugin-shim/pom.xml | 17 +-- security-admin/pom.xml | 143 ++++++++++++++---- storm-agent/pom.xml | 72 +++------ ugsync/pom.xml | 29 +--- unixauthclient/pom.xml | 23 --- unixauthservice/pom.xml | 25 +-- 63 files changed, 557 insertions(+), 923 deletions(-) diff --git a/agents-audit/pom.xml b/agents-audit/pom.xml index dbb1502e6d..e57f42cc96 100644 --- a/agents-audit/pom.xml +++ b/agents-audit/pom.xml @@ -42,52 +42,29 @@ commons-lang ${commons.lang.version} + + org.apache.commons + commons-lang3 + ${commons.lang3.version} + + + commons-collections + commons-collections + ${commons.collections.version} + org.apache.hadoop - hadoop-common + hadoop-client-api ${hadoop.version} - - - com.google.guava - guava - - - net.minidev - json-smart - - - io.netty - netty-handler - - - io.netty - netty-transport-native-epoll - - - org.apache.commons - commons-text - - - org.apache.commons - commons-configuration2 - - - log4j - * - - - org.slf4j - * - - - org.apache.commons - commons-compress - - - org.apache.zookeeper - zookeeper - - + + + org.apache.hadoop + hadoop-client-runtime + ${hadoop.version} + + + com.google.code.gson + gson org.apache.commons @@ -421,5 +398,11 @@ ${slf4j.version} test + + com.google.code.findbugs + jsr305 + ${jsr305.version} + compile + diff --git a/agents-common/pom.xml b/agents-common/pom.xml index 502a1f716d..b5550814fa 100644 --- a/agents-common/pom.xml +++ b/agents-common/pom.xml @@ -59,6 +59,26 @@ commons-lang ${commons.lang.version} + + javax.servlet + javax.servlet-api + ${javax.servlet.version} + + + commons-io + commons-io + ${commons.io.version} + + + org.apache.hadoop + hadoop-client-api + ${hadoop.version} + + + org.apache.hadoop + hadoop-client-runtime + ${hadoop.version} + com.nimbusds nimbus-jose-jwt @@ -69,49 +89,6 @@ json-smart ${jsonsmart.version} - - org.apache.hadoop - hadoop-common - ${hadoop.version} - - - net.minidev - json-smart - - - io.netty - netty-handler - - - io.netty - netty-transport-native-epoll - - - org.apache.commons - commons-text - - - org.apache.commons - commons-configuration2 - - - log4j - * - - - org.slf4j - * - - - org.apache.commons - commons-compress - - - org.apache.zookeeper - zookeeper - - - org.apache.commons commons-text diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerCache.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerCache.java index 9769aaa522..42ba3aefa9 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerCache.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerCache.java @@ -17,7 +17,7 @@ package org.apache.ranger.plugin.util; -import com.sun.istack.NotNull; +import javax.annotation.Nonnull; import org.apache.ranger.plugin.util.AutoClosableLock.AutoClosableTryLock; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -363,7 +363,7 @@ private ThreadFactory createThreadFactory() { private final AtomicInteger number = new AtomicInteger(1); @Override - public Thread newThread(@NotNull Runnable r) { + public Thread newThread(@Nonnull Runnable r) { Thread t = new Thread(r, namePrefix + number.getAndIncrement()); if (!t.isDaemon()) { diff --git a/agents-cred/pom.xml b/agents-cred/pom.xml index 0d2850a39f..d45b775d13 100644 --- a/agents-cred/pom.xml +++ b/agents-cred/pom.xml @@ -43,52 +43,24 @@ org.apache.hadoop - hadoop-common + hadoop-client-api + ${hadoop.version} + + + org.apache.hadoop + hadoop-client-runtime ${hadoop.version} - - - net.minidev - json-smart - - - io.netty - netty-handler - - - io.netty - netty-transport-native-epoll - - - org.apache.commons - commons-text - - - org.apache.commons - commons-configuration2 - - - log4j - * - - - org.slf4j - * - - - org.apache.commons - commons-compress - - - org.apache.zookeeper - zookeeper - - org.apache.commons commons-text ${commons.text.version} + + org.apache.httpcomponents + httpclient + ${httpcomponents.httpclient.version} + org.apache.commons commons-configuration2 diff --git a/build_ranger_using_docker.sh b/build_ranger_using_docker.sh index 9aadfb5f72..4be8861cbd 100755 --- a/build_ranger_using_docker.sh +++ b/build_ranger_using_docker.sh @@ -56,13 +56,13 @@ if [ $build_image -eq 1 ]; then docker rmi -f $image_name docker build -t $image_name - < tmp.sha1 RUN cat apache-maven-3.6.3-bin.tar.gz.sha512 | cut -f 1 -d " " > tmp.sha1.download @@ -99,12 +99,11 @@ ENV PATH /tools/maven/bin:$PATH ENV MAVEN_OPTS "-Xmx2048m -XX:MaxPermSize=512m" # Setup gosu for easier command execution -RUN gpg --keyserver pool.sks-keyservers.net --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4 \ - && curl -o /usr/local/bin/gosu -SL "https://github.com/tianon/gosu/releases/download/1.10/gosu-amd64" \ - && curl -o /usr/local/bin/gosu.asc -SL "https://github.com/tianon/gosu/releases/download/1.10/gosu-amd64.asc" \ +RUN gpg --keyserver hkps://keys.openpgp.org --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4 \ + && curl -o /usr/local/bin/gosu -SL "https://github.com/tianon/gosu/releases/download/1.16/gosu-amd64" \ + && curl -o /usr/local/bin/gosu.asc -SL "https://github.com/tianon/gosu/releases/download/1.16/gosu-amd64.asc" \ && gpg --verify /usr/local/bin/gosu.asc \ && rm /usr/local/bin/gosu.asc \ - && rm -r /root/.gnupg/ \ && chmod +x /usr/local/bin/gosu RUN useradd -ms /bin/bash builder diff --git a/credentialbuilder/pom.xml b/credentialbuilder/pom.xml index e54c933918..8038e3e688 100644 --- a/credentialbuilder/pom.xml +++ b/credentialbuilder/pom.xml @@ -38,6 +38,12 @@ commons-collections ${commons.collections.version} + + commons-io + commons-io + ${commons.io.version} + test + org.apache.commons commons-configuration2 @@ -65,71 +71,19 @@ org.apache.hadoop - hadoop-common + hadoop-client-api + ${hadoop.version} + + + org.apache.hadoop + hadoop-client-runtime ${hadoop.version} - - - org.apache.commons - commons-compress - - - org.apache.commons - commons-lang3 - - - io.netty - netty-handler - - - io.netty - netty-transport-native-epoll - - - org.apache.commons - commons-text - - - org.apache.commons - commons-configuration2 - - - log4j - * - - - org.slf4j - * - - - org.apache.zookeeper - zookeeper - - org.apache.commons commons-text ${commons.text.version} - - org.apache.hadoop - hadoop-auth - ${hadoop.version} - - - log4j - * - - - org.slf4j - * - - - org.apache.zookeeper - zookeeper - - - org.apache.commons commons-compress diff --git a/distro/src/main/assembly/admin-web.xml b/distro/src/main/assembly/admin-web.xml index 54fba59ba7..e284017168 100644 --- a/distro/src/main/assembly/admin-web.xml +++ b/distro/src/main/assembly/admin-web.xml @@ -78,7 +78,6 @@ org.apache.hive:hive-exec:jar:${hive.version} org.apache.hive:hive-jdbc:jar:${hive.version} org.apache.hive:hive-metastore:jar:${hive.version} - org.apache.thrift:libfb303:jar:${libfb303.version} org.apache.thrift:libthrift:jar:${libthrift.version} @@ -162,8 +161,8 @@ 755 644 - org.apache.hadoop:hadoop-yarn-api:jar:${hadoop.version} - org.apache.hadoop:hadoop-yarn-common:jar:${hadoop.version} + org.apache.hadoop:hadoop-client-api:jar:${hadoop.version} + org.apache.hadoop:hadoop-client-runtime:jar:${hadoop.version} @@ -238,7 +237,7 @@ org.apache.hadoop:hadoop-auth:jar:${hadoop.version} org.apache.ranger:ranger-plugins-common org.slf4j:slf4j-api:jar:${slf4j.version} - org.apache.hadoop:hadoop-common + org.apache.hadoop:hadoop-common:jar:${hadoop.version} commons-logging:commons-logging com.sun.jersey.contribs:jersey-multipart commons-collections:commons-collections @@ -272,7 +271,7 @@ com.carrotsearch:hppc joda-time:joda-time org.apache.ranger:ranger-plugins-cred - org.apache.ranger:ugsyn-util + org.apache.ranger:ugsync-util org.eclipse.jetty:jetty-client:jar:${jetty-client.version} org.apache.commons:commons-compress:jar:${commons.compress.version} org.slf4j:log4j-over-slf4j:jar:${slf4j.version} @@ -303,8 +302,10 @@ commons-logging:commons-logging org.slf4j:slf4j-api:jar:${slf4j.version} org.apache.commons:commons-lang3 - org.apache.hadoop:hadoop-common - org.apache.hadoop:hadoop-auth + org.apache.hadoop:hadoop-client-api:jar:${hadoop.version} + org.apache.hadoop:hadoop-client-runtime:jar:${hadoop.version} + org.apache.htrace:htrace-core4:jar:${htrace-core.version} + org.apache.hadoop:hadoop-auth:jar:${hadoop.version} com.fasterxml.woodstox:woodstox-core:jar:${fasterxml.woodstox.version} org.codehaus.woodstox:stax2-api:jar:${codehaus.woodstox.stax2api.version} org.apache.commons:commons-configuration2:jar:${commons.configuration.version} diff --git a/distro/src/main/assembly/hbase-agent.xml b/distro/src/main/assembly/hbase-agent.xml index 37e2903a46..10ada9c1c3 100644 --- a/distro/src/main/assembly/hbase-agent.xml +++ b/distro/src/main/assembly/hbase-agent.xml @@ -107,8 +107,8 @@ org.apache.commons:commons-lang3:jar:${commons.lang3.version} org.apache.commons:commons-compress:jar:${commons.compress.version} org.slf4j:slf4j-api:jar:${slf4j-api.version} - org.apache.hadoop:hadoop-common:jar:${hadoop.version} - org.apache.hadoop:hadoop-auth:jar:${hadoop.version} + org.apache.hadoop:hadoop-client-api:jar:${hadoop.version} + org.apache.hadoop:hadoop-client-runtime:jar:${hadoop.version} org.codehaus.woodstox:stax2-api com.fasterxml.woodstox:woodstox-core org.apache.hadoop.thirdparty:hadoop-shaded-guava:jar:${hadoop-shaded-guava.version} diff --git a/distro/src/main/assembly/hdfs-agent.xml b/distro/src/main/assembly/hdfs-agent.xml index 8b133d993f..b5cba5bcf3 100644 --- a/distro/src/main/assembly/hdfs-agent.xml +++ b/distro/src/main/assembly/hdfs-agent.xml @@ -58,8 +58,8 @@ commons-lang:commons-lang commons-logging:commons-logging:jar:${commons.logging.version} org.slf4j:slf4j-api:jar:${slf4j-api.version} - org.apache.hadoop:hadoop-common:jar:${hadoop.version} - org.apache.hadoop:hadoop-auth:jar:${hadoop.version} + org.apache.hadoop:hadoop-client-api:jar:${hadoop.version} + org.apache.hadoop:hadoop-client-runtime:jar:${hadoop.version} org.codehaus.woodstox:stax2-api com.fasterxml.woodstox:woodstox-core org.apache.hadoop.thirdparty:hadoop-shaded-guava:jar:${hadoop-shaded-guava.version} diff --git a/distro/src/main/assembly/hive-agent.xml b/distro/src/main/assembly/hive-agent.xml index 9b9bc5b3af..22e31e47f8 100644 --- a/distro/src/main/assembly/hive-agent.xml +++ b/distro/src/main/assembly/hive-agent.xml @@ -99,8 +99,9 @@ commons-lang:commons-lang commons-logging:commons-logging:jar:${commons.logging.version} org.slf4j:slf4j-api:jar:${slf4j-api.version} - org.apache.hadoop:hadoop-common:jar:${hadoop.version} - org.apache.hadoop:hadoop-auth:jar:${hadoop.version} + org.apache.hadoop:hadoop-client-api:jar:${hadoop.version} + org.apache.hadoop:hadoop-client-runtime:jar:${hadoop.version} + org.apache.htrace:htrace-core4 org.codehaus.woodstox:stax2-api com.fasterxml.woodstox:woodstox-core org.apache.hadoop.thirdparty:hadoop-shaded-guava:jar:${hadoop-shaded-guava.version} diff --git a/distro/src/main/assembly/kms.xml b/distro/src/main/assembly/kms.xml index 46c0edb00b..779267ae1b 100755 --- a/distro/src/main/assembly/kms.xml +++ b/distro/src/main/assembly/kms.xml @@ -38,9 +38,9 @@ ews/webapp/WEB-INF/lib/ false - org.apache.hadoop:hadoop-common:jar:${hadoop.version} + org.apache.hadoop:hadoop-client-api:jar:${hadoop.version} + org.apache.hadoop:hadoop-client-runtime:jar:${hadoop.version} org.apache.hadoop.thirdparty:hadoop-shaded-guava:jar:${hadoop-shaded-guava.version} - org.apache.hadoop:hadoop-auth:jar:${hadoop.version} org.eclipse.persistence:eclipselink org.eclipse.persistence:javax.persistence com.googlecode.log4jdbc:log4jdbc @@ -69,7 +69,7 @@ org.apache.commons:commons-math3 commons-net:commons-net:jar:${commons.net.version} org.apache.curator:curator-recipes - org.apache.hadoop:hadoop-annotations + org.apache.hadoop:hadoop-annotations:jar:${hadoop.version} org.apache.httpcomponents:httpcore javax.xml.bind:jaxb-api com.sun.xml.bind:jaxb-impl @@ -85,7 +85,6 @@ org.apache.httpcomponents:httpmime:jar:${httpcomponents.httpmime.version} org.apache.httpcomponents:httpclient:jar:${kms.httpcomponents.httpclient.version} org.noggit:noggit:jar:${noggit.version} - org.apache.hadoop:hadoop-hdfs:jar:${hadoop.version} com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:jar:${fasterxml.jackson.version} com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:jar:${fasterxml.jackson.version} org.apache.kerby:kerb-core:jar:${kerby.version} @@ -209,9 +208,8 @@ org.eclipse.jdt.core.compiler:ecj:jar:P20140317-1600 org.eclipse.jetty:jetty-client:jar:${jetty-client.version} com.google.protobuf:protobuf-java:jar:${gcp.protobuf-java.version} - org.apache.hadoop:hadoop-hdfs:jar:${hadoop.version} - org.apache.hadoop:hadoop-common:jar:${hadoop.version} - org.apache.hadoop:hadoop-auth:jar:${hadoop.version} + org.apache.hadoop:hadoop-client-api:jar:${hadoop.version} + org.apache.hadoop:hadoop-client-runtime:jar:${hadoop.version} org.apache.solr:solr-solrj:jar:${solr.version} org.apache.ranger:ranger-plugins-common com.kstruct:gethostname4j:jar:${kstruct.gethostname4j.version} @@ -341,8 +339,8 @@ commons-lang:commons-lang:jar:${commons.lang.version} commons-logging:commons-logging:jar:${commons.logging.version} org.slf4j:slf4j-api:jar:${slf4j-api.version} - org.apache.hadoop:hadoop-common:jar:${hadoop.version} - org.apache.hadoop:hadoop-auth:jar:${hadoop.version} + org.apache.hadoop:hadoop-client-api:jar:${hadoop.version} + org.apache.hadoop:hadoop-client-runtime:jar:${hadoop.version} org.apache.ranger:ranger-plugins-cred org.apache.ranger:credentialbuilder org.codehaus.woodstox:stax2-api @@ -387,8 +385,8 @@ commons-lang:commons-lang commons-logging:commons-logging org.slf4j:slf4j-api - org.apache.hadoop:hadoop-common - org.apache.hadoop:hadoop-auth + org.apache.hadoop:hadoop-client-api:jar:${hadoop.version} + org.apache.hadoop:hadoop-client-runtime:jar:${hadoop.version} org.codehaus.woodstox:stax2-api com.fasterxml.woodstox:woodstox-core org.apache.commons:commons-compress:jar:${commons.compress.version} diff --git a/distro/src/main/assembly/knox-agent.xml b/distro/src/main/assembly/knox-agent.xml index a7906fe4f8..e1536c5a5a 100644 --- a/distro/src/main/assembly/knox-agent.xml +++ b/distro/src/main/assembly/knox-agent.xml @@ -62,8 +62,8 @@ org.apache.httpcomponents:httpcore:jar:${httpcomponents.httpcore.version} org.noggit:noggit:jar:${noggit.version} com.google.protobuf:protobuf-java:jar:${protobuf-java.version} - org.apache.hadoop:hadoop-hdfs:jar:${hadoop.version} - org.apache.hadoop:hadoop-common:jar:${hadoop.version} + org.apache.hadoop:hadoop-client-api:jar:${hadoop.version} + org.apache.hadoop:hadoop-client-runtime:jar:${hadoop.version} com.fasterxml.jackson.core:jackson-annotations:jar:${fasterxml.jackson.version} com.fasterxml.jackson.core:jackson-core:jar:${fasterxml.jackson.version} com.fasterxml.jackson.core:jackson-databind:jar:${fasterxml.jackson.version} @@ -111,8 +111,8 @@ commons-lang:commons-lang commons-logging:commons-logging:jar:${commons.logging.version} org.slf4j:slf4j-api:jar:${slf4j-api.version} - org.apache.hadoop:hadoop-common:jar:${hadoop.version} - org.apache.hadoop:hadoop-auth:jar:${hadoop.version} + org.apache.hadoop:hadoop-client-api:jar:${hadoop.version} + org.apache.hadoop:hadoop-client-runtime:jar:${hadoop.version} org.codehaus.woodstox:stax2-api com.fasterxml.woodstox:woodstox-core org.apache.commons:commons-compress:jar:${commons.compress.version} diff --git a/distro/src/main/assembly/plugin-atlas.xml b/distro/src/main/assembly/plugin-atlas.xml index 59b229c18e..25c5333e67 100644 --- a/distro/src/main/assembly/plugin-atlas.xml +++ b/distro/src/main/assembly/plugin-atlas.xml @@ -116,8 +116,8 @@ commons-lang:commons-lang:jar:${commons.lang.version} commons-logging:commons-logging org.slf4j:slf4j-api:jar:${slf4j-api.version} - org.apache.hadoop:hadoop-common:jar:${hadoop.version} - org.apache.hadoop:hadoop-auth:jar:${hadoop.version} + org.apache.hadoop:hadoop-client-api:jar:${hadoop.version} + org.apache.hadoop:hadoop-client-runtime:jar:${hadoop.version} org.apache.ranger:ranger-plugins-cred org.apache.ranger:credentialbuilder com.fasterxml.woodstox:woodstox-core:jar:${fasterxml.woodstox.version} diff --git a/distro/src/main/assembly/plugin-elasticsearch.xml b/distro/src/main/assembly/plugin-elasticsearch.xml index 64a68b649c..f3a409c1c8 100644 --- a/distro/src/main/assembly/plugin-elasticsearch.xml +++ b/distro/src/main/assembly/plugin-elasticsearch.xml @@ -60,10 +60,8 @@ 644 commons-configuration:commons-configuration:jar:${commons.configuration.version} - org.apache.hadoop:hadoop-common:jar:${hadoop.version} - org.apache.hadoop:hadoop-common-plus:jar:${hadoop.version} - org.apache.hadoop:hadoop-auth:jar:${hadoop.version} - org.apache.hadoop:hadoop-hdfs:jar:${hadoop.version} + org.apache.hadoop:hadoop-client-api:jar:${hadoop.version} + org.apache.hadoop:hadoop-client-runtime:jar:${hadoop.version} com.google.code.gson:gson org.eclipse.jetty:jetty-client:jar:${jetty-client.version} com.sun.jersey:jersey-bundle @@ -126,8 +124,8 @@ commons-lang:commons-lang:jar:${commons.lang.version} commons-logging:commons-logging:jar:${commons.logging.version} org.slf4j:slf4j-api:jar:${slf4j-api.version} - org.apache.hadoop:hadoop-common:jar:${hadoop.version} - org.apache.hadoop:hadoop-auth:jar:${hadoop.version} + org.apache.hadoop:hadoop-client-api:jar:${hadoop.version} + org.apache.hadoop:hadoop-client-runtime:jar:${hadoop.version} org.apache.ranger:ranger-plugins-cred org.apache.ranger:credentialbuilder org.apache.hadoop.thirdparty:hadoop-shaded-guava:jar:${hadoop-shaded-guava.version} diff --git a/distro/src/main/assembly/plugin-kafka.xml b/distro/src/main/assembly/plugin-kafka.xml index 4fe600cd91..20b4a0eee4 100644 --- a/distro/src/main/assembly/plugin-kafka.xml +++ b/distro/src/main/assembly/plugin-kafka.xml @@ -50,10 +50,8 @@ 644 org.apache.commons:commons-configuration2:jar:${commons.configuration.version} - org.apache.hadoop:hadoop-common:jar:${hadoop.version} - org.apache.hadoop:hadoop-common-plus:jar:${hadoop.version} - org.apache.hadoop:hadoop-auth:jar:${hadoop.version} - org.apache.hadoop:hadoop-hdfs:jar:${hadoop.version} + org.apache.hadoop:hadoop-client-api:jar:${hadoop.version} + org.apache.hadoop:hadoop-client-runtime:jar:${hadoop.version} com.google.code.gson:gson org.eclipse.jetty:jetty-client:jar:${jetty-client.version} commons-collections:commons-collections @@ -121,8 +119,8 @@ commons-lang:commons-lang:jar:${commons.lang.version} commons-logging:commons-logging org.slf4j:slf4j-api:jar:${slf4j-api.version} - org.apache.hadoop:hadoop-common:jar:${hadoop.version} - org.apache.hadoop:hadoop-auth:jar:${hadoop.version} + org.apache.hadoop:hadoop-client-api:jar:${hadoop.version} + org.apache.hadoop:hadoop-client-runtime:jar:${hadoop.version} org.apache.ranger:ranger-plugins-cred org.apache.ranger:credentialbuilder org.codehaus.woodstox:stax2-api diff --git a/distro/src/main/assembly/plugin-kms.xml b/distro/src/main/assembly/plugin-kms.xml index 605bdeff9b..b53aa1328c 100755 --- a/distro/src/main/assembly/plugin-kms.xml +++ b/distro/src/main/assembly/plugin-kms.xml @@ -54,7 +54,8 @@ 644 org.apache.commons:commons-configuration2:jar:${commons.configuration.version} - org.apache.hadoop:hadoop-common:jar:${hadoop.version} + org.apache.hadoop:hadoop-client-api:jar:${hadoop.version} + org.apache.hadoop:hadoop-client-runtime:jar:${hadoop.version} com.google.code.gson:gson org.eclipse.jetty:jetty-client:jar:${jetty-client.version} commons-collections:commons-collections @@ -99,8 +100,8 @@ commons-lang:commons-lang:jar:${commons.lang.version} commons-logging:commons-logging:jar:${commons.logging.version} org.slf4j:slf4j-api:jar:${slf4j-api.version} - org.apache.hadoop:hadoop-common:jar:${hadoop.version} - org.apache.hadoop:hadoop-auth:jar:${hadoop.version} + org.apache.hadoop:hadoop-client-api:jar:${hadoop.version} + org.apache.hadoop:hadoop-client-runtime:jar:${hadoop.version} org.apache.ranger:ranger-plugins-cred org.apache.ranger:credentialbuilder org.codehaus.woodstox:stax2-api diff --git a/distro/src/main/assembly/plugin-kylin.xml b/distro/src/main/assembly/plugin-kylin.xml index b4a0076163..3e21c56508 100644 --- a/distro/src/main/assembly/plugin-kylin.xml +++ b/distro/src/main/assembly/plugin-kylin.xml @@ -97,8 +97,8 @@ commons-lang:commons-lang:jar:${commons.lang.version} commons-logging:commons-logging:jar:${commons.logging.version} org.slf4j:slf4j-api:jar:${slf4j-api.version} - org.apache.hadoop:hadoop-common:jar:${hadoop.version} - org.apache.hadoop:hadoop-auth:jar:${hadoop.version} + org.apache.hadoop:hadoop-client-api:jar:${hadoop.version} + org.apache.hadoop:hadoop-client-runtime:jar:${hadoop.version} org.apache.ranger:ranger-plugins-cred org.apache.ranger:credentialbuilder org.codehaus.woodstox:stax2-api diff --git a/distro/src/main/assembly/plugin-ozone.xml b/distro/src/main/assembly/plugin-ozone.xml index 5f1e6a5387..f5a05a2ebd 100644 --- a/distro/src/main/assembly/plugin-ozone.xml +++ b/distro/src/main/assembly/plugin-ozone.xml @@ -57,8 +57,8 @@ commons-lang:commons-lang commons-logging:commons-logging:jar:${commons.logging.version} org.slf4j:slf4j-api:jar:${slf4j-api.version} - org.apache.hadoop:hadoop-common:jar:${hadoop.version} - org.apache.hadoop:hadoop-auth:jar:${hadoop.version} + org.apache.hadoop:hadoop-client-api:jar:${hadoop.version} + org.apache.hadoop:hadoop-client-runtime:jar:${hadoop.version} com.fasterxml.woodstox:woodstox-core:jar:${fasterxml.woodstox.version} org.codehaus.woodstox:stax2-api:jar:${codehaus.woodstox.stax2api.version} com.sun.jersey:jersey-core diff --git a/distro/src/main/assembly/plugin-presto.xml b/distro/src/main/assembly/plugin-presto.xml index e4101237a2..4657951325 100644 --- a/distro/src/main/assembly/plugin-presto.xml +++ b/distro/src/main/assembly/plugin-presto.xml @@ -84,10 +84,8 @@ com.google.protobuf:protobuf-java:jar:${protobuf-java.version} com.google.re2j:re2j:jar:${presto.re2j.version} org.apache.commons:commons-configuration2:jar:${commons.configuration.version} - org.apache.hadoop:hadoop-common:jar:${hadoop.version} - org.apache.hadoop:hadoop-common-plus:jar:${hadoop.version} - org.apache.hadoop:hadoop-auth:jar:${hadoop.version} - org.apache.hadoop:hadoop-hdfs:jar:${hadoop.version} + org.apache.hadoop:hadoop-client-api:jar:${hadoop.version} + org.apache.hadoop:hadoop-client-runtime:jar:${hadoop.version} org.apache.ranger:ranger-plugins-cred org.apache.ranger:credentialbuilder org.codehaus.woodstox:stax2-api @@ -142,8 +140,8 @@ commons-lang:commons-lang commons-logging:commons-logging:jar:${commons.logging.version} org.slf4j:slf4j-api:jar:${slf4j-api.version} - org.apache.hadoop:hadoop-common:jar:${hadoop.version} - org.apache.hadoop:hadoop-auth:jar:${hadoop.version} + org.apache.hadoop:hadoop-client-api:jar:${hadoop.version} + org.apache.hadoop:hadoop-client-runtime:jar:${hadoop.version} org.codehaus.woodstox:stax2-api com.fasterxml.woodstox:woodstox-core org.apache.hadoop.thirdparty:hadoop-shaded-guava:jar:${hadoop-shaded-guava.version} diff --git a/distro/src/main/assembly/plugin-schema-registry.xml b/distro/src/main/assembly/plugin-schema-registry.xml index 03f01aec75..df7195c9c7 100644 --- a/distro/src/main/assembly/plugin-schema-registry.xml +++ b/distro/src/main/assembly/plugin-schema-registry.xml @@ -1,21 +1,21 @@ + - - + schema-registry-plugin jar diff --git a/distro/src/main/assembly/plugin-solr.xml b/distro/src/main/assembly/plugin-solr.xml index b1b1104211..fd6cb38d17 100644 --- a/distro/src/main/assembly/plugin-solr.xml +++ b/distro/src/main/assembly/plugin-solr.xml @@ -97,8 +97,8 @@ commons-lang:commons-lang:jar:${commons.lang.version} commons-logging:commons-logging org.slf4j:slf4j-api:jar:${slf4j-api.version} - org.apache.hadoop:hadoop-common:jar:${hadoop.version} - org.apache.hadoop:hadoop-auth:jar:${hadoop.version} + org.apache.hadoop:hadoop-client-api:jar:${hadoop.version} + org.apache.hadoop:hadoop-client-runtime:jar:${hadoop.version} org.apache.ranger:ranger-plugins-cred org.apache.ranger:credentialbuilder org.apache.ranger:ranger-solr-plugin diff --git a/distro/src/main/assembly/plugin-sqoop.xml b/distro/src/main/assembly/plugin-sqoop.xml index 2230d90672..a229f92fef 100644 --- a/distro/src/main/assembly/plugin-sqoop.xml +++ b/distro/src/main/assembly/plugin-sqoop.xml @@ -102,8 +102,8 @@ commons-lang:commons-lang:jar:${commons.lang.version} commons-logging:commons-logging:jar:${commons.logging.version} org.slf4j:slf4j-api:jar:${slf4j-api.version} - org.apache.hadoop:hadoop-common:jar:${hadoop.version} - org.apache.hadoop:hadoop-auth:jar:${hadoop.version} + org.apache.hadoop:hadoop-client-api:jar:${hadoop.version} + org.apache.hadoop:hadoop-client-runtime:jar:${hadoop.version} org.apache.ranger:ranger-plugins-cred org.apache.ranger:credentialbuilder org.codehaus.woodstox:stax2-api diff --git a/distro/src/main/assembly/plugin-trino.xml b/distro/src/main/assembly/plugin-trino.xml index 16e2ddec7d..9fa563dc62 100644 --- a/distro/src/main/assembly/plugin-trino.xml +++ b/distro/src/main/assembly/plugin-trino.xml @@ -76,10 +76,8 @@ com.google.protobuf:protobuf-java:jar:${protobuf-java.version} com.google.re2j:re2j:jar:${trino.re2j.version} org.apache.commons:commons-configuration2:jar:${commons.configuration.version} - org.apache.hadoop:hadoop-common:jar:${hadoop.version} - org.apache.hadoop:hadoop-common-plus:jar:${hadoop.version} - org.apache.hadoop:hadoop-auth:jar:${hadoop.version} - org.apache.hadoop:hadoop-hdfs:jar:${hadoop.version} + org.apache.hadoop:hadoop-client-api:jar:${hadoop.version} + org.apache.hadoop:hadoop-client-runtime:jar:${hadoop.version} org.apache.ranger:ranger-plugins-cred org.apache.ranger:credentialbuilder org.codehaus.woodstox:stax2-api @@ -137,8 +135,8 @@ org.apache.commons:commons-compress:jar:${commons.compress.version} commons-logging:commons-logging:jar:${commons.logging.version} org.slf4j:slf4j-api:jar:${slf4j-api.version} - org.apache.hadoop:hadoop-common:jar:${hadoop.version} - org.apache.hadoop:hadoop-auth:jar:${hadoop.version} + org.apache.hadoop:hadoop-client-api:jar:${hadoop.version} + org.apache.hadoop:hadoop-client-runtime:jar:${hadoop.version} org.codehaus.woodstox:stax2-api com.fasterxml.woodstox:woodstox-core org.apache.hadoop.thirdparty:hadoop-shaded-guava:jar:${hadoop-shaded-guava.version} diff --git a/distro/src/main/assembly/plugin-yarn.xml b/distro/src/main/assembly/plugin-yarn.xml index 98e2b39cb2..714f22e24c 100644 --- a/distro/src/main/assembly/plugin-yarn.xml +++ b/distro/src/main/assembly/plugin-yarn.xml @@ -104,8 +104,8 @@ commons-lang:commons-lang:jar:${commons.lang.version} commons-logging:commons-logging:jar:${commons.logging.version} org.slf4j:slf4j-api:jar:${slf4j-api.version} - org.apache.hadoop:hadoop-common:jar:${hadoop.version} - org.apache.hadoop:hadoop-auth:jar:${hadoop.version} + org.apache.hadoop:hadoop-client-api:jar:${hadoop.version} + org.apache.hadoop:hadoop-client-runtime:jar:${hadoop.version} org.apache.ranger:ranger-plugins-cred org.apache.ranger:credentialbuilder org.codehaus.woodstox:stax2-api diff --git a/distro/src/main/assembly/ranger-tools.xml b/distro/src/main/assembly/ranger-tools.xml index f12159ff9f..72d9b9b550 100644 --- a/distro/src/main/assembly/ranger-tools.xml +++ b/distro/src/main/assembly/ranger-tools.xml @@ -45,10 +45,8 @@ org.apache.commons:commons-lang3:jar:${commons.lang3.version} log4j:log4j org.apache.commons:commons-configuration2:jar:${commons.configuration.version} - org.apache.hadoop:hadoop-common:jar:${hadoop.version} - org.apache.hadoop:hadoop-common-plus:jar:${hadoop.version} - org.apache.hadoop:hadoop-auth:jar:${hadoop.version} - org.apache.hadoop:hadoop-hdfs:jar:${hadoop.version} + org.apache.hadoop:hadoop-client-api:jar:${hadoop.version} + org.apache.hadoop:hadoop-client-runtime:jar:${hadoop.version} org.eclipse.persistence:eclipselink org.eclipse.persistence:javax.persistence commons-collections:commons-collections diff --git a/distro/src/main/assembly/sample-client.xml b/distro/src/main/assembly/sample-client.xml index 56f3677c2a..6df9bfdfbb 100644 --- a/distro/src/main/assembly/sample-client.xml +++ b/distro/src/main/assembly/sample-client.xml @@ -48,8 +48,8 @@ org.slf4j:slf4j-log4j12 log4j:log4j org.apache.commons:commons-configuration2:jar:${commons.configuration.version} - org.apache.hadoop:hadoop-common:jar:${hadoop.version} - org.apache.hadoop:hadoop-auth:jar:${hadoop.version} + org.apache.hadoop:hadoop-client-api:jar:${hadoop.version} + org.apache.hadoop:hadoop-client-runtime:jar:${hadoop.version} org.eclipse.persistence:eclipselink org.eclipse.persistence:javax.persistence commons-collections:commons-collections diff --git a/distro/src/main/assembly/storm-agent.xml b/distro/src/main/assembly/storm-agent.xml index 083bbbf225..01b559965b 100644 --- a/distro/src/main/assembly/storm-agent.xml +++ b/distro/src/main/assembly/storm-agent.xml @@ -58,8 +58,8 @@ false commons-configuration:commons-configuration - org.apache.hadoop:hadoop-common - org.apache.hadoop:hadoop-common-plus + org.apache.hadoop:hadoop-client-api:jar:${hadoop.version} + org.apache.hadoop:hadoop-client-runtime:jar:${hadoop.version} com.google.code.gson:gson org.eclipse.jetty:jetty-client:jar:${jetty-client.version} commons-collections:commons-collections @@ -69,7 +69,7 @@ org.apache.httpcomponents:httpmime:jar:${httpcomponents.httpmime.version} org.noggit:noggit:jar:${noggit.version} com.google.protobuf:protobuf-java:jar:${protobuf-java.version} - org.apache.hadoop:hadoop-hdfs + org.apache.hadoop:hadoop-hdfs:jar:${hadoop.version} org.apache.htrace:htrace-core:jar:${htrace-core.version} commons-cli:commons-cli:jar:${commons.cli.version} commons-lang:commons-lang:jar:${commons.lang.version} @@ -118,8 +118,8 @@ commons-lang:commons-lang:jar:${commons.lang.version} commons-logging:commons-logging org.slf4j:slf4j-api:jar:${slf4j-api.version} - org.apache.hadoop:hadoop-common:jar - org.apache.hadoop:hadoop-auth:jar + org.apache.hadoop:hadoop-client-api:jar:${hadoop.version} + org.apache.hadoop:hadoop-client-runtime:jar:${hadoop.version} org.apache.ranger:ranger-plugins-cred org.apache.ranger:credentialbuilder org.apache.hadoop.thirdparty:hadoop-shaded-guava:jar:${hadoop-shaded-guava.version} diff --git a/distro/src/main/assembly/tagsync.xml b/distro/src/main/assembly/tagsync.xml index da30a2d827..7996b865a1 100644 --- a/distro/src/main/assembly/tagsync.xml +++ b/distro/src/main/assembly/tagsync.xml @@ -50,8 +50,8 @@ org.apache.atlas:atlas-client-v2:jar:${atlas.version} org.apache.atlas:atlas-client-common:jar:${atlas.version} org.apache.atlas:atlas-common:jar:${atlas.version} - org.apache.hadoop:hadoop-auth - org.apache.hadoop:hadoop-common + org.apache.hadoop:hadoop-client-api:jar:${hadoop.version} + org.apache.hadoop:hadoop-client-runtime:jar:${hadoop.version} org.apache.commons:commons-compress org.apache.kafka:kafka-clients:jar:${kafka.version} org.apache.ranger:credentialbuilder diff --git a/distro/src/main/assembly/usersync.xml b/distro/src/main/assembly/usersync.xml index e6874a29a6..1eee30ea87 100644 --- a/distro/src/main/assembly/usersync.xml +++ b/distro/src/main/assembly/usersync.xml @@ -46,9 +46,10 @@ commons-collections:commons-collections commons-lang:commons-lang commons-logging:commons-logging - org.apache.hadoop:hadoop-auth + org.apache.hadoop:hadoop-auth:jar:${hadoop.version} org.slf4j:slf4j-api:jar:${slf4j.version} - org.apache.hadoop:hadoop-common + org.apache.hadoop:hadoop-client-api:jar:${hadoop.version} + org.apache.hadoop:hadoop-client-runtime:jar:${hadoop.version} org.apache.commons:commons-csv org.apache.ranger:credentialbuilder org.apache.ranger:ranger-util diff --git a/embeddedwebserver/pom.xml b/embeddedwebserver/pom.xml index f94a37815f..c4fa87160e 100644 --- a/embeddedwebserver/pom.xml +++ b/embeddedwebserver/pom.xml @@ -108,6 +108,11 @@ commons-lang3 ${commons.lang3.version} + + com.google.protobuf + protobuf-java + ${protobuf-java.version} + org.apache.commons commons-compress diff --git a/hbase-agent/pom.xml b/hbase-agent/pom.xml index e9a603af6d..a31dfcd60b 100644 --- a/hbase-agent/pom.xml +++ b/hbase-agent/pom.xml @@ -169,37 +169,6 @@ httpcore ${httpcomponents.httpcore.version} - - org.apache.hadoop - hadoop-common - ${hadoop.version} - - - org.apache.commons - commons-text - - - org.apache.commons - commons-configuration2 - - - log4j - * - - - org.slf4j - * - - - org.apache.commons - commons-compress - - - org.apache.zookeeper - zookeeper - - - org.apache.commons commons-text @@ -355,71 +324,34 @@ org.apache.hadoop - hadoop-minicluster + hadoop-client-api ${hadoop.version} - test - - - io.netty - netty-all - - - io.netty - netty - - - org.slf4j - * - - - log4j - * - - - org.apache.avro - avro - - org.apache.hadoop - hadoop-auth + hadoop-client-runtime ${hadoop.version} - - - log4j - * - - - org.slf4j - * - - - test org.apache.hadoop - hadoop-hdfs + hadoop-client-minicluster ${hadoop.version} test - - - io.netty - netty-all - - - io.netty - netty - - - log4j - * - - - org.slf4j - * - - + + + org.apache.commons + commons-text + ${commons.text.version} + + + org.apache.commons + commons-configuration2 + ${commons.configuration.version} + + + org.glassfish + javax.el + ${javax.el.version} org.slf4j diff --git a/hdfs-agent/pom.xml b/hdfs-agent/pom.xml index 901c5a22af..f513df79fe 100644 --- a/hdfs-agent/pom.xml +++ b/hdfs-agent/pom.xml @@ -31,6 +31,11 @@ .. + + javax.servlet + javax.servlet-api + ${javax.servlet.version} + javax.servlet javax.servlet-api @@ -71,16 +76,6 @@ - - org.apache.commons - commons-text - ${commons.text.version} - - - org.apache.commons - commons-configuration2 - ${commons.configuration.version} - org.apache.hadoop hadoop-hdfs @@ -94,6 +89,10 @@ io.netty netty + + ch.qos.reload4j + * + log4j * @@ -104,6 +103,16 @@ + + org.apache.commons + commons-text + ${commons.text.version} + + + org.apache.commons + commons-configuration2 + ${commons.configuration.version} + io.netty netty-all diff --git a/hive-agent/pom.xml b/hive-agent/pom.xml index 530aea73e3..0cb8d98c72 100644 --- a/hive-agent/pom.xml +++ b/hive-agent/pom.xml @@ -240,23 +240,15 @@ org.apache.hadoop - hadoop-mapreduce-client-core + hadoop-client-api + ${hadoop.version} + test + + + org.apache.hadoop + hadoop-client-runtime ${hadoop.version} test - - - io.netty - netty - - - log4j - * - - - org.apache.avro - avro - - diff --git a/kms/pom.xml b/kms/pom.xml index 4a901843be..c0fc088e7c 100644 --- a/kms/pom.xml +++ b/kms/pom.xml @@ -54,46 +54,18 @@ org.apache.hadoop - hadoop-common + hadoop-client-api + ${hadoop.version} + + + org.apache.hadoop + hadoop-client-runtime + ${hadoop.version} + + + org.apache.hadoop + hadoop-client-minicluster ${hadoop.version} - - - log4j - * - - - org.slf4j - * - - - org.apache.commons - commons-compress - - - org.apache.commons - commons-lang3 - - - com.google.protobuf - protobuf-java - - - org.apache.commons - commons-text - - - org.apache.commons - commons-configuration2 - - - log4j - * - - - org.slf4j - slf4j-log4j12 - - org.apache.commons @@ -176,30 +148,6 @@ jsr250-api provided - - org.apache.hadoop - hadoop-auth - ${hadoop.version} - compile - - - com.nimbusds - nimbus-jose-jwt - - - net.minidev - json-smart - - - log4j - * - - - org.slf4j - * - - - com.sun.jersey jersey-core @@ -335,21 +283,6 @@ httpmime ${httpcomponents.httpmime.version} - - org.apache.hadoop - hadoop-hdfs - ${hadoop.version} - - - io.netty - netty - - - log4j - * - - - org.apache.ranger ranger-plugins-common diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java index ed37df6346..006d3d02f9 100644 --- a/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java +++ b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java @@ -32,14 +32,14 @@ import org.apache.hadoop.http.HtmlQuoting; import org.apache.ranger.kms.metrics.KMSMetrics; -import javax.servlet.FilterChain; -import javax.servlet.FilterConfig; -import javax.servlet.ServletException; -import javax.servlet.ServletRequest; -import javax.servlet.ServletResponse; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import javax.servlet.http.HttpServletResponseWrapper; +import org.apache.hadoop.shaded.javax.servlet.FilterChain; +import org.apache.hadoop.shaded.javax.servlet.FilterConfig; +import org.apache.hadoop.shaded.javax.servlet.ServletException; +import org.apache.hadoop.shaded.javax.servlet.ServletRequest; +import org.apache.hadoop.shaded.javax.servlet.ServletResponse; +import org.apache.hadoop.shaded.javax.servlet.http.HttpServletRequest; +import org.apache.hadoop.shaded.javax.servlet.http.HttpServletResponse; +import org.apache.hadoop.shaded.javax.servlet.http.HttpServletResponseWrapper; import java.io.IOException; import java.util.Map; import java.util.Properties; diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java index b5be765e6b..5e420cb281 100644 --- a/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java +++ b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java @@ -29,9 +29,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.ws.rs.core.Response; -import javax.ws.rs.ext.ExceptionMapper; -import javax.ws.rs.ext.Provider; +import org.apache.hadoop.shaded.javax.ws.rs.core.Response; +import org.apache.hadoop.shaded.javax.ws.rs.ext.ExceptionMapper; +import org.apache.hadoop.shaded.javax.ws.rs.ext.Provider; import java.io.IOException; diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJMXServlet.java b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJMXServlet.java index 6918015a90..7936fb5934 100644 --- a/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJMXServlet.java +++ b/kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJMXServlet.java @@ -20,8 +20,8 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.jmx.JMXJsonServlet; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; +import org.apache.hadoop.shaded.javax.servlet.http.HttpServletRequest; +import org.apache.hadoop.shaded.javax.servlet.http.HttpServletResponse; import java.io.IOException; diff --git a/knox-agent/pom.xml b/knox-agent/pom.xml index fb631725b0..7b8a5ab378 100644 --- a/knox-agent/pom.xml +++ b/knox-agent/pom.xml @@ -61,6 +61,10 @@ org.apache.zookeeper zookeeper + + javax.ws.rs + jsr311-api + @@ -114,26 +118,13 @@ org.apache.hadoop - hadoop-hdfs + hadoop-client-api + ${hadoop.version} + + + org.apache.hadoop + hadoop-client-runtime ${hadoop.version} - - - io.netty - netty-all - - - io.netty - netty - - - log4j - * - - - org.slf4j - * - - org.apache.httpcomponents diff --git a/plugin-kafka/pom.xml b/plugin-kafka/pom.xml index 631e47814d..b36892e0bf 100644 --- a/plugin-kafka/pom.xml +++ b/plugin-kafka/pom.xml @@ -59,26 +59,13 @@ org.apache.hadoop - hadoop-hdfs + hadoop-client-api + ${hadoop.version} + + + org.apache.hadoop + hadoop-client-runtime ${hadoop.version} - - - io.netty - netty-all - - - io.netty - netty - - - log4j - * - - - org.slf4j - * - - org.apache.httpcomponents diff --git a/plugin-nifi-registry/src/test/java/org/apache/ranger/services/nifi/registry/client/TestNiFiRegistryClient.java b/plugin-nifi-registry/src/test/java/org/apache/ranger/services/nifi/registry/client/TestNiFiRegistryClient.java index 7db646f4dd..71a6229805 100644 --- a/plugin-nifi-registry/src/test/java/org/apache/ranger/services/nifi/registry/client/TestNiFiRegistryClient.java +++ b/plugin-nifi-registry/src/test/java/org/apache/ranger/services/nifi/registry/client/TestNiFiRegistryClient.java @@ -18,7 +18,6 @@ */ package org.apache.ranger.services.nifi.registry.client; -import com.google.common.io.Resources; import com.sun.jersey.api.client.ClientResponse; import com.sun.jersey.api.client.WebResource; import org.apache.ranger.plugin.service.ResourceLookupContext; @@ -28,8 +27,10 @@ import org.mockito.Mockito; import javax.ws.rs.core.Response; +import java.io.BufferedReader; import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.InputStreamReader; import java.net.URL; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -45,8 +46,15 @@ public class TestNiFiRegistryClient { @Before public void setup() throws IOException { final URL responseFile = TestNiFiRegistryClient.class.getResource("/resources-response.json"); - final String resourcesResponse = Resources.toString(responseFile, StandardCharsets.UTF_8); - registryClient = new MockNiFiRegistryClient(resourcesResponse, 200); + StringBuilder resourcesResponse = new StringBuilder(); + try (BufferedReader reader = new BufferedReader(new InputStreamReader(responseFile.openStream(), StandardCharsets.UTF_8))) { + String line; + while ((line = reader.readLine()) != null) { + resourcesResponse.append(line).append("\n"); + } + } + + registryClient = new MockNiFiRegistryClient(resourcesResponse.toString(), 200); } @Test diff --git a/plugin-ozone/pom.xml b/plugin-ozone/pom.xml index 0c91e96590..bb0f39de41 100644 --- a/plugin-ozone/pom.xml +++ b/plugin-ozone/pom.xml @@ -62,26 +62,13 @@ limitations under the License. org.apache.hadoop - hadoop-hdfs + hadoop-client-api + ${hadoop.version} + + + org.apache.hadoop + hadoop-client-runtime ${hadoop.version} - - - io.netty - netty-all - - - io.netty - netty - - - log4j - * - - - org.slf4j - * - - org.apache.httpcomponents diff --git a/plugin-schema-registry/pom.xml b/plugin-schema-registry/pom.xml index 46cb1140b4..67b336695d 100644 --- a/plugin-schema-registry/pom.xml +++ b/plugin-schema-registry/pom.xml @@ -44,84 +44,20 @@ org.apache.hadoop - hadoop-hdfs + hadoop-client-api ${hadoop.version} - - - log4j - * - - - com.sun.jersey - jersey-core - - - com.sun.jersey - jersey-json - - - org.slf4j - * - - org.apache.hadoop - hadoop-client + hadoop-client-runtime ${hadoop.version} - - - log4j - * - - - org.slf4j - * - - - com.sun.jersey - jersey-core - - - javax.servlet - servlet-api - - - com.sun.jersey - jersey-client - - - org.apache.hadoop - hadoop-yarn-api - - - org.apache.hadoop - hadoop-yarn-common - - - org.apache.hadoop - hadoop-mapreduce-client - - - org.apache.hadoop - hadoop-mapreduce-client-app - - - org.apache.hadoop - hadoop-mapreduce-client-core - - - org.apache.hadoop - hadoop-mapreduce-client-jobclient - - - - org.slf4j - log4j-over-slf4j - ${slf4j.version} - test - + + org.slf4j + log4j-over-slf4j + ${slf4j.version} + test + org.apache.kafka kafka-clients @@ -131,10 +67,10 @@ log4j * - - org.slf4j - * - + + org.slf4j + * + diff --git a/plugin-yarn/pom.xml b/plugin-yarn/pom.xml index 3c36e71047..260a2a58ff 100644 --- a/plugin-yarn/pom.xml +++ b/plugin-yarn/pom.xml @@ -53,22 +53,12 @@ org.apache.hadoop - hadoop-yarn-common + hadoop-client-api ${hadoop.version} - - - log4j - * - - - org.slf4j - * - - org.apache.hadoop - hadoop-yarn-api + hadoop-client-runtime ${hadoop.version} diff --git a/ranger-authn/pom.xml b/ranger-authn/pom.xml index 9c0ad8dc59..77a1bcaf9a 100644 --- a/ranger-authn/pom.xml +++ b/ranger-authn/pom.xml @@ -33,32 +33,21 @@ + + javax.servlet + javax.servlet-api + ${javax.servlet.version} + + + + org.apache.hadoop + hadoop-client-api + ${hadoop.version} + org.apache.hadoop - hadoop-common + hadoop-client-runtime ${hadoop.version} - - - io.netty - netty-all - - - org.apache.commons - commons-text - - - org.apache.commons - commons-configuration2 - - - org.apache.commons - commons-compress - - - net.minidev - json-smart - - @@ -73,6 +62,12 @@ ${commons.configuration.version} + + org.slf4j + log4j-over-slf4j + ${slf4j.version} + + commons-lang commons-lang diff --git a/ranger-common-ha/pom.xml b/ranger-common-ha/pom.xml index 68aa6ea968..2a05a5d51e 100644 --- a/ranger-common-ha/pom.xml +++ b/ranger-common-ha/pom.xml @@ -79,30 +79,13 @@ org.apache.hadoop - hadoop-common + hadoop-client-api + ${hadoop.version} + + + org.apache.hadoop + hadoop-client-runtime ${hadoop.version} - - - net.minidev - json-smart - - - io.netty - netty-all - - - log4j - * - - - org.slf4j - * - - - org.apache.commons - commons-compress - - commons-lang diff --git a/ranger-examples/distro/src/main/assembly/plugin-sampleapp.xml b/ranger-examples/distro/src/main/assembly/plugin-sampleapp.xml index ff45067d0f..a812aea7b4 100644 --- a/ranger-examples/distro/src/main/assembly/plugin-sampleapp.xml +++ b/ranger-examples/distro/src/main/assembly/plugin-sampleapp.xml @@ -36,8 +36,8 @@ org.apache.ranger:ranger-plugins-audit org.apache.httpcomponents:httpmime:jar:${httpcomponent.httpmime.version} org.noggit:noggit:jar:${noggit.version} - org.apache.hadoop:hadoop-common:jar:${hadoop.version} - org.apache.hadoop:hadoop-auth:jar:${hadoop.version} + org.apache.hadoop:hadoop-client-api:jar:${hadoop.version} + org.apache.hadoop:hadoop-client-runtime:jar:${hadoop.version} commons-collections:commons-collections commons-configuration:commons-configuration:jar:${commons.configuration.version} commons-io:commons-io:jar:${commons.io.version} diff --git a/ranger-examples/distro/src/main/assembly/sample-client.xml b/ranger-examples/distro/src/main/assembly/sample-client.xml index 02bf867ac3..b2ba6a6f77 100644 --- a/ranger-examples/distro/src/main/assembly/sample-client.xml +++ b/ranger-examples/distro/src/main/assembly/sample-client.xml @@ -45,8 +45,8 @@ org.slf4j:slf4j-log4j12 log4j:log4j org.apache.commons:commons-configuration2:jar:${commons.configuration.version} - org.apache.hadoop:hadoop-common:jar:${hadoop.version} - org.apache.hadoop:hadoop-auth:jar:${hadoop.version} + org.apache.hadoop:hadoop-client-api:jar:${hadoop.version} + org.apache.hadoop:hadoop-client-runtime:jar:${hadoop.version} commons-collections:commons-collections com.sun.jersey:jersey-bundle commons-io:commons-io diff --git a/ranger-examples/plugin-sampleapp/pom.xml b/ranger-examples/plugin-sampleapp/pom.xml index 03e5a0a3ef..e8711ed9c3 100644 --- a/ranger-examples/plugin-sampleapp/pom.xml +++ b/ranger-examples/plugin-sampleapp/pom.xml @@ -39,42 +39,13 @@ org.apache.hadoop - hadoop-common + hadoop-client-api + ${hadoop.version} + + + org.apache.hadoop + hadoop-client-runtime ${hadoop.version} - - - io.netty - netty-handler - - - io.netty - netty-transport-native-epoll - - - org.apache.commons - commons-text - - - org.apache.commons - commons-configuration2 - - - log4j - * - - - org.slf4j - * - - - org.apache.commons - commons-compress - - - org.apache.zookeeper - zookeeper - - org.apache.commons diff --git a/ranger-hdfs-plugin-shim/pom.xml b/ranger-hdfs-plugin-shim/pom.xml index 6dc1aa3d86..d0cbb1173f 100644 --- a/ranger-hdfs-plugin-shim/pom.xml +++ b/ranger-hdfs-plugin-shim/pom.xml @@ -31,6 +31,26 @@ .. + + org.apache.hadoop + hadoop-client-api + ${hadoop.version} + + + org.apache.hadoop + hadoop-client-runtime + ${hadoop.version} + + + org.apache.commons + commons-text + ${commons.text.version} + + + org.apache.commons + commons-configuration2 + ${commons.configuration.version} + org.apache.hadoop hadoop-hdfs @@ -44,6 +64,10 @@ io.netty netty + + ch.qos.reload4j + * + log4j * diff --git a/ranger-metrics/pom.xml b/ranger-metrics/pom.xml index 2e72328ba0..d1edb36d4b 100644 --- a/ranger-metrics/pom.xml +++ b/ranger-metrics/pom.xml @@ -36,26 +36,13 @@ org.apache.hadoop - hadoop-common + hadoop-client-api + ${hadoop.version} + + + org.apache.hadoop + hadoop-client-runtime ${hadoop.version} - - - io.netty - netty-all - - - log4j - * - - - org.slf4j - * - - - org.apache.commons - commons-compress - - org.apache.ranger diff --git a/ranger-metrics/src/main/java/org/apache/ranger/metrics/sink/RangerMetricsJsonSink.java b/ranger-metrics/src/main/java/org/apache/ranger/metrics/sink/RangerMetricsJsonSink.java index 4e23120ddf..8191720f6a 100644 --- a/ranger-metrics/src/main/java/org/apache/ranger/metrics/sink/RangerMetricsJsonSink.java +++ b/ranger-metrics/src/main/java/org/apache/ranger/metrics/sink/RangerMetricsJsonSink.java @@ -24,7 +24,7 @@ import java.util.Objects; import java.util.Set; -import org.apache.commons.configuration2.SubsetConfiguration; +import org.apache.hadoop.shaded.org.apache.commons.configuration2.SubsetConfiguration; import org.apache.hadoop.metrics2.AbstractMetric; import org.apache.hadoop.metrics2.MetricType; import org.apache.hadoop.metrics2.MetricsRecord; diff --git a/ranger-metrics/src/main/java/org/apache/ranger/metrics/sink/RangerMetricsPrometheusSink.java b/ranger-metrics/src/main/java/org/apache/ranger/metrics/sink/RangerMetricsPrometheusSink.java index 141fa372a1..c3b4cc27aa 100644 --- a/ranger-metrics/src/main/java/org/apache/ranger/metrics/sink/RangerMetricsPrometheusSink.java +++ b/ranger-metrics/src/main/java/org/apache/ranger/metrics/sink/RangerMetricsPrometheusSink.java @@ -28,7 +28,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.regex.Pattern; -import org.apache.commons.configuration2.SubsetConfiguration; +import org.apache.hadoop.shaded.org.apache.commons.configuration2.SubsetConfiguration; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.metrics2.AbstractMetric; import org.apache.hadoop.metrics2.MetricType; diff --git a/ranger-metrics/src/main/java/org/apache/ranger/metrics/source/RangerMetricsContainerSource.java b/ranger-metrics/src/main/java/org/apache/ranger/metrics/source/RangerMetricsContainerSource.java index d380d9e66e..38308451b2 100644 --- a/ranger-metrics/src/main/java/org/apache/ranger/metrics/source/RangerMetricsContainerSource.java +++ b/ranger-metrics/src/main/java/org/apache/ranger/metrics/source/RangerMetricsContainerSource.java @@ -19,7 +19,7 @@ package org.apache.ranger.metrics.source; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.ranger.metrics.RangerMetricsInfo; import org.apache.ranger.server.tomcat.EmbeddedServer; import org.apache.ranger.server.tomcat.EmbeddedServerMetricsCollector; diff --git a/ranger-ozone-plugin-shim/pom.xml b/ranger-ozone-plugin-shim/pom.xml index 0ed739b228..1f965b465f 100644 --- a/ranger-ozone-plugin-shim/pom.xml +++ b/ranger-ozone-plugin-shim/pom.xml @@ -49,6 +49,26 @@ credentialbuilder ${project.version} + + org.apache.hadoop + hadoop-client-api + ${hadoop.version} + + + org.apache.hadoop + hadoop-client-runtime + ${hadoop.version} + + + org.apache.httpcomponents + httpcore + ${httpcomponents.httpcore.version} + + + org.bouncycastle + bcpkix-jdk15on + ${org.bouncycastle.bcpkix-jdk15on} + org.apache.ozone ozone-common diff --git a/ranger-presto-plugin-shim/pom.xml b/ranger-presto-plugin-shim/pom.xml index c78fafd20c..a831d92ca1 100644 --- a/ranger-presto-plugin-shim/pom.xml +++ b/ranger-presto-plugin-shim/pom.xml @@ -85,26 +85,13 @@ org.apache.hadoop - hadoop-hdfs + hadoop-client-api + ${hadoop.version} + + + org.apache.hadoop + hadoop-client-runtime ${hadoop.version} - - - io.netty - netty-all - - - io.netty - netty - - - log4j - * - - - org.slf4j - * - - diff --git a/ranger-storm-plugin-shim/pom.xml b/ranger-storm-plugin-shim/pom.xml index 98c985fd76..e4003823d6 100644 --- a/ranger-storm-plugin-shim/pom.xml +++ b/ranger-storm-plugin-shim/pom.xml @@ -61,6 +61,26 @@ credentialbuilder ${project.version} + + org.apache.hadoop + hadoop-client-api + ${hadoop.version} + + + org.apache.hadoop + hadoop-client-runtime + ${hadoop.version} + + + org.apache.commons + commons-text + ${commons.text.version} + + + org.apache.commons + commons-configuration2 + ${commons.configuration.version} + org.apache.ranger ranger-plugin-classloader diff --git a/ranger-util/pom.xml b/ranger-util/pom.xml index 6a4b581b62..a1d5913f74 100644 --- a/ranger-util/pom.xml +++ b/ranger-util/pom.xml @@ -28,7 +28,12 @@ org.apache.hadoop - hadoop-annotations + hadoop-client-api + ${hadoop.version} + + + org.apache.hadoop + hadoop-client-runtime ${hadoop.version} diff --git a/ranger-yarn-plugin-shim/pom.xml b/ranger-yarn-plugin-shim/pom.xml index 67026635f8..82fa652ab7 100644 --- a/ranger-yarn-plugin-shim/pom.xml +++ b/ranger-yarn-plugin-shim/pom.xml @@ -48,18 +48,13 @@ org.apache.hadoop - hadoop-yarn-common + hadoop-client-api + ${hadoop.version} + + + org.apache.hadoop + hadoop-client-runtime ${hadoop.version} - - - log4j - * - - - org.slf4j - * - - org.apache.ranger diff --git a/security-admin/pom.xml b/security-admin/pom.xml index f7f8a8175e..ea24a77076 100644 --- a/security-admin/pom.xml +++ b/security-admin/pom.xml @@ -415,7 +415,7 @@ zookeeper-jute - + com.googlecode.log4jdbc log4jdbc @@ -494,6 +494,16 @@ org.apache.ranger ranger-util ${project.version} + + + org.apache.hadoop + hadoop-client-api + + + org.apache.hadoop + hadoop-client-runtime + + org.apache.thrift @@ -515,11 +525,63 @@ protobuf-java ${protobuf-java.version} + + org.apache.hadoop + hadoop-auth + ${hadoop.version} + + + commons-codec + * + + + javax.servlet + * + + + com.nimbusds + * + + + net.minidev + * + + + org.apache.zookeeper + * + + + ch.qos.reload4j + * + + + log4j + * + + + org.slf4j + * + + + + + org.apache.hadoop + hadoop-annotations + ${hadoop.version} + org.apache.hadoop hadoop-common ${hadoop.version} + + org.apache.hadoop + hadoop-auth + + + org.apache.hadoop + hadoop-annotations + commons-httpclient commons-httpclient @@ -600,6 +662,10 @@ org.apache.commons commons-configuration2 + + ch.qos.reload4j + * + log4j * @@ -621,50 +687,49 @@ ${netty-all.version} - org.apache.hadoop - hadoop-hdfs - ${hadoop.version} + org.apache.ranger + unixauthclient + ${project.version} - com.sun.jersey - jersey-core - - - com.sun.jersey - jersey-server - - - com.google.guava - guava - - - io.netty - netty + org.apache.hadoop + hadoop-client-api - log4j - * - - - org.slf4j - * + org.apache.hadoop + hadoop-client-runtime - - org.apache.ranger - unixauthclient - ${project.version} - org.apache.ranger ranger-plugins-common ${project.version} + + + org.apache.hadoop + hadoop-client-api + + + org.apache.hadoop + hadoop-client-runtime + + org.apache.ranger ugsync-util ${project.version} + + + org.apache.hadoop + hadoop-client-api + + + org.apache.hadoop + hadoop-client-runtime + + com.nimbusds @@ -792,11 +857,31 @@ org.apache.ranger ranger-authn ${project.version} + + + org.apache.hadoop + hadoop-client-api + + + org.apache.hadoop + hadoop-client-runtime + + org.apache.ranger ranger-metrics ${project.version} + + + org.apache.hadoop + hadoop-client-api + + + org.apache.hadoop + hadoop-client-runtime + + diff --git a/storm-agent/pom.xml b/storm-agent/pom.xml index f2f6f3d5f1..341c344008 100644 --- a/storm-agent/pom.xml +++ b/storm-agent/pom.xml @@ -57,7 +57,11 @@ org.apache.hadoop - hadoop-common + hadoop-client-api + + + org.apache.hadoop + hadoop-client-runtime @@ -72,7 +76,11 @@ org.apache.hadoop - hadoop-common + hadoop-client-api + + + org.apache.hadoop + hadoop-client-runtime io.netty @@ -93,6 +101,14 @@ org.slf4j * + + org.apache.hadoop + hadoop-client-api + + + org.apache.hadoop + hadoop-client-runtime + @@ -102,61 +118,13 @@ org.apache.hadoop - hadoop-hdfs + hadoop-client-api ${hadoop.version} - - - io.netty - netty-all - - - io.netty - netty - - - log4j - * - - - org.slf4j - * - - org.apache.hadoop - hadoop-common + hadoop-client-runtime ${hadoop.version} - - - io.netty - netty-handler - - - io.netty - netty-transport-native-epoll - - - org.apache.commons - commons-text - - - org.apache.commons - commons-configuration2 - - - log4j - * - - - org.slf4j - * - - - org.apache.zookeeper - zookeeper - - org.apache.commons diff --git a/ugsync/pom.xml b/ugsync/pom.xml index 873b0605d0..a6543479b7 100644 --- a/ugsync/pom.xml +++ b/ugsync/pom.xml @@ -75,30 +75,13 @@ org.apache.hadoop - hadoop-auth + hadoop-client-api + ${hadoop.version} + + + org.apache.hadoop + hadoop-client-runtime ${hadoop.version} - - - log4j - * - - - org.slf4j - * - - - org.apache.curator - * - - - org.apache.zookeeper - * - - - net.minidev - json-smart - - org.apache.commons diff --git a/unixauthclient/pom.xml b/unixauthclient/pom.xml index 23e78e305f..ef0ce36ed6 100644 --- a/unixauthclient/pom.xml +++ b/unixauthclient/pom.xml @@ -74,29 +74,6 @@ commons-text ${commons.text.version} - - org.apache.hadoop - hadoop-auth - ${hadoop.version} - - - log4j - * - - - org.slf4j - * - - - net.minidev - json-smart - - - org.apache.zookeeper - zookeeper - - - org.slf4j log4j-over-slf4j diff --git a/unixauthservice/pom.xml b/unixauthservice/pom.xml index 17984666d0..41175e112e 100644 --- a/unixauthservice/pom.xml +++ b/unixauthservice/pom.xml @@ -81,26 +81,13 @@ org.apache.hadoop - hadoop-auth + hadoop-client-api + ${hadoop.version} + + + org.apache.hadoop + hadoop-client-runtime ${hadoop.version} - - - log4j - * - - - org.slf4j - * - - - net.minidev - json-smart - - - org.apache.zookeeper - zookeeper - - org.slf4j