tool
/
atlas
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ATLAS-3820: fix import-kafka.sh failures; updated docker setup for Kafka bridge 

(cherry picked from commit 2c2d9197e68b809198d6e9f1232473d1c2d9e960)
This commit is contained in:
Madhan Neethiraj 2022-11-15 14:22:04 -08:00
parent 7f9e980f6a
commit fcbc016451
7 changed files with 112 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
addons/kafka-bridge/pom.xml
View File

@ -125,7 +125,7 @@
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>12.0.1</version>
<version>${guava.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
@ -223,6 +223,21 @@
<artifactId>atlas-common</artifactId>
<version>${project.version}</version>
</artifactItem>
<artifactItem>
<groupId>commons-collections</groupId>
<artifactId>commons-collections</artifactId>
<version>${commons-collections.version}</version>
</artifactItem>
<artifactItem>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
<version>${commons-logging.version}</version>
</artifactItem>
<artifactItem>
<groupId>commons-lang</groupId>
<artifactId>commons-lang</artifactId>
<version>${commons-lang.version}</version>
</artifactItem>
<artifactItem>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
@ -233,11 +248,66 @@
<artifactId>jersey-multipart</artifactId>
<version>${jersey.version}</version>
</artifactItem>
<artifactItem>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>${jackson.databind.version}</version>
</artifactItem>
<artifactItem>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
<version>${jackson.version}</version>
</artifactItem>
<artifactItem>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
<version>${jackson.version}</version>
</artifactItem>
<artifactItem>
<groupId>com.fasterxml.jackson.jaxrs</groupId>
<artifactId>jackson-jaxrs-json-provider</artifactId>
<version>${jackson.version}</version>
</artifactItem>
<artifactItem>
<groupId>com.fasterxml.jackson.jaxrs</groupId>
<artifactId>jackson-jaxrs-base</artifactId>
<version>${jackson.version}</version>
</artifactItem>
<artifactItem>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-bundle</artifactId>
<version>${jersey.version}</version>
</artifactItem>
<artifactItem>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-auth</artifactId>
<version>${hadoop.version}</version>
</artifactItem>
<artifactItem>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>${guava.version}</version>
</artifactItem>
<artifactItem>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>${httpcomponents-httpclient.version}</version>
</artifactItem>
<artifactItem>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcore</artifactId>
<version>${httpcomponents-httpcore.version}</version>
</artifactItem>
<artifactItem>
<groupId>commons-configuration</groupId>
<artifactId>commons-configuration</artifactId>
<version>${commons-conf.version}</version>
</artifactItem>
<artifactItem>
<groupId>com.googlecode.json-simple</groupId>
<artifactId>json-simple</artifactId>
<version>${json-simple.version}</version>
</artifactItem>
</artifactItems>
</configuration>
</execution>

18
addons/kafka-bridge/src/bin/import-kafka.sh
View File

@ -99,25 +99,12 @@ fi
KAFKA_CP="${KAFKA_CONF}"
for i in "${KAFKA_HOME}/libs/kafka-clients"*.jar "${KAFKA_HOME}/libs/slf4j-api"*.jar; do
for i in "${KAFKA_HOME}/libs/kafka-clients"*.jar "${KAFKA_HOME}/libs/slf4j-"*.jar "${KAFKA_HOME}/libs/log4j-"*.jar "${KAFKA_HOME}/libs/commons-"*.jar "${KAFKA_HOME}/libs/jackson-module-jaxb-annotations"*.jar; do
KAFKA_CP="${KAFKA_CP}:$i"
done
#Add hadoop conf in classpath
if [ ! -z "$HADOOP_CLASSPATH" ]; then
HADOOP_CP=$HADOOP_CLASSPATH
elif [ ! -z "$HADOOP_HOME" ]; then
HADOOP_CP=`$HADOOP_HOME/bin/hadoop classpath`
elif [ $(command -v hadoop) ]; then
HADOOP_CP=`hadoop classpath`
#echo $HADOOP_CP
else
echo "Environment variable HADOOP_CLASSPATH or HADOOP_HOME need to be set"
exit 1
fi
CP="${ATLASCPPATH}:${HADOOP_CP}:${KAFKA_CP}"
CP="${ATLASCPPATH}:${KAFKA_CP}"
# If running in cygwin, convert pathnames and classpath to Windows format.
if [ "${CYGWIN}" == "true" ]
@ -125,7 +112,6 @@ then
ATLAS_LOG_DIR=`cygpath -w ${ATLAS_LOG_DIR}`
LOGFILE=`cygpath -w ${LOGFILE}`
KAFKA_CP=`cygpath -w ${KAFKA_CP}`
HADOOP_CP=`cygpath -w ${HADOOP_CP}`
CP=`cygpath -w -p ${CP}`
fi

2
addons/kafka-bridge/src/main/java/org/apache/atlas/kafka/bridge/KafkaBridge.java
View File

@ -134,7 +134,7 @@ public class KafkaBridge {
KafkaBridge importer = new KafkaBridge(atlasConf, atlasClientV2, kafkaUtils);
if(!KAFKA_SCHEMA_REGISTRY_ENV_VARIABLE.isEmpty()){
if(StringUtils.isNotEmpty(KAFKA_SCHEMA_REGISTRY_ENV_VARIABLE)) {
KAFKA_SCHEMA_REGISTRY_HOSTNAME = KAFKA_SCHEMA_REGISTRY_ENV_VARIABLE;
}

1
dev-support/atlas-docker/.dockerignore
View File

@ -2,6 +2,7 @@
!config
!dist/apache-atlas-*-hbase-hook.tar.gz
!dist/apache-atlas-*-hive-hook.tar.gz
!dist/apache-atlas-*-kafka-hook.tar.gz
!dist/apache-atlas-*-server.tar.gz
!downloads/*
!scripts/*

19
dev-support/atlas-docker/Dockerfile.atlas-kafka
View File

@ -20,17 +20,24 @@ ARG ATLAS_VERSION
ARG KAFKA_VERSION
COPY ./downloads/kafka_2.12-${KAFKA_VERSION}.tgz /home/atlas/dist/
COPY ./dist/apache-atlas-${ATLAS_VERSION}-kafka-hook.tar.gz /home/atlas/dist/
COPY ./downloads/kafka_2.12-${KAFKA_VERSION}.tgz /home/atlas/dist/
COPY ./scripts/atlas-kafka-setup.sh /home/atlas/scripts/
COPY ./scripts/atlas-kafka.sh /home/atlas/scripts/
COPY ./scripts/atlas-kafka-setup.sh /home/atlas/scripts/
COPY ./scripts/atlas-kafka.sh /home/atlas/scripts/
COPY ./scripts/atlas-kafka-application.properties /home/atlas/scripts/
RUN tar xvfz /home/atlas/dist/kafka_2.12-${KAFKA_VERSION}.tgz --directory=/opt/ && \
ln -s /opt/kafka_2.12-${KAFKA_VERSION} /opt/kafka && \
rm -f /home/atlas/dist/kafka_2.12-${KAFKA_VERSION}.tgz
rm -f /home/atlas/dist/kafka_2.12-${KAFKA_VERSION}.tgz && \
tar xvfz /home/atlas/dist/apache-atlas-${ATLAS_VERSION}-kafka-hook.tar.gz --directory=/opt/ && \
ln -s /opt/apache-atlas-kafka-hook-${ATLAS_VERSION} /opt/apache-atlas-kafka-hook && \
rm -f /home/atlas/dist/apache-atlas-${ATLAS_VERSION}-kafka-hook.tar.gz && \
cp -f /home/atlas/scripts/atlas-kafka-application.properties /opt/kafka/config/atlas-application.properties
ENV KAFKA_HOME /opt/kafka
ENV PATH /usr/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/kafka/bin
ENV KAFKA_HOME /opt/kafka
ENV KAFKA_CONF_DIR /opt/kafka/config
ENV PATH /usr/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/kafka/bin
EXPOSE 9092

1
dev-support/atlas-docker/scripts/atlas-build.sh
View File

@ -89,3 +89,4 @@ mvn ${ARG_PROFILES} ${ARG_SKIPTESTS} -DskipDocs clean package
mv -f distro/target/apache-atlas-${ATLAS_VERSION}-server.tar.gz /home/atlas/dist/
mv -f distro/target/apache-atlas-${ATLAS_VERSION}-hive-hook.tar.gz /home/atlas/dist/
mv -f distro/target/apache-atlas-${ATLAS_VERSION}-hbase-hook.tar.gz /home/atlas/dist/
mv -f distro/target/apache-atlas-${ATLAS_VERSION}-kafka-hook.tar.gz /home/atlas/dist/

23
dev-support/atlas-docker/scripts/atlas-kafka-application.properties Normal file
View File

@ -0,0 +1,23 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
atlas.rest.address=http://atlas.example.com:21000
atlas.kafka.zookeeper.connect=atlas-zk.example.com:2181
atlas.kafka.bootstrap.servers=atlas-kafka.example.com:9092