tool
/
atlas
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ATLAS-4878: utility to analyze hook notifications

This commit is contained in:
Madhan Neethiraj 2024-06-09 18:36:56 -07:00
parent 70817c27c1
commit 978087a882
9 changed files with 796 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
distro/pom.xml
View File

@ -136,6 +136,7 @@ atlas.graph.storage.hbase.regions-per-server=1
<descriptor>src/main/assemblies/atlas-repair-index-package.xml</descriptor>
<!--<descriptor>src/main/assemblies/migration-exporter.xml</descriptor>-->
<descriptor>src/main/assemblies/classification-updater.xml</descriptor>
<descriptor>src/main/assemblies/notification-analyzer.xml</descriptor>
</descriptors>
<finalName>apache-atlas-${project.version}</finalName>
<tarLongFileMode>gnu</tarLongFileMode>

71
distro/src/main/assemblies/notification-analyzer.xml Normal file
View File

@ -0,0 +1,71 @@
<!--
**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*
-->
<assembly>
<id>notification-analyzer</id>
<formats>
<format>zip</format>
</formats>
<baseDirectory>notification-analyzer</baseDirectory>
<fileSets>
<fileSet>
<includes>
<include>README*</include>
</includes>
</fileSet>
<fileSet>
<directory>../tools/notification-analyzer/target/dependency</directory>
<outputDirectory>.</outputDirectory>
</fileSet>
<fileSet>
<directory>../tools/notification-analyzer/scripts</directory>
<outputDirectory>.</outputDirectory>
<includes>
<include>*.sh</include>
</includes>
<fileMode>0755</fileMode>
<directoryMode>0755</directoryMode>
</fileSet>
<fileSet>
<directory>../tools/notification-analyzer/src/main/resources</directory>
<outputDirectory>.</outputDirectory>
<includes>
<include>atlas-log4j.xml</include>
<include>atlas-application.properties</include>
</includes>
</fileSet>
<fileSet>
<directory>../tools/notification-analyzer</directory>
<outputDirectory>.</outputDirectory>
<includes>
<include>README</include>
</includes>
</fileSet>
<fileSet>
<directory>../tools/notification-analyzer/target</directory>
<outputDirectory>.</outputDirectory>
<includes>
<include>atlas-notification-analyzer-${project.version}.jar</include>
</includes>
</fileSet>
</fileSets>
</assembly>

1
pom.xml
View File

@ -846,6 +846,7 @@
<module>addons/kafka-bridge</module>
<module>tools/classification-updater</module>
<module>tools/atlas-index-repair</module>
<module>tools/notification-analyzer</module>
<module>addons/impala-hook-api</module>
<module>addons/impala-bridge-shim</module>
<module>addons/impala-bridge</module>

119
tools/notification-analyzer/README Normal file
View File

@ -0,0 +1,119 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
Introduction
This utility analyzes hook notification messages stored in JSON format
in a file, and reports following details:
- number of notifications per notification type
- number of entities created/updated
- number of entity references in notifications per entity type
- number of entity operations performed
- number of entity operations performed per entity type
Setup
- All libraries necessary to run the utility are packaged in following file:
distro/target/apache-atlas-<version>-notification-analyzer.zip
- Unzip the file in the directory where the tool needs to be installed.
- Update log configurations in atlas-log4j.xml
Running the utility
- Execute following command to start the utility:
./notification-analyzer.sh -m message_file.json [-o output_file]
- Progress will be printed in the following format in the output file (if specified) or on stdout:
PROGRESS #1: analyzed 1000 notifications, 1071 messages
PROGRESS #2: analyzed 2000 notifications, 2131 messages
PROGRESS #3: analyzed 3000 notifications, 3194 messages
...
Completed analyzing 114755 notification, 120816 messages. Time taken: 234 seconds
- Note that the number of notifications might be less than the number
of messages in the file in case some notifications were split into
multiple messages due to their size.
- Logs will be printed in file /tmp/atlas-notification-analyzer.log. The location of
the log file can be configured using following environment variables:
LOGFILE_DIR
LOGFILE_NAME
Sample Result:
The utility will print the result of analysis in the following format:
{
"notifications": 114755,
"notificationLengthAvg": 74331,
"notificationLengthMax": 101148684,
"splitNotifications": 453,
"splitNotificationLengthAvg": 13901446,
"splitNotificationLengthMax": 101148684,
"entities": 598435,
"notificationEntities": 2575347,
"notificationByType": {
"ENTITY_CREATE_V2": 49428,
"ENTITY_FULL_UPDATE_V2": 1597,
"ENTITY_PARTIAL_UPDATE_V2": 36561,
"ENTITY_DELETE_V2": 27169
},
"notificationEntityByType": {
"hdfs_path": 16417,
"hive_db": 20471,
"hive_table": 57143,
"hive_storagedesc": 30018,
"hive_column": 685384,
"hive_process": 41512
"hive_column_lineage": 1724402,
},
"entityOperations": {
"CREATE": 598435,
"UPDATE": 1913182
"PARTIAL_UPDATE": 36561,
"DELETE": 27169
},
"entityOperationsByType": {
"CREATE": {
"hdfs_path": 10940,
"hive_db": 224,
"hive_table": 22154,
"hive_storagedesc": 15280,
"hive_column": 332332,
"hive_process": 23462,
"hive_column_lineage": 194043
},
"UPDATE" {
"hdfs_path": 5477,
"hive_column": 319559,
"hive_column_lineage": 1530359,
"hive_db": 20203,
"hive_process": 18050,
"hive_storagedesc": 13204,
"hive_table": 6330
},
"PARTIAL_UPDATE": {
"hive_column": 33493,
"hive_storagedesc": 1534,
"hive_table": 1534
},
"DELETE": {
"hive_db": 44,
"hive_table": 27125
}
}
}

192
tools/notification-analyzer/pom.xml Normal file
View File

@ -0,0 +1,192 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
~ regarding copyright ownership. The ASF licenses this file
~ to you under the Apache License, Version 2.0 (the
~ "License"); you may not use this file except in compliance
~ with the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>apache-atlas</artifactId>
<groupId>org.apache.atlas</groupId>
<version>3.0.0-SNAPSHOT</version>
<relativePath>../../</relativePath>
</parent>
<artifactId>atlas-notification-analyzer</artifactId>
<description>Apache Atlas Notification Analyzer</description>
<name>Apache Atlas Notification Analyzer</name>
<packaging>jar</packaging>
<dependencies>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>${slf4j.version}</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>${slf4j.version}</version>
</dependency>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-notification</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
<version>${commons-cli.version}</version>
</dependency>
<dependency>
<groupId>commons-collections</groupId>
<artifactId>commons-collections</artifactId>
<version>${commons-collections.version}</version>
</dependency>
</dependencies>
<profiles>
<profile>
<id>dist</id>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>copy-binaries</id>
<phase>package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/dependency</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<artifactItems>
<artifactItem>
<groupId>${project.groupId}</groupId>
<artifactId>${project.artifactId}</artifactId>
<version>${project.version}</version>
</artifactItem>
<artifactItem>
<groupId>${project.groupId}</groupId>
<artifactId>atlas-intg</artifactId>
<version>${project.version}</version>
</artifactItem>
<artifactItem>
<groupId>${project.groupId}</groupId>
<artifactId>atlas-notification</artifactId>
<version>${project.version}</version>
</artifactItem>
<artifactItem>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
<version>${commons-cli.version}</version>
</artifactItem>
<artifactItem>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
<version>${commons-codec.version}</version>
</artifactItem>
<artifactItem>
<groupId>commons-collections</groupId>
<artifactId>commons-collections</artifactId>
<version>${commons-collections.version}</version>
</artifactItem>
<artifactItem>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>${commons-io.version}</version>
</artifactItem>
<artifactItem>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</artifactItem>
<artifactItem>
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
<version>1.26.2</version>
</artifactItem>
<artifactItem>
<groupId>commons-configuration</groupId>
<artifactId>commons-configuration</artifactId>
<version>${commons-conf.version}</version>
</artifactItem>
<artifactItem>
<groupId>commons-lang</groupId>
<artifactId>commons-lang</artifactId>
<version>${commons-lang.version}</version>
</artifactItem>
<artifactItem>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
<version>${commons-logging.version}</version>
</artifactItem>
<artifactItem>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
<version>${jackson.version}</version>
</artifactItem>
<artifactItem>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
<version>${jackson.version}</version>
</artifactItem>
<artifactItem>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>${jackson.databind.version}</version>
</artifactItem>
<artifactItem>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
<version>${log4j.version}</version>
</artifactItem>
<artifactItem>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
<version>${log4j2.version}</version>
</artifactItem>
<artifactItem>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
<version>${log4j2.version}</version>
</artifactItem>
<artifactItem>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>${slf4j.version}</version>
</artifactItem>
<artifactItem>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>${slf4j.version}</version>
</artifactItem>
</artifactItems>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</project>

28
tools/notification-analyzer/scripts/notification-analyzer.sh Normal file
View File

@ -0,0 +1,28 @@
#!/bin/bash
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License. See accompanying LICENSE file.
#
# resolve links - $0 may be a softlink
M2_REPO=~/.m2/repository
LIB_DIR=$(pwd)
LOGFILE_DIR="${LOGFILE_DIR:-/tmp/}"
LOGFILE_NAME="${LOGFILE_NAME:-atlas-notification-analyzer.log}"
CP=.
for i in "${LIB_DIR}/"*.jar; do
CP="${CP}:$i"
done
java -cp ${CP} -Dlog4j.configuration=atlas-log4j.xml -Datlas.log.dir=${LOGFILE_DIR} -Datlas.log.file=${LOGFILE_NAME} org.apache.atlas.tools.NotificationAnalyzer $*

322
tools/notification-analyzer/src/main/java/org/apache/atlas/tools/NotificationAnalyzer.java Normal file
View File

@ -0,0 +1,322 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.tools;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.model.notification.HookNotification;
import org.apache.atlas.notification.AtlasNotificationMessageDeserializer;
import org.apache.atlas.notification.NotificationInterface.NotificationType;
import org.apache.atlas.utils.AtlasJson;
import org.apache.atlas.v1.model.instance.Referenceable;
import org.apache.atlas.v1.model.notification.HookNotificationV1.EntityCreateRequest;
import org.apache.atlas.v1.model.notification.HookNotificationV1.EntityDeleteRequest;
import org.apache.atlas.v1.model.notification.HookNotificationV1.EntityPartialUpdateRequest;
import org.apache.atlas.v1.model.notification.HookNotificationV1.EntityUpdateRequest;
import org.apache.atlas.model.notification.HookNotification.EntityCreateRequestV2;
import org.apache.atlas.model.notification.HookNotification.EntityDeleteRequestV2;
import org.apache.atlas.model.notification.HookNotification.EntityPartialUpdateRequestV2;
import org.apache.atlas.model.notification.HookNotification.EntityUpdateRequestV2;
import org.apache.commons.cli.BasicParser;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.Options;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.*;
import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;
public class NotificationAnalyzer {
private static final Logger LOG = LoggerFactory.getLogger(NotificationAnalyzer.class);
private final String msgFile;
private final String outputFile;
private final AtlasNotificationMessageDeserializer deserializer;
private final Map<String, AtomicInteger> notificationCountByType = new HashMap<>();
private final AtomicInteger entityCount = new AtomicInteger();
private final Map<String, AtomicInteger> entityCountByType = new HashMap<>();
private final Map<String, AtomicInteger> entityOperCount = new HashMap<>();
private final Map<String, Map<String, AtomicInteger>> entityOperByTypeCount = new HashMap<>();
private final Set<String> knownEntities = new HashSet<>();
private final IntSummaryStatistics notificationStats = new IntSummaryStatistics();
private final IntSummaryStatistics splitNotificationStats = new IntSummaryStatistics();
public static void main(String[] args) {
CommandLineParser parser = new BasicParser();
Options options = new Options();
options.addOption("m", "message-file", true, "Messages file");
options.addOption("o", "output-file", true, "Output file");
try {
CommandLine cmd = parser.parse(options, args);
String msgFile = cmd.getOptionValue("m");
String outFile = cmd.getOptionValue("o");
if (msgFile == null || msgFile.isEmpty()) {
msgFile = "ATLAS_HOOK.json";
}
NotificationAnalyzer analyzer = new NotificationAnalyzer(msgFile, outFile, NotificationType.HOOK);
analyzer.analyze();
} catch (Throwable t) {
throw new RuntimeException(t);
}
}
public NotificationAnalyzer(String msgFile, String outputFile, NotificationType notificationType) {
this.msgFile = msgFile;
this.outputFile = outputFile;
this.deserializer = notificationType.getDeserializer();
}
public void analyze() throws Exception {
long startTimeMs = System.currentTimeMillis();
try (BufferedReader reader = getInputReader(); PrintWriter writer = getOutputWriter()) {
int msgCount = 0;
int notificationSize = 0;
for (String msg = reader.readLine(); msg != null; msg = reader.readLine()) {
msgCount++;
notificationSize += msg.length();
HookNotification notification = (HookNotification) deserializer.deserialize(msg);
if (notification == null) { // split notification, continue
continue;
}
notificationStats.accept(notificationSize);
if (notificationSize > msg.length()) {
splitNotificationStats.accept(notificationSize);
}
notificationSize = 0;
notificationCountByType.computeIfAbsent(notification.getType().name(), e -> new AtomicInteger()).incrementAndGet();
switch (notification.getType()) {
case ENTITY_CREATE:
handleEntityCreate((EntityCreateRequest) notification);
break;
case ENTITY_PARTIAL_UPDATE:
handleEntityPartialUpdate((EntityPartialUpdateRequest) notification);
break;
case ENTITY_FULL_UPDATE:
handleEntityUpdate((EntityUpdateRequest) notification);
break;
case ENTITY_DELETE:
handleEntityDelete((EntityDeleteRequest) notification);
break;
case ENTITY_CREATE_V2:
handleEntityCreateV2((EntityCreateRequestV2) notification);
break;
case ENTITY_PARTIAL_UPDATE_V2:
handleEntityPartialUpdateV2((EntityPartialUpdateRequestV2) notification);
break;
case ENTITY_FULL_UPDATE_V2:
handleEntityUpdateV2((EntityUpdateRequestV2) notification);
break;
case ENTITY_DELETE_V2:
handleEntityDeleteV2((EntityDeleteRequestV2) notification);
break;
}
if ((notificationStats.getCount() % 1000) == 0) {
LOG.info("PROGRESS #{}: analyzed {} notifications, {} messages", (msgCount / 1000), notificationStats.getCount(), msgCount);
writer.printf("PROGRESS #%1$d: analyzed %2$d notifications, %3$d messages%n", (msgCount / 1000), notificationStats.getCount(), msgCount);
writer.flush();
}
}
long timeTakenSeconds = (System.currentTimeMillis() - startTimeMs) / 1000;
LOG.info("Completed analyzing {}. Time taken: {} seconds", msgFile, timeTakenSeconds);
writer.printf("Completed analyzing %1$s. Time taken: %2$d seconds%n", msgFile, timeTakenSeconds);
writer.flush();
Map<String, Object> results = new LinkedHashMap<>();
results.put("messages", msgCount);
results.put("notifications", notificationStats.getCount());
results.put("notificationLengthAvg", (int) notificationStats.getAverage());
results.put("notificationLengthMax", notificationStats.getMax());
results.put("splitNotifications", splitNotificationStats.getCount());
results.put("splitNotificationLengthAvg", (int) splitNotificationStats.getAverage());
results.put("splitNotificationLengthMax", splitNotificationStats.getMax());
results.put("entities", entityCount);
results.put("notificationEntities", entityCountByType.values().stream().mapToInt(AtomicInteger::get).sum());
results.put("notificationByType", notificationCountByType);
results.put("notificationEntityByType", entityCountByType);
results.put("entityOperations", entityOperCount);
results.put("entityOperationsByType", entityOperByTypeCount);
String msg = AtlasJson.toJson(results);
LOG.info(msg);
writer.println(msg);
writer.flush();
}
}
private void handleEntityCreate(EntityCreateRequest request) {
if (request.getEntities() != null) {
for (Referenceable entity : request.getEntities()) {
recordEntity(entity);
}
}
}
private void handleEntityUpdate(EntityUpdateRequest request) {
if (request.getEntities() != null) {
for (Referenceable entity : request.getEntities()) {
recordEntity(entity);
}
}
}
private void handleEntityPartialUpdate(EntityPartialUpdateRequest request) {
recordEntityOperation(request.getTypeName(), "PARTIAL_UPDATE");
}
private void handleEntityDelete(EntityDeleteRequest request) {
knownEntities.remove(getEntityKey(request));
recordEntityOperation(request.getTypeName(), "DELETE");
}
private void handleEntityCreateV2(EntityCreateRequestV2 request) {
if (request.getEntities() != null) {
if (request.getEntities().getEntities() != null) {
for (AtlasEntity entity : request.getEntities().getEntities()) {
recordEntity(entity);
}
}
if (request.getEntities().getReferredEntities() != null) {
for (AtlasEntity entity : request.getEntities().getReferredEntities().values()) {
recordEntity(entity);
}
}
}
}
private void handleEntityUpdateV2(EntityUpdateRequestV2 request) {
if (request.getEntities() != null) {
if (request.getEntities().getEntities() != null) {
for (AtlasEntity entity : request.getEntities().getEntities()) {
recordEntity(entity);
}
}
if (request.getEntities().getReferredEntities() != null) {
for (AtlasEntity entity : request.getEntities().getReferredEntities().values()) {
recordEntity(entity);
}
}
}
}
private void handleEntityPartialUpdateV2(EntityPartialUpdateRequestV2 request) {
if (request.getEntity() != null && request.getEntity().getEntity() != null) {
AtlasEntity entity = request.getEntity().getEntity();
recordEntityOperation(entity.getTypeName(), "PARTIAL_UPDATE");
}
}
private void handleEntityDeleteV2(EntityDeleteRequestV2 request) {
if (request.getEntities() != null) {
for (AtlasObjectId objId : request.getEntities()) {
knownEntities.remove(getEntityKey(objId));
recordEntityOperation(objId.getTypeName(), "DELETE");
}
}
}
private void recordEntity(AtlasEntity entity) {
final String operation;
if (knownEntities.add(getEntityKey(entity))) {
operation = "CREATE";
} else {
operation = "UPDATE";
}
recordEntityOperation(entity.getTypeName(), operation);
}
private void recordEntity(Referenceable entity) {
final String operation;
if (knownEntities.add(getEntityKey(entity))) {
operation = "CREATE";
} else {
operation = "UPDATE";
}
recordEntityOperation(entity.getTypeName(), operation);
}
private void recordEntityOperation(String entityTypeName, String operation) {
if (operation.equals("CREATE")) {
entityCount.incrementAndGet();
}
entityCountByType.computeIfAbsent(entityTypeName, c -> new AtomicInteger()).incrementAndGet();
entityOperCount.computeIfAbsent(operation, c -> new AtomicInteger()).incrementAndGet();
entityOperByTypeCount.computeIfAbsent(operation, c -> new TreeMap<>()).computeIfAbsent(entityTypeName, c -> new AtomicInteger()).incrementAndGet();
}
private String getEntityKey(AtlasEntity entity) {
return entity.getTypeName() + ":" + getUniqueKey(entity.getAttributes());
}
private String getEntityKey(AtlasObjectId objectId) {
return objectId.getTypeName() + ":" + getUniqueKey(objectId.getUniqueAttributes());
}
private String getEntityKey(Referenceable entity) {
return entity.getTypeName() + ":" + getUniqueKey(entity.getValues());
}
private String getEntityKey(EntityDeleteRequest request) {
return request.getTypeName() + ":" + request.getAttributeValue();
}
private Object getUniqueKey(Map<String, Object> attributes) {
Object ret = attributes.get("qualifiedName");
return ret == null ? attributes.get("name") : ret;
}
private BufferedReader getInputReader() throws IOException {
return new BufferedReader(new FileReader(msgFile));
}
private PrintWriter getOutputWriter() throws IOException {
return (outputFile == null || outputFile.isEmpty()) ? new PrintWriter(System.out) : new PrintWriter(new FileWriter(outputFile, true));
}
}

18
tools/notification-analyzer/src/main/resources/atlas-application.properties Normal file
View File

@ -0,0 +1,18 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

44
tools/notification-analyzer/src/main/resources/atlas-log4j.xml Executable file
View File

@ -0,0 +1,44 @@
<?xml version="1.0" encoding="UTF-8" ?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
~ regarding copyright ownership. The ASF licenses this file
~ to you under the Apache License, Version 2.0 (the
~ "License"); you may not use this file except in compliance
~ with the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
<appender name="console" class="org.apache.log4j.ConsoleAppender">
<param name="Target" value="System.out"/>
<layout class="org.apache.log4j.PatternLayout">
<param name="ConversionPattern" value="%d %-5p - [%t:%x] ~ %m (%C{1}:%L)%n"/>
</layout>
</appender>
<appender name="FILE" class="org.apache.log4j.RollingFileAppender">
<param name="File" value="${atlas.log.dir}/${atlas.log.file}"/>
<param name="Append" value="true"/>
<param name="maxFileSize" value="100MB" />
<param name="maxBackupIndex" value="20" />
<layout class="org.apache.log4j.PatternLayout">
<param name="ConversionPattern" value="%d %-5p - [%t:%x] ~ %m (%C{1}:%L)%n"/>
</layout>
</appender>
<root>
<priority value="info"/>
<appender-ref ref="FILE"/>
</root>
</log4j:configuration>