tool
/
atlas
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ATLAS-4454 : Upgrade janusgraph version to 0.6.0 

Signed-off-by: Pinal Shah <pinal.shah@freestoneinfotech.com>
This commit is contained in:
Pinal Shah 2021-10-29 22:03:30 +05:30
parent 7581672b7a
commit 6721c0d949
12 changed files with 98 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
graphdb/janus-hbase2/pom.xml
View File

@ -34,7 +34,7 @@
<dependency>
<groupId>org.janusgraph</groupId>
<artifactId>janusgraph-core</artifactId>
<version>${janus.version}</version>
<version>${janusgraph.version}</version>
<exclusions>
<exclusion>
<groupId>com.codahale.metrics</groupId>

7
graphdb/janus-hbase2/src/main/java/org/janusgraph/diskstorage/hbase2/HBaseKeyColumnValueStore.java
View File

@ -40,7 +40,9 @@ import org.janusgraph.diskstorage.keycolumnvalue.KeyColumnValueStore;
import org.janusgraph.diskstorage.keycolumnvalue.KeyIterator;
import org.janusgraph.diskstorage.keycolumnvalue.KeyRangeQuery;
import org.janusgraph.diskstorage.keycolumnvalue.KeySliceQuery;
import org.janusgraph.diskstorage.keycolumnvalue.KeySlicesIterator;
import org.janusgraph.diskstorage.keycolumnvalue.SliceQuery;
import org.janusgraph.diskstorage.keycolumnvalue.MultiSlicesQuery;
import org.janusgraph.diskstorage.keycolumnvalue.StoreTransaction;
import org.janusgraph.diskstorage.util.RecordIterator;
import org.janusgraph.diskstorage.util.StaticArrayBuffer;
@ -150,6 +152,11 @@ public class HBaseKeyColumnValueStore implements KeyColumnValueStore {
return executeKeySliceQuery(new FilterList(FilterList.Operator.MUST_PASS_ALL), query);
}
@Override
public KeySlicesIterator getKeys(MultiSlicesQuery queries, StoreTransaction txh) throws BackendException {
throw new UnsupportedOperationException();
}
public static Filter getFilter(SliceQuery query) {
byte[] colStartBytes = query.getSliceStart().length() > 0 ? query.getSliceStart().as(StaticBuffer.ARRAY_FACTORY) : null;
byte[] colEndBytes = query.getSliceEnd().length() > 0 ? query.getSliceEnd().as(StaticBuffer.ARRAY_FACTORY) : null;

2
graphdb/janus-hbase2/src/main/java/org/janusgraph/diskstorage/hbase2/HBaseStoreManager.java
View File

@ -440,7 +440,7 @@ public class HBaseStoreManager extends DistributedStoreManager implements KeyCol
throw new TemporaryBackendException(e);
}
sleepAfterWrite(txh, commitTime);
this.sleepAfterWrite(commitTime);
}
@Override

29
graphdb/janus/pom.xml
View File

@ -65,15 +65,15 @@
</dependency>
<dependency>
<groupId>commons-configuration</groupId>
<artifactId>commons-configuration</artifactId>
<scope>provided</scope>
<groupId>org.apache.commons</groupId>
<artifactId>commons-configuration2</artifactId>
<version>${commons-conf2.version}</version>
</dependency>
<dependency>
<groupId>org.janusgraph</groupId>
<artifactId>janusgraph-server</artifactId>
<version>${janus.version}</version>
<version>${janusgraph.version}</version>
<exclusions>
<exclusion>
<groupId>com.codahale.metrics</groupId>
@ -101,7 +101,7 @@
<dependency>
<groupId>org.janusgraph</groupId>
<artifactId>janusgraph-berkeleyje</artifactId>
<version>${janus.version}</version>
<version>${janusgraph.version}</version>
<exclusions>
<exclusion>
<groupId>ch.qos.logback</groupId>
@ -113,7 +113,7 @@
<dependency>
<groupId>org.janusgraph</groupId>
<artifactId>janusgraph-cassandra</artifactId>
<version>${janus.version}</version>
<version>${janusgraph.cassandra.version}</version>
<exclusions>
<exclusion>
<groupId>ch.qos.logback</groupId>
@ -129,7 +129,7 @@
<dependency>
<groupId>org.janusgraph</groupId>
<artifactId>janusgraph-cql</artifactId>
<version>${janus.version}</version>
<version>${janusgraph.version}</version>
<exclusions>
<exclusion>
<groupId>ch.qos.logback</groupId>
@ -147,7 +147,7 @@
<dependency>
<groupId>org.janusgraph</groupId>
<artifactId>janusgraph-es</artifactId>
<version>${janus.version}</version>
<version>${janusgraph.version}</version>
<exclusions>
<exclusion>
<groupId>ch.qos.logback</groupId>
@ -159,7 +159,7 @@
<dependency>
<groupId>org.janusgraph</groupId>
<artifactId>janusgraph-hbase</artifactId>
<version>${janus.version}</version>
<version>${janusgraph.version}</version>
<exclusions>
<exclusion>
<groupId>ch.qos.logback</groupId>
@ -171,7 +171,7 @@
<dependency>
<groupId>org.janusgraph</groupId>
<artifactId>janusgraph-solr</artifactId>
<version>${janus.version}</version>
<version>${janusgraph.version}</version>
<exclusions>
<exclusion>
<groupId>ch.qos.logback</groupId>
@ -191,7 +191,13 @@
<dependency>
<groupId>org.janusgraph</groupId>
<artifactId>janusgraph-lucene</artifactId>
<version>${janus.version}</version>
<version>${janusgraph.version}</version>
<exclusions>
<exclusion>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-backward-codecs</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
@ -269,7 +275,6 @@
<artifactId>mockito-all</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>

34
graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphDatabase.java
View File

@ -29,6 +29,7 @@ import org.apache.atlas.repository.graphdb.janus.serializer.BigIntegerSerializer
import org.apache.atlas.repository.graphdb.janus.serializer.TypeCategorySerializer;
import org.apache.atlas.typesystem.types.DataTypes.TypeCategory;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.ConfigurationConverter;
import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONMapper;
import org.janusgraph.core.JanusGraph;
import org.janusgraph.core.JanusGraphException;
@ -51,6 +52,7 @@ import java.time.Duration;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
import static org.apache.atlas.ApplicationProperties.DEFAULT_INDEX_RECOVERY;
import static org.apache.atlas.ApplicationProperties.INDEX_RECOVERY_CONF;
@ -100,18 +102,18 @@ public class AtlasJanusGraphDatabase implements GraphDatabase<AtlasJanusVertex,
Configuration janusConfig = ApplicationProperties.getSubsetConfiguration(configProperties, GRAPH_PREFIX);
//add serializers for non-standard property value types that Atlas uses
janusConfig.addProperty("attributes.custom.attribute1.attribute-class", TypeCategory.class.getName());
janusConfig.addProperty("attributes.custom.attribute1.serializer-class", TypeCategorySerializer.class.getName());
janusConfig.setProperty("attributes.custom.attribute1.attribute-class", TypeCategory.class.getName());
janusConfig.setProperty("attributes.custom.attribute1.serializer-class", TypeCategorySerializer.class.getName());
//not ideal, but avoids making large changes to Atlas
janusConfig.addProperty("attributes.custom.attribute2.attribute-class", ArrayList.class.getName());
janusConfig.addProperty("attributes.custom.attribute2.serializer-class", SerializableSerializer.class.getName());
janusConfig.setProperty("attributes.custom.attribute2.attribute-class", ArrayList.class.getName());
janusConfig.setProperty("attributes.custom.attribute2.serializer-class", SerializableSerializer.class.getName());
janusConfig.addProperty("attributes.custom.attribute3.attribute-class", BigInteger.class.getName());
janusConfig.addProperty("attributes.custom.attribute3.serializer-class", BigIntegerSerializer.class.getName());
janusConfig.setProperty("attributes.custom.attribute3.attribute-class", BigInteger.class.getName());
janusConfig.setProperty("attributes.custom.attribute3.serializer-class", BigIntegerSerializer.class.getName());
janusConfig.addProperty("attributes.custom.attribute4.attribute-class", BigDecimal.class.getName());
janusConfig.addProperty("attributes.custom.attribute4.serializer-class", BigDecimalSerializer.class.getName());
janusConfig.setProperty("attributes.custom.attribute4.attribute-class", BigDecimal.class.getName());
janusConfig.setProperty("attributes.custom.attribute4.serializer-class", BigDecimalSerializer.class.getName());
return janusConfig;
}
@ -188,14 +190,16 @@ public class AtlasJanusGraphDatabase implements GraphDatabase<AtlasJanusVertex,
@VisibleForTesting
static JanusGraph initJanusGraph(Configuration config) {
org.apache.commons.configuration2.Configuration conf2 = createConfiguration2(config);
try {
return JanusGraphFactory.open(config);
return JanusGraphFactory.open(conf2);
} catch (JanusGraphException e) {
LOG.warn("JanusGraphException: {}", e.getMessage());
if (e.getMessage().startsWith(OLDER_STORAGE_EXCEPTION)) {
LOG.info("Newer client is being used with older janus storage version. Setting allow-upgrade=true and reattempting connection");
config.addProperty("graph.allow-upgrade", true);
return JanusGraphFactory.open(config);
return JanusGraphFactory.open(conf2);
} else {
throw new RuntimeException(e);
}
@ -252,7 +256,9 @@ public class AtlasJanusGraphDatabase implements GraphDatabase<AtlasJanusVertex,
try {
Configuration cfg = getConfiguration();
cfg.setProperty("storage.batch-loading", true);
return JanusGraphFactory.open(cfg);
org.apache.commons.configuration2.Configuration conf2 = createConfiguration2(cfg);
return JanusGraphFactory.open(conf2);
} catch (IllegalArgumentException ex) {
LOG.error("getBulkLoadingGraphInstance: Failed!", ex);
} catch (AtlasException ex) {
@ -262,6 +268,12 @@ public class AtlasJanusGraphDatabase implements GraphDatabase<AtlasJanusVertex,
return null;
}
private static org.apache.commons.configuration2.Configuration createConfiguration2(Configuration conf) {
Properties properties = ConfigurationConverter.getProperties(conf);
return org.apache.commons.configuration2.ConfigurationConverter.getConfiguration(properties);
}
public static void unload() {
synchronized (AtlasJanusGraphDatabase.class) {

2
graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/AtlasGraphSONReader.java
View File

@ -262,7 +262,7 @@ public final class AtlasGraphSONReader {
}
final GraphSONMapper.Builder builder = GraphSONMapper.build();
final GraphSONMapper mapper = builder.typeInfo(TypeInfo.NO_TYPES).create();
final GraphSONMapper mapper = builder.create();
return new AtlasGraphSONReader(mapper.createMapper(), relationshipCache, graph, bulkLoadGraph,
numWorkers, batchSize, suppliedStartIndex);

2
graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/query/NativeJanusGraphQuery.java
View File

@ -236,7 +236,7 @@ public class NativeJanusGraphQuery implements NativeTinkerpopGraphQuery<AtlasJan
@Override
public void orderBy(final String propertyName, final AtlasGraphQuery.SortOrder sortOrder) {
Order order = sortOrder == AtlasGraphQuery.SortOrder.ASC ? Order.incr : Order.decr;
Order order = sortOrder == AtlasGraphQuery.SortOrder.ASC ? Order.asc : Order.desc;
query.orderBy(propertyName, order);
}

56
graphdb/janus/src/main/java/org/janusgraph/diskstorage/solr/Solr6Index.java
View File

@ -672,6 +672,27 @@ public class Solr6Index implements IndexProvider {
doc -> doc.getFieldValue(keyIdField).toString());
}
@Override
public Long queryCount(IndexQuery query, KeyInformation.IndexRetriever information, BaseTransaction tx) throws BackendException {
try {
String collection = query.getStore();
String keyIdField = this.getKeyFieldId(collection);
SolrQuery solrQuery = new SolrQuery("*:*");
solrQuery.set("fl", new String[]{keyIdField});
String queryFilter = this.buildQueryFilter(query.getCondition(), information.get(collection));
solrQuery.addFilterQuery(new String[]{queryFilter});
QueryResponse response = this.solrClient.query(collection, solrQuery);
logger.debug("Executed query [{}] in {} ms", query, response.getElapsedTime());
return response.getResults().getNumFound();
} catch (IOException ex) {
logger.error("Query did not complete : ", ex);
throw new PermanentBackendException(ex);
} catch (SolrServerException ex) {
logger.error("Unable to query Solr index.", ex);
throw new PermanentBackendException(ex);
}
}
private void addOrderToQuery(SolrQuery solrQuery, List<IndexQuery.OrderEntry> orders) {
for (final IndexQuery.OrderEntry order1 : orders) {
final String item = order1.getKey();
@ -765,10 +786,12 @@ public class Solr6Index implements IndexProvider {
final String key = atom.getKey();
final JanusGraphPredicate predicate = atom.getPredicate();
if (value instanceof Number) {
if (value == null && predicate == Cmp.NOT_EQUAL) {
return key + ":*";
} else if (value instanceof Number) {
final String queryValue = escapeValue(value);
Preconditions.checkArgument(predicate instanceof Cmp,
"Relation not supported on numeric types: " + predicate);
"Relation not supported on numeric types: %s", predicate);
final Cmp numRel = (Cmp) predicate;
switch (numRel) {
case EQUAL:
@ -804,18 +827,18 @@ public class Solr6Index implements IndexProvider {
return (key + ":" + escapeValue(value) + "*");
} else if (predicate == Text.REGEX || predicate == Text.CONTAINS_REGEX) {
return (key + ":/" + value + "/");
} else if (predicate == Cmp.EQUAL) {
} else if (predicate == Cmp.EQUAL || predicate == Cmp.NOT_EQUAL) {
final String tokenizer =
ParameterType.STRING_ANALYZER.findParameter(information.get(key).getParameters(), null);
if(tokenizer != null){
return tokenize(information, value, key, predicate,tokenizer);
} else {
if (tokenizer != null) {
return tokenize(information, value, key, predicate, tokenizer);
} else if (predicate == Cmp.EQUAL) {
return (key + ":\"" + escapeValue(value) + "\"");
} else { // Cmp.NOT_EQUAL case
return ("-" + key + ":\"" + escapeValue(value) + "\"");
}
} else if (predicate == Cmp.NOT_EQUAL) {
return ("-" + key + ":\"" + escapeValue(value) + "\"");
} else if (predicate == Text.FUZZY || predicate == Text.CONTAINS_FUZZY) {
return (key + ":"+escapeValue(value)+"~");
return (key + ":"+escapeValue(value)+"~"+Text.getMaxEditDistance(value.toString()));
} else if (predicate == Cmp.LESS_THAN) {
return (key + ":[* TO \"" + escapeValue(value) + "\"}");
} else if (predicate == Cmp.LESS_THAN_EQUAL) {
@ -830,9 +853,9 @@ public class Solr6Index implements IndexProvider {
} else if (value instanceof Geoshape) {
final Mapping map = Mapping.getMapping(information.get(key));
Preconditions.checkArgument(predicate instanceof Geo && predicate != Geo.DISJOINT,
"Relation not supported on geo types: " + predicate);
"Relation not supported on geo types: %s", predicate);
Preconditions.checkArgument(map == Mapping.PREFIX_TREE || predicate == Geo.WITHIN || predicate == Geo.INTERSECT,
"Relation not supported on geopoint types: " + predicate);
"Relation not supported on geopoint types: %s", predicate);
final Geoshape geo = (Geoshape)value;
if (geo.getType() == Geoshape.Type.CIRCLE && (predicate == Geo.INTERSECT || map == Mapping.DEFAULT)) {
final Geoshape.Point center = geo.getPoint();
@ -852,8 +875,7 @@ public class Solr6Index implements IndexProvider {
} else if (value instanceof Date || value instanceof Instant) {
final String s = value.toString();
final String queryValue = escapeValue(value instanceof Date ? toIsoDate((Date) value) : value.toString());
Preconditions.checkArgument(predicate instanceof Cmp, "Relation not supported on date types: "
+ predicate);
Preconditions.checkArgument(predicate instanceof Cmp, "Relation not supported on date types: %s", predicate);
final Cmp numRel = (Cmp) predicate;
switch (numRel) {
@ -944,7 +966,11 @@ public class Solr6Index implements IndexProvider {
if (terms.isEmpty()) {
return "";
} else if (terms.size() == 1) {
return (key + ":(" + escapeValue(terms.get(0)) + ")");
if (janusgraphPredicate == Cmp.NOT_EQUAL) {
return ("-" + key + ":(" + escapeValue(terms.get(0)) + ")");
} else {
return (key + ":(" + escapeValue(terms.get(0)) + ")");
}
} else {
final And<JanusGraphElement> andTerms = new And<>();
for (final String term : terms) {
@ -999,7 +1025,7 @@ public class Solr6Index implements IndexProvider {
@Override
public void close() throws BackendException {
logger.trace("Shutting down connection to Solr", solrClient);
logger.trace("Shutting down connection to Solr {}", solrClient);
try {
solrClient.close();
} catch (final IOException e) {

2
graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/BaseUtils.java
View File

@ -73,7 +73,7 @@ public class BaseUtils {
private JsonNode getEntityNode(String json) throws IOException {
GraphSONMapper.Builder builder = GraphSONMapper.build();
final ObjectMapper mapper = builder.typeInfo(TypeInfo.NO_TYPES).create().createMapper();
final ObjectMapper mapper = builder.create().createMapper();
return mapper.readTree(json);
}

5
pom.xml
View File

@ -717,7 +717,8 @@
<httpcomponents-httpcore.version>4.4.13</httpcomponents-httpcore.version>
<jackson.databind.version>2.11.3</jackson.databind.version>
<jackson.version>2.11.3</jackson.version>
<janus.version>0.5.3</janus.version>
<janusgraph.version>0.6.0</janusgraph.version>
<janusgraph.cassandra.version>0.5.3</janusgraph.cassandra.version>
<javax-inject.version>1</javax-inject.version>
<javax.servlet.version>3.1.0</javax.servlet.version>
<jersey-spring.version>1.19.4</jersey-spring.version>
@ -767,7 +768,7 @@
<surefire.forkCount>2C</surefire.forkCount>
<surefire.version>3.0.0-M5</surefire.version>
<testng.version>6.9.4</testng.version>
<tinkerpop.version>3.4.10</tinkerpop.version>
<tinkerpop.version>3.5.1</tinkerpop.version>
<woodstox-core.version>5.0.3</woodstox-core.version>
<zookeeper.version>3.5.7</zookeeper.version>
</properties>

2
repository/src/test/java/org/apache/atlas/query/TraversalComposerTest.java
View File

@ -53,7 +53,7 @@ public class TraversalComposerTest extends BaseDSLComposer {
"[JanusGraphStep([],[__typeName.eq(DB)]), DedupGlobalStep@[d], RangeGlobalStep(0,25)]");
verify("Table groupby(owner) select name, owner, clusterName orderby name",
"[JanusGraphStep([],[__typeName.eq(Table)]), TraversalFilterStep([JanusGraphPropertiesStep([Table.owner],property)]), GroupStep(value(Table.owner),[FoldStep]), DedupGlobalStep, RangeGlobalStep(0,25)]");
"[JanusGraphStep([],[__typeName.eq(Table), Table.owner.neq]), GroupStep(value(Table.owner),[FoldStep]), DedupGlobalStep, RangeGlobalStep(0,25)]");
}
private void verify(String dsl, String expected) {

2
tools/atlas-index-repair/pom.xml
View File

@ -53,7 +53,7 @@
<dependency>
<groupId>org.janusgraph</groupId>
<artifactId>janusgraph-core</artifactId>
<version>${janus.version}</version>
<version>${janusgraph.version}</version>
</dependency>
</dependencies>