tool
/
atlas
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Cache of compiled DSL queries

This commit is contained in:
Jeff Hagelberg 2017-01-30 14:31:53 +05:30 committed by Vimal Sharma
parent d204df78e4
commit 37c8a4d1a5
11 changed files with 688 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
common/pom.xml
View File

@ -70,7 +70,6 @@
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>${guava.version}</version>
<scope>test</scope>
</dependency>
</dependencies>
</project>

97
common/src/main/java/org/apache/atlas/utils/LruCache.java Normal file
View File

@ -0,0 +1,97 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.utils;
import java.text.DateFormat;
import java.util.Date;
import java.util.LinkedHashMap;
import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Fixed size LRU Cache.
*
*/
public class LruCache<K, V> extends LinkedHashMap<K, V>{
private static final long serialVersionUID = 8715233786643882558L;
private static final Logger LOGGER = LoggerFactory.getLogger(LruCache.class.getName());
/**
* Specifies the number evictions that pass before a warning is logged.
*/
private final int evictionWarningThrottle;
// The number of evictions since the last warning was logged.
private long evictionsSinceWarning = 0;
// When the last eviction warning was issued.
private Date lastEvictionWarning = new Date();
// The maximum number of entries the cache holds.
private final int capacity;
/**
*
* @param cacheSize The size of the cache.
* @param evictionWarningThrottle The number evictions that pass before a warning is logged.
*/
public LruCache(int cacheSize, int evictionWarningThrottle) {
super(cacheSize, 0.75f, true);
this.evictionWarningThrottle = evictionWarningThrottle;
this.capacity = cacheSize;
}
@Override
protected boolean removeEldestEntry(Map.Entry<K,V> eldest) {
if( size() > capacity) {
evictionWarningIfNeeded();
return true;
}
return false;
}
/**
* Logs a warning if a threshold number of evictions has occurred since the
* last warning.
*/
private void evictionWarningIfNeeded() {
// If not logging eviction warnings, just return.
if (evictionWarningThrottle <= 0) {
return;
}
evictionsSinceWarning++;
if (evictionsSinceWarning >= evictionWarningThrottle) {
DateFormat dateFormat = DateFormat.getDateTimeInstance();
if (LOGGER.isInfoEnabled()) {
LOGGER.info("There have been " + evictionsSinceWarning
+ " evictions from the cache since "
+ dateFormat.format(lastEvictionWarning));
}
evictionsSinceWarning = 0;
lastEvictionWarning = new Date();
}
}
}

233
common/src/test/java/org/apache/atlas/utils/LruCacheTest.java Normal file
View File

@ -0,0 +1,233 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.utils;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertFalse;
import static org.testng.Assert.assertNull;
import static org.testng.Assert.assertTrue;
import java.util.HashMap;
import java.util.Map;
import org.apache.commons.lang.RandomStringUtils;
import org.testng.annotations.Test;
/**
* Tests the LruCache.
*/
@Test
public class LruCacheTest {
/**
* Tests the basic operations on the cache.
*/
@Test
public void testBasicOps() throws Exception {
LruCache<String, String> cache = new LruCache<>(1000, 0);
// Get the static cache and populate it. Its size and other
// characteristics depend on the bootstrap properties that are hard to
// control in a test. So it is hard to see that if we add more entries
// than the size of the cache one is evicted, or that it gets reaped at
// the right time. However, a lot of this type of functionality is
// tested by the underlying LruCache's test.
// Note that query handle IDs are of the form sessionID::queryID
String h1 = createHandle("s1::", "1::");
String q1 = createQuery();
String h2 = createHandle("s1::", "2::");
String q2 = createQuery();
String h3 = createHandle("s2::", "1::");
String q3 = createQuery();
String h4 = createHandle("s1::", "3::");
String q4 = createQuery();
String h5 = createHandle("s3::", null);
String q5 = createQuery();
String h5b = createHandle("s3::", null);
String q5b = createQuery();
String h6 = createHandle(null, "3::");
String q6 = createQuery();
String h6b = createHandle(null, "3::");
String q6b = createQuery();
// Test put and get.
cache.put(h1, q1);
cache.put(h2, q2);
cache.put(h3, q3);
cache.put(h4, q4);
cache.put(h5, q5);
cache.put(h6, q6);
assertEquals(cache.get(h1), q1);
assertEquals(cache.get(h2), q2);
assertEquals(cache.get(h3), q3);
assertEquals(cache.get(h4), q4);
assertEquals(cache.get(h5), q5);
assertEquals(cache.remove(h1), q1);
assertEquals(cache.remove(h2), q2);
assertEquals(cache.remove(h3), q3);
assertEquals(cache.remove(h4), q4);
assertEquals(cache.remove(h5), q5);
assertNull(cache.remove(h5b));
assertEquals(q6, cache.remove(h6));
assertNull(cache.remove(h6b));
cache.put(h5b, q5b);
cache.put(h6b, q6b);
assertEquals(q5b, cache.remove(h5));
assertNull(cache.remove(h5b));
assertEquals(q6b, cache.remove(h6));
assertNull(cache.remove(h6b));
}
@Test
public void testMapOperations() {
Map<String, String> reference = new HashMap<>();
reference.put("name", "Fred");
reference.put("occupation", "student");
reference.put("height", "5'11");
reference.put("City", "Littleton");
reference.put("State", "MA");
LruCache<String, String> map = new LruCache<>(10, 10);
map.putAll(reference);
assertEquals(map.size(), reference.size());
assertEquals(map.keySet().size(), reference.keySet().size());
assertTrue(map.keySet().containsAll(reference.keySet()));
assertTrue(reference.keySet().containsAll(map.keySet()));
assertEquals(reference.entrySet().size(), map.entrySet().size());
for(Map.Entry<String, String> entry : map.entrySet()) {
assertTrue(reference.containsKey(entry.getKey()));
assertEquals(entry.getValue(), reference.get(entry.getKey()));
assertTrue(map.containsKey(entry.getKey()));
assertTrue(map.containsValue(entry.getValue()));
assertTrue(map.values().contains(entry.getValue()));
}
assertTrue(reference.equals(map));
assertTrue(map.equals(reference));
}
@Test
public void testReplaceValueInMap() {
LruCache<String, String> map = new LruCache<>(10, 10);
map.put("name", "Fred");
map.put("name", "George");
assertEquals(map.get("name"), "George");
assertEquals(map.size(), 1);
}
@Test
public void testOrderUpdatedWhenAddExisting() {
LruCache<String, String> map = new LruCache<>(2, 10);
map.put("name", "Fred");
map.put("age", "15");
map.put("name", "George");
//age should be evicted
map.put("height", "5'3\"");
//age is now least recently used
assertFalse(map.containsKey("age"));
}
@Test
public void testMapRemove() {
LruCache<String, String> map = new LruCache<>(10, 10);
map.put("name", "Fred");
map.put("occupation", "student");
map.put("height", "5'11");
map.put("City", "Littleton");
map.put("State", "MA");
assertMapHasSize(map, 5);
assertTrue(map.containsKey("State"));
map.remove("State");
assertMapHasSize(map, 4);
assertFalse(map.containsKey("State"));
}
private void assertMapHasSize(LruCache<String, String> map, int size) {
assertEquals(map.size(), size);
assertEquals(map.keySet().size(), size);
assertEquals(map.values().size(), size);
assertEquals(map.entrySet().size(), size);
}
@Test
public void testEvict() {
LruCache<String, String> map = new LruCache<>(5, 10);
map.put("name", "Fred");
map.put("occupation", "student");
map.put("height", "5'11");
map.put("City", "Littleton");
map.put("State", "MA");
assertMapHasSize(map, 5);
//name should be evicted next
assertTrue(map.containsKey("name"));
map.put("zip", "01460");
assertFalse(map.containsKey("name"));
assertMapHasSize(map, 5);
map.get("occupation");
//height should be evicted next
assertTrue(map.containsKey("height"));
map.put("country", "USA");
assertFalse(map.containsKey("height"));
assertMapHasSize(map, 5);
}
/**
* Create a fake query handle for testing.
*
* @param queryPrefix
* @param pkgPrefix
* @return a new query handle.
*/
private String createHandle(String s1, String s2) {
return s1 + ": " + s2 + ":select x from x in y";
}
/**
* Create a mock IInternalQuery.
*
* @return a mock IInternalQuery.
* @throws QueryException
*/
private String createQuery() {
return RandomStringUtils.randomAlphabetic(10);
}
}

14
distro/src/conf/atlas-application.properties
View File

@ -218,3 +218,17 @@ atlas.metric.query.cache.ttlInSecs=900
#atlas.metric.query.entity.entityTagged=
#
#atlas.metric.query.tags.entityTags=
######### Compiled Query Cache Configuration #########
# The size of the compiled query cache. Older queries will be evicted from the cache
# when we reach the capacity.
#atlas.CompiledQueryCache.capacity=1000
# Allows notifications when items are evicted from the compiled query
# cache because it has become full. A warning will be issued when
# the specified number of evictions have occurred. If the eviction
# warning threshold <= 0, no eviction warnings will be issued.
#atlas.CompiledQueryCache.evictionWarningThrottle=0

1
release-log.txt
View File

@ -9,6 +9,7 @@ ATLAS-1060 Add composite indexes for exact match performance improvements for al
ATLAS-1127 Modify creation and modification timestamps to Date instead of Long(sumasai)
ALL CHANGES:
ATLAS-1387 Compiled Query Cache (jnhagelberg@us.ibm.com via svimal2106)
ATLAS-1312 Update QuickStart to use the v2 APIs for types and entities creation (sarath.kum4r@gmail.com via mneethiraj)
ATLAS-1498 added unit-tests to validate handling of array/map/struct attributes in entity create/update (sumasai via mneethiraj)
ATLAS-1114 Performance improvements for create/update entity (jnhagelb)

97
repository/src/main/java/org/apache/atlas/discovery/graph/GraphBackedDiscoveryService.java
View File

@ -18,6 +18,17 @@
package org.apache.atlas.discovery.graph;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import javax.inject.Inject;
import javax.inject.Singleton;
import javax.script.ScriptException;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.GraphTransaction;
import org.apache.atlas.discovery.DiscoveryException;
@ -38,24 +49,17 @@ import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.util.CompiledQueryCacheKey;
import org.apache.atlas.util.NoopGremlinQuery;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import scala.util.Either;
import scala.util.parsing.combinator.Parsers;
import javax.inject.Inject;
import javax.inject.Singleton;
import javax.script.ScriptException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
/**
* Graph backed implementation of Search.
*/
@ -124,42 +128,57 @@ public class GraphBackedDiscoveryService implements DiscoveryService {
}
public GremlinQueryResult evaluate(String dslQuery, QueryParams queryParams) throws DiscoveryException {
if (LOG.isDebugEnabled()) {
if(LOG.isDebugEnabled()) {
LOG.debug("Executing dsl query={}", dslQuery);
}
try {
Either<Parsers.NoSuccess, Expressions.Expression> either = QueryParser.apply(dslQuery, queryParams);
if (either.isRight()) {
Expressions.Expression expression = either.right().get();
return evaluate(dslQuery, expression);
} else {
throw new DiscoveryException("Invalid expression : " + dslQuery + ". " + either.left());
GremlinQuery gremlinQuery = parseAndTranslateDsl(dslQuery, queryParams);
if(gremlinQuery instanceof NoopGremlinQuery) {
return new GremlinQueryResult(dslQuery, ((NoopGremlinQuery)gremlinQuery).getDataType(), Collections.emptyList());
}
return new GremlinEvaluator(gremlinQuery, graphPersistenceStrategy, graph).evaluate();
} catch (Exception e) { // unable to catch ExpressionException
throw new DiscoveryException("Invalid expression : " + dslQuery, e);
}
}
private GremlinQueryResult evaluate(String dslQuery, Expressions.Expression expression) {
Expressions.Expression validatedExpression = QueryProcessor.validate(expression);
private GremlinQuery parseAndTranslateDsl(String dslQuery, QueryParams queryParams) throws DiscoveryException {
//If the final limit is 0, don't launch the query, return with 0 rows
if (validatedExpression instanceof Expressions.LimitExpression
&& ((Integer)((Expressions.LimitExpression) validatedExpression).limit().rawValue()) == 0) {
return new GremlinQueryResult(dslQuery, validatedExpression.dataType(), Collections.emptyList());
CompiledQueryCacheKey entry = new CompiledQueryCacheKey(dslQuery, queryParams);
GremlinQuery gremlinQuery = QueryProcessor.compiledQueryCache().get(entry);
if(gremlinQuery == null) {
Expressions.Expression validatedExpression = parseQuery(dslQuery, queryParams);
//If the final limit is 0, don't launch the query, return with 0 rows
if (validatedExpression instanceof Expressions.LimitExpression
&& ((Integer)((Expressions.LimitExpression) validatedExpression).limit().rawValue()) == 0) {
gremlinQuery = new NoopGremlinQuery(validatedExpression.dataType());
}
else {
gremlinQuery = new GremlinTranslator(validatedExpression, graphPersistenceStrategy).translate();
if (LOG.isDebugEnabled()) {
LOG.debug("Query = {}", validatedExpression);
LOG.debug("Expression Tree = {}", validatedExpression.treeString());
LOG.debug("Gremlin Query = {}", gremlinQuery.queryStr());
}
}
QueryProcessor.compiledQueryCache().put(entry, gremlinQuery);
}
return gremlinQuery;
}
private Expressions.Expression parseQuery(String dslQuery, QueryParams queryParams) throws DiscoveryException {
Either<Parsers.NoSuccess, Expressions.Expression> either = QueryParser.apply(dslQuery, queryParams);
if (either.isRight()) {
Expressions.Expression expression = either.right().get();
Expressions.Expression validatedExpression = QueryProcessor.validate(expression);
return validatedExpression;
} else {
throw new DiscoveryException("Invalid expression : " + dslQuery + ". " + either.left());
}
GremlinQuery gremlinQuery = new GremlinTranslator(validatedExpression, graphPersistenceStrategy).translate();
if (LOG.isDebugEnabled()) {
LOG.debug("Query = {}", validatedExpression);
LOG.debug("Expression Tree = {}", validatedExpression.treeString());
LOG.debug("Gremlin Query = {}", gremlinQuery.queryStr());
}
return new GremlinEvaluator(gremlinQuery, graphPersistenceStrategy, graph).evaluate();
}
/**
@ -182,12 +201,12 @@ public class GraphBackedDiscoveryService implements DiscoveryService {
throw new DiscoveryException(se);
}
}
private List<Map<String, String>> extractResult(final Object o) throws DiscoveryException {
List<Map<String, String>> result = new ArrayList<>();
if (o instanceof List) {
List l = (List) o;
for (Object value : l) {
Map<String, String> oRow = new HashMap<>();
if (value instanceof Map) {
@ -205,7 +224,7 @@ public class GraphBackedDiscoveryService implements DiscoveryService {
oRow.put(key, propertyValue.toString());
}
}
} else if (value instanceof String) {
oRow.put("", value.toString());
} else if(value instanceof AtlasEdge) {
@ -220,14 +239,14 @@ public class GraphBackedDiscoveryService implements DiscoveryService {
} else {
throw new DiscoveryException(String.format("Cannot process result %s", String.valueOf(value)));
}
result.add(oRow);
}
}
else {
result.add(new HashMap<String, String>() {{
put("result", o.toString());
}});
}});
}
return result;
}

48
repository/src/main/java/org/apache/atlas/util/AtlasRepositoryConfiguration.java
View File

@ -43,6 +43,9 @@ public class AtlasRepositoryConfiguration {
private static Logger LOG = LoggerFactory.getLogger(AtlasRepositoryConfiguration.class);
public static final int DEFAULT_COMPILED_QUERY_CACHE_EVICTION_WARNING_THROTTLE = 0;
public static final int DEFAULT_COMPILED_QUERY_CACHE_CAPACITY = 1000;
public static final String TYPE_CACHE_IMPLEMENTATION_PROPERTY = "atlas.TypeCache.impl";
public static final String AUDIT_EXCLUDED_OPERATIONS = "atlas.audit.excludes";
private static List<String> skippedOperations = null;
@ -70,7 +73,7 @@ public class AtlasRepositoryConfiguration {
public static Class<? extends EntityAuditRepository> getAuditRepositoryImpl() {
try {
Configuration config = ApplicationProperties.get();
return ApplicationProperties.getClass(config,
return ApplicationProperties.getClass(config,
AUDIT_REPOSITORY_IMPLEMENTATION_PROPERTY, HBaseBasedAuditRepository.class.getName(), EntityAuditRepository.class);
} catch (AtlasException e) {
throw new RuntimeException(e);
@ -83,7 +86,7 @@ public class AtlasRepositoryConfiguration {
public static Class<? extends DeleteHandler> getDeleteHandlerImpl() {
try {
Configuration config = ApplicationProperties.get();
return ApplicationProperties.getClass(config,
return ApplicationProperties.getClass(config,
DELETE_HANDLER_IMPLEMENTATION_PROPERTY, SoftDeleteHandler.class.getName(), DeleteHandler.class);
} catch (AtlasException e) {
throw new RuntimeException(e);
@ -99,15 +102,50 @@ public class AtlasRepositoryConfiguration {
throw new RuntimeException(e);
}
}
public static final String COMPILED_QUERY_CACHE_CAPACITY = "atlas.CompiledQueryCache.capacity";
/**
* Get the configuration property that specifies the size of the compiled query
* cache. This is an optional property. A default is used if it is not
* present.
*
* @return the size to be used when creating the compiled query cache.
*/
public static int getCompiledQueryCacheCapacity() {
try {
return ApplicationProperties.get().getInt(COMPILED_QUERY_CACHE_CAPACITY, DEFAULT_COMPILED_QUERY_CACHE_CAPACITY);
} catch (AtlasException e) {
throw new RuntimeException(e);
}
}
public static final String COMPILED_QUERY_CACHE_EVICTION_WARNING_THROTTLE = "atlas.CompiledQueryCache.evictionWarningThrottle";
/**
* Get the configuration property that specifies the number evictions that pass
* before a warning is logged. This is an optional property. A default is
* used if it is not present.
*
* @return the number of evictions before a warning is logged.
*/
public static int getCompiledQueryCacheEvictionWarningThrottle() {
try {
return ApplicationProperties.get().getInt(COMPILED_QUERY_CACHE_EVICTION_WARNING_THROTTLE, DEFAULT_COMPILED_QUERY_CACHE_EVICTION_WARNING_THROTTLE);
} catch (AtlasException e) {
throw new RuntimeException(e);
}
}
private static final String GRAPH_DATABASE_IMPLEMENTATION_PROPERTY = "atlas.graphdb.backend";
private static final String DEFAULT_GRAPH_DATABASE_IMPLEMENTATION_CLASS = "org.apache.atlas.repository.graphdb.titan0.Titan0GraphDatabase";
@SuppressWarnings("unchecked")
public static Class<? extends GraphDatabase> getGraphDatabaseImpl() {
try {
Configuration config = ApplicationProperties.get();
return ApplicationProperties.getClass(config,
return ApplicationProperties.getClass(config,
GRAPH_DATABASE_IMPLEMENTATION_PROPERTY, DEFAULT_GRAPH_DATABASE_IMPLEMENTATION_CLASS, GraphDatabase.class);
} catch (AtlasException e) {
throw new RuntimeException(e);

87
repository/src/main/java/org/apache/atlas/util/CompiledQueryCacheKey.java Normal file
View File

@ -0,0 +1,87 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.util;
import org.apache.atlas.query.QueryParams;
/**
* Represents a key for an entry in the compiled query cache.
*
*/
public class CompiledQueryCacheKey {
private final String dslQuery;
private final QueryParams queryParams;
public CompiledQueryCacheKey(String dslQuery, QueryParams queryParams) {
super();
this.dslQuery = dslQuery;
this.queryParams = queryParams;
}
public CompiledQueryCacheKey(String dslQuery) {
super();
this.dslQuery = dslQuery;
this.queryParams = null;
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((dslQuery == null) ? 0 : dslQuery.hashCode());
result = prime * result + ((queryParams == null) ? 0 : queryParams.hashCode());
return result;
}
@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null) {
return false;
}
if (!(obj instanceof CompiledQueryCacheKey)) {
return false;
}
CompiledQueryCacheKey other = (CompiledQueryCacheKey) obj;
if (! equals(dslQuery, other.dslQuery)) {
return false;
}
if (! equals(queryParams, other.queryParams)) {
return false;
}
return true;
}
private static boolean equals(Object o1, Object o2) {
if(o1 == o2) {
return true;
}
if(o1 == null) {
return o2 == null;
}
return o1.equals(o2);
}
}

39
repository/src/main/java/org/apache/atlas/util/NoopGremlinQuery.java Normal file
View File

@ -0,0 +1,39 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.util;
import org.apache.atlas.query.GremlinQuery;
import org.apache.atlas.typesystem.types.IDataType;
/**
* Represents a query that we know will have no results.
*
*/
public class NoopGremlinQuery extends GremlinQuery {
private final IDataType dataType;
public NoopGremlinQuery(IDataType dataType) {
super(null, null, null);
this.dataType = dataType;
}
public IDataType getDataType() {
return dataType;
}
}

35
repository/src/main/scala/org/apache/atlas/query/QueryProcessor.scala Normal file → Executable file
View File

@ -21,10 +21,18 @@ package org.apache.atlas.query
import org.apache.atlas.repository.graphdb.AtlasGraph
import org.apache.atlas.query.Expressions._
import org.slf4j.{Logger, LoggerFactory}
import org.apache.atlas.util.AtlasRepositoryConfiguration
import org.apache.atlas.utils.LruCache
import org.apache.atlas.util.CompiledQueryCacheKey
import java.util.Collections
object QueryProcessor {
val LOG : Logger = LoggerFactory.getLogger("org.apache.atlas.query.QueryProcessor")
val compiledQueryCache = Collections.synchronizedMap(new LruCache[CompiledQueryCacheKey, GremlinQuery](
AtlasRepositoryConfiguration.getCompiledQueryCacheCapacity(),
AtlasRepositoryConfiguration.getCompiledQueryCacheEvictionWarningThrottle()));
def evaluate(e: Expression, g: AtlasGraph[_,_], gP : GraphPersistenceStrategies = null):
GremlinQueryResult = {
@ -33,11 +41,28 @@ object QueryProcessor {
strategy = GraphPersistenceStrategy1(g);
}
val e1 = validate(e)
val q = new GremlinTranslator(e1, strategy).translate()
LOG.debug("Query: " + e1)
LOG.debug("Expression Tree:\n" + e1.treeString)
LOG.debug("Gremlin Query: " + q.queryStr)
//convert the query expression to DSL so we can check whether or not it is in the compiled
//query cache and avoid validating/translating it again if it is.
val dsl = e.toString();
val cacheKey = new CompiledQueryCacheKey(dsl);
var q = compiledQueryCache.get(cacheKey);
if(q == null) {
//query was not found in the compiled query cache. Validate
//and translate it, then cache the result.
val e1 = validate(e)
q = new GremlinTranslator(e1, strategy).translate()
compiledQueryCache.put(cacheKey, q);
if(LOG.isDebugEnabled()) {
LOG.debug("Validated Query: " + e1)
LOG.debug("Expression Tree:\n" + e1.treeString);
}
}
if(LOG.isDebugEnabled()) {
LOG.debug("DSL Query: " + dsl);
LOG.debug("Gremlin Query: " + q.queryStr)
}
new GremlinEvaluator(q, strategy, g).evaluate()
}

86
repository/src/test/java/org/apache/atlas/util/CompiledQueryCacheKeyTest.java Normal file
View File

@ -0,0 +1,86 @@
package org.apache.atlas.util;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotSame;
import org.apache.atlas.query.QueryParams;
import org.testng.annotations.Test;
/**
* Tests hashcode/equals behavior of CompiledQueryCacheKey
*
*
*/
public class CompiledQueryCacheKeyTest {
@Test
public void testNoQueryParams() {
CompiledQueryCacheKey e1 = new CompiledQueryCacheKey("query 1");
CompiledQueryCacheKey e2 = new CompiledQueryCacheKey("query 1");
CompiledQueryCacheKey e3 = new CompiledQueryCacheKey("query 2");
assertKeysEqual(e1, e2);
assertKeysDifferent(e2, e3);
}
@Test
public void testWithQueryParams() {
CompiledQueryCacheKey e1 = new CompiledQueryCacheKey("query 1", new QueryParams(10,10));
CompiledQueryCacheKey e2 = new CompiledQueryCacheKey("query 1", new QueryParams(10,10));
CompiledQueryCacheKey e3 = new CompiledQueryCacheKey("query 2", new QueryParams(10,10));
assertKeysEqual(e1, e2);
assertKeysDifferent(e2, e3);
}
@Test
public void testOnlyQueryParamsDifferent() {
CompiledQueryCacheKey e1 = new CompiledQueryCacheKey("query 1", new QueryParams(10,10));
CompiledQueryCacheKey e2 = new CompiledQueryCacheKey("query 1", new QueryParams(20,10));
assertKeysDifferent(e1, e2);
}
@Test
public void testOnlyDslDifferent() {
CompiledQueryCacheKey e1 = new CompiledQueryCacheKey("query 1", new QueryParams(10,10));
CompiledQueryCacheKey e2 = new CompiledQueryCacheKey("query 2", new QueryParams(10,10));
assertKeysDifferent(e1, e2);
}
@Test
public void testMixOfQueryParamsAndNone() {
CompiledQueryCacheKey e1 = new CompiledQueryCacheKey("query 1", new QueryParams(10,10));
CompiledQueryCacheKey e2 = new CompiledQueryCacheKey("query 1");
assertKeysDifferent(e1, e2);
}
private void assertKeysEqual(CompiledQueryCacheKey e1, CompiledQueryCacheKey e2) {
assertEquals(e1.hashCode(), e2.hashCode());
assertEquals(e1, e2);
assertEquals(e2, e1);
}
private void assertKeysDifferent(CompiledQueryCacheKey e1, CompiledQueryCacheKey e2) {
assertNotSame(e1.hashCode(), e2.hashCode());
assertNotSame(e1, e2);
assertNotSame(e2, e1);
}
}