tool
/
atlas
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ATLAS-2932: DSL Refactoring for using Traversal - improved readability #2

This commit is contained in:
Madhan Neethiraj 2021-01-20 18:20:45 -08:00
parent b5014c606e
commit 810d705314
13 changed files with 431 additions and 284 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphTraversal.java
View File

@ -106,30 +106,37 @@ public class AtlasJanusGraphTraversal extends AtlasGraphTraversal<AtlasJanusVert
@Override
public Map<String, Collection<AtlasJanusVertex>> getAtlasVertexMap() {
List list = getResultList();
if (CollectionUtils.isEmpty(list) || !(list.get(0) instanceof Map)) {
return Collections.emptyMap();
}
List list = getResultList();
Map<String, Collection<AtlasJanusVertex>> ret;
Map<String, Collection<AtlasJanusVertex>> ret = new HashMap<>();
Map map = (Map) list.get(0);
for (Object key : map.keySet()) {
if (!(key instanceof String)) {
continue;
}
if (CollectionUtils.isNotEmpty(list) && (list.get(0) instanceof Map)) {
Map map = (Map) list.get(0);
Object value = map.get(key);
if (value instanceof List) {
Collection<AtlasJanusVertex> values = new ArrayList<>();
for (Object o : (List) value) {
if (o instanceof Vertex) {
values.add(GraphDbObjectFactory.createVertex((AtlasJanusGraph) atlasGraph, (Vertex) o));
} else {
LOG.warn("{} is not a vertex.", o.getClass().getSimpleName());
}
ret = new HashMap<>(map.size());
for (Object key : map.keySet()) {
if (!(key instanceof String)) {
continue;
}
Object value = map.get(key);
if (value instanceof List) {
Collection<AtlasJanusVertex> values = new ArrayList<>();
for (Object o : (List) value) {
if (o instanceof Vertex) {
values.add(GraphDbObjectFactory.createVertex((AtlasJanusGraph) atlasGraph, (Vertex) o));
} else {
LOG.warn("{} is not a vertex.", o.getClass().getSimpleName());
}
}
ret.put((String) key, values);
}
ret.put((String) key, values);
}
} else {
ret = Collections.emptyMap();
}
return ret;

14
repository/src/main/java/org/apache/atlas/query/AtlasDSL.java
View File

@ -121,7 +121,7 @@ public class AtlasDSL {
}
public GremlinQuery translate() throws AtlasBaseException {
QueryMetadata queryMetadata = new QueryMetadata(queryContext);
QueryMetadata queryMetadata = new QueryMetadata(queryContext);
GremlinQueryComposer queryComposer = new GremlinQueryComposer(typeRegistry, queryMetadata, limit, offset);
queryContext.accept(new DSLVisitor(queryComposer));
@ -132,13 +132,13 @@ public class AtlasDSL {
}
private void processErrorList(GremlinQueryComposer gremlinQueryComposer) throws AtlasBaseException {
if (CollectionUtils.isEmpty(gremlinQueryComposer.getErrorList())) {
return;
}
if (CollectionUtils.isNotEmpty(gremlinQueryComposer.getErrorList())) {
final String errorMessage = StringUtils.join(gremlinQueryComposer.getErrorList(), ", ");
final String errorMessage = StringUtils.join(gremlinQueryComposer.getErrorList(), ", ");
LOG.warn("DSL Errors: {}", errorMessage);
throw new AtlasBaseException(AtlasErrorCode.INVALID_DSL_QUERY, this.query, errorMessage);
LOG.warn("DSL Errors: {}", errorMessage);
throw new AtlasBaseException(AtlasErrorCode.INVALID_DSL_QUERY, this.query, errorMessage);
}
}
}

115
repository/src/main/java/org/apache/atlas/query/DSLVisitor.java
View File

@ -71,6 +71,7 @@ public class DSLVisitor extends AtlasDSLParserBaseVisitor<Void> {
gremlinQueryComposer.addLimit(ctx.limitClause().NUMBER().getText(),
(ctx.offsetClause() == null ? "0" : ctx.offsetClause().NUMBER().getText()));
return super.visitLimitOffset(ctx);
}
@ -85,8 +86,8 @@ public class DSLVisitor extends AtlasDSLParserBaseVisitor<Void> {
// Groupby attr also represent select expr, no processing is needed in that case
// visit groupBy would handle the select expr appropriately
if (!(ctx.getParent() instanceof GroupByExpressionContext)) {
String[] items = new String[ctx.selectExpression().size()];
String[] labels = new String[ctx.selectExpression().size()];
String[] items = new String[ctx.selectExpression().size()];
String[] labels = new String[ctx.selectExpression().size()];
int countIdx = -1;
int sumIdx = -1;
int minIdx = -1;
@ -123,6 +124,7 @@ public class DSLVisitor extends AtlasDSLParserBaseVisitor<Void> {
gremlinQueryComposer.addSelect(selectClauseComposer);
}
return super.visitSelectExpr(ctx);
}
@ -134,7 +136,9 @@ public class DSLVisitor extends AtlasDSLParserBaseVisitor<Void> {
// Extract the attribute from parentheses
String text = ctx.expr().getText().replace("(", "").replace(")", "");
gremlinQueryComposer.addOrderBy(text, (ctx.sortOrder() != null && ctx.sortOrder().getText().equalsIgnoreCase("desc")));
return super.visitOrderByExpr(ctx);
}
@ -145,7 +149,9 @@ public class DSLVisitor extends AtlasDSLParserBaseVisitor<Void> {
}
ExprContext expr = ctx.expr();
processExpr(expr, gremlinQueryComposer);
return super.visitWhereClause(ctx);
}
@ -167,6 +173,7 @@ public class DSLVisitor extends AtlasDSLParserBaseVisitor<Void> {
gremlinQueryComposer.addFrom(fromSrc.literal().getText());
}
}
return super.visitFromExpression(ctx);
}
@ -192,7 +199,9 @@ public class DSLVisitor extends AtlasDSLParserBaseVisitor<Void> {
}
String s = ctx.selectExpr().getText();
gremlinQueryComposer.addGroupBy(s);
return super.visitGroupByExpression(ctx);
}
@ -202,16 +211,19 @@ public class DSLVisitor extends AtlasDSLParserBaseVisitor<Void> {
}
gqc.addIsA(ctx.arithE().getText(), ctx.identifier().getText());
return super.visitIsClause(ctx);
}
private void visitHasClause(GremlinQueryComposer gqc, HasClauseContext ctx) {
gqc.addFromProperty(ctx.arithE().getText(), ctx.identifier().getText());
super.visitHasClause(ctx);
}
private void visitHasTermClause(GremlinQueryComposer gqc, HasTermClauseContext ctx) {
gqc.addHasTerm(ctx.arithE().getText(), ctx.identifier().getText());
super.visitHasTermClause(ctx);
}
@ -226,6 +238,7 @@ public class DSLVisitor extends AtlasDSLParserBaseVisitor<Void> {
if (ctx.expr().compE() != null && ctx.expr().compE().isClause() != null && ctx.expr().compE().isClause().arithE() != null) {
gremlinQueryComposer.addFrom(ctx.expr().compE().isClause().arithE().getText());
return;
}
@ -235,6 +248,7 @@ public class DSLVisitor extends AtlasDSLParserBaseVisitor<Void> {
if (ctx.expr().compE() != null && ctx.expr().compE().hasTermClause() != null && ctx.expr().compE().hasTermClause().arithE() != null) {
gremlinQueryComposer.addFrom(ctx.expr().compE().hasTermClause().arithE().getText());
return;
}
}
@ -248,10 +262,9 @@ public class DSLVisitor extends AtlasDSLParserBaseVisitor<Void> {
}
private void processExprRight(final ExprContext expr, GremlinQueryComposer gremlinQueryComposer) {
GremlinQueryComposer nestedProcessor = gremlinQueryComposer.createNestedProcessor();
List<GremlinQueryComposer> nestedQueries = new ArrayList<>();
String prev = null;
GremlinQueryComposer nestedProcessor = gremlinQueryComposer.createNestedProcessor();
List<GremlinQueryComposer> nestedQueries = new ArrayList<>();
String prev = null;
// Process first expression then proceed with the others
// expr -> compE exprRight*
@ -266,10 +279,10 @@ public class DSLVisitor extends AtlasDSLParserBaseVisitor<Void> {
// AND expression
if (exprRight.K_AND() != null) {
if (prev == null) prev = AND;
if (OR.equalsIgnoreCase(prev)) {
// Change of context
GremlinQueryComposer orClause = nestedProcessor.createNestedProcessor();
orClause.addOrClauses(nestedQueries);
nestedQueries.clear();
nestedQueries.add(orClause);
@ -277,14 +290,16 @@ public class DSLVisitor extends AtlasDSLParserBaseVisitor<Void> {
// Record all processed attributes
gremlinQueryComposer.addProcessedAttributes(orClause.getAttributesProcessed());
}
prev = AND;
}
// OR expression
if (exprRight.K_OR() != null) {
if (prev == null) prev = OR;
if (AND.equalsIgnoreCase(prev)) {
// Change of context
GremlinQueryComposer andClause = nestedProcessor.createNestedProcessor();
andClause.addAndClauses(nestedQueries);
nestedQueries.clear();
nestedQueries.add(andClause);
@ -292,65 +307,74 @@ public class DSLVisitor extends AtlasDSLParserBaseVisitor<Void> {
// Record all processed attributes
gremlinQueryComposer.addProcessedAttributes(andClause.getAttributesProcessed());
}
prev = OR;
}
processExpr(exprRight.compE(), nestedProcessor);
nestedQueries.add(nestedProcessor);
// Record all processed attributes
gremlinQueryComposer.addProcessedAttributes(nestedProcessor.getAttributesProcessed());
}
if (AND.equalsIgnoreCase(prev)) {
gremlinQueryComposer.addAndClauses(nestedQueries);
}
if (OR.equalsIgnoreCase(prev)) {
} else if (OR.equalsIgnoreCase(prev)) {
gremlinQueryComposer.addOrClauses(nestedQueries);
}
}
private void processExpr(final CompEContext compE, final GremlinQueryComposer gremlinQueryComposer) {
if (compE != null && compE.isClause() == null && compE.hasClause() == null && compE.hasTermClause() == null) {
ComparisonClauseContext comparisonClause = compE.comparisonClause();
if (compE != null) {
IsClauseContext isClause = compE.isClause();
HasClauseContext hasClause = compE.hasClause();
HasTermClauseContext hasTermClause = compE.hasTermClause();
// The nested expression might have ANDs/ORs
if (comparisonClause == null) {
ExprContext exprContext = compE.arithE().multiE().atomE().expr();
// Only extract comparison clause if there are no nested exprRight clauses
if (CollectionUtils.isEmpty(exprContext.exprRight())) {
comparisonClause = exprContext.compE().comparisonClause();
}
if (isClause != null) {
visitIsClause(gremlinQueryComposer, isClause);
}
if (comparisonClause != null) {
String lhs = comparisonClause.arithE(0).getText();
String op, rhs;
AtomEContext atomECtx = comparisonClause.arithE(1).multiE().atomE();
if (atomECtx.literal() == null ||
(atomECtx.literal() != null && atomECtx.literal().valueArray() == null)) {
op = comparisonClause.operator().getText().toUpperCase();
rhs = comparisonClause.arithE(1).getText();
if (hasClause != null) {
visitHasClause(gremlinQueryComposer, hasClause);
}
if (hasTermClause != null) {
visitHasTermClause(gremlinQueryComposer, hasTermClause);
}
if (isClause == null && hasClause == null && hasTermClause == null) {
ComparisonClauseContext comparisonClause = compE.comparisonClause();
// The nested expression might have ANDs/ORs
if (comparisonClause == null) {
ExprContext exprContext = compE.arithE().multiE().atomE().expr();
// Only extract comparison clause if there are no nested exprRight clauses
if (CollectionUtils.isEmpty(exprContext.exprRight())) {
comparisonClause = exprContext.compE().comparisonClause();
}
}
if (comparisonClause != null) {
String lhs = comparisonClause.arithE(0).getText();
AtomEContext atomECtx = comparisonClause.arithE(1).multiE().atomE();
String op, rhs;
if (atomECtx.literal() == null || (atomECtx.literal() != null && atomECtx.literal().valueArray() == null)) {
op = comparisonClause.operator().getText().toUpperCase();
rhs = comparisonClause.arithE(1).getText();
} else {
op = "in";
rhs = getInClause(atomECtx);
}
gremlinQueryComposer.addWhere(lhs, op, rhs);
} else {
op = "in";
rhs = getInClause(atomECtx);
processExpr(compE.arithE().multiE().atomE().expr(), gremlinQueryComposer);
}
gremlinQueryComposer.addWhere(lhs, op, rhs);
} else {
processExpr(compE.arithE().multiE().atomE().expr(), gremlinQueryComposer);
}
}
if (compE != null && compE.isClause() != null) {
visitIsClause(gremlinQueryComposer, compE.isClause());
}
if (compE != null && compE.hasClause() != null) {
visitHasClause(gremlinQueryComposer, compE.hasClause());
}
if (compE != null && compE.hasTermClause() != null) {
visitHasTermClause(gremlinQueryComposer, compE.hasTermClause());
}
}
private String getInClause(AtomEContext atomEContext) {
@ -358,6 +382,7 @@ public class DSLVisitor extends AtlasDSLParserBaseVisitor<Void> {
ValueArrayContext valueArrayContext = atomEContext.literal().valueArray();
int startIdx = 1;
int endIdx = valueArrayContext.children.size() - 1;
for (int i = startIdx; i < endIdx; i++) {
sb.append(valueArrayContext.getChild(i));
}

6
repository/src/main/java/org/apache/atlas/query/GremlinQuery.java
View File

@ -30,9 +30,9 @@ public class GremlinQuery {
private AtlasGraphTraversal traversal;
public GremlinQuery(String gremlinQuery, AtlasDSL.QueryMetadata queryMetadata, GremlinClauseList clauses, SelectClauseComposer selectComposer) {
this.queryStr = gremlinQuery;
this.queryMetadata = queryMetadata;
this.clauses = clauses;
this.queryStr = gremlinQuery;
this.queryMetadata = queryMetadata;
this.clauses = clauses;
this.selectComposer = selectComposer;
}

163
repository/src/main/java/org/apache/atlas/query/GremlinQueryComposer.java
View File

@ -62,9 +62,8 @@ public class GremlinQueryComposer {
private static final int DEFAULT_QUERY_RESULT_OFFSET = 0;
private static final ThreadLocal<DateFormat[]> DSL_DATE_FORMAT = ThreadLocal.withInitial(() -> {
final String formats[] = { ISO8601_FORMAT, ISO8601_DATE_FORMAT };
DateFormat[] dfs = new DateFormat[formats.length];
final String[] formats = { ISO8601_FORMAT, ISO8601_DATE_FORMAT };
final DateFormat[] dfs = new DateFormat[formats.length];
for (int i = 0; i < formats.length; i++) {
dfs[i] = new SimpleDateFormat(formats[i]);
@ -79,7 +78,6 @@ public class GremlinQueryComposer {
private final Set<String> attributesProcessed = new HashSet<>();
private final Lookup lookup;
private final AtlasDSL.QueryMetadata queryMetadata;
private final int providedLimit;
private final int providedOffset;
private final Context context;
@ -165,9 +163,9 @@ public class GremlinQueryComposer {
}
public void addHasTerm(String typeName, String termName) {
String attributeToSearch;
String qualifiedAttributeSeperator = String.valueOf(GlossaryUtils.invalidNameChars[0]);
String[] terms = termName.split(qualifiedAttributeSeperator);
String qualifiedAttributeSeperator = String.valueOf(GlossaryUtils.invalidNameChars[0]);
String[] terms = termName.split(qualifiedAttributeSeperator);
String attributeToSearch;
if (terms.length > 1) {
attributeToSearch = GlossaryUtils.QUALIFIED_NAME_ATTR;;
@ -181,13 +179,15 @@ public class GremlinQueryComposer {
public void addWhere(String lhs, String operator, String rhs) {
String currentType = context.getActiveTypeName();
IdentifierHelper.Info org = null;
IdentifierHelper.Info lhsI = createInfo(lhs);
if (!lhsI.isPrimitive()) {
introduceType(lhsI);
org = lhsI;
org = lhsI;
lhsI = createInfo(lhs);
lhsI.setTypeName(org.getTypeName());
}
@ -204,11 +204,12 @@ public class GremlinQueryComposer {
rhs = addQuotesIfNecessary(lhsI, rhs);
SearchParameters.Operator op = SearchParameters.Operator.fromString(operator);
if (StringUtils.equals(lhsI.getAttributeName(), Constants.IS_INCOMPLETE_PROPERTY_KEY)) {
addForIsIncompleteClause(lhsI, op, rhs);
} else {
if (op == SearchParameters.Operator.LIKE) {
final AtlasStructType.AtlasAttribute attribute = context.getActiveEntityType().getAttribute(lhsI.getAttributeName());
final AtlasStructType.AtlasAttribute attribute = context.getActiveEntityType().getAttribute(lhsI.getAttributeName());
final AtlasStructDef.AtlasAttributeDef.IndexType indexType = attribute.getAttributeDef().getIndexType();
if (indexType == AtlasStructDef.AtlasAttributeDef.IndexType.STRING || !containsNumberAndLettersOnly(rhs)) {
@ -220,68 +221,78 @@ public class GremlinQueryComposer {
add(GremlinClause.HAS_OPERATOR, getPropertyForClause(lhsI), "within", rhs);
} else {
Object normalizedRhs = getNormalizedAttrVal(lhsI, IdentifierHelper.removeQuotes(rhs));
addWithNormalizedValue(GremlinClause.HAS_OPERATOR, getPropertyForClause(lhsI), op.getSymbols()[1], normalizedRhs, rhs);
}
}
// record that the attribute has been processed so that the select clause doesn't add a attr presence check
attributesProcessed.add(lhsI.getQualifiedName());
if (org != null && org.isReferredType()) {
add(GremlinClause.DEDUP);
if (org.getEdgeDirection() != null) {
GremlinClause gremlinClauseForEdgeLabel = org.getEdgeDirection().equals(IN) ? GremlinClause.OUT : GremlinClause.IN;
add(gremlinClauseForEdgeLabel, org.getEdgeLabel());
} else {
add(GremlinClause.OUT, org.getEdgeLabel());
}
context.registerActive(currentType);
}
}
private void addForIsIncompleteClause(IdentifierHelper.Info lhsI,SearchParameters.Operator op, String rhs ) {
GremlinClause clause = GremlinClause.HAS_OPERATOR;
rhs = rhs.replace("'", "").replace("\"", "");
switch (op) {
case EQ:
if (IdentifierHelper.isCompleteValue(rhs)) {
clause = GremlinClause.HAS_NOT_PROPERTY;
} else if (IdentifierHelper.isInCompleteValue(rhs)) {
rhs = Constants.INCOMPLETE_ENTITY_VALUE.toString();
rhs = Constants.INCOMPLETE_ENTITY_VALUE.toString();
}
break;
case NEQ:
if (IdentifierHelper.isCompleteValue(rhs)) {
op = SearchParameters.Operator.EQ;
rhs = Constants.INCOMPLETE_ENTITY_VALUE.toString();
op = SearchParameters.Operator.EQ;
rhs = Constants.INCOMPLETE_ENTITY_VALUE.toString();
} else if (IdentifierHelper.isInCompleteValue(rhs)) {
clause = GremlinClause.HAS_NOT_PROPERTY;
}
break;
}
Object normalizedRhs = getNormalizedAttrVal(lhsI, IdentifierHelper.removeQuotes(rhs));
addWithNormalizedValue(clause, getPropertyForClause(lhsI), op.getSymbols()[1], normalizedRhs, rhs);
}
private Object getNormalizedAttrVal(IdentifierHelper.Info attrInfo, String attrVal) {
Object ret = attrVal;
AtlasEntityType entityType = context.getActiveEntityType();
String attrName = attrInfo.getAttributeName();
if (entityType == null || StringUtils.isEmpty(attrVal)) {
return attrVal;
if (entityType != null && StringUtils.isNotEmpty(attrVal)) {
String attrName = attrInfo.getAttributeName();
AtlasType attributeType = entityType.getAttributeType(attrName);
if (attributeType != null) {
Object normalizedValue = attributeType.getNormalizedValue(attrVal);
if (normalizedValue != null && attributeType instanceof AtlasBuiltInTypes.AtlasDateType) {
ret = ((Date) normalizedValue).getTime();
} else {
ret = normalizedValue;
}
}
}
AtlasType attributeType = entityType.getAttributeType(attrName);
if (attributeType == null) {
return attrVal;
}
Object normalizedValue = attributeType.getNormalizedValue(attrVal);
if (normalizedValue != null && attributeType instanceof AtlasBuiltInTypes.AtlasDateType) {
return ((Date) normalizedValue).getTime();
}
return normalizedValue;
return ret;
}
private boolean containsNumberAndLettersOnly(String rhs) {
@ -294,11 +305,13 @@ public class GremlinQueryComposer {
public void addAndClauses(List<GremlinQueryComposer> queryComposers) {
List<String> clauses = addToSubClause(queryComposers);
add(GremlinClause.AND, String.join(",", clauses));
}
public void addOrClauses(List<GremlinQueryComposer> queryComposers) {
List<String> clauses = addToSubClause(queryComposers);
add(GremlinClause.OR, String.join(",", clauses));
}
@ -317,6 +330,7 @@ public class GremlinQueryComposer {
if (!(queryMetadata.hasOrderBy() && queryMetadata.hasGroupBy())) {
addSelectTransformation(selectClauseComposer, null, false);
}
this.context.setSelectClauseComposer(selectClauseComposer);
}
@ -340,6 +354,7 @@ public class GremlinQueryComposer {
public void addLimit(String limit, String offset) {
SelectClauseComposer scc = context.getSelectClauseComposer();
if (scc == null) {
addLimitHelper(limit, offset);
} else {
@ -356,11 +371,10 @@ public class GremlinQueryComposer {
public String get() {
close();
boolean mustTransform = !isNestedQuery() && queryMetadata.needTransformation();
String items[] = getFormattedClauses(mustTransform);
String s = mustTransform ?
getTransformedClauses(items) :
String.join(".", items);
boolean mustTransform = !isNestedQuery() && queryMetadata.needTransformation();
String[] items = getFormattedClauses(mustTransform);
String s = mustTransform ? getTransformedClauses(items) : String.join(".", items);
return s;
}
@ -370,6 +384,7 @@ public class GremlinQueryComposer {
public void addOrderBy(String name, boolean isDesc) {
IdentifierHelper.Info ia = createInfo(name);
if (queryMetadata.hasSelect() && queryMetadata.hasGroupBy()) {
addSelectTransformation(this.context.selectClauseComposer, getPropertyForClause(ia), isDesc);
} else if (queryMetadata.hasGroupBy()) {
@ -381,8 +396,7 @@ public class GremlinQueryComposer {
}
public boolean hasFromClause() {
return queryClauses.contains(GremlinClause.HAS_TYPE) != -1 ||
queryClauses.contains(GremlinClause.HAS_TYPE_WITHIN) != -1;
return queryClauses.contains(GremlinClause.HAS_TYPE) != -1 || queryClauses.contains(GremlinClause.HAS_TYPE_WITHIN) != -1;
}
private void addWithNormalizedValue(GremlinClause clause, String propertyForClause, String symbol, Object normalizedRhs, String strValue) {
@ -390,15 +404,16 @@ public class GremlinQueryComposer {
}
private long getDateFormat(String s) {
for (DateFormat dateFormat : DSL_DATE_FORMAT.get()) {
try {
return dateFormat.parse(s).getTime();
} catch (ParseException ignored) {
// ignore
}
}
context.validator.check(false, AtlasErrorCode.INVALID_DSL_INVALID_DATE, s);
return -1;
}
@ -412,6 +427,7 @@ public class GremlinQueryComposer {
private String getPropertyForClause(IdentifierHelper.Info ia) {
String vertexPropertyName = lookup.getVertexPropertyName(ia.getTypeName(), ia.getAttributeName());
if (StringUtils.isNotEmpty(vertexPropertyName)) {
return vertexPropertyName;
}
@ -433,11 +449,13 @@ public class GremlinQueryComposer {
if(StringUtils.isEmpty(ia.getQualifiedName())) {
context.getErrorList().add("Unable to find qualified name for " + ia.getAttributeName());
continue;
}
if (scc.isAggregatorWithArgument(i) && !ia.isPrimitive()) {
context.check(false, AtlasErrorCode.INVALID_DSL_SELECT_INVALID_AGG, ia.getQualifiedName());
return;
}
@ -450,6 +468,7 @@ public class GremlinQueryComposer {
}
scc.setIsSelectNoop(hasNoopCondition(ia));
if (scc.getIsSelectNoop()) {
return;
}
@ -457,6 +476,7 @@ public class GremlinQueryComposer {
if (introduceType(ia)) {
scc.incrementTypesIntroduced();
scc.setIsSelectNoop(!ia.hasParts());
if (ia.hasParts()) {
scc.assign(i, getPropertyForClause(createInfo(ia.get())), GremlinClause.INLINE_GET_PROPERTY);
}
@ -494,6 +514,7 @@ public class GremlinQueryComposer {
} else {
ret = queryClauses.getValue(0) + funCall;
}
return ret;
}
@ -505,6 +526,7 @@ public class GremlinQueryComposer {
for (int i = startIdx; i < endIdx; i++) {
items[i] = queryClauses.getValue(i);
}
return items;
}
@ -512,6 +534,7 @@ public class GremlinQueryComposer {
final String orderByQualifiedAttrName,
final boolean isDesc) {
GremlinClause gremlinClause;
if (selectClauseComposer.getIsSelectNoop()) {
gremlinClause = GremlinClause.SELECT_NOOP_FN;
} else if (queryMetadata.hasGroupBy()) {
@ -519,6 +542,7 @@ public class GremlinQueryComposer {
} else {
gremlinClause = selectClauseComposer.onlyAggregators() ? GremlinClause.SELECT_ONLY_AGG_FN : GremlinClause.SELECT_FN;
}
if (StringUtils.isEmpty(orderByQualifiedAttrName)) {
add(0, gremlinClause,
selectClauseComposer.getLabelHeader(),
@ -528,10 +552,13 @@ public class GremlinQueryComposer {
} else {
int itemIdx = selectClauseComposer.getAttrIndex(orderByQualifiedAttrName);
GremlinClause sortClause = GremlinClause.INLINE_DEFAULT_TUPLE_SORT;
if (itemIdx != -1) {
sortClause = isDesc ? GremlinClause.INLINE_TUPLE_SORT_DESC : GremlinClause.INLINE_TUPLE_SORT_ASC;
}
String idxStr = String.valueOf(itemIdx);
add(0, gremlinClause,
selectClauseComposer.getLabelHeader(),
selectClauseComposer.getAssignmentExprString(),
@ -544,17 +571,23 @@ public class GremlinQueryComposer {
}
private String addQuotesIfNecessary(IdentifierHelper.Info rhsI, String rhs) {
if(rhsI.isNumeric()) return rhs;
if (IdentifierHelper.isTrueOrFalse(rhs)) return rhs;
if (IdentifierHelper.isQuoted(rhs)) return rhs;
if(rhsI.isNumeric()) {
return rhs;
}
if (IdentifierHelper.isTrueOrFalse(rhs)) {
return rhs;
}
if (IdentifierHelper.isQuoted(rhs)) {
return rhs;
}
return IdentifierHelper.getQuoted(rhs);
}
private String parseDate(String rhs) {
String s = IdentifierHelper.isQuoted(rhs) ?
IdentifierHelper.removeQuotes(rhs) :
rhs;
String s = IdentifierHelper.isQuoted(rhs) ? IdentifierHelper.removeQuotes(rhs) : rhs;
return String.format("'%d'", getDateFormat(s));
}
@ -579,6 +612,7 @@ public class GremlinQueryComposer {
if (queryClauses.isEmpty()) {
queryClauses.clear();
return;
}
@ -597,16 +631,19 @@ public class GremlinQueryComposer {
private void moveToLast(GremlinClause clause) {
int index = queryClauses.contains(clause);
if (-1 == index) {
return;
}
GremlinClauseValue gcv = queryClauses.remove(index);
queryClauses.add(gcv);
}
public void remove(GremlinClause clause) {
int index = queryClauses.contains(clause);
if (-1 == index) {
return;
}
@ -631,10 +668,12 @@ public class GremlinQueryComposer {
if (ia.isReferredType()) {
if (ia.getEdgeDirection() != null) {
GremlinClause gremlinClauseForEdgeLabel = ia.getEdgeDirection().equals(OUT) ? GremlinClause.OUT : GremlinClause.IN;
add(gremlinClauseForEdgeLabel, ia.getEdgeLabel());
} else {
add(GremlinClause.OUT, ia.getEdgeLabel());
}
context.registerActive(ia);
}
@ -659,6 +698,7 @@ public class GremlinQueryComposer {
private void addGroupByClause(String name) {
IdentifierHelper.Info ia = createInfo(name);
add(GremlinClause.GROUP_BY, ia);
}
@ -705,16 +745,16 @@ public class GremlinQueryComposer {
private final Object rawValue;
public GremlinClauseValue(GremlinClause clause, String property, String operator, Object rawValue, String str) {
this.clause = clause;
this.value = clause.get(property, operator, str);
this.values = new String[] {property, operator, str};
this.clause = clause;
this.value = clause.get(property, operator, str);
this.values = new String[] {property, operator, str};
this.rawValue = rawValue;
}
public GremlinClauseValue(GremlinClause clause, String... values) {
this.clause = clause;
this.value = clause.get(values);
this.values = values;
this.clause = clause;
this.value = clause.get(values);
this.values = values;
this.rawValue = null;
}
@ -744,15 +784,15 @@ public class GremlinQueryComposer {
static class Context {
private static final AtlasStructType UNKNOWN_TYPE = new AtlasStructType(new AtlasStructDef());
private final Lookup lookup;
private final ClauseValidator validator;
private final Map<String, String> aliasMap = new HashMap<>();
private AtlasType activeType;
private SelectClauseComposer selectClauseComposer;
private String numericTypeFormatter = "";
private final Lookup lookup;
private final ClauseValidator validator;
private final Map<String, String> aliasMap = new HashMap<>();
private AtlasType activeType;
private SelectClauseComposer selectClauseComposer;
private String numericTypeFormatter = "";
public Context(Lookup lookup) {
this.lookup = lookup;
this.lookup = lookup;
this.validator = new ClauseValidator();
}
@ -760,9 +800,11 @@ public class GremlinQueryComposer {
if (shouldRegister(typeName)) {
try {
activeType = lookup.getType(typeName);
aliasMap.put(typeName, typeName);
} catch (AtlasBaseException e) {
validator.check(e, AtlasErrorCode.INVALID_DSL_UNKNOWN_TYPE, typeName);
activeType = UNKNOWN_TYPE;
}
}
@ -778,9 +820,7 @@ public class GremlinQueryComposer {
}
public AtlasEntityType getActiveEntityType() {
return (activeType instanceof AtlasEntityType) ?
(AtlasEntityType) activeType :
null;
return (activeType instanceof AtlasEntityType) ? (AtlasEntityType) activeType : null;
}
public String getActiveTypeName() {
@ -824,6 +864,7 @@ public class GremlinQueryComposer {
public void addAlias(String alias, String typeName) {
if (aliasMap.containsKey(alias)) {
check(false, AtlasErrorCode.INVALID_DSL_DUPLICATE_ALIAS, alias, getActiveTypeName());
return;
}
@ -852,7 +893,7 @@ public class GremlinQueryComposer {
}
private static class ClauseValidator {
List<String> errorList = new ArrayList<>();
final List<String> errorList = new ArrayList<>();
public ClauseValidator() {
}
@ -886,6 +927,7 @@ public class GremlinQueryComposer {
public boolean check(Exception ex, AtlasErrorCode vm, String... args) {
String[] extraArgs = getExtraSlotArgs(args, ex.getMessage());
return check(false, vm, extraArgs);
}
@ -911,8 +953,11 @@ public class GremlinQueryComposer {
private String[] getExtraSlotArgs(String[] args, String s) {
String[] argsPlus1 = new String[args.length + 1];
System.arraycopy(args, 0, argsPlus1, 0, args.length);
argsPlus1[args.length] = s;
return argsPlus1;
}
}

87
repository/src/main/java/org/apache/atlas/query/SelectClauseComposer.java
View File

@ -34,11 +34,9 @@ public class SelectClauseComposer {
private static final String MAX_STR = "max";
private static final String SUM_STR = "sum";
private final String[] labels;
private final String[] attributes;
private final String[] items;
private final int countIdx;
private final int sumIdx;
private final int minIdx;
@ -51,6 +49,10 @@ public class SelectClauseComposer {
private boolean isSelectNoop = false;
private int introducedTypesCount = 0;
public enum AggregatorFlag {
NONE, COUNT, MIN, MAX, SUM
}
public SelectClauseComposer(String[] labels, String[] attributes, String[] items, int countIdx, int sumIdx, int minIdx, int maxIdx) {
this.labels = labels;
this.attributes = Arrays.copyOf(attributes, attributes.length);
@ -59,22 +61,26 @@ public class SelectClauseComposer {
this.sumIdx = sumIdx;
this.minIdx = minIdx;
this.maxIdx = maxIdx;
int aggCount = 0;
if (countIdx != -1) {
this.aggregatorFlags.put(countIdx, AggregatorFlag.COUNT);
aggCount++;
}
if (sumIdx != -1) {
this.aggregatorFlags.put(sumIdx, AggregatorFlag.SUM);
aggCount++;
}
if (maxIdx != -1) {
this.aggregatorFlags.put(maxIdx, AggregatorFlag.MAX);
aggCount++;
}
if (minIdx != -1) {
this.aggregatorFlags.put(minIdx, AggregatorFlag.MIN);
aggCount++;
}
@ -83,9 +89,9 @@ public class SelectClauseComposer {
public static boolean isKeyword(String s) {
return COUNT_STR.equals(s) ||
MIN_STR.equals(s) ||
MAX_STR.equals(s) ||
SUM_STR.equals(s);
MIN_STR.equals(s) ||
MAX_STR.equals(s) ||
SUM_STR.equals(s);
}
public String[] getItems() {
@ -95,21 +101,27 @@ public class SelectClauseComposer {
public boolean updateAsApplicable(int currentIndex, String propertyForClause, String qualifiedName) {
boolean ret = false;
if (currentIndex == getCountIdx()) {
ret = assign(currentIndex, COUNT_STR, GremlinClause.INLINE_COUNT.get(), GremlinClause.INLINE_ASSIGNMENT);
this.isNumericAggregator.add(currentIndex);
} else if (currentIndex == getMinIdx()) {
ret = assign(currentIndex, MIN_STR, propertyForClause, GremlinClause.INLINE_ASSIGNMENT, GremlinClause.INLINE_MIN);
this.isNumericAggregator.add(currentIndex);
} else if (currentIndex == getMaxIdx()) {
ret = assign(currentIndex, MAX_STR, propertyForClause, GremlinClause.INLINE_ASSIGNMENT, GremlinClause.INLINE_MAX);
this.isNumericAggregator.add(currentIndex);
} else if (currentIndex == getSumIdx()) {
ret = assign(currentIndex, SUM_STR, propertyForClause, GremlinClause.INLINE_ASSIGNMENT, GremlinClause.INLINE_SUM);
this.isNumericAggregator.add(currentIndex);
}
attributes[currentIndex] = qualifiedName;
return ret;
}
@ -117,9 +129,9 @@ public class SelectClauseComposer {
return attributes;
}
public boolean assign(int i, String qualifiedName, GremlinClause clause) {
items[i] = clause.get(qualifiedName, qualifiedName);
return true;
}
@ -161,57 +173,32 @@ public class SelectClauseComposer {
public int getAttrIndex(String attr) {
int ret = -1;
for (int i = 0; i < attributes.length; i++) {
if (attributes[i].equals(attr)) {
ret = i;
break;
}
}
return ret;
}
private boolean assign(String item, String assignExpr) {
itemAssignmentExprs.put(item, assignExpr);
return true;
}
private boolean assign(int i, String s, String p1, GremlinClause clause) {
items[i] = s;
return assign(items[i], clause.get(s, p1));
}
private boolean assign(int i, String s, String p1, GremlinClause inline, GremlinClause clause) {
items[i] = s;
return assign(items[i], inline.get(s, clause.get(p1, p1)));
}
public int getCountIdx() {
return countIdx;
}
public int getSumIdx() {
return sumIdx;
}
public int getMaxIdx() {
return maxIdx;
}
public int getMinIdx() {
return minIdx;
}
private String getJoinedQuotedStr(String[] elements) {
StringJoiner joiner = new StringJoiner(",");
Arrays.stream(elements)
.map(x -> x.contains("'") ? "\"" + x + "\"" : "'" + x + "'")
.forEach(joiner::add);
return joiner.toString();
}
public boolean isAggregatorWithArgument(int i) {
@ -241,6 +228,7 @@ public class SelectClauseComposer {
public boolean getIsSelectNoop() {
return this.isSelectNoop;
}
public void setIsSelectNoop(boolean isSelectNoop) {
this.isSelectNoop = isSelectNoop;
}
@ -273,7 +261,32 @@ public class SelectClauseComposer {
this.isPrimitiveAttr.add(i);
}
public enum AggregatorFlag {
NONE, COUNT, MIN, MAX, SUM
private boolean assign(String item, String assignExpr) {
itemAssignmentExprs.put(item, assignExpr);
return true;
}
private boolean assign(int i, String s, String p1, GremlinClause clause) {
items[i] = s;
return assign(items[i], clause.get(s, p1));
}
private boolean assign(int i, String s, String p1, GremlinClause inline, GremlinClause clause) {
items[i] = s;
return assign(items[i], inline.get(s, clause.get(p1, p1)));
}
private String getJoinedQuotedStr(String[] elements) {
StringJoiner joiner = new StringJoiner(",");
Arrays.stream(elements)
.map(x -> x.contains("'") ? "\"" + x + "\"" : "'" + x + "'")
.forEach(joiner::add);
return joiner.toString();
}
}

27
repository/src/main/java/org/apache/atlas/query/executors/GremlinClauseToTraversalTranslator.java
View File

@ -57,10 +57,13 @@ public class GremlinClauseToTraversalTranslator {
public AtlasGraphTraversal process(GremlinClauseList clauseList) {
Stack<List<AtlasGraphTraversal>> subTraversals = new Stack<>();
AtlasGraphTraversal ret = process(null, subTraversals, clauseList);
AtlasGraphTraversal ret = process(null, subTraversals, clauseList);
if (!subTraversals.isEmpty()) {
String errorMessage = "Sub-traversals found not to be empty! " + subTraversals.toString();
LOG.warn(errorMessage);
throw new RuntimeException(errorMessage);
}
@ -71,12 +74,13 @@ public class GremlinClauseToTraversalTranslator {
Stack<List<AtlasGraphTraversal>> collected,
GremlinClauseList clauseList) {
int size = clauseList.getList().size();
for (int index = 0; index < size; index++) {
for (int index = 0; index < size; index++) {
if (clauseList.hasSubClause(index)) {
List<GremlinClauseList> subClauses = clauseList.getSubClauses(index);
collected.push(new ArrayList<>());
for (GremlinClauseList sc : subClauses) {
process(traversal, collected, sc);
}
@ -91,9 +95,9 @@ public class GremlinClauseToTraversalTranslator {
private AtlasGraphTraversal traverse(AtlasGraphTraversal traversal,
Stack<List<AtlasGraphTraversal>> trLists,
GremlinQueryComposer.GremlinClauseValue clauseValue) {
GremlinClause clause = clauseValue.getClause();
String[] values = clauseValue.getValues();
String[] values = clauseValue.getValues();
switch (clause) {
case G:
break;
@ -109,6 +113,7 @@ public class GremlinClauseToTraversalTranslator {
case AND: {
if (trLists != null && !trLists.peek().isEmpty()) {
List<AtlasGraphTraversal> subTraversals = trLists.pop();
traversal.and(subTraversals.toArray(new Traversal[0]));
} else {
throw new RuntimeException("subTraversals not expected to be NULL: " + clause.toString());
@ -119,6 +124,7 @@ public class GremlinClauseToTraversalTranslator {
case OR: {
if (trLists != null && !trLists.peek().isEmpty()) {
List<AtlasGraphTraversal> subTraversals = trLists.pop();
traversal.or(subTraversals.toArray(new Traversal[0]));
} else {
throw new RuntimeException("subTraversals not expected to be NULL: " + clause.toString());
@ -136,6 +142,7 @@ public class GremlinClauseToTraversalTranslator {
case HAS_OPERATOR:
P predicate = getPredicate(values[1], values[2], clauseValue.getRawValue());
traversal.has(values[0], predicate);
break;
@ -185,11 +192,13 @@ public class GremlinClauseToTraversalTranslator {
case NESTED_START:
traversal = traversal.startAnonymousTraversal();
trLists.peek().add(traversal);
break;
case HAS_TYPE_WITHIN:
String[] subTypes = StringUtils.split(removeRedundantQuotes(values[0]), ',');
traversal.has("__typeName", P.within(subTypes));
break;
@ -215,8 +224,10 @@ public class GremlinClauseToTraversalTranslator {
case RANGE:
traversal.dedup();
long low = Long.parseLong(values[1]);
long high = low + Long.parseLong(values[2]);
traversal.range(Scope.global, low, high);
break;
@ -229,11 +240,12 @@ public class GremlinClauseToTraversalTranslator {
case TERM:
String term = String.format("AtlasGlossaryTerm.%s", values[0]);
traversal.and(
traversal.startAnonymousTraversal()
.in(org.apache.atlas.repository.Constants.TERM_ASSIGNMENT_LABEL)
.has(term, P.eq(values[1]))
.dedup()
.in(org.apache.atlas.repository.Constants.TERM_ASSIGNMENT_LABEL)
.has(term, P.eq(values[1]))
.dedup()
);
break;
@ -276,6 +288,7 @@ public class GremlinClauseToTraversalTranslator {
private String[] csvToArray(String strRhs) {
String csvRow = StringUtils.replaceEach(strRhs, new String[]{"[", "]", "'"}, new String[]{"", "", ""});
return csvRow.split(",");
}

30
repository/src/main/java/org/apache/atlas/query/executors/ScriptEngineBasedExecutor.java
View File

@ -19,6 +19,8 @@ package org.apache.atlas.query.executors;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.discovery.AtlasSearchResult;
import org.apache.atlas.model.discovery.AtlasSearchResult.AttributeSearchResult;
import org.apache.atlas.model.discovery.AtlasSearchResult.AtlasQueryType;
import org.apache.atlas.query.AtlasDSL;
import org.apache.atlas.query.GremlinQuery;
import org.apache.atlas.query.QueryParams;
@ -37,6 +39,7 @@ import java.util.Map;
public class ScriptEngineBasedExecutor implements DSLQueryExecutor {
private static final Logger LOG = LoggerFactory.getLogger(ScriptEngineBasedExecutor.class);
private final AtlasTypeRegistry typeRegistry;
private final AtlasGraph graph;
private final EntityGraphRetriever entityRetriever;
@ -49,11 +52,11 @@ public class ScriptEngineBasedExecutor implements DSLQueryExecutor {
@Override
public AtlasSearchResult execute(String dslQuery, int limit, int offset) throws AtlasBaseException {
AtlasSearchResult ret = new AtlasSearchResult(dslQuery, AtlasSearchResult.AtlasQueryType.DSL);
GremlinQuery gremlinQuery = toGremlinQuery(dslQuery, limit, offset);
AtlasSearchResult ret = new AtlasSearchResult(dslQuery, AtlasQueryType.DSL);
GremlinQuery gremlinQuery = toGremlinQuery(dslQuery, limit, offset);
String queryStr = gremlinQuery.queryStr();
Object result = graph.executeGremlinScript(queryStr, false);
Object result = graph.executeGremlinScript(queryStr, false);
if (result instanceof List && CollectionUtils.isNotEmpty((List)result)) {
List queryResult = (List) result;
Object firstElement = queryResult.get(0);
@ -79,6 +82,7 @@ public class ScriptEngineBasedExecutor implements DSLQueryExecutor {
if (value instanceof List && CollectionUtils.isNotEmpty((List)value)) {
for (Object o : (List) value) {
Object entry = o;
if (entry instanceof AtlasVertex) {
ret.addEntity(entityRetriever.toAtlasEntityHeader((AtlasVertex) entry));
}
@ -96,8 +100,8 @@ public class ScriptEngineBasedExecutor implements DSLQueryExecutor {
}
private GremlinQuery toGremlinQuery(String query, int limit, int offset) throws AtlasBaseException {
QueryParams params = QueryParams.getNormalizedParams(limit, offset);
GremlinQuery gremlinQuery = new AtlasDSL.Translator(query, typeRegistry, params.offset(), params.limit()).translate();
QueryParams params = QueryParams.getNormalizedParams(limit, offset);
GremlinQuery gremlinQuery = new AtlasDSL.Translator(query, typeRegistry, params.offset(), params.limit()).translate();
if (LOG.isDebugEnabled()) {
LOG.debug("Translated Gremlin Query: {}", gremlinQuery.queryStr());
@ -106,13 +110,14 @@ public class ScriptEngineBasedExecutor implements DSLQueryExecutor {
return gremlinQuery;
}
private AtlasSearchResult.AttributeSearchResult toAttributesResult(List results, GremlinQuery query) {
AtlasSearchResult.AttributeSearchResult ret = new AtlasSearchResult.AttributeSearchResult();
List<String> names = (List<String>) results.get(0);
List<List<Object>> values = extractValues(results.subList(1, results.size()));
private AttributeSearchResult toAttributesResult(List results, GremlinQuery query) {
AttributeSearchResult ret = new AttributeSearchResult();
List<String> names = (List<String>) results.get(0);
List<List<Object>> values = extractValues(results.subList(1, results.size()));
ret.setName(names);
ret.setValues(values);
return ret;
}
@ -121,22 +126,25 @@ public class ScriptEngineBasedExecutor implements DSLQueryExecutor {
for (Object obj : results) {
if (obj instanceof Map) {
Map map = (Map) obj;
Map map = (Map) obj;
List<Object> list = new ArrayList<>();
if (MapUtils.isNotEmpty(map)) {
for (Object key : map.keySet()) {
Object vals = map.get(key);
if(vals instanceof List) {
List l = (List) vals;
list.addAll(l);
}
}
values.add(list);
}
} else if (obj instanceof List) {
List list = (List) obj;
if (CollectionUtils.isNotEmpty(list)) {
values.add(list);
}

80
repository/src/main/java/org/apache/atlas/query/executors/SelectClauseProjections.java
View File

@ -19,6 +19,7 @@ package org.apache.atlas.query.executors;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.discovery.AtlasSearchResult;
import org.apache.atlas.model.discovery.AtlasSearchResult.AttributeSearchResult;
import org.apache.atlas.query.GremlinQuery;
import org.apache.atlas.query.SelectClauseComposer;
import org.apache.atlas.repository.graphdb.AtlasVertex;
@ -46,13 +47,13 @@ public class SelectClauseProjections {
public static AtlasSearchResult usingList(final GremlinQuery queryInfo,
final EntityGraphRetriever entityRetriever,
final Collection<AtlasVertex> resultList) throws AtlasBaseException {
AtlasSearchResult ret = new AtlasSearchResult();
SelectClauseComposer SelectClauseInfo = queryInfo.getSelectComposer();
AtlasSearchResult.AttributeSearchResult attributeSearchResult = new AtlasSearchResult.AttributeSearchResult();
AtlasSearchResult ret = new AtlasSearchResult();
SelectClauseComposer selectClauseInfo = queryInfo.getSelectComposer();
AttributeSearchResult attributeSearchResult = new AttributeSearchResult();
attributeSearchResult.setName(Arrays.stream(SelectClauseInfo.getLabels()).collect(Collectors.toList()));
attributeSearchResult.setName(Arrays.stream(selectClauseInfo.getLabels()).collect(Collectors.toList()));
Collection<List<Object>> values = getProjectionRows(resultList, SelectClauseInfo, entityRetriever);
Collection<List<Object>> values = getProjectionRows(resultList, selectClauseInfo, entityRetriever);
if (values instanceof List) {
attributeSearchResult.setValues((List) values);
@ -68,32 +69,37 @@ public class SelectClauseProjections {
public static AtlasSearchResult usingMap(final GremlinQuery gremlinQuery,
final EntityGraphRetriever entityRetriever,
final Map<String, Collection<AtlasVertex>> resultMap) throws AtlasBaseException {
AtlasSearchResult ret = new AtlasSearchResult();
SelectClauseComposer selectClauseInfo = gremlinQuery.getSelectComposer();
AtlasSearchResult.AttributeSearchResult attributeSearchResult = new AtlasSearchResult.AttributeSearchResult();
AtlasSearchResult ret = new AtlasSearchResult();
SelectClauseComposer selectClauseInfo = gremlinQuery.getSelectComposer();
AttributeSearchResult attributeSearchResult = new AttributeSearchResult();
attributeSearchResult.setName(Arrays.stream(selectClauseInfo.getLabels()).collect(Collectors.toList()));
List<List<Object>> values = new ArrayList<>();
for (Collection<AtlasVertex> value : resultMap.values()) {
Collection<List<Object>> projectionRows = getProjectionRows(value, selectClauseInfo, entityRetriever);
values.addAll(projectionRows);
}
attributeSearchResult.setValues(getSublistForGroupBy(gremlinQuery, values));
ret.setAttributes(attributeSearchResult);
return ret;
}
private static List<List<Object>> getSublistForGroupBy(GremlinQuery gremlinQuery, List<List<Object>> values) {
int startIndex = gremlinQuery.getQueryMetadata().getResolvedOffset() - 1 ;
int startIndex = gremlinQuery.getQueryMetadata().getResolvedOffset() - 1;
if (startIndex < 0) {
startIndex = 0;
}
int endIndex = startIndex + gremlinQuery.getQueryMetadata().getResolvedLimit();
if (startIndex >= values.size()) {
endIndex = 0;
endIndex = 0;
startIndex = 0;
}
@ -124,6 +130,7 @@ public class SelectClauseProjections {
} else {
if (selectClauseComposer.isPrimitiveAttribute(idx)) {
String propertyName = selectClauseComposer.getAttribute(idx);
row.add(vertex.getProperty(propertyName, Object.class));
} else {
row.add(entityRetriever.toAtlasEntityHeaderWithClassifications(vertex));
@ -143,15 +150,18 @@ public class SelectClauseProjections {
}
final String propertyName = selectClauseComposer.getAttribute(idx);
double sum = 0;
double sum = 0;
for (AtlasVertex vertex : vertices) {
Number value = vertex.getProperty(propertyName, Number.class);
if (value != null) {
sum += value.doubleValue();
} else {
LOG.warn("Property: {} for vertex: {} not found!", propertyName, vertex.getId());
}
}
return sum;
}
@ -160,6 +170,7 @@ public class SelectClauseProjections {
if (selectClauseComposer.isNumericAggregator(idx)) {
AtlasVertex maxV = Collections.max(vertices, new VertexPropertyComparator(propertyName));
return maxV.getProperty(propertyName, Object.class);
} else {
return Collections.max(vertices.stream().map(v -> v.getProperty(propertyName, String.class))
@ -173,6 +184,7 @@ public class SelectClauseProjections {
if (selectClauseComposer.isNumericAggregator(idx)) {
AtlasVertex minV = Collections.min(vertices, new VertexPropertyComparator(propertyName));
return minV.getProperty(propertyName, Object.class);
} else {
return Collections.min(vertices.stream()
@ -200,34 +212,24 @@ public class SelectClauseProjections {
return -1;
} else if (p2 == null) {
return 1;
}
if (p1 instanceof String && p2 instanceof String) {
return ((String) p1).compareTo((String) p2);
}
if (p1 instanceof Byte && p2 instanceof Byte) {
return ((Byte) p1).compareTo((Byte) p2);
}
if (p1 instanceof Short && p2 instanceof Short) {
return ((Short) p1).compareTo((Short) p2);
}
if (p1 instanceof Integer && p2 instanceof Integer) {
return ((Integer) p1).compareTo((Integer) p2);
}
if (p1 instanceof Float && p2 instanceof Float) {
return ((Float) p1).compareTo((Float) p2);
}
if (p1 instanceof Double && p2 instanceof Double) {
return ((Double) p1).compareTo((Double) p2);
}
if (p1 instanceof Long && p2 instanceof Long) {
return ((Long) p1).compareTo((Long) p2);
}
if (p1 instanceof BigInteger && p2 instanceof BigInteger) {
return ((BigInteger) p1).compareTo((BigInteger) p2);
}
if (p1 instanceof BigDecimal && p2 instanceof BigDecimal) {
return ((BigDecimal) p1).compareTo((BigDecimal) p2);
} else if (p1 instanceof String) {
return (p2 instanceof String) ? ((String) p1).compareTo((String) p2) : 0;
} else if (p1 instanceof Integer) {
return (p2 instanceof Integer) ? ((Integer) p1).compareTo((Integer) p2) : 0;
} else if (p1 instanceof Long) {
return (p2 instanceof Long) ? ((Long) p1).compareTo((Long) p2) : 0;
} else if (p1 instanceof Short) {
return (p2 instanceof Short) ? ((Short) p1).compareTo((Short) p2) : 0;
} else if (p1 instanceof Float) {
return (p2 instanceof Float) ? ((Float) p1).compareTo((Float) p2) : 0;
} else if (p1 instanceof Double) {
return (p2 instanceof Double) ? ((Double) p1).compareTo((Double) p2) : 0;
} else if (p1 instanceof Byte) {
return (p2 instanceof Byte) ? ((Byte) p1).compareTo((Byte) p2) : 0;
} else if (p1 instanceof BigInteger) {
return (p2 instanceof BigInteger) ? ((BigInteger) p1).compareTo((BigInteger) p2) : 0;
} else if (p1 instanceof BigDecimal) {
return (p2 instanceof BigDecimal) ? ((BigDecimal) p1).compareTo((BigDecimal) p2) : 0;
}
return 0;

10
repository/src/main/java/org/apache/atlas/query/executors/TraversalBasedExecutor.java
View File

@ -56,8 +56,8 @@ public class TraversalBasedExecutor implements DSLQueryExecutor {
@Override
public AtlasSearchResult execute(String dslQuery, int limit, int offset) throws AtlasBaseException {
AtlasSearchResult ret = new AtlasSearchResult(dslQuery, AtlasSearchResult.AtlasQueryType.DSL);
GremlinQuery gremlinQuery = toTraversal(dslQuery, limit, offset);
AtlasSearchResult ret = new AtlasSearchResult(dslQuery, AtlasSearchResult.AtlasQueryType.DSL);
GremlinQuery gremlinQuery = toTraversal(dslQuery, limit, offset);
AtlasGraphTraversal<AtlasVertex, AtlasEdge> graphTraversal = gremlinQuery.getTraversal();
if (LOG.isDebugEnabled()) {
@ -65,6 +65,7 @@ public class TraversalBasedExecutor implements DSLQueryExecutor {
}
List<AtlasVertex> resultList = graphTraversal.getAtlasVertexList();
return (CollectionUtils.isNotEmpty(resultList))
? getSearchResult(ret, gremlinQuery, resultList)
: getSearchResult(ret, gremlinQuery, graphTraversal.getAtlasVertexMap());
@ -106,12 +107,13 @@ public class TraversalBasedExecutor implements DSLQueryExecutor {
private GremlinQuery toTraversal(String query, int limit, int offset) throws AtlasBaseException {
QueryParams params = QueryParams.getNormalizedParams(limit, offset);
query = getStringWithLimitOffset(query, params);
AtlasDSL.Translator dslTranslator = new AtlasDSL.Translator(query, typeRegistry, params.offset(), params.limit());
GremlinQuery gremlinQuery = dslTranslator.translate();
GremlinQuery gremlinQuery = dslTranslator.translate();
AtlasGraphTraversal result = GremlinClauseToTraversalTranslator.run(this.graph, gremlinQuery.getClauses());
AtlasGraphTraversal result = GremlinClauseToTraversalTranslator.run(this.graph, gremlinQuery.getClauses());
gremlinQuery.setResult(result);
return gremlinQuery;

68
repository/src/test/java/org/apache/atlas/query/BaseDSLComposer.java
View File

@ -38,20 +38,22 @@ import static org.mockito.Mockito.when;
import static org.testng.FileAssert.fail;
public class BaseDSLComposer {
protected AtlasTypeRegistry registry = mock(AtlasTypeRegistry.class);
protected final AtlasTypeRegistry registry = mock(AtlasTypeRegistry.class);
protected AtlasDSLParser.QueryContext getParsedQuery(String query) {
AtlasDSLParser.QueryContext queryContext = null;
try {
queryContext = AtlasDSL.Parser.parse(query);
} catch (AtlasBaseException e) {
fail(e.getMessage());
}
return queryContext;
}
public static class TestLookup implements org.apache.atlas.query.Lookup {
AtlasTypeRegistry registry;
final AtlasTypeRegistry registry;
TestLookup(AtlasTypeRegistry typeRegistry) {
this.registry = typeRegistry;
@ -59,34 +61,38 @@ public class BaseDSLComposer {
@Override
public AtlasType getType(String typeName) throws AtlasBaseException {
AtlasType type;
final AtlasType type;
if(typeName.equals("PII") || typeName.equals("Dimension")) {
type = mock(AtlasType.class);
when(type.getTypeCategory()).thenReturn(TypeCategory.CLASSIFICATION);
} else {
type = mock(AtlasEntityType.class);
when(type.getTypeCategory()).thenReturn(TypeCategory.ENTITY);
AtlasStructType.AtlasAttribute attr = mock(AtlasStructType.AtlasAttribute.class);
AtlasStructDef.AtlasAttributeDef def = mock(AtlasStructDef.AtlasAttributeDef.class);
AtlasStructType.AtlasAttribute attr = mock(AtlasStructType.AtlasAttribute.class);
AtlasStructDef.AtlasAttributeDef def = mock(AtlasStructDef.AtlasAttributeDef.class);
when(def.getIndexType()).thenReturn(AtlasStructDef.AtlasAttributeDef.IndexType.DEFAULT);
when(attr.getAttributeDef()).thenReturn(def);
AtlasStructType.AtlasAttribute attr_s = mock(AtlasStructType.AtlasAttribute.class);
AtlasStructDef.AtlasAttributeDef def_s = mock(AtlasStructDef.AtlasAttributeDef.class);
when(def_s.getIndexType()).thenReturn(AtlasStructDef.AtlasAttributeDef.IndexType.STRING);
when(attr_s.getAttributeDef()).thenReturn(def_s);
when(((AtlasEntityType) type).getAttribute(anyString())).thenReturn(attr);
when(((AtlasEntityType) type).getAttribute(eq("name"))).thenReturn(attr_s);
}
if(typeName.equals("PIII")) {
throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_NOT_FOUND);
}
when(type.getTypeName()).thenReturn(typeName);
return type;
}
@ -100,13 +106,15 @@ public class BaseDSLComposer {
throw new AtlasBaseException("Invalid attribute");
}
if(name.contains("."))
if(name.contains(".")) {
return name;
}
if(!context.getActiveTypeName().equals(name))
if(!context.getActiveTypeName().equals(name)) {
return String.format("%s.%s", context.getActiveTypeName(), name);
else
} else {
return name;
}
}
@Override
@ -122,21 +130,23 @@ public class BaseDSLComposer {
@Override
public String getRelationshipEdgeLabel(GremlinQueryComposer.Context context, String attributeName) {
if (attributeName.equalsIgnoreCase("columns"))
if (attributeName.equalsIgnoreCase("columns")) {
return "__Table.columns";
if (attributeName.equalsIgnoreCase("db"))
} else if (attributeName.equalsIgnoreCase("db")) {
return "__Table.db";
if (attributeName.equalsIgnoreCase("meanings"))
} else if (attributeName.equalsIgnoreCase("meanings")) {
return "r:AtlasGlossarySemanticAssignment";
else
} else {
return "__DB.Table";
}
}
@Override
public AtlasRelationshipEdgeDirection getRelationshipEdgeDirection(GremlinQueryComposer.Context context, String attributeName) {
if (attributeName.equalsIgnoreCase("meanings")){
if (attributeName.equalsIgnoreCase("meanings")) {
return IN;
}
return OUT;
}
@ -173,6 +183,7 @@ public class BaseDSLComposer {
@Override
public String getTypeAndSubTypes(GremlinQueryComposer.Context context) {
String[] str = new String[]{"'Asset'", "'Table'"};
return StringUtils.join(str, ",");
}
@ -183,30 +194,21 @@ public class BaseDSLComposer {
@Override
public String getTypeFromEdge(GremlinQueryComposer.Context context, String item) {
if(context.getActiveTypeName().equals("DB") && item.equals("Table")) {
if (context.getActiveTypeName().equals("DB") && item.equals("Table")) {
return "Table";
}
if(context.getActiveTypeName().equals("Table") && item.equals("Column")) {
} else if (context.getActiveTypeName().equals("Table") && item.equals("Column")) {
return "Column";
}
if(context.getActiveTypeName().equals("Table") && item.equals("db")) {
} else if (context.getActiveTypeName().equals("Table") && item.equals("db")) {
return "DB";
}
if(context.getActiveTypeName().equals("Table") && item.equals("columns")) {
} else if (context.getActiveTypeName().equals("Table") && item.equals("columns")) {
return "Column";
}
if(context.getActiveTypeName().equals("Table") && item.equals("meanings")) {
} else if (context.getActiveTypeName().equals("Table") && item.equals("meanings")) {
return "AtlasGlossaryTerm";
}
if(context.getActiveTypeName().equals(item)) {
} else if (context.getActiveTypeName().equals(item)) {
return null;
} else {
return context.getActiveTypeName();
}
return context.getActiveTypeName();
}
@Override

54
repository/src/test/java/org/apache/atlas/query/DSLQueriesTest.java
View File

@ -55,6 +55,7 @@ public class DSLQueriesTest extends BasicTestSetup {
private static final Logger LOG = LoggerFactory.getLogger(DSLQueriesTest.class);
private final int DEFAULT_LIMIT = 25;
@Inject
private EntityDiscoveryService discoveryService;
@ -78,38 +79,44 @@ public class DSLQueriesTest extends BasicTestSetup {
{"Employee", 4},
};
int pollingAttempts = 5;
int pollingBackoff = 0; // in msecs
int pollingAttempts = 5;
int pollingBackoff = 0; // in msecs
boolean success;
for (int attempt = 0; attempt < pollingAttempts; attempt++, pollingBackoff += attempt * 5000) {
LOG.debug("Polling -- Attempt {}, Backoff {}", attempt, pollingBackoff);
success = false;
for (Object[] verificationQuery : basicVerificationQueries) {
String query = (String) verificationQuery[0];
int expected = (int) verificationQuery[1];
String query = (String) verificationQuery[0];
int expected = (int) verificationQuery[1];
try {
AtlasSearchResult result = discoveryService.searchUsingDslQuery(query, 25, 0);
if (result.getEntities() == null || result.getEntities().isEmpty()) {
LOG.warn("DSL {} returned no entities", query);
success = false;
} else if (result.getEntities().size() != expected) {
LOG.warn("DSL {} returned unexpected number of entities. Expected {} Actual {}", query, expected, result.getEntities().size());
success = false;
} else {
success = true;
}
} catch (AtlasBaseException e) {
LOG.error("Got exception for DSL {}, errorCode: {}", query, e.getAtlasErrorCode());
waitOrBailout(pollingAttempts, pollingBackoff, attempt);
}
}
// DSL queries were successful
if (success) {
LOG.info("Polling was success");
break;
} else {
waitOrBailout(pollingAttempts, pollingBackoff, attempt);
@ -120,9 +127,11 @@ public class DSLQueriesTest extends BasicTestSetup {
private void waitOrBailout(final int pollingAttempts, final int pollingBackoff, final int attempt) throws InterruptedException {
if (attempt == pollingAttempts - 1) {
LOG.error("Polling failed after {} attempts", pollingAttempts);
throw new SkipException("Polling for test data was unsuccessful");
} else {
LOG.warn("Waiting for {} before polling again", pollingBackoff);
Thread.sleep(pollingBackoff);
}
}
@ -197,9 +206,11 @@ public class DSLQueriesTest extends BasicTestSetup {
@Test(dataProvider = "comparisonQueriesProvider")
public void comparison(String query, int expected) throws AtlasBaseException {
AtlasSearchResult searchResult = discoveryService.searchUsingDslQuery(query, DEFAULT_LIMIT, 0);
assertSearchResult(searchResult, expected, query);
AtlasSearchResult searchResult2 = discoveryService.searchUsingDslQuery(query.replace("where", " "), DEFAULT_LIMIT, 0);
assertSearchResult(searchResult2, expected, query);
}
@ -222,6 +233,7 @@ public class DSLQueriesTest extends BasicTestSetup {
@Test(dataProvider = "glossaryTermQueries")
public void glossaryTerm(String query, int expected, ListValidator lvExpected) throws AtlasBaseException {
AtlasSearchResult result = queryAssert(query, expected, DEFAULT_LIMIT, 0);
if (lvExpected == null) {
return;
}
@ -301,7 +313,9 @@ public class DSLQueriesTest extends BasicTestSetup {
private AtlasSearchResult queryAssert(String query, final int expected, final int limit, final int offset) throws AtlasBaseException {
AtlasSearchResult searchResult = discoveryService.searchUsingDslQuery(query, limit, offset);
assertSearchResult(searchResult, expected, query);
return searchResult;
}
@ -494,9 +508,11 @@ public class DSLQueriesTest extends BasicTestSetup {
@Test(dataProvider = "orderByProvider")
public void orderBy(String query, int expected, String attributeName, boolean ascending) throws AtlasBaseException {
AtlasSearchResult searchResult = queryAssert(query, expected, DEFAULT_LIMIT, 0);
assertSortOrder(query, attributeName, ascending, searchResult.getEntities());
searchResult = queryAssert(query.replace("where", " "), expected, DEFAULT_LIMIT, 0);
assertSortOrder(query, attributeName, ascending, searchResult.getEntities());
}
@ -506,15 +522,16 @@ public class DSLQueriesTest extends BasicTestSetup {
}
AtlasEntityHeader prev = null;
for (AtlasEntityHeader current : entities) {
if (prev != null && current.hasAttribute(attributeName)) {
String lhs = (String) prev.getAttribute(attributeName);
String rhs = (String) current.getAttribute(attributeName);
int compResult = lhs.compareTo(rhs);
String lhs = (String) prev.getAttribute(attributeName);
String rhs = (String) current.getAttribute(attributeName);
int compResult = lhs.compareTo(rhs);
if (ascending) {
assertTrue(compResult <= 0, query);
}
else {
} else {
assertTrue(compResult >= 0, query);
}
}
@ -690,11 +707,13 @@ public class DSLQueriesTest extends BasicTestSetup {
@Test(dataProvider = "errorQueriesProvider", expectedExceptions = { AtlasBaseException.class })
public void errorQueries(String query) throws AtlasBaseException {
LOG.debug(query);
discoveryService.searchUsingDslQuery(query, DEFAULT_LIMIT, 0);
}
private void queryAssert(String query, TableValidator fv) throws AtlasBaseException {
AtlasSearchResult searchResult = discoveryService.searchUsingDslQuery(query, DEFAULT_LIMIT, 0);
assertNotNull(searchResult);
assertNull(searchResult.getEntities());
@ -703,6 +722,7 @@ public class DSLQueriesTest extends BasicTestSetup {
private void assertSearchResult(AtlasSearchResult searchResult, int expected, String query) {
assertNotNull(searchResult);
if(expected == 0) {
assertTrue(searchResult.getAttributes() == null || CollectionUtils.isEmpty(searchResult.getAttributes().getValues()));
assertNull(searchResult.getEntities(), query);
@ -724,7 +744,7 @@ public class DSLQueriesTest extends BasicTestSetup {
}
}
public String[] fieldNames;
public String[] fieldNames;
public List<NameValueEntry> values = new ArrayList<>();
public TableValidator() {
@ -736,16 +756,19 @@ public class DSLQueriesTest extends BasicTestSetup {
public TableValidator header(String... fieldNames) {
this.fieldNames = fieldNames;
return this;
}
public TableValidator row(Object... values) {
NameValueEntry obj = new NameValueEntry();
for (int i = 0; i < fieldNames.length; i++) {
obj.setFieldValue(fieldNames[i], values[i]);
}
this.values.add(obj);
return this;
}
@ -755,14 +778,15 @@ public class DSLQueriesTest extends BasicTestSetup {
assertEquals(actual.values.size(), expected.values.size());
Map<String, Object> actualKeyItemsForCompare = new HashMap<>();
Map<String, Object> expectedItemsForCompare = new HashMap<>();
Map<String, Object> expectedItemsForCompare = new HashMap<>();
for (int i = 0; i < actual.values.size(); i++) {
getMapFrom(expectedItemsForCompare, expected.values.get(i).items);
getMapFrom(actualKeyItemsForCompare, actual.values.get(i).items);
}
for (String key : actualKeyItemsForCompare.keySet()) {
Object actualValue = actualKeyItemsForCompare.get(key);
Object actualValue = actualKeyItemsForCompare.get(key);
Object expectedValue = expectedItemsForCompare.get(key);
assertNotNull(actualValue, "Key: " + key + ": Failed!");
@ -773,6 +797,7 @@ public class DSLQueriesTest extends BasicTestSetup {
private static Map<String, Object> getMapFrom(Map<String, Object> valuesMap, Map<String, Object> linkedHashMap) {
for (Map.Entry<String, Object> entry : linkedHashMap.entrySet()) {
String key = entry.getValue().toString();
valuesMap.put(key, linkedHashMap);
break;
}
@ -782,10 +807,12 @@ public class DSLQueriesTest extends BasicTestSetup {
public static TableValidator from(AtlasSearchResult.AttributeSearchResult searchResult) {
TableValidator fv = new TableValidator();
fv.header(searchResult.getName().toArray(new String[]{}));
for (int i = 0; i < searchResult.getValues().size(); i++) {
List list = searchResult.getValues().get(i);
fv.row(list.toArray());
}
@ -803,6 +830,7 @@ public class DSLQueriesTest extends BasicTestSetup {
String errorMessage = String.format("Expected: %s\r\nActual: %s", expected.values, actual.values);
assertEquals(actual.values.size(), expected.values.size(), errorMessage);
if (expected.values.size() > 0) {
for (String expectedVal : expected.values) {
assertTrue(actual.values.contains(expectedVal), errorMessage);

14
repository/src/test/java/org/apache/atlas/query/TraversalComposerTest.java
View File

@ -58,22 +58,24 @@ public class TraversalComposerTest extends BaseDSLComposer {
private void verify(String dsl, String expected) {
AtlasDSLParser.QueryContext queryContext = getParsedQuery(dsl);
String actual = getTraversalAsStr(queryContext);
String actual = getTraversalAsStr(queryContext);
assertEquals(actual, expected, dsl);
}
private String getTraversalAsStr(AtlasDSLParser.QueryContext queryContext) {
org.apache.atlas.query.Lookup lookup = new TestLookup(registry);
GremlinQueryComposer.Context context = new GremlinQueryComposer.Context(lookup);
AtlasDSL.QueryMetadata queryMetadata = new AtlasDSL.QueryMetadata(queryContext);
org.apache.atlas.query.Lookup lookup = new TestLookup(registry);
GremlinQueryComposer.Context context = new GremlinQueryComposer.Context(lookup);
AtlasDSL.QueryMetadata queryMetadata = new AtlasDSL.QueryMetadata(queryContext);
GremlinQueryComposer gremlinQueryComposer = new GremlinQueryComposer(lookup, context, queryMetadata);
DSLVisitor qv = new DSLVisitor(gremlinQueryComposer);
GremlinQueryComposer gremlinQueryComposer = new GremlinQueryComposer(lookup, context, queryMetadata);
DSLVisitor qv = new DSLVisitor(gremlinQueryComposer);
qv.visit(queryContext);
gremlinQueryComposer.get();
AtlasGraphTraversal traversal = GremlinClauseToTraversalTranslator.run(graph, gremlinQueryComposer.clauses());
return traversal.toString();
}
}