tool
/
atlas
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ATLAS-4671: Basic Search : Exclude Header attributes of entities from the response 

Signed-off-by: Pinal Shah <pinal.shah@freestoneinfotech.com>
This commit is contained in:
Pinal Shah 2022-09-14 15:36:39 +05:30
parent e3a6d238e3
commit 37c242755c
7 changed files with 210 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
intg/src/main/java/org/apache/atlas/model/discovery/QuickSearchParameters.java
View File

@ -50,6 +50,7 @@ public class QuickSearchParameters implements Serializable {
private Set<String> attributes;
private String sortBy;
private SortOrder sortOrder;
private boolean excludeHeaderAttributes;
/**
* for framework use.
@ -158,4 +159,12 @@ public class QuickSearchParameters implements Serializable {
public void setSortOrder(SortOrder sortOrder) {
this.sortOrder = sortOrder;
}
public boolean getExcludeHeaderAttributes() {
return excludeHeaderAttributes;
}
public void setExcludeHeaderAttributes(boolean excludeHeaderAttributes) {
this.excludeHeaderAttributes = excludeHeaderAttributes;
}
}

14
intg/src/main/java/org/apache/atlas/model/discovery/SearchParameters.java
View File

@ -50,6 +50,8 @@ public class SearchParameters implements Serializable {
private boolean includeClassificationAttributes;
private boolean includeSubTypes = true;
private boolean includeSubClassifications = true;
private boolean excludeHeaderAttributes = false;
private int limit;
private int offset;
private String marker;
@ -258,6 +260,14 @@ public class SearchParameters implements Serializable {
this.tagFilters = tagFilters;
}
public boolean getExcludeHeaderAttributes() {
return excludeHeaderAttributes;
}
public void setExcludeHeaderAttributes(boolean excludeHeaderAttributes) {
this.excludeHeaderAttributes = excludeHeaderAttributes;
}
/**
* Attribute values included in the results
* @return
@ -307,6 +317,7 @@ public class SearchParameters implements Serializable {
includeClassificationAttributes == that.includeClassificationAttributes &&
includeSubTypes == that.includeSubTypes &&
includeSubClassifications == that.includeSubClassifications &&
excludeHeaderAttributes == that.excludeHeaderAttributes &&
limit == that.limit &&
offset == that.offset &&
Objects.equals(query, that.query) &&
@ -323,7 +334,7 @@ public class SearchParameters implements Serializable {
@Override
public int hashCode() {
return Objects.hash(query, typeName, classification, termName, includeSubTypes, includeSubClassifications,
excludeDeletedEntities, includeClassificationAttributes, limit, offset, entityFilters,
excludeDeletedEntities, includeClassificationAttributes, excludeHeaderAttributes, limit, offset, entityFilters,
tagFilters, attributes, sortBy, sortOrder);
}
@ -341,6 +352,7 @@ public class SearchParameters implements Serializable {
sb.append(", includeSubClassifications='").append(includeSubClassifications).append('\'');
sb.append(", excludeDeletedEntities=").append(excludeDeletedEntities);
sb.append(", includeClassificationAttributes=").append(includeClassificationAttributes);
sb.append(", excludeHeaderAttributes=").append(excludeHeaderAttributes);
sb.append(", limit=").append(limit);
sb.append(", offset=").append(offset);
sb.append(", entityFilters=").append(entityFilters);

27
repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
View File

@ -468,6 +468,32 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
ret.setNextMarker(nextMarker);
}
//If excludeHeaderAttributes is true, only primitive attributes requested in 'attributes' field will be sent in the response
Set<String> attributes = searchParameters.getAttributes();
if (searchContext.excludeHeaderAttributes()) {
AtlasSearchResult.AttributeSearchResult attributeSearchResult = new AtlasSearchResult.AttributeSearchResult();
attributeSearchResult.setName(new ArrayList<>(attributes));
Collection<List<Object>> values = new ArrayList<>();
for (AtlasVertex vertex : resultList) {
List<Object> row = new ArrayList<>();
for (String attrName : attributes) {
AtlasEntityType entityType = searchContext.getEntityTypes().iterator().next();
AtlasAttribute attribute = entityType.getAttribute(attrName);
Object value = vertex.getProperty(attribute.getVertexPropertyName(), Object.class);
row.add(value != null ? value : StringUtils.EMPTY);
}
values.add(row);
}
attributeSearchResult.setValues(new ArrayList<>(values));
ret.setAttributes(attributeSearchResult);
return ret;
}
// By default any attribute that shows up in the search parameter should be sent back in the response
// If additional values are requested then the entityAttributes will be a superset of the all search attributes
// and the explicitly requested attribute(s)
@ -922,6 +948,7 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
searchParameters.setAttributes(quickSearchParameters.getAttributes());
searchParameters.setSortBy(quickSearchParameters.getSortBy());
searchParameters.setSortOrder(quickSearchParameters.getSortOrder());
searchParameters.setExcludeHeaderAttributes(quickSearchParameters.getExcludeHeaderAttributes());
return searchParameters;
}

38
repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
View File

@ -32,10 +32,13 @@ import org.apache.atlas.repository.graphdb.AtlasGraphQuery;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2;
import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever;
import org.apache.atlas.type.AtlasArrayType;
import org.apache.atlas.type.AtlasBuiltInTypes;
import org.apache.atlas.type.AtlasClassificationType;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasStructType;
import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.util.AtlasRepositoryConfiguration;
import org.apache.commons.collections.CollectionUtils;
@ -86,6 +89,7 @@ public class SearchContext {
private boolean terminateSearch = false;
private SearchProcessor searchProcessor;
private Integer marker;
private boolean hasRelationshipAttributes = false;
public final static AtlasClassificationType MATCH_ALL_WILDCARD_CLASSIFICATION = new AtlasClassificationType(new AtlasClassificationDef(WILDCARD_CLASSIFICATIONS));
public final static AtlasClassificationType MATCH_ALL_CLASSIFIED = new AtlasClassificationType(new AtlasClassificationDef(ALL_CLASSIFICATIONS));
@ -146,6 +150,9 @@ public class SearchContext {
//remove other types if builtin type is present
filterStructTypes();
//validate 'attributes' field
validateAttributes();
//gather all classifications and its corresponding subtypes
Set<String> classificationTypeAndSubTypes = new HashSet<>();
String classificationTypeAndSubTypesQryStr = null;
@ -345,6 +352,37 @@ public class SearchContext {
}
}
private void validateAttributes() throws AtlasBaseException {
Set<String> attributes = searchParameters.getAttributes();
if (CollectionUtils.isNotEmpty(attributes) && CollectionUtils.isNotEmpty(entityTypes)) {
AtlasEntityType entityType = entityTypes.iterator().next();
for (String attr : attributes) {
AtlasAttribute attribute = entityType.getAttribute(attr);
if (attribute == null) {
attribute = entityType.getRelationshipAttribute(attr, null);
hasRelationshipAttributes = attribute != null;
}
if (attribute == null) {
throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_ATTRIBUTE, attr, entityType.getTypeName());
}
}
}
}
public boolean excludeHeaderAttributes() {
if (CollectionUtils.isNotEmpty(entityTypes) &&
searchParameters.getExcludeHeaderAttributes() &&
CollectionUtils.isNotEmpty(searchParameters.getAttributes()) &&
!hasRelationshipAttributes){
return true;
}
return false;
}
public boolean hasAttributeFilter(FilterCriteria filterCriteria) {
return filterCriteria != null &&
(CollectionUtils.isNotEmpty(filterCriteria.getCriterion()) || StringUtils.isNotEmpty(filterCriteria.getAttributeName()));

117
repository/src/test/java/org/apache/atlas/discovery/AtlasDiscoveryServiceTest.java
View File

@ -31,6 +31,7 @@ import org.apache.atlas.model.discovery.AtlasAggregationEntry;
import org.apache.atlas.model.instance.AtlasClassification;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasEntityHeader;
import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.model.instance.EntityMutationResponse;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.store.graph.v2.AtlasEntityStream;
@ -42,6 +43,7 @@ import org.testng.annotations.*;
import javax.inject.Inject;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@ -929,6 +931,109 @@ public class AtlasDiscoveryServiceTest extends BasicTestSetup {
Assert.assertTrue(list.get(0).getDisplayText().equalsIgnoreCase("time_id"));
}
//test excludeHeaderAttributes
@Test
public void excludeHeaderAttributesStringAttr() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTypeName(HIVE_TABLE_TYPE);
params.setExcludeHeaderAttributes(true);
SearchParameters.FilterCriteria filterCriteria = getSingleFilterCondition("tableType", Operator.EQ, "Managed");
params.setEntityFilters(filterCriteria);
params.setSortBy("name");
params.setAttributes(new HashSet<String>() {{ add("name");}});
params.setLimit(1);
AtlasSearchResult searchResult = discoveryService.searchWithParameters(params);
AtlasSearchResult.AttributeSearchResult expected = new AtlasSearchResult.AttributeSearchResult();
expected.setName(Arrays.asList("name"));
expected.setValues(Arrays.asList(Arrays.asList("log_fact_daily_mv")));
assertSearchResult(searchResult,expected);
}
@Test
public void excludeHeaderAttributesRelationAttr() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTypeName(HIVE_TABLE_TYPE);
params.setExcludeHeaderAttributes(true);
SearchParameters.FilterCriteria filterCriteria = getSingleFilterCondition("name", Operator.EQ, "time_dim");
params.setEntityFilters(filterCriteria);
params.setAttributes(new HashSet<String>() {{ add("name"); add("db");}});
params.setLimit(1);
AtlasSearchResult searchResult = discoveryService.searchWithParameters(params);
assertNotNull(searchResult);
assertNotNull(searchResult.getEntities());
assertNotNull(searchResult.getReferredEntities());
}
@Test
public void excludeHeaderAttributesSystemAttr() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTypeName(HIVE_TABLE_TYPE);
params.setExcludeHeaderAttributes(true);
params.setAttributes(new HashSet<String>() {{ add("name"); add("__state");}});
params.setLimit(1);
SearchParameters.FilterCriteria filterCriteria = getSingleFilterCondition("tableType", Operator.EQ, "Managed");
params.setEntityFilters(filterCriteria);
params.setSortBy("name");
AtlasSearchResult searchResult = discoveryService.searchWithParameters(params);
AtlasSearchResult.AttributeSearchResult expected = new AtlasSearchResult.AttributeSearchResult();
expected.setName(Arrays.asList("name","__state"));
expected.setValues(Arrays.asList(Arrays.asList("log_fact_daily_mv","ACTIVE")));
assertSearchResult(searchResult,expected);
}
@Test
public void excludeHeaderAttributesAllEntityTypeSysAttr() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTypeName(HIVE_TABLE_TYPE+","+ALL_ENTITY_TYPES);
params.setExcludeHeaderAttributes(true);
params.setAttributes(new HashSet<String>() {{ add("__state");}});
params.setLimit(2);
AtlasSearchResult searchResult = discoveryService.searchWithParameters(params);
AtlasSearchResult.AttributeSearchResult expected = new AtlasSearchResult.AttributeSearchResult();
expected.setName(Arrays.asList("__state"));
expected.setValues(Arrays.asList(Arrays.asList("ACTIVE"), Arrays.asList("ACTIVE")));
assertSearchResult(searchResult,expected);
}
@Test
public void excludeHeaderAttributesAllEntityTypeSysAttrs() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTypeName(HIVE_TABLE_TYPE+","+ALL_ENTITY_TYPES);
params.setExcludeHeaderAttributes(true);
params.setAttributes(new HashSet<String>() {{ add("__state"); add("__guid");}});
params.setLimit(2);
AtlasSearchResult searchResult = discoveryService.searchWithParameters(params);
assertEquals(searchResult.getAttributes().getValues().size(), 2);
}
@Test(expectedExceptions = AtlasBaseException.class, expectedExceptionsMessageRegExp = "Attribute name not found for type __ENTITY_ROOT")
public void excludeHeaderAttributesAllEntityType() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTypeName(HIVE_TABLE_TYPE+","+ALL_ENTITY_TYPES);
params.setExcludeHeaderAttributes(true);
params.setAttributes(new HashSet<String>() {{ add("name");}});
params.setLimit(1);
discoveryService.searchWithParameters(params);
}
@Test(expectedExceptions = AtlasBaseException.class, expectedExceptionsMessageRegExp = "Attribute name1 not found for type hive_table")
public void excludeHeaderAttributesInvalidAttr() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTypeName(HIVE_TABLE_TYPE);
params.setExcludeHeaderAttributes(true);
params.setAttributes(new HashSet<String>() {{ add("name1");}});
params.setLimit(1);
discoveryService.searchWithParameters(params);
}
private String gethiveTableSalesFactGuid() throws AtlasBaseException {
if (salesFactGuid == null) {
SearchParameters params = new SearchParameters();
@ -958,6 +1063,18 @@ public class AtlasDiscoveryServiceTest extends BasicTestSetup {
}
}
private void assertSearchResult(AtlasSearchResult searchResult, AtlasSearchResult.AttributeSearchResult expected) {
assertNotNull(searchResult);
AtlasSearchResult.AttributeSearchResult result = searchResult.getAttributes();
assertNotNull(result);
assertTrue(result.getName().containsAll(expected.getName()));
int i = 0;
for (List<Object> value : result.getValues()) {
assertTrue(value.containsAll(expected.getValues().get(i)));
i++;
}
}
private void assertAggregationMetrics(AtlasQuickSearchResult searchResult) {
Map<String, List<AtlasAggregationEntry>> agg = searchResult.getAggregationMetrics();
Assert.assertTrue(CollectionUtils.isNotEmpty(agg.get("__typeName")));

12
webapp/src/test/resources/json/search-parameters/combination-filters.json
View File

@ -10,7 +10,7 @@
"offset": 0,
"entityFilters": null,
"tagFilters": null,
"attributes": [""]
"attributes": []
},
"expectedCount": 1
},
@ -25,7 +25,7 @@
"offset": 0,
"entityFilters": null,
"tagFilters": null,
"attributes": [""]
"attributes": []
},
"expectedCount": 0
},
@ -129,7 +129,7 @@
]
},
"tagFilters": null,
"attributes": [""]
"attributes": []
},
"expectedCount": 1
},
@ -144,7 +144,7 @@
"offset": 0,
"entityFilters": null,
"tagFilters": null,
"attributes": [""]
"attributes": []
},
"expectedCount": 1
},
@ -159,7 +159,7 @@
"offset": 0,
"entityFilters": null,
"tagFilters": null,
"attributes": [""]
"attributes": []
},
"expectedCount": 3
},
@ -178,7 +178,7 @@
"attributeValue" : "+"
},
"tagFilters": null,
"attributes": [""]
"attributes": []
},
"expectedCount": 1
}

13
webapp/src/test/resources/json/search-parameters/entity-filters.json
View File

@ -15,7 +15,6 @@
},
"tagFilters": null,
"attributes": [
""
]
},
"expectedCount": 3
@ -38,7 +37,6 @@
},
"tagFilters": null,
"attributes": [
""
]
},
"expectedCount": 3
@ -59,7 +57,6 @@
},
"tagFilters": null,
"attributes": [
""
]
},
"expectedCount": 2
@ -81,7 +78,6 @@
},
"tagFilters": null,
"attributes": [
""
]
},
"expectedCount": 1
@ -113,7 +109,6 @@
},
"tagFilters": null,
"attributes": [
""
]
},
"expectedCount": 3
@ -145,7 +140,6 @@
},
"tagFilters": null,
"attributes": [
""
]
},
"expectedCount": 3
@ -177,7 +171,6 @@
},
"tagFilters": null,
"attributes": [
""
]
},
"expectedCount": 3
@ -199,7 +192,6 @@
},
"tagFilters": null,
"attributes": [
""
]
},
"expectedCount": 3
@ -231,7 +223,6 @@
},
"tagFilters": null,
"attributes": [
""
]
},
"expectedCount": 1
@ -263,7 +254,6 @@
},
"tagFilters": null,
"attributes": [
""
]
},
"expectedCount": 18
@ -295,7 +285,6 @@
},
"tagFilters": null,
"attributes": [
""
]
},
"expectedCount": 3
@ -327,7 +316,6 @@
},
"tagFilters": null,
"attributes": [
""
]
},
"expectedCount": 3
@ -359,7 +347,6 @@
},
"tagFilters": null,
"attributes": [
""
]
},
"expectedCount": 3