tool
/
atlas
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ATLAS-3327-2 Create internal vertex property for classification names and propagated classification names

This commit is contained in:
Le Ma 2019-08-01 14:59:13 -07:00 committed by Sarath Subramanian
parent f912aa8990
commit 8127189c4e
8 changed files with 113 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
common/src/main/java/org/apache/atlas/repository/Constants.java
View File

@ -180,6 +180,7 @@ public final class Constants {
public static final String CLASSIFICATION_EDGE_IS_PROPAGATED_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "isPropagated");
public static final String CLASSIFICATION_EDGE_STATE_PROPERTY_KEY = STATE_PROPERTY_KEY;
public static final String CLASSIFICATION_LABEL = "classifiedAs";
public static final String CLASSIFICATION_NAME_DELIMITER = "|";
public static final String TERM_ASSIGNMENT_LABEL = "r:AtlasGlossarySemanticAssignment";
public static final String ATTRIBUTE_INDEX_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "index");
public static final String ATTRIBUTE_KEY_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "key");

2
repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
View File

@ -319,6 +319,8 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
createCommonVertexIndex(management, CREATED_BY_KEY, UniqueKind.NONE, String.class, SINGLE, false, false);
createCommonVertexIndex(management, CLASSIFICATION_TEXT_KEY, UniqueKind.NONE, String.class, SINGLE, false, false);
createCommonVertexIndex(management, MODIFIED_BY_KEY, UniqueKind.NONE, String.class, SINGLE, false, false);
createCommonVertexIndex(management, CLASSIFICATION_NAMES_KEY, UniqueKind.NONE, String.class, SINGLE, true, false);
createCommonVertexIndex(management, PROPAGATED_CLASSIFICATION_NAMES_KEY, UniqueKind.NONE, String.class, SINGLE, true, false);
createCommonVertexIndex(management, TRAIT_NAMES_PROPERTY_KEY, UniqueKind.NONE, String.class, SET, true, true);
createCommonVertexIndex(management, PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, UniqueKind.NONE, String.class, LIST, true, true);

26
repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
View File

@ -85,6 +85,8 @@ import static org.apache.atlas.repository.Constants.CLASSIFICATION_ENTITY_GUID;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_ENTITY_STATUS;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_LABEL;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_EDGE_NAME_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_NAME_DELIMITER;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_VERTEX_NAME_KEY;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_VERTEX_PROPAGATE_KEY;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_VERTEX_REMOVE_PROPAGATIONS_KEY;
@ -92,8 +94,6 @@ import static org.apache.atlas.repository.Constants.CREATED_BY_KEY;
import static org.apache.atlas.repository.Constants.ENTITY_TYPE_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.MODIFIED_BY_KEY;
import static org.apache.atlas.repository.Constants.PROPAGATED_CLASSIFICATION_NAMES_KEY;
import static org.apache.atlas.repository.Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.RELATIONSHIPTYPE_BLOCKED_PROPAGATED_CLASSIFICATIONS_KEY;
import static org.apache.atlas.repository.Constants.RELATIONSHIPTYPE_TAG_PROPAGATION_KEY;
import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY;
@ -659,18 +659,6 @@ public final class GraphHelper {
return element.toString();
}
public static void addToPropagatedClassificationAndTraitNames(AtlasVertex entityVertex, String classificationName) {
if (LOG.isDebugEnabled()) {
LOG.debug("Adding property {} = \"{}\" to vertex {}", PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationName, string(entityVertex));
}
entityVertex.addListProperty(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationName);
String propClsNames = entityVertex.getProperty(PROPAGATED_CLASSIFICATION_NAMES_KEY, String.class);
propClsNames = StringUtils.isEmpty(propClsNames)? classificationName : propClsNames + "|" + classificationName;
entityVertex.setProperty(PROPAGATED_CLASSIFICATION_NAMES_KEY, propClsNames);
}
/**
* Remove the specified edge from the graph.
*
@ -1849,4 +1837,14 @@ public final class GraphHelper {
public static boolean getDefaultRemovePropagations() {
return removePropagations;
}
public static String getDelimitedClassificationNames(Collection<String> classificationNames) {
String ret = null;
if (CollectionUtils.isNotEmpty(classificationNames)) {
ret = CLASSIFICATION_NAME_DELIMITER + StringUtils.join(classificationNames, CLASSIFICATION_NAME_DELIMITER)
+ CLASSIFICATION_NAME_DELIMITER;
}
return ret;
}
}

3
repository/src/main/java/org/apache/atlas/repository/patches/ClassificationTextPatch.java
View File

@ -71,8 +71,7 @@ public class ClassificationTextPatch extends AtlasPatchHandler {
LOG.debug("processItem(typeName={}, vertexId={})", typeName, vertexId);
}
getEntityGraphMapper().updateClassificationText(vertex);
getEntityGraphMapper().setClassificationNames(vertex);
getEntityGraphMapper().updateClassificationTextAndNames(vertex);
if(LOG.isDebugEnabled()) {
LOG.debug("processItem(typeName={}, vertexId={}): Done!", typeName, vertexId);

46
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java
View File

@ -62,6 +62,7 @@ import static org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags.
import static org.apache.atlas.repository.Constants.CLASSIFICATION_EDGE_NAME_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_ENTITY_STATUS;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_LABEL;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_NAME_DELIMITER;
import static org.apache.atlas.repository.Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.MODIFIED_BY_KEY;
import static org.apache.atlas.repository.Constants.PROPAGATED_CLASSIFICATION_NAMES_KEY;
@ -450,7 +451,7 @@ public abstract class DeleteHandlerV1 {
graphHelper.addClassificationEdge(propagatedEntityVertex, classificationVertex, true);
addToPropagatedClassificationAndTraitNames(propagatedEntityVertex, classificationName);
addToPropagatedClassificationNames(propagatedEntityVertex, classificationName);
// record add propagation details to send notifications at the end
RequestContext context = RequestContext.get();
@ -614,7 +615,7 @@ public abstract class DeleteHandlerV1 {
graphHelper.removeEdge(propagatedEdge);
removeFromPropagatedClassificationAndTraitNames(impactedEntityVertex, classificationName);
removeFromPropagatedClassificationNames(impactedEntityVertex, classificationName);
} else {
if (LOG.isDebugEnabled()) {
LOG.debug(" --> Not removing propagated classification edge from [{}] --> [{}][{}] using edge label: [{}], since edge doesn't exist",
@ -666,7 +667,7 @@ public abstract class DeleteHandlerV1 {
getTypeName(entityVertex), GraphHelper.getGuid(entityVertex), CLASSIFICATION_LABEL);
}
removeFromPropagatedClassificationAndTraitNames(entityVertex, classificationName);
removeFromPropagatedClassificationNames(entityVertex, classificationName);
deleteEdge(edge, true);
@ -987,18 +988,47 @@ public abstract class DeleteHandlerV1 {
return Objects.nonNull(entityType) && entityType.isInternalType();
}
private void removeFromPropagatedClassificationAndTraitNames(AtlasVertex entityVertex, String classificationName) {
private void addToPropagatedClassificationNames(AtlasVertex entityVertex, String classificationName) {
if (LOG.isDebugEnabled()) {
LOG.debug("Adding property {} = \"{}\" to vertex {}", PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationName, string(entityVertex));
}
entityVertex.addListProperty(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationName);
String propClsNames = entityVertex.getProperty(PROPAGATED_CLASSIFICATION_NAMES_KEY, String.class);
propClsNames = StringUtils.isEmpty(propClsNames) ? CLASSIFICATION_NAME_DELIMITER + classificationName
: (propClsNames + classificationName);
propClsNames = propClsNames + CLASSIFICATION_NAME_DELIMITER;
entityVertex.setProperty(PROPAGATED_CLASSIFICATION_NAMES_KEY, propClsNames);
}
private void removeFromPropagatedClassificationNames(AtlasVertex entityVertex, String classificationName) {
if (entityVertex != null && StringUtils.isNotEmpty(classificationName)) {
List<String> propagatedTraitNames = getTraitNames(entityVertex, true);
propagatedTraitNames.remove(classificationName);
entityVertex.removeProperty(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY);
entityVertex.removeProperty(PROPAGATED_CLASSIFICATION_NAMES_KEY);
setPropagatedClassificationNames(entityVertex, propagatedTraitNames);
}
}
for (String propagatedTraitName : propagatedTraitNames) {
addToPropagatedClassificationAndTraitNames(entityVertex, propagatedTraitName);
private void setPropagatedClassificationNames(AtlasVertex entityVertex, List<String> classificationNames) {
if (LOG.isDebugEnabled()) {
LOG.debug("Adding property {} = \"{}\" to vertex {}", PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationNames, string(entityVertex));
}
entityVertex.removeProperty(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY);
entityVertex.removeProperty(PROPAGATED_CLASSIFICATION_NAMES_KEY);
if (CollectionUtils.isNotEmpty(classificationNames)) {
for (String classificationName : classificationNames) {
entityVertex.addListProperty(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationName);
}
String propClsName = CLASSIFICATION_NAME_DELIMITER + StringUtils.join(classificationNames, CLASSIFICATION_NAME_DELIMITER) + CLASSIFICATION_NAME_DELIMITER;
entityVertex.setProperty(PROPAGATED_CLASSIFICATION_NAMES_KEY, propClsName);
}
}

60
repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
View File

@ -81,11 +81,11 @@ import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.PA
import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.UPDATE;
import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinality.SET;
import static org.apache.atlas.repository.Constants.*;
import static org.apache.atlas.repository.graph.GraphHelper.DEFAULT_REMOVE_PROPAGATIONS_ON_ENTITY_DELETE;
import static org.apache.atlas.repository.graph.GraphHelper.getCollectionElementsUsingRelationship;
import static org.apache.atlas.repository.graph.GraphHelper.getClassificationEdge;
import static org.apache.atlas.repository.graph.GraphHelper.getClassificationVertex;
import static org.apache.atlas.repository.graph.GraphHelper.getDefaultRemovePropagations;
import static org.apache.atlas.repository.graph.GraphHelper.getDelimitedClassificationNames;
import static org.apache.atlas.repository.graph.GraphHelper.getMapElementsProperty;
import static org.apache.atlas.repository.graph.GraphHelper.getStatus;
import static org.apache.atlas.repository.graph.GraphHelper.getTraitLabel;
@ -1512,7 +1512,7 @@ public class EntityGraphMapper {
LOG.debug("Adding classification [{}] to [{}] using edge label: [{}]", classificationName, entityType.getTypeName(), getTraitLabel(classificationName));
}
addClassificationAndTraitNames(entityVertex, classificationName);
addToClassificationNames(entityVertex, classificationName);
// add a new AtlasVertex for the struct or trait instance
AtlasVertex classificationVertex = createClassificationVertex(classification);
@ -1676,47 +1676,67 @@ public class EntityGraphMapper {
traitNames.remove(classificationName);
updateClassificationAndTraitNames(entityVertex, traitNames);
setClassificationNames(entityVertex, traitNames);
updateModificationMetadata(entityVertex);
for (Map.Entry<AtlasVertex, List<AtlasClassification>> entry : removedClassifications.entrySet()) {
AtlasEntity entity = updateClassificationText(entry.getKey());
List<AtlasClassification> deletedClassificationNames = entry.getValue();
entityChangeNotifier.onClassificationDeletedFromEntity(entity, deletedClassificationNames);
}
}
public AtlasEntity updateClassificationText(AtlasVertex vertex) throws AtlasBaseException {
String guid = GraphHelper.getGuid(vertex);
AtlasEntity entity = instanceConverter.getAndCacheEntity(guid);
private AtlasEntity updateClassificationText(AtlasVertex vertex) throws AtlasBaseException {
String guid = GraphHelper.getGuid(vertex);
AtlasEntity entity = instanceConverter.getAndCacheEntity(guid);
vertex.setProperty(CLASSIFICATION_TEXT_KEY, fullTextMapperV2.getClassificationTextForEntity(entity));
return entity;
}
public void setClassificationNames(AtlasVertex vertex) {
List<String> clsNamesList = getTraitNames(vertex, false);
List<String> propClsNamesList = getTraitNames(vertex, true);
String clsNames = StringUtils.join(clsNamesList, CLASSIFICATION_NAME_DELIMITER);
String propClsNames = StringUtils.join(propClsNamesList, CLASSIFICATION_NAME_DELIMITER);
public void updateClassificationTextAndNames(AtlasVertex vertex) throws AtlasBaseException {
String guid = GraphHelper.getGuid(vertex);
AtlasEntity entity = instanceConverter.getAndCacheEntity(guid);
vertex.setProperty(CLASSIFICATION_NAMES_KEY, clsNames);
vertex.setProperty(PROPAGATED_CLASSIFICATION_NAMES_KEY, propClsNames);
if (CollectionUtils.isEmpty(entity.getClassifications())) return;
List<String> classificationNames = new ArrayList<>();
List<String> propagatedClassificationNames = new ArrayList<>();
for (AtlasClassification classification : entity.getClassifications()) {
if (isPropagatedClassification(classification, guid)) {
propagatedClassificationNames.add(classification.getTypeName());
} else {
classificationNames.add(classification.getTypeName());
}
}
vertex.setProperty(CLASSIFICATION_NAMES_KEY, getDelimitedClassificationNames(classificationNames));
vertex.setProperty(PROPAGATED_CLASSIFICATION_NAMES_KEY, getDelimitedClassificationNames(propagatedClassificationNames));
vertex.setProperty(CLASSIFICATION_TEXT_KEY, fullTextMapperV2.getClassificationTextForEntity(entity));
}
private void addClassificationAndTraitNames(AtlasVertex entityVertex, String classificationName) {
private boolean isPropagatedClassification(AtlasClassification classification, String guid) {
String classificationEntityGuid = classification.getEntityGuid();
return StringUtils.isNotEmpty(classificationEntityGuid) && !StringUtils.equals(classificationEntityGuid, guid);
}
private void addToClassificationNames(AtlasVertex entityVertex, String classificationName) {
AtlasGraphUtilsV2.addEncodedProperty(entityVertex, TRAIT_NAMES_PROPERTY_KEY, classificationName);
String clsNames = entityVertex.getProperty(CLASSIFICATION_NAMES_KEY, String.class);
clsNames = StringUtils.isEmpty(clsNames) ? classificationName : clsNames + CLASSIFICATION_NAME_DELIMITER + classificationName;
clsNames = StringUtils.isEmpty(clsNames) ? CLASSIFICATION_NAME_DELIMITER + classificationName : clsNames + classificationName;
clsNames = clsNames + CLASSIFICATION_NAME_DELIMITER;
entityVertex.setProperty(CLASSIFICATION_NAMES_KEY, clsNames);
}
private void updateClassificationAndTraitNames(AtlasVertex entityVertex, List<String> traitNames) {
private void setClassificationNames(AtlasVertex entityVertex, List<String> traitNames) {
if (entityVertex != null) {
entityVertex.removeProperty(TRAIT_NAMES_PROPERTY_KEY);
entityVertex.removeProperty(CLASSIFICATION_NAMES_KEY);
@ -1725,7 +1745,11 @@ public class EntityGraphMapper {
AtlasGraphUtilsV2.addEncodedProperty(entityVertex, TRAIT_NAMES_PROPERTY_KEY, traitName);
}
entityVertex.setProperty(CLASSIFICATION_NAMES_KEY, StringUtils.join(traitNames, CLASSIFICATION_NAME_DELIMITER));
String clsNames = StringUtils.join(traitNames, CLASSIFICATION_NAME_DELIMITER);
clsNames = StringUtils.isEmpty(clsNames) ? clsNames : CLASSIFICATION_NAME_DELIMITER + clsNames + CLASSIFICATION_NAME_DELIMITER;
entityVertex.setProperty(CLASSIFICATION_NAMES_KEY, clsNames);
}
}

8
repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityStateChecker.java
View File

@ -34,6 +34,7 @@ import org.apache.atlas.type.AtlasClassificationType;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
@ -48,6 +49,7 @@ import java.util.List;
import java.util.Map;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_EDGE_IS_PROPAGATED_PROPERTY_KEY;
import static org.apache.atlas.repository.graph.GraphHelper.getDelimitedClassificationNames;
@Component
public final class EntityStateChecker {
@ -266,10 +268,13 @@ public final class EntityStateChecker {
}
entityVertex.removeProperty(Constants.TRAIT_NAMES_PROPERTY_KEY);
entityVertex.removeProperty(Constants.CLASSIFICATION_NAMES_KEY);
for (String classificationName : traitVertexNames) {
AtlasGraphUtilsV2.addEncodedProperty(entityVertex, Constants.TRAIT_NAMES_PROPERTY_KEY, classificationName);
}
entityVertex.setProperty(Constants.CLASSIFICATION_NAMES_KEY, getDelimitedClassificationNames(traitVertexNames));
}
if (propagatedTraitNamesToAdd != null || propagatedTraitNamesToRemove != null) {
@ -282,10 +287,13 @@ public final class EntityStateChecker {
}
entityVertex.removeProperty(Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY);
entityVertex.removeProperty(Constants.PROPAGATED_CLASSIFICATION_NAMES_KEY);
for (String classificationName : propagatedTraitVertexNames) {
AtlasGraphUtilsV2.addEncodedProperty(entityVertex, Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationName);
}
entityVertex.setProperty(Constants.PROPAGATED_CLASSIFICATION_NAMES_KEY,getDelimitedClassificationNames(propagatedTraitVertexNames));
}
AtlasGraphUtilsV2.setEncodedProperty(entityVertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, RequestContext.get().getRequestTime());

16
webapp/src/test/java/org/apache/atlas/web/adapters/TestEntityREST.java
View File

@ -131,11 +131,12 @@ public class TestEntityREST {
Assert.assertEquals(retrievedClassification, testClassification);
// For ATLAS-3327 to test internal properties are added properly.
AtlasVertex vertex = AtlasGraphUtilsV2.findByGuid(dbEntity.getGuid());
AtlasVertex vertex = AtlasGraphUtilsV2.findByGuid(dbEntity.getGuid());
String expectedClsName = Constants.CLASSIFICATION_NAME_DELIMITER + TestUtilsV2.CLASSIFICATION + Constants.CLASSIFICATION_NAME_DELIMITER ;
String classificationNames = vertex.getProperty(Constants.CLASSIFICATION_NAMES_KEY, String.class);
Assert.assertNotNull(classificationNames);
Assert.assertEquals(classificationNames, TestUtilsV2.CLASSIFICATION);
Assert.assertEquals(classificationNames, expectedClsName);
}
@Test(dependsOnMethods = "testGetEntityById")
@ -167,8 +168,9 @@ public class TestEntityREST {
}
// For ATLAS-3327 to test internal properties are added properly.
String expectedClsNames = TestUtilsV2.CLASSIFICATION + "|" + TestUtilsV2.PHI;
AtlasVertex vertex = AtlasGraphUtilsV2.findByGuid(dbEntity.getGuid());
String expectedClsNames = Constants.CLASSIFICATION_NAME_DELIMITER + TestUtilsV2.CLASSIFICATION +
Constants.CLASSIFICATION_NAME_DELIMITER + TestUtilsV2.PHI + Constants.CLASSIFICATION_NAME_DELIMITER;
AtlasVertex vertex = AtlasGraphUtilsV2.findByGuid(dbEntity.getGuid());
String classificationNames = vertex.getProperty(Constants.CLASSIFICATION_NAMES_KEY, String.class);
Assert.assertNotNull(classificationNames);
@ -204,8 +206,8 @@ public class TestEntityREST {
deleteClassification(dbEntity.getGuid(), TestUtilsV2.PHI);
expectedClsNames = TestUtilsV2.CLASSIFICATION;
vertex = AtlasGraphUtilsV2.findByGuid(dbEntity.getGuid());
expectedClsNames = Constants.CLASSIFICATION_NAME_DELIMITER + TestUtilsV2.CLASSIFICATION + Constants.CLASSIFICATION_NAME_DELIMITER;
vertex = AtlasGraphUtilsV2.findByGuid(dbEntity.getGuid());
classificationNames = vertex.getProperty(Constants.CLASSIFICATION_NAMES_KEY, String.class);
Assert.assertNotNull(classificationNames);
@ -257,7 +259,7 @@ public class TestEntityREST {
Assert.assertEquals(classifications, retrievedClassificationsList);
String expectedClsNames = TestUtilsV2.CLASSIFICATION;
String expectedClsNames = Constants.CLASSIFICATION_NAME_DELIMITER + TestUtilsV2.CLASSIFICATION + Constants.CLASSIFICATION_NAME_DELIMITER;
AtlasVertex vertex = AtlasGraphUtilsV2.findByGuid(dbEntity.getGuid());
String classificationNames = vertex.getProperty(Constants.CLASSIFICATION_NAMES_KEY, String.class);