tool
/
atlas
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ATLAS-3327 Create internal vertex property for classification names and propagated classification names

This commit is contained in:
Le Ma 2019-07-18 14:42:40 -07:00 committed by Sarath Subramanian
parent 7054e2b1ad
commit 854a8a012f
6 changed files with 113 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
common/src/main/java/org/apache/atlas/repository/Constants.java
View File

@ -39,8 +39,8 @@ public final class Constants {
public static final String GUID_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "guid");
public static final String RELATIONSHIP_GUID_PROPERTY_KEY = encodePropertyKey(RELATIONSHIP_PROPERTY_KEY_PREFIX + GUID_PROPERTY_KEY);
public static final String HISTORICAL_GUID_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "historicalGuids");
public static final String FREETEXT_REQUEST_HANDLER = "/freetext";
public static final String TERMS_REQUEST_HANDLER = "/terms";
public static final String FREETEXT_REQUEST_HANDLER = "/freetext";
public static final String TERMS_REQUEST_HANDLER = "/terms";
/**
* Entity type name property key.
@ -82,15 +82,17 @@ public final class Constants {
/**
* Trait names property key and index name.
*/
public static final String TRAIT_NAMES_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "traitNames");
public static final String PROPAGATED_TRAIT_NAMES_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "propagatedTraitNames");
public static final String TRAIT_NAMES_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "traitNames");
public static final String PROPAGATED_TRAIT_NAMES_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "propagatedTraitNames");
public static final String VERSION_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "version");
public static final String STATE_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "state");
public static final String CREATED_BY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "createdBy");
public static final String CLASSIFICATION_TEXT_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "classificationsText");
public static final String VERSION_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "version");
public static final String STATE_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "state");
public static final String CREATED_BY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "createdBy");
public static final String CLASSIFICATION_TEXT_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "classificationsText");
public static final String CLASSIFICATION_NAMES_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "classificationNames");
public static final String PROPAGATED_CLASSIFICATION_NAMES_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "propagatedClassificationNames");
public static final String MODIFIED_BY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "modifiedBy");
public static final String MODIFIED_BY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "modifiedBy");
/**
* Patch vertices property keys.

7
repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
View File

@ -92,6 +92,7 @@ import static org.apache.atlas.repository.Constants.CREATED_BY_KEY;
import static org.apache.atlas.repository.Constants.ENTITY_TYPE_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.MODIFIED_BY_KEY;
import static org.apache.atlas.repository.Constants.PROPAGATED_CLASSIFICATION_NAMES_KEY;
import static org.apache.atlas.repository.Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.RELATIONSHIPTYPE_BLOCKED_PROPAGATED_CLASSIFICATIONS_KEY;
import static org.apache.atlas.repository.Constants.RELATIONSHIPTYPE_TAG_PROPAGATION_KEY;
@ -658,12 +659,16 @@ public final class GraphHelper {
return element.toString();
}
public static void addToPropagatedTraitNames(AtlasVertex entityVertex, String classificationName) {
public static void addToPropagatedClassificationAndTraitNames(AtlasVertex entityVertex, String classificationName) {
if (LOG.isDebugEnabled()) {
LOG.debug("Adding property {} = \"{}\" to vertex {}", PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationName, string(entityVertex));
}
entityVertex.addListProperty(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationName);
String propClsNames = entityVertex.getProperty(PROPAGATED_CLASSIFICATION_NAMES_KEY, String.class);
propClsNames = StringUtils.isEmpty(propClsNames)? classificationName : propClsNames + "|" + classificationName;
entityVertex.setProperty(PROPAGATED_CLASSIFICATION_NAMES_KEY, propClsNames);
}
/**

1
repository/src/main/java/org/apache/atlas/repository/patches/ClassificationTextPatch.java
View File

@ -72,6 +72,7 @@ public class ClassificationTextPatch extends AtlasPatchHandler {
}
getEntityGraphMapper().updateClassificationText(vertex);
getEntityGraphMapper().setClassificationNames(vertex);
if(LOG.isDebugEnabled()) {
LOG.debug("processItem(typeName={}, vertexId={}): Done!", typeName, vertexId);

12
repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java
View File

@ -64,6 +64,7 @@ import static org.apache.atlas.repository.Constants.CLASSIFICATION_ENTITY_STATUS
import static org.apache.atlas.repository.Constants.CLASSIFICATION_LABEL;
import static org.apache.atlas.repository.Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.MODIFIED_BY_KEY;
import static org.apache.atlas.repository.Constants.PROPAGATED_CLASSIFICATION_NAMES_KEY;
import static org.apache.atlas.repository.Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.RELATIONSHIP_GUID_PROPERTY_KEY;
import static org.apache.atlas.repository.graph.GraphHelper.*;
@ -449,7 +450,7 @@ public abstract class DeleteHandlerV1 {
graphHelper.addClassificationEdge(propagatedEntityVertex, classificationVertex, true);
addToPropagatedTraitNames(propagatedEntityVertex, classificationName);
addToPropagatedClassificationAndTraitNames(propagatedEntityVertex, classificationName);
// record add propagation details to send notifications at the end
RequestContext context = RequestContext.get();
@ -613,7 +614,7 @@ public abstract class DeleteHandlerV1 {
graphHelper.removeEdge(propagatedEdge);
removeFromPropagatedTraitNames(impactedEntityVertex, classificationName);
removeFromPropagatedClassificationAndTraitNames(impactedEntityVertex, classificationName);
} else {
if (LOG.isDebugEnabled()) {
LOG.debug(" --> Not removing propagated classification edge from [{}] --> [{}][{}] using edge label: [{}], since edge doesn't exist",
@ -665,7 +666,7 @@ public abstract class DeleteHandlerV1 {
getTypeName(entityVertex), GraphHelper.getGuid(entityVertex), CLASSIFICATION_LABEL);
}
removeFromPropagatedTraitNames(entityVertex, classificationName);
removeFromPropagatedClassificationAndTraitNames(entityVertex, classificationName);
deleteEdge(edge, true);
@ -986,16 +987,17 @@ public abstract class DeleteHandlerV1 {
return Objects.nonNull(entityType) && entityType.isInternalType();
}
private void removeFromPropagatedTraitNames(AtlasVertex entityVertex, String classificationName) {
private void removeFromPropagatedClassificationAndTraitNames(AtlasVertex entityVertex, String classificationName) {
if (entityVertex != null && StringUtils.isNotEmpty(classificationName)) {
List<String> propagatedTraitNames = getTraitNames(entityVertex, true);
propagatedTraitNames.remove(classificationName);
entityVertex.removeProperty(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY);
entityVertex.removeProperty(PROPAGATED_CLASSIFICATION_NAMES_KEY);
for (String propagatedTraitName : propagatedTraitNames) {
addToPropagatedTraitNames(entityVertex, propagatedTraitName);
addToPropagatedClassificationAndTraitNames(entityVertex, propagatedTraitName);
}
}
}

55
repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
View File

@ -81,6 +81,7 @@ import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.PA
import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.UPDATE;
import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinality.SET;
import static org.apache.atlas.repository.Constants.*;
import static org.apache.atlas.repository.graph.GraphHelper.DEFAULT_REMOVE_PROPAGATIONS_ON_ENTITY_DELETE;
import static org.apache.atlas.repository.graph.GraphHelper.getCollectionElementsUsingRelationship;
import static org.apache.atlas.repository.graph.GraphHelper.getClassificationEdge;
import static org.apache.atlas.repository.graph.GraphHelper.getClassificationVertex;
@ -105,8 +106,9 @@ import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelation
public class EntityGraphMapper {
private static final Logger LOG = LoggerFactory.getLogger(EntityGraphMapper.class);
private static final String SOFT_REF_FORMAT = "%s:%s";
private static final int INDEXED_STR_SAFE_LEN = AtlasConfiguration.GRAPHSTORE_INDEXED_STRING_SAFE_LENGTH.getInt();
private static final String SOFT_REF_FORMAT = "%s:%s";
private static final int INDEXED_STR_SAFE_LEN = AtlasConfiguration.GRAPHSTORE_INDEXED_STRING_SAFE_LENGTH.getInt();
private static final String CLASSIFICATION_NAME_DELIMITER = "|";
private final GraphHelper graphHelper = GraphHelper.getInstance();
private final AtlasGraph graph;
@ -116,7 +118,7 @@ public class EntityGraphMapper {
private final AtlasEntityChangeNotifier entityChangeNotifier;
private final AtlasInstanceConverter instanceConverter;
private final EntityGraphRetriever entityRetriever;
private final FullTextMapperV2 fullTextMapperV2;
private final FullTextMapperV2 fullTextMapperV2;
@Inject
public EntityGraphMapper(DeleteHandlerDelegate deleteDelegate, AtlasTypeRegistry typeRegistry, AtlasGraph atlasGraph,
@ -1510,7 +1512,7 @@ public class EntityGraphMapper {
LOG.debug("Adding classification [{}] to [{}] using edge label: [{}]", classificationName, entityType.getTypeName(), getTraitLabel(classificationName));
}
AtlasGraphUtilsV2.addEncodedProperty(entityVertex, TRAIT_NAMES_PROPERTY_KEY, classificationName);
addClassificationAndTraitNames(entityVertex, classificationName);
// add a new AtlasVertex for the struct or trait instance
AtlasVertex classificationVertex = createClassificationVertex(classification);
@ -1575,7 +1577,6 @@ public class EntityGraphMapper {
AtlasEntity entity = instanceConverter.getAndCacheEntity(entityGuid);
List<AtlasClassification> addedClassifications = StringUtils.equals(entityGuid, guid) ? addClassifications : propagations.get(vertex);
vertex.setProperty(CLASSIFICATION_TEXT_KEY, fullTextMapperV2.getClassificationTextForEntity(entity));
if (CollectionUtils.isNotEmpty(addedClassifications)) {
entityChangeNotifier.onClassificationAddedToEntity(entity, addedClassifications);
@ -1675,7 +1676,7 @@ public class EntityGraphMapper {
traitNames.remove(classificationName);
updateTraitNamesProperty(entityVertex, traitNames);
updateClassificationAndTraitNames(entityVertex, traitNames);
updateModificationMetadata(entityVertex);
@ -1696,6 +1697,38 @@ public class EntityGraphMapper {
return entity;
}
public void setClassificationNames(AtlasVertex vertex) {
List<String> clsNamesList = getTraitNames(vertex, false);
List<String> propClsNamesList = getTraitNames(vertex, true);
String clsNames = StringUtils.join(clsNamesList, CLASSIFICATION_NAME_DELIMITER);
String propClsNames = StringUtils.join(propClsNamesList, CLASSIFICATION_NAME_DELIMITER);
vertex.setProperty(CLASSIFICATION_NAMES_KEY, clsNames);
vertex.setProperty(PROPAGATED_CLASSIFICATION_NAMES_KEY, propClsNames);
}
private void addClassificationAndTraitNames(AtlasVertex entityVertex, String classificationName) {
AtlasGraphUtilsV2.addEncodedProperty(entityVertex, TRAIT_NAMES_PROPERTY_KEY, classificationName);
String clsNames = entityVertex.getProperty(CLASSIFICATION_NAMES_KEY, String.class);
clsNames = StringUtils.isEmpty(clsNames) ? classificationName : clsNames + CLASSIFICATION_NAME_DELIMITER + classificationName;
entityVertex.setProperty(CLASSIFICATION_NAMES_KEY, clsNames);
}
private void updateClassificationAndTraitNames(AtlasVertex entityVertex, List<String> traitNames) {
if (entityVertex != null) {
entityVertex.removeProperty(TRAIT_NAMES_PROPERTY_KEY);
entityVertex.removeProperty(CLASSIFICATION_NAMES_KEY);
for (String traitName : traitNames) {
AtlasGraphUtilsV2.addEncodedProperty(entityVertex, TRAIT_NAMES_PROPERTY_KEY, traitName);
}
entityVertex.setProperty(CLASSIFICATION_NAMES_KEY, StringUtils.join(traitNames, CLASSIFICATION_NAME_DELIMITER));
}
}
public void updateClassifications(EntityMutationContext context, String guid, List<AtlasClassification> classifications) throws AtlasBaseException {
if (CollectionUtils.isEmpty(classifications)) {
throw new AtlasBaseException(AtlasErrorCode.INVALID_CLASSIFICATION_PARAMS, "update", guid);
@ -1918,16 +1951,6 @@ public class EntityGraphMapper {
}
}
private void updateTraitNamesProperty(AtlasVertex entityVertex, List<String> traitNames) {
if (entityVertex != null) {
entityVertex.removeProperty(TRAIT_NAMES_PROPERTY_KEY);
for (String traitName : traitNames) {
AtlasGraphUtilsV2.addEncodedProperty(entityVertex, TRAIT_NAMES_PROPERTY_KEY, traitName);
}
}
}
private void validateClassificationExists(List<String> existingClassifications, List<String> suppliedClassifications) throws AtlasBaseException {
Set<String> existingNames = new HashSet<>(existingClassifications);
for (String classificationName : suppliedClassifications) {

53
webapp/src/test/java/org/apache/atlas/web/adapters/TestEntityREST.java
View File

@ -30,6 +30,9 @@ import org.apache.atlas.model.instance.AtlasEntityHeader;
import org.apache.atlas.model.instance.EntityMutationResponse;
import org.apache.atlas.model.instance.EntityMutations;
import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2;
import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasTypeUtil;
import org.apache.atlas.web.rest.EntityREST;
@ -62,7 +65,7 @@ public class TestEntityREST {
private AtlasEntity dbEntity;
private AtlasClassification testClassification;
private AtlasClassification phiClassification;
@BeforeClass
@ -108,7 +111,7 @@ public class TestEntityREST {
}
@Test(dependsOnMethods = "testGetEntityById")
public void testAddAndGetClassification() throws Exception {
public void testAddAndGetClassification() throws Exception {
List<AtlasClassification> classifications = new ArrayList<>();
testClassification = new AtlasClassification(TestUtilsV2.CLASSIFICATION, new HashMap<String, Object>() {{ put("tag", "tagName"); }});
@ -126,6 +129,13 @@ public class TestEntityREST {
Assert.assertNotNull(retrievedClassification);
Assert.assertEquals(retrievedClassification, testClassification);
// For ATLAS-3327 to test internal properties are added properly.
AtlasVertex vertex = AtlasGraphUtilsV2.findByGuid(dbEntity.getGuid());
String classificationNames = vertex.getProperty(Constants.CLASSIFICATION_NAMES_KEY, String.class);
Assert.assertNotNull(classificationNames);
Assert.assertEquals(classificationNames, TestUtilsV2.CLASSIFICATION);
}
@Test(dependsOnMethods = "testGetEntityById")
@ -156,6 +166,14 @@ public class TestEntityREST {
Assert.assertEquals(retrievedClassification.getAttribute(attrName), phiClassification.getAttribute(attrName));
}
// For ATLAS-3327 to test internal properties are added properly.
String expectedClsNames = TestUtilsV2.CLASSIFICATION + "|" + TestUtilsV2.PHI;
AtlasVertex vertex = AtlasGraphUtilsV2.findByGuid(dbEntity.getGuid());
String classificationNames = vertex.getProperty(Constants.CLASSIFICATION_NAMES_KEY, String.class);
Assert.assertNotNull(classificationNames);
Assert.assertEquals(classificationNames, expectedClsNames);
// update multiple tags attributes
phiClassification = new AtlasClassification(TestUtilsV2.PHI, new HashMap<String, Object>() {{
put("stringAttr", "sample_string_v2");
@ -178,11 +196,24 @@ public class TestEntityREST {
Assert.assertNotNull(updatedClassification);
Assert.assertEquals(updatedClassification.getAttribute("tag"), testClassification.getAttribute("tag"));
vertex = AtlasGraphUtilsV2.findByGuid(dbEntity.getGuid());
classificationNames = vertex.getProperty(Constants.CLASSIFICATION_NAMES_KEY, String.class);
Assert.assertNotNull(classificationNames);
Assert.assertEquals(classificationNames, expectedClsNames);
deleteClassification(dbEntity.getGuid(), TestUtilsV2.PHI);
expectedClsNames = TestUtilsV2.CLASSIFICATION;
vertex = AtlasGraphUtilsV2.findByGuid(dbEntity.getGuid());
classificationNames = vertex.getProperty(Constants.CLASSIFICATION_NAMES_KEY, String.class);
Assert.assertNotNull(classificationNames);
Assert.assertEquals(classificationNames, expectedClsNames);
}
@Test(dependsOnMethods = "testAddAndGetClassification")
public void testGetEntityWithAssociations() throws Exception {
public void testGetEntityWithAssociations() throws Exception {
AtlasEntityWithExtInfo entity = entityREST.getById(dbEntity.getGuid(), false, false);
final List<AtlasClassification> retrievedClassifications = entity.getEntity().getClassifications();
@ -192,11 +223,18 @@ public class TestEntityREST {
}
@Test(dependsOnMethods = "testGetEntityWithAssociations")
public void testDeleteClassification() throws Exception {
public void testDeleteClassification() throws Exception {
deleteClassification(dbEntity.getGuid(), TestUtilsV2.CLASSIFICATION);
final AtlasClassification.AtlasClassifications retrievedClassifications = entityREST.getClassifications(dbEntity.getGuid());
String expectedClsNames = "";
AtlasVertex vertex = AtlasGraphUtilsV2.findByGuid(dbEntity.getGuid());
String classificationNames = vertex.getProperty(Constants.CLASSIFICATION_NAMES_KEY, String.class);
Assert.assertNotNull(classificationNames);
Assert.assertEquals(classificationNames, expectedClsNames);
Assert.assertNotNull(retrievedClassifications);
Assert.assertEquals(retrievedClassifications.getList().size(), 0);
}
@ -219,6 +257,13 @@ public class TestEntityREST {
Assert.assertEquals(classifications, retrievedClassificationsList);
String expectedClsNames = TestUtilsV2.CLASSIFICATION;
AtlasVertex vertex = AtlasGraphUtilsV2.findByGuid(dbEntity.getGuid());
String classificationNames = vertex.getProperty(Constants.CLASSIFICATION_NAMES_KEY, String.class);
Assert.assertNotNull(classificationNames);
Assert.assertEquals(classificationNames, expectedClsNames);
final AtlasClassification retrievedClassification = entityREST.getClassification(dbEntity.getGuid(), TestUtilsV2.CLASSIFICATION);
Assert.assertNotNull(retrievedClassification);