tool
/
atlas
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ATLAS-4892: Export/Import: Unable to import shell entities 

Signed-off-by: Pinal Shah <pinal.shah@freestoneinfotech.com>
This commit is contained in:
priyanshi-shah26 2024-08-23 19:14:16 +05:30 committed by Pinal Shah
parent c28930cf4a
commit 1ca1c4a8a5
6 changed files with 47 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java
View File

@ -1190,6 +1190,9 @@ public class AtlasEntityType extends AtlasStructType {
}
if (value == null) {
if (entityObj.getIsIncomplete() != null && entityObj.getIsIncomplete()) {// In case of import shell entities, avoid checking of mandatory attributes
continue;
}
ret = false;
messages.add(objName + "." + attributeName + ": mandatory attribute value missing in type " + getTypeName());
}

3
intg/src/main/java/org/apache/atlas/type/AtlasStructType.java
View File

@ -421,6 +421,9 @@ public class AtlasStructType extends AtlasType {
AtlasEntity entityObj = (AtlasEntity) structObj;
if (entityObj.getRelationshipAttribute(attrName) == null) {
if (entityObj.getIsIncomplete() != null && entityObj.getIsIncomplete()) { // In case of import shell entities, avoid checking of mandatory attributes
continue;
}
ret = false;
messages.add(fieldName + ": mandatory attribute value missing in type " + getTypeName());
}

2
repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityGraphDiscoveryV2.java
View File

@ -192,7 +192,7 @@ public class AtlasEntityGraphDiscoveryV2 implements EntityGraphDiscovery {
protected void resolveReferences() throws AtlasBaseException {
MetricRecorder metric = RequestContext.get().startMetricRecord("resolveReferences");
EntityResolver[] entityResolvers = new EntityResolver[] { new IDBasedEntityResolver(this.graph, typeRegistry),
EntityResolver[] entityResolvers = new EntityResolver[] { new IDBasedEntityResolver(this.graph, typeRegistry, entityGraphMapper),
new UniqAttrBasedEntityResolver(this.graph, typeRegistry, entityGraphMapper)
};

13
repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java
View File

@ -1254,6 +1254,10 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
// change entity 'isInComplete' to 'false' during full update
if (isEntityIncomplete(vertex)) {
if (RequestContext.get().isImportInProgress() && (entity.getIsIncomplete() != null && entity.getIsIncomplete())) {
continue;
}
vertex.removeProperty(IS_INCOMPLETE_PROPERTY_KEY);
entity.setIsIncomplete(FALSE);
@ -1269,8 +1273,9 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
requestContext.recordEntityGuidUpdate(entity, guid);
}
context.addUpdated(guid, entity, entityType, vertex);
if (!isEntityIncomplete(vertex)) { // In case of an import shell entities, skip updating to entitiesCreated, to avoid mapAttributesAndClassification // In case of hook shell entities, it will not reach to this case
context.addUpdated(guid, entity, entityType, vertex);
}
} else {
graphDiscoverer.validateAndNormalize(entity);
@ -1291,7 +1296,9 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
requestContext.recordEntityGuidUpdate(entity, guid);
context.addCreated(guid, entity, entityType, vertex);
if (!isEntityIncomplete(vertex)) { // In case of an import shell entities, skip adding to entitiesUpdated, to avoid mapAttributesAndClassification // In case of hook shell entities, it will not reach to this case
context.addCreated(guid, entity, entityType, vertex);
}
}
// during import, update the system attributes

10
repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
View File

@ -194,7 +194,10 @@ public class EntityGraphMapper {
LOG.debug("==> createShellEntityVertex({})", objectId.getTypeName());
}
final String guid = UUID.randomUUID().toString();
String guid = objectId.getGuid();
if (!AtlasTypeUtil.isAssignedGuid(guid)) {
guid = UUID.randomUUID().toString();
}
AtlasEntityType entityType = typeRegistry.getEntityTypeByName(objectId.getTypeName());
AtlasVertex ret = createStructVertex(objectId);
@ -225,6 +228,11 @@ public class EntityGraphMapper {
return ret;
}
public AtlasVertex createShellEntityVertex(AtlasEntity entity, EntityGraphDiscoveryContext context) throws AtlasBaseException {
AtlasObjectId objectId = new AtlasObjectId(entity.getGuid(), entity.getTypeName(), entity.getAttributes());
return createShellEntityVertex(objectId, context);
}
public AtlasVertex createVertexWithGuid(AtlasEntity entity, String guid) throws AtlasBaseException {
if (LOG.isDebugEnabled()) {
LOG.debug("==> createVertexWithGuid({})", entity.getTypeName());

30
repository/src/main/java/org/apache/atlas/repository/store/graph/v2/IDBasedEntityResolver.java
View File

@ -38,10 +38,16 @@ public class IDBasedEntityResolver implements EntityResolver {
private final AtlasGraph graph;
private final AtlasTypeRegistry typeRegistry;
private final EntityGraphMapper entityGraphMapper;
public IDBasedEntityResolver(AtlasGraph graph, AtlasTypeRegistry typeRegistry) {
this(graph, typeRegistry, null);
}
public IDBasedEntityResolver(AtlasGraph graph, AtlasTypeRegistry typeRegistry, EntityGraphMapper entityGraphMapper) {
this.graph = graph;
this.typeRegistry = typeRegistry;
this.entityGraphMapper = entityGraphMapper;
}
public EntityGraphDiscoveryContext resolveEntityReferences(EntityGraphDiscoveryContext context) throws AtlasBaseException {
@ -55,19 +61,25 @@ public class IDBasedEntityResolver implements EntityResolver {
boolean isAssignedGuid = AtlasTypeUtil.isAssignedGuid(guid);
AtlasVertex vertex = isAssignedGuid ? AtlasGraphUtilsV2.findByGuid(this.graph, guid) : null;
if (vertex == null && !RequestContext.get().isImportInProgress()) { // if not found in the store, look if the entity is present in the stream
if (vertex == null) { // if not found in the store, look if the entity is present in the stream
AtlasEntity entity = entityStream.getByGuid(guid);
if (!RequestContext.get().isImportInProgress()) {
if (entity != null) { // look for the entity in the store using unique-attributes
AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entity.getTypeName());
if (entity != null) { // look for the entity in the store using unique-attributes
AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entity.getTypeName());
if (entityType == null) {
throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_INVALID, TypeCategory.ENTITY.name(), entity.getTypeName());
if (entityType == null) {
throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_INVALID, TypeCategory.ENTITY.name(), entity.getTypeName());
}
vertex = AtlasGraphUtilsV2.findByUniqueAttributes(this.graph, entityType, entity.getAttributes());
} else if (!isAssignedGuid) { // for local-guids, entity must be in the stream
throw new AtlasBaseException(AtlasErrorCode.REFERENCED_ENTITY_NOT_FOUND, guid);
}
} else if (entity != null && entity.getIsIncomplete() != null && entity.getIsIncomplete()) {
if (entityGraphMapper != null) {
vertex = entityGraphMapper.createShellEntityVertex(entity, context);
}
vertex = AtlasGraphUtilsV2.findByUniqueAttributes(this.graph, entityType, entity.getAttributes());
} else if (!isAssignedGuid) { // for local-guids, entity must be in the stream
throw new AtlasBaseException(AtlasErrorCode.REFERENCED_ENTITY_NOT_FOUND, guid);
}
}