tool
/
atlas
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ATLAS-3654: enable support for solr in standalone mode 

Signed-off-by: Madhan Neethiraj <madhan@apache.org>
This commit is contained in:
Damian Warszawski 2020-05-19 23:57:56 +02:00 committed by Madhan Neethiraj
parent 7718c5eb49
commit 6248e361fc
4 changed files with 91 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
distro/src/bin/atlas_config.py
View File

@ -66,7 +66,9 @@ CONF_FILE="atlas-application.properties"
STORAGE_BACKEND_CONF="atlas.graph.storage.backend"
HBASE_STORAGE_LOCAL_CONF_ENTRY="atlas.graph.storage.hostname\s*=\s*localhost"
SOLR_INDEX_CONF_ENTRY="atlas.graph.index.search.backend\s*=\s*solr"
SOLR_INDEX_LOCAL_CONF_ENTRY="atlas.graph.index.search.solr.zookeeper-url\s*=\s*localhost"
SOLR_INDEX_MODE_CONF_ENTRY="atlas.graph.index.search.solr.mode"
SOLR_INDEX_LOCAL_STANDALONE_CONF_ENTRY="atlas.graph.index.search.solr.http-urls\s*=(http|https)://localhost"
SOLR_INDEX_LOCAL_CLOUD_CONF_ENTRY="atlas.graph.index.search.solr.zookeeper-url\s*=\s*localhost"
SOLR_INDEX_ZK_URL="atlas.graph.index.search.solr.zookeeper-url"
TOPICS_TO_CREATE="atlas.notification.topics"
ATLAS_HTTP_PORT="atlas.server.http.port"
@ -453,7 +455,14 @@ def is_solr_local(confdir):
return False
confdir = os.path.join(confdir, CONF_FILE)
return grep(confdir, SOLR_INDEX_CONF_ENTRY) is not None and grep(confdir, SOLR_INDEX_LOCAL_CONF_ENTRY) is not None
return is_solr_local_cloud_mode(confdir) or is_solr_local_standalone_mode(confdir)
def is_solr_local_cloud_mode(confdir):
return grep(confdir, SOLR_INDEX_CONF_ENTRY) is not None and getConfig(confdir, SOLR_INDEX_MODE_CONF_ENTRY) == 'cloud' and grep(confdir, SOLR_INDEX_LOCAL_CLOUD_CONF_ENTRY) is not None
def is_solr_local_standalone_mode(confdir):
return grep(confdir, SOLR_INDEX_CONF_ENTRY) is not None and getConfig(confdir, SOLR_INDEX_MODE_CONF_ENTRY) == 'http' and grep(confdir, SOLR_INDEX_LOCAL_STANDALONE_CONF_ENTRY) is not None
def is_elasticsearch_local():
if os.environ.get(MANAGE_LOCAL_ELASTICSEARCH, "False").lower() == 'false':

17
docs/src/documents/Setup/InstallationInstruction.md
View File

@ -150,7 +150,7 @@ By default, Apache Atlas uses JanusGraph as the graph repository and is the only
* Start Apache Solr in cloud mode.
SolrCloud mode uses a ZooKeeper Service as a highly available, central location for cluster management. For a small cluster, running with an existing ZooKeeper quorum should be fine. For larger clusters, you would want to run separate multiple ZooKeeper quorum with at least 3 servers.
Note: Apache Atlas currently supports Apache Solr in "cloud" mode only. "http" mode is not supported. For more information, refer Apache Solr documentation - https://cwiki.apache.org/confluence/display/solr/SolrCloud
For more information, refer Apache Solr documentation - https://cwiki.apache.org/confluence/display/solr/SolrCloud
* For e.g., to bring up an Apache Solr node listening on port 8983 on a machine, you can use the command:
@ -197,6 +197,21 @@ Pre-requisites for running Apache Solr in cloud mode
* SolrCloud has support for replication and sharding. It is highly recommended to use SolrCloud with at least two Apache Solr nodes running on different servers with replication enabled.
If using SolrCloud, then you also need ZooKeeper installed and configured with 3 or 5 ZooKeeper nodes
* Start Apache Solr in http mode - alternative setup to Solr in cloud mode.
Solr Standalone is used for a single instance, and it keeps configuration information on the file system. It does not require zookeeper and provides high performance for medium size index.
Can be consider as a good option for fast prototyping as well as valid configuration for development environments. In some cases it demonstrates a better performance than solr cloud mode in production grade setup of Atlas.
* Change ATLAS configuration to point to Standalone Apache Solr instance setup. Please make sure the following configurations are set to the below values in ATLAS_HOME/conf/atlas-application.properties
<SyntaxHighlighter wrapLines={true} language="powershell" style={theme.dark}>
{`atlas.graph.index.search.backend=solr
atlas.graph.index.search.solr.mode=http
atlas.graph.index.search.solr.http-urls=<a single or list of URLs for the Solr instances must be provided.> eg: localhost:2181,10.1.6.5:2181`}
</SyntaxHighlighter>
Note: Solr standalone can be run in embedded mode using `embedded-hbase-solr` profile.
*Configuring Elasticsearch as the indexing backend for the Graph Repository (Tech Preview)*
By default, Apache Atlas uses [JanusGraph](https://janusgraph.org/) as the graph repository and is the only graph repository implementation available currently. For configuring [JanusGraph](https://janusgraph.org/) to work with Elasticsearch, please follow the instructions below

74
graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphIndexClient.java
View File

@ -32,14 +32,17 @@ import org.apache.commons.lang.StringUtils;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrResponse;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.request.GenericSolrRequest;
import org.apache.solr.client.solrj.request.V2Request;
import org.apache.solr.client.solrj.response.V2Response;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.client.solrj.response.FacetField;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.client.solrj.response.TermsResponse;
import org.apache.solr.client.solrj.util.ClientUtils;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.util.ContentStream;
import org.apache.solr.common.util.NamedList;
import org.janusgraph.diskstorage.solr.Solr6Index;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -79,6 +82,8 @@ public class AtlasJanusGraphIndexClient implements AtlasGraphIndexClient {
return;
}
Solr6Index.Mode solrMode = Solr6Index.getSolrMode();
//1) try updating request handler
//2) if update fails, try creating request handler
@ -100,7 +105,7 @@ public class AtlasJanusGraphIndexClient implements AtlasGraphIndexClient {
try {
LOG.info("Attempting to update free text request handler {} for collection {}", FREETEXT_REQUEST_HANDLER, collectionName);
updateFreeTextRequestHandler(solrClient, collectionName, indexFieldName2SearchWeightMap);
updateFreeTextRequestHandler(solrClient, collectionName, indexFieldName2SearchWeightMap, solrMode);
LOG.info("Successfully updated free text request handler {} for collection {}..", FREETEXT_REQUEST_HANDLER, collectionName);
@ -114,7 +119,7 @@ public class AtlasJanusGraphIndexClient implements AtlasGraphIndexClient {
try {
LOG.info("Attempting to create free text request handler {} for collection {}", FREETEXT_REQUEST_HANDLER, collectionName);
createFreeTextRequestHandler(solrClient, collectionName, indexFieldName2SearchWeightMap);
createFreeTextRequestHandler(solrClient, collectionName, indexFieldName2SearchWeightMap, solrMode);
LOG.info("Successfully created free text request handler {} for collection {}", FREETEXT_REQUEST_HANDLER, collectionName);
return;
@ -256,10 +261,13 @@ public class AtlasJanusGraphIndexClient implements AtlasGraphIndexClient {
return;
}
Solr6Index.Mode solrMode = Solr6Index.getSolrMode();
//update the request handler
performRequestHandlerAction(collectionName,
solrClient,
generatePayLoadForSuggestions(generateSuggestionsString(suggestionProperties)));
generatePayLoadForSuggestions(generateSuggestionsString(suggestionProperties)),
solrMode);
} catch (Throwable t) {
String msg = String.format("Error encountered in creating the request handler '%s' for collection '%s'", Constants.TERMS_REQUEST_HANDLER, collectionName);
@ -458,35 +466,53 @@ public class AtlasJanusGraphIndexClient implements AtlasGraphIndexClient {
return ret.toString();
}
private V2Response updateFreeTextRequestHandler(SolrClient solrClient, String collectionName,
Map<String, Integer> indexFieldName2SearchWeightMap) throws IOException, SolrServerException, AtlasBaseException {
private SolrResponse updateFreeTextRequestHandler(SolrClient solrClient, String collectionName,
Map<String, Integer> indexFieldName2SearchWeightMap,
Solr6Index.Mode mode) throws IOException, SolrServerException, AtlasBaseException {
String searchWeightString = generateSearchWeightString(indexFieldName2SearchWeightMap);
String payLoadString = generatePayLoadForFreeText("update-requesthandler", searchWeightString);
return performRequestHandlerAction(collectionName, solrClient, payLoadString);
return performRequestHandlerAction(collectionName, solrClient, payLoadString, mode);
}
private V2Response createFreeTextRequestHandler(SolrClient solrClient, String collectionName,
Map<String, Integer> indexFieldName2SearchWeightMap) throws IOException, SolrServerException, AtlasBaseException {
private SolrResponse createFreeTextRequestHandler(SolrClient solrClient, String collectionName,
Map<String, Integer> indexFieldName2SearchWeightMap,
Solr6Index.Mode mode) throws IOException, SolrServerException, AtlasBaseException {
String searchWeightString = generateSearchWeightString(indexFieldName2SearchWeightMap);
String payLoadString = generatePayLoadForFreeText("create-requesthandler", searchWeightString);
return performRequestHandlerAction(collectionName, solrClient, payLoadString);
return performRequestHandlerAction(collectionName, solrClient, payLoadString, mode);
}
private V2Response performRequestHandlerAction(String collectionName,
private SolrResponse performRequestHandlerAction(String collectionName,
SolrClient solrClient,
String actionPayLoad) throws IOException, SolrServerException, AtlasBaseException {
V2Request v2Request = new V2Request.Builder(String.format("/collections/%s/config", collectionName))
.withMethod(SolrRequest.METHOD.POST)
.withPayload(actionPayLoad)
.build();
String actionPayLoad,
Solr6Index.Mode mode) throws IOException, SolrServerException, AtlasBaseException {
switch (mode) {
case CLOUD:
V2Request v2request = new V2Request.Builder(String.format("/collections/%s/config", collectionName))
.withMethod(SolrRequest.METHOD.POST)
.withPayload(actionPayLoad)
.build();
return validateResponseForSuccess(v2Request.process(solrClient));
return validateResponseForSuccess(v2request.process(solrClient));
case HTTP:
Collection<ContentStream> contentStreams = ClientUtils.toContentStreams(actionPayLoad, "application/json; charset=UTF-8");
GenericSolrRequest request = new GenericSolrRequest(SolrRequest.METHOD.POST, String.format("/%s/config", collectionName), null);
request.setContentStreams(contentStreams);
request.setUseV2(false);
return validateResponseForSuccess(request.process(solrClient));
default:
throw new IllegalArgumentException("Unsupported Solr operation mode: " + mode);
}
}
private V2Response validateResponseForSuccess(V2Response v2Response) throws AtlasBaseException {
if(v2Response == null) {
private SolrResponse validateResponseForSuccess(SolrResponse solrResponse) throws AtlasBaseException {
if(solrResponse == null) {
String msg = "Received null response .";
LOG.error(msg);
@ -495,10 +521,10 @@ public class AtlasJanusGraphIndexClient implements AtlasGraphIndexClient {
}
if (LOG.isDebugEnabled()) {
LOG.debug("V2 Response is {}", v2Response.toString());
LOG.debug("V2 Response is {}", solrResponse.toString());
}
NamedList<Object> response = v2Response.getResponse();
NamedList<Object> response = solrResponse.getResponse();
if(response != null) {
Object errorMessages = response.get("errorMessages");
@ -523,7 +549,7 @@ public class AtlasJanusGraphIndexClient implements AtlasGraphIndexClient {
throw new AtlasBaseException(msg);
} else {
if(LOG.isDebugEnabled()) {
LOG.debug("Successfully performed response handler action. V2 Response is {}", v2Response.toString());
LOG.debug("Successfully performed response handler action. V2 Response is {}", solrResponse.toString());
}
}
@ -533,7 +559,7 @@ public class AtlasJanusGraphIndexClient implements AtlasGraphIndexClient {
}
}
return v2Response;
return solrResponse;
}
static final class TermFreq {

16
graphdb/janus/src/main/java/org/janusgraph/diskstorage/solr/Solr6Index.java
View File

@ -143,7 +143,7 @@ public class Solr6Index implements IndexProvider {
private static Solr6Index instance = null;
public static final ConfigOption<Boolean> CREATE_SOLR_CLIENT_PER_REQUEST = new ConfigOption(SOLR_NS, "create-client-per-request", "when false, allows the sharing of solr client across other components.", org.janusgraph.diskstorage.configuration.ConfigOption.Type.LOCAL, false);
private enum Mode {
public enum Mode {
HTTP, CLOUD;
public static Mode parse(String mode) {
@ -183,7 +183,6 @@ public class Solr6Index implements IndexProvider {
private final boolean waitSearcher;
private final boolean kerberosEnabled;
public Solr6Index(final Configuration config) throws BackendException {
// Add Kerberos-enabled SolrHttpClientBuilder
HttpClientUtil.setHttpClientBuilder(new Krb5HttpClientBuilder().getBuilder());
@ -217,6 +216,19 @@ public class Solr6Index implements IndexProvider {
Solr6Index.instance = this;
}
public static Mode getSolrMode() {
Solr6Index solr6Index = Solr6Index.instance;
Mode ret = (solr6Index != null) ? Mode.parse(solr6Index.configuration.get(SOLR_MODE)) : null;
if (ret == null) {
logger.warn("SolrMode is not set. Assuming {}", Mode.CLOUD);
ret = Mode.CLOUD;
}
return ret;
}
public static SolrClient getSolrClient() {
if (Solr6Index.instance != null) {
if (createSolrClientPerRequest) {