tool
/
atlas
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ATLAS-4340: Set Solr wait-searcher property to false by default to make Solr commits async

This commit is contained in:
Sarath Subramanian 2021-07-27 14:59:48 -07:00
parent 98c6cea014
commit a7ac64d549
12 changed files with 38 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
addons/falcon-bridge/src/test/resources/atlas-application.properties
View File

@ -24,6 +24,7 @@ atlas.rest.address=http://localhost:31000
# Graph database implementation. Value inserted by maven.
atlas.graphdb.backend=org.apache.atlas.repository.graphdb.janus.AtlasJanusGraphDatabase
atlas.graph.index.search.solr.wait-searcher=true
# Graph Storage
atlas.graph.storage.backend=berkeleyje

1
addons/hbase-bridge/src/test/resources/atlas-application.properties
View File

@ -24,6 +24,7 @@ atlas.rest.address=http://localhost:31000
# Graph database implementation. Value inserted by maven.
atlas.graphdb.backend=org.apache.atlas.repository.graphdb.janus.AtlasJanusGraphDatabase
atlas.graph.index.search.solr.wait-searcher=true
# Graph Storage
atlas.graph.storage.backend=berkeleyje

1
addons/kafka-bridge/src/test/resources/atlas-application.properties
View File

@ -24,6 +24,7 @@ atlas.rest.address=http://localhost:31000
# Graph database implementation. Value inserted by maven.
atlas.graphdb.backend=org.apache.atlas.repository.graphdb.janus.AtlasJanusGraphDatabase
atlas.graph.index.search.solr.wait-searcher=true
# Graph Storage
atlas.graph.storage.backend=berkeleyje

3
authorization/src/test/resources/atlas-application.properties
View File

@ -18,8 +18,7 @@
#system property
atlas.data=${sys:user.dir}/target/data
atlas.graph.index.search.solr.wait-searcher=true
#re-use existing property
atlas.graph.data=${atlas.data}/graph

8
distro/pom.xml
View File

@ -41,7 +41,7 @@ atlas.graph.index.search.solr.mode=cloud
atlas.graph.index.search.solr.zookeeper-url=
atlas.graph.index.search.solr.zookeeper-connect-timeout=60000
atlas.graph.index.search.solr.zookeeper-session-timeout=60000
atlas.graph.index.search.solr.wait-searcher=true
atlas.graph.index.search.solr.wait-searcher=false
#Solr http mode properties
#atlas.graph.index.search.solr.mode=http
@ -162,7 +162,7 @@ atlas.graph.index.search.solr.mode=cloud
atlas.graph.index.search.solr.zookeeper-url=localhost:2181
atlas.graph.index.search.solr.zookeeper-connect-timeout=60000
atlas.graph.index.search.solr.zookeeper-session-timeout=60000
atlas.graph.index.search.solr.wait-searcher=true
atlas.graph.index.search.solr.wait-searcher=false
#Solr http mode properties
#atlas.graph.index.search.solr.mode=http
@ -321,7 +321,7 @@ atlas.graph.index.search.solr.mode=cloud
atlas.graph.index.search.solr.zookeeper-url=localhost:2181
atlas.graph.index.search.solr.zookeeper-connect-timeout=60000
atlas.graph.index.search.solr.zookeeper-session-timeout=60000
atlas.graph.index.search.solr.wait-searcher=true
atlas.graph.index.search.solr.wait-searcher=false
#Solr http mode properties
#atlas.graph.index.search.solr.mode=http
@ -411,7 +411,7 @@ atlas.graph.index.search.solr.mode=cloud
atlas.graph.index.search.solr.zookeeper-url=localhost:2181
atlas.graph.index.search.solr.zookeeper-connect-timeout=60000
atlas.graph.index.search.solr.zookeeper-session-timeout=60000
atlas.graph.index.search.solr.wait-searcher=true
atlas.graph.index.search.solr.wait-searcher=false
#Solr http mode properties
#atlas.graph.index.search.solr.mode=http

6
distro/src/bin/atlas_config.py
View File

@ -125,7 +125,7 @@ def solrHomeDir(dir):
return os.environ.get(SOLR_HOME, os.path.join(dir, "data", "solr"))
def solrConfDir(dir):
return os.environ.get(SOLR_CONF, os.path.join(dir, "solr", CONFIG_SETS_CONF))
return os.environ.get(SOLR_CONF, os.path.join(dir, "conf", "solr"))
def solrPort():
return os.environ.get(SOLR_PORT, DEFAULT_SOLR_PORT)
@ -516,7 +516,7 @@ def get_atlas_url_port(confdir):
else:
port = getConfigWithDefault(confdir, ATLAS_HTTP_PORT, DEFAULT_ATLAS_HTTP_PORT)
print("starting atlas on port %s" % port)
print("Starting Atlas server on port: %s" % port)
return port
def get_atlas_url_host(confdir):
@ -524,7 +524,7 @@ def get_atlas_url_host(confdir):
host = getConfigWithDefault(confdir, ATLAS_SERVER_BIND_ADDRESS, DEFAULT_ATLAS_SERVER_HOST)
if (host == '0.0.0.0'):
host = DEFAULT_ATLAS_SERVER_HOST
print("starting atlas on host %s" % host)
print("\nStarting Atlas server on host: %s" % host)
return host
def wait_for_startup(confdir, wait):

18
distro/src/bin/atlas_start.py
View File

@ -110,14 +110,17 @@ def main():
mc.server_pid_not_running(pid)
if is_hbase and mc.is_hbase_local(confdir):
print("configured for local hbase.")
print("\nConfigured for local HBase.")
mc.configure_hbase(atlas_home)
print("Starting local HBase...")
mc.run_hbase_action(mc.hbaseBinDir(atlas_home), "start", hbase_conf_dir, logdir)
print("hbase started.")
print("Local HBase started!")
#solr setup
if mc.is_solr_local(confdir):
print("configured for local solr.")
print("\nConfigured for local Solr.")
if mc.is_cassandra_local(confdir):
print("Cassandra embedded configured.")
@ -128,12 +131,17 @@ def main():
mc.run_zookeeper(mc.zookeeperBinDir(atlas_home), "start", logdir)
print("zookeeper started.")
print("Starting local Solr...")
mc.run_solr(mc.solrBinDir(atlas_home), "start", mc.get_solr_zk_url(confdir), mc.solrPort(), logdir, True, mc.solrHomeDir(atlas_home))
print("solr started.")
print("setting up solr collections...")
print("Local Solr started!")
print("\nCreating Solr collections for Atlas using config: " + mc.solrConfDir(atlas_home))
mc.create_solr_collection(mc.solrBinDir(atlas_home), mc.solrConfDir(atlas_home), "vertex_index", logdir)
mc.create_solr_collection(mc.solrBinDir(atlas_home), mc.solrConfDir(atlas_home), "edge_index", logdir)
mc.create_solr_collection(mc.solrBinDir(atlas_home), mc.solrConfDir(atlas_home), "fulltext_index", logdir)
#elasticsearch setup

6
distro/src/test/python/scripts/TestMetadata.py
View File

@ -109,13 +109,13 @@ class TestMetadata(unittest.TestCase):
call(['atlas_home/solr/bin/solr', 'start', '-z', 'localhost:9838', '-p', '9838', '-s', 'atlas_home/data/solr'], 'atlas_home/logs',
False, True),
call(['atlas_home/solr/bin/solr', 'create', '-c', 'vertex_index', '-d',
'atlas_home/solr/server/solr/configsets/_default/conf', '-shards', '1', '-replicationFactor',
'atlas_home/conf/solr', '-shards', '1', '-replicationFactor',
'1'], 'atlas_home/logs', False, True),
call(['atlas_home/solr/bin/solr', 'create', '-c', 'edge_index', '-d',
'atlas_home/solr/server/solr/configsets/_default/conf', '-shards', '1', '-replicationFactor',
'atlas_home/conf/solr', '-shards', '1', '-replicationFactor',
'1'], 'atlas_home/logs', False, True),
call(['atlas_home/solr/bin/solr', 'create', '-c', 'fulltext_index', '-d',
'atlas_home/solr/server/solr/configsets/_default/conf', '-shards', '1', '-replicationFactor',
'atlas_home/conf/solr', '-shards', '1', '-replicationFactor',
'1'], 'atlas_home/logs', False, True)]
runProcess_mock.assert_has_calls(calls)

1
graphdb/janus/src/test/resources/atlas-application.properties
View File

@ -18,6 +18,7 @@
######### Graph Database to Use #########
atlas.graphdb.backend=org.apache.atlas.repository.graphdb.janus.AtlasJanusGraphDatabase
atlas.graph.index.search.solr.wait-searcher=true
######### Atlas Server Configs #########
atlas.rest.address=http://localhost:31000

14
intg/src/main/java/org/apache/atlas/ApplicationProperties.java
View File

@ -64,7 +64,7 @@ public final class ApplicationProperties extends PropertiesConfiguration {
public static final String LDAP_BIND_PASSWORD = "atlas.authentication.method.ldap.bind.password";
public static final String MASK_LDAP_PASSWORD = "********";
public static final String DEFAULT_GRAPHDB_BACKEND = GRAPHBD_BACKEND_JANUS;
public static final boolean DEFAULT_SOLR_WAIT_SEARCHER = true;
public static final boolean DEFAULT_SOLR_WAIT_SEARCHER = false;
public static final boolean DEFAULT_INDEX_MAP_NAME = false;
public static final AtlasRunMode DEFAULT_ATLAS_RUN_MODE = AtlasRunMode.PROD;
public static final String INDEX_SEARCH_MAX_RESULT_SET_SIZE = "atlas.graph.index.search.max-result-set-size";
@ -342,11 +342,12 @@ public final class ApplicationProperties extends PropertiesConfiguration {
if (indexBackend.equalsIgnoreCase(INDEX_BACKEND_SOLR)) {
LOG.info("Atlas is running in MODE: {}.", runMode.name());
if(runMode == AtlasRunMode.PROD) {
//we do not want these configurations to be over written in Production mode.
clearPropertyDirect(SOLR_WAIT_SEARCHER_CONF);
addPropertyDirect(SOLR_WAIT_SEARCHER_CONF, DEFAULT_SOLR_WAIT_SEARCHER);
LOG.info("Setting solr-wait-searcher property '" + DEFAULT_SOLR_WAIT_SEARCHER + "'");
if (runMode == AtlasRunMode.PROD) {
if (!containsKey(SOLR_WAIT_SEARCHER_CONF)) {
addPropertyDirect(SOLR_WAIT_SEARCHER_CONF, DEFAULT_SOLR_WAIT_SEARCHER);
}
LOG.info("Setting solr.wait-searcher property '" + getBoolean(SOLR_WAIT_SEARCHER_CONF) + "'");
clearPropertyDirect(INDEX_MAP_NAME_CONF);
addPropertyDirect(INDEX_MAP_NAME_CONF, DEFAULT_INDEX_MAP_NAME);
@ -361,6 +362,7 @@ public final class ApplicationProperties extends PropertiesConfiguration {
addPropertyDirect(INDEX_SEARCH_MAX_RESULT_SET_SIZE, indexMaxResultSetSize);
LOG.info("Setting " + INDEX_SEARCH_MAX_RESULT_SET_SIZE + " = " + indexMaxResultSetSize);
LOG.info("Setting " + SOLR_WAIT_SEARCHER_CONF + " = " + getBoolean(SOLR_WAIT_SEARCHER_CONF));
setDbCacheConfDefaults();
}

3
intg/src/test/resources/atlas-application.properties
View File

@ -18,8 +18,7 @@
#system property
atlas.data=${sys:user.dir}/target/data
atlas.graph.index.search.solr.wait-searcher=true
#re-use existing property
atlas.graph.data=${atlas.data}/graph

2
webapp/src/test/resources/atlas-application.properties
View File

@ -20,7 +20,7 @@
atlas.rest.address=http://localhost:31000
######### Graph Database Configs #########
atlas.graph.index.search.solr.wait-searcher=true
# Graph database implementation. Value inserted by maven.
atlas.graphdb.backend=org.apache.atlas.repository.graphdb.janus.AtlasJanusGraphDatabase