JMX issue with DataStax OpsCenter Agent -
we're running 8 node dse cluster (just cassandra) divided among 2 datacenters. working fine apart agent on 1 node stubbornly refuses cooperate.
here version info:
cassandra 3.0.9.1346
dse 5.0.3
opscenter 6.03
all nodes have upgraded sstables , have been repaired.
here log:
info [async-dispatch-47] 2016-11-08 14:33:13,811 starting system. info [async-dispatch-47] 2016-11-08 14:33:13,832 starting dynamicenvironmentcomponent warn [async-dispatch-47] 2016-11-08 14:33:13,845 exception while processing jmx data: java.lang.nullpointerexception error [async-dispatch-47] 2016-11-08 14:33:13,845 error starting dynamicenvironmentcomponent. java.lang.nullpointerexception @ clojure.java.io$as_relative_path.invoke(io.clj:404) @ clojure.java.io$file.invoke(io.clj:416) @ opsagent.environment.collection$cassandra_yaml_location__gt_install_location.invoke(collection.clj:118) @ opsagent.environment.dynamic$dynamic_env_state.invoke(dynamic.clj:61) @ clojure.core$partial$fn__4527.invoke(core.clj:2492) @ opsagent.jmx$create_jmx_pool_with_config$wrapper__11504.doinvoke(jmx.clj:221) @ clojure.lang.restfn.invoke(restfn.java:410) @ clojure.lang.afn.applytohelper(afn.java:154) @ clojure.lang.restfn.applyto(restfn.java:132) @ clojure.core$apply.invoke(core.clj:630) @ opsagent.environment.dynamic$add_dynamic_state.invoke(dynamic.clj:143) @ opsagent.environment.dynamic.dynamicenvironmentcomponent.start(dynamic.clj:168) @ com.stuartsierra.component$fn__8838$g__8832__8840.invoke(component.clj:4) @ com.stuartsierra.component$fn__8838$g__8831__8843.invoke(component.clj:4) @ clojure.lang.var.invoke(var.java:379) @ clojure.lang.afn.applytohelper(afn.java:154) @ clojure.lang.var.applyto(var.java:700) @ clojure.core$apply.invoke(core.clj:632) @ com.stuartsierra.component$try_action.invoke(component.clj:116) @ clojure.lang.var.invoke(var.java:401) @ opsagent.config_service$update_system$fn__20056.invoke(config_service.clj:200) @ clojure.lang.arrayseq.reduce(arrayseq.java:114) @ clojure.core$reduce.invoke(core.clj:6518) @ opsagent.config_service$update_system.doinvoke(config_service.clj:194) @ clojure.lang.restfn.invoke(restfn.java:425) @ opsagent.config_service$start_system_bang_.invoke(config_service.clj:219) @ opsagent.config_service$fn__20133$fn__20134$state_machine__4719__auto____20135$fn__20137.invoke(config_service.clj:245) @ opsagent.config_service$fn__20133$fn__20134$state_machine__4719__auto____20135.invoke(config_service.clj:242) @ clojure.core.async.impl.ioc_macros$run_state_machine.invoke(ioc_macros.clj:940) @ clojure.core.async.impl.ioc_macros$run_state_machine_wrapped.invoke(ioc_macros.clj:944) @ clojure.core.async$ioc_alts_bang_$fn__4884.invoke(async.clj:362) @ clojure.core.async$do_alts$fn__4838$fn__4841.invoke(async.clj:231) @ clojure.core.async.impl.channels.manytomanychannel$fn__1215.invoke(channels.clj:262) @ clojure.lang.afn.run(afn.java:22) @ java.util.concurrent.threadpoolexecutor.runworker(threadpoolexecutor.java:1142) @ java.util.concurrent.threadpoolexecutor$worker.run(threadpoolexecutor.java:617) @ java.lang.thread.run(thread.java:745) info [async-dispatch-47] 2016-11-08 14:33:13,848 finished starting system. any ideas?
edit: here requested additional info.
the node's addresses follows: listen: 10.2.2.22 rpc: 10.1.2.22
all ips private , ssl disabled because dcs connected via vpn.
# nodetool status status=up/down |/ state=normal/leaving/joining/moving -- address load tokens owns host id rack un 192.168.1.72 12.14 gb 256 ? a15c57e1-3c53-4d4f-9df9-29945b9f1c88 rac1 un 192.168.1.92 11.36 gb 256 ? 9820b96a-a3c6-460f-839b-5dabc89313a0 rac1 un 192.168.1.82 11.67 gb 256 ? f9c13cb0-ee44-4ce2-ac7e-14ec1f7c1d23 rac1 datacenter: dc2 =============== status=up/down |/ state=normal/leaving/joining/moving -- address load tokens owns host id rack un 10.2.2.32 11.04 gb 256 ? bfe86bb3-d272-4946-ac8a-e176fe9f8e64 rac3 un 10.2.2.22 11.29 gb 256 ? c8694e93-0d8a-41b4-82a3-6c450497e8ec rac2 un 10.2.2.52 11.46 gb 256 ? e941faf1-ad5b-46a7-8857-bdf9dd2a3459 rac5 un 10.2.2.42 10.9 gb 256 ? 7bbd2397-a3bc-4cfe-9c03-334186e7e0dd rac4 un 10.2.2.12 5.29 gb 256 ? bf7a0587-2b09-47d6-b6d5-24e1422318b9 rac1 # address.yaml
stomp_interface: 192.168.1.31
use_ssl: 0
cassandara_conf: /etc/dse/cassandra
root@anc-t2:~# netstat -lnt active internet connections (only servers) proto recv-q send-q local address foreign address
state tcp 0 0 10.1.2.22:9160 0.0.0.0:*
listen tcp 0 0 10.1.2.22:9042 0.0.0.0:*
listen tcp 0 0 0.0.0.0:22 0.0.0.0:*
listen tcp 0 0 10.2.2.22:7000 0.0.0.0:*
listen tcp 0 0 0.0.0.0:17500 0.0.0.0:*
listen tcp 0 0 127.0.0.1:7199 0.0.0.0:*
listen tcp 0 0 127.0.0.1:17600 0.0.0.0:*
listen tcp 0 0 10.2.2.22:8609 0.0.0.0:*
listen tcp 0 0 127.0.0.1:54882 0.0.0.0:*
listen tcp 0 0 127.0.0.1:17603 0.0.0.0:*
listen tcp6 0 0 :::6900 :::*
listen tcp6 0 0 :::61621 :::*
listen tcp6 0 0 :::9910 :::*
listen tcp6 0 0 :::22 :::*
listen tcp6 0 0 :::17500 :::*
listen
# opscenterd.conf [webserver] port = 8888 interface = 0.0.0.0 # following settings can used enable ssl support opscenter # web application. change these values point ssl certificate , key # wish use opscenter install, port # serve ssl traffic from. #ssl_keyfile = /var/lib/opscenter/ssl/opscenter.key #ssl_certfile = /var/lib/opscenter/ssl/opscenter.pem #ssl_port = 8443 [authentication] # set option true enable opscenter authentication. default admin # account created username "admin" , password "admin". # accounts , roles can created , modified within web ui. enabled = false # better understand needs of users , improve opscenter, opscenter # reports information , clusters manages central datastax # server. information reported anonymously, , potentially sensitive # information, such ip addresses, hashed in non-reversible way: # http://www.datastax.com/documentation/opscenter/help/statsreporterproperties.html [stat_reporter] # interval setting determines how statistics reported. disable # reporting, set 0 # interval = 86400 # 24 hours # cluster.conf [jmx] username = password = port = 7199 [agents] [cassandra] username = seed_hosts = 192.168.1.72 password = cql_port = 9042 cqlsh:opscenter> desc keyspace; create keyspace "opscenter" replication = {'class': 'networktopologystrategy', 'dc1': '1', 'dc2': '1'} , durable_writes = true; create table "opscenter".rollup_state ( node text, name text, res int, avg float, histogram blob, max float, min float, ts timestamp, type int, value float, weight float, primary key (node, name, res) ) clustering order (name asc, res asc) , bloom_filter_fp_chance = 0.1 , caching = {'keys': 'all', 'rows_per_partition': 'none'} , comment = '{"version": [5, 2, 1], "info": "opscenter management data."}' , compaction = {'class': 'org.apache.cassandra.db.compaction.leveledcompactionstrategy'} , compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.lz4compressor'} , crc_check_chance = 1.0 , dclocal_read_repair_chance = 0.0 , default_time_to_live = 0 , gc_grace_seconds = 0 , max_index_interval = 2048 , memtable_flush_period_in_ms = 0 , min_index_interval = 128 , read_repair_chance = 0.0 , speculative_retry = '99percentile'; create table "opscenter".events_timeline ( key text, column1 bigint, value blob, primary key (key, column1) ) compact storage , clustering order (column1 asc) , bloom_filter_fp_chance = 0.01 , caching = {'keys': 'all', 'rows_per_partition': 'none'} , comment = '{"info": "opscenter management data.", "version": [5, 2, 1]}' , compaction = {'class': 'org.apache.cassandra.db.compaction.sizetieredcompactionstrategy', 'max_threshold': '8', 'min_threshold': '4'} , compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.lz4compressor'} , crc_check_chance = 1.0 , dclocal_read_repair_chance = 0.1 , default_time_to_live = 0 , gc_grace_seconds = 864000 , max_index_interval = 2048 , memtable_flush_period_in_ms = 0 , min_index_interval = 128 , read_repair_chance = 0.25 , speculative_retry = '99percentile'; create table "opscenter".settings ( key blob, column1 blob, value blob, primary key (key, column1) ) compact storage , clustering order (column1 asc) , bloom_filter_fp_chance = 0.01 , caching = {'keys': 'all', 'rows_per_partition': 'none'} , comment = '{"info": "opscenter management data.", "version": [5, 2, 1]}' , compaction = {'class': 'org.apache.cassandra.db.compaction.sizetieredcompactionstrategy', 'max_threshold': '12', 'min_threshold': '8'} , compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.lz4compressor'} , crc_check_chance = 1.0 , dclocal_read_repair_chance = 0.1 , default_time_to_live = 0 , gc_grace_seconds = 864000 , max_index_interval = 2048 , memtable_flush_period_in_ms = 0 , min_index_interval = 128 , read_repair_chance = 1.0 , speculative_retry = '99percentile'; create table "opscenter".rollups60 ( key text, timestamp varint, value blob, primary key (key, timestamp) ) compact storage , clustering order (timestamp asc) , bloom_filter_fp_chance = 0.01 , caching = {'keys': 'all', 'rows_per_partition': 'none'} , comment = '{"info": "opscenter management data.", "version": [5, 2, 1]}' , compaction = {'class': 'org.apache.cassandra.db.compaction.sizetieredcompactionstrategy', 'max_threshold': '32', 'min_threshold': '4'} , compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.lz4compressor'} , crc_check_chance = 1.0 , dclocal_read_repair_chance = 0.1 , default_time_to_live = 0 , gc_grace_seconds = 0 , max_index_interval = 2048 , memtable_flush_period_in_ms = 0 , min_index_interval = 128 , read_repair_chance = 0.25 , speculative_retry = '99percentile'; create table "opscenter".backup_reports ( week text, event_time timestamp, backup_id text, type text, destination text, deleted_at timestamp, full_status text, keyspaces text, status text, primary key (week, event_time, backup_id, type, destination) ) clustering order (event_time desc, backup_id asc, type asc, destination asc) , bloom_filter_fp_chance = 0.01 , caching = {'keys': 'all', 'rows_per_partition': 'none'} , comment = '{"info": "opscenter management data.", "version": [5, 2, 1]}' , compaction = {'class': 'org.apache.cassandra.db.compaction.sizetieredcompactionstrategy', 'max_threshold': '32', 'min_threshold': '4'} , compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.lz4compressor'} , crc_check_chance = 1.0 , dclocal_read_repair_chance = 0.1 , default_time_to_live = 0 , gc_grace_seconds = 864000 , max_index_interval = 2048 , memtable_flush_period_in_ms = 0 , min_index_interval = 128 , read_repair_chance = 0.0 , speculative_retry = '99percentile'; create table "opscenter".rollups86400 ( key text, timestamp varint, value blob, primary key (key, timestamp) ) compact storage , clustering order (timestamp asc) , bloom_filter_fp_chance = 0.01 , caching = {'keys': 'all', 'rows_per_partition': 'none'} , comment = '{"info": "opscenter management data.", "version": [5, 2, 1]}' , compaction = {'class': 'org.apache.cassandra.db.compaction.sizetieredcompactionstrategy', 'max_threshold': '8', 'min_threshold': '2'} , compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.lz4compressor'} , crc_check_chance = 1.0 , dclocal_read_repair_chance = 0.1 , default_time_to_live = 0 , gc_grace_seconds = 0 , max_index_interval = 2048 , memtable_flush_period_in_ms = 0 , min_index_interval = 128 , read_repair_chance = 0.25 , speculative_retry = '99percentile'; create table "opscenter".bestpractice_results ( key text, column1 varint, value blob, primary key (key, column1) ) compact storage , clustering order (column1 desc) , bloom_filter_fp_chance = 0.01 , caching = {'keys': 'all', 'rows_per_partition': 'none'} , comment = '{"info": "opscenter management data.", "version": [5, 2, 1]}' , compaction = {'class': 'org.apache.cassandra.db.compaction.sizetieredcompactionstrategy', 'max_threshold': '32', 'min_threshold': '4'} , compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.lz4compressor'} , crc_check_chance = 1.0 , dclocal_read_repair_chance = 0.1 , default_time_to_live = 0 , gc_grace_seconds = 0 , max_index_interval = 2048 , memtable_flush_period_in_ms = 0 , min_index_interval = 128 , read_repair_chance = 0.25 , speculative_retry = '99percentile'; create table "opscenter".pdps ( key text, column1 text, value blob, primary key (key, column1) ) compact storage , clustering order (column1 asc) , bloom_filter_fp_chance = 0.01 , caching = {'keys': 'all', 'rows_per_partition': 'none'} , comment = '{"info": "opscenter management data.", "version": [5, 2, 1]}' , compaction = {'class': 'org.apache.cassandra.db.compaction.sizetieredcompactionstrategy', 'max_threshold': '32', 'min_threshold': '4'} , compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.lz4compressor'} , crc_check_chance = 1.0 , dclocal_read_repair_chance = 0.1 , default_time_to_live = 0 , gc_grace_seconds = 0 , max_index_interval = 2048 , memtable_flush_period_in_ms = 0 , min_index_interval = 128 , read_repair_chance = 0.25 , speculative_retry = '99percentile'; create table "opscenter".rollups7200 ( key text, timestamp varint, value blob, primary key (key, timestamp) ) compact storage , clustering order (timestamp asc) , bloom_filter_fp_chance = 0.01 , caching = {'keys': 'all', 'rows_per_partition': 'none'} , comment = '{"info": "opscenter management data.", "version": [5, 2, 1]}' , compaction = {'class': 'org.apache.cassandra.db.compaction.sizetieredcompactionstrategy', 'max_threshold': '8', 'min_threshold': '2'} , compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.lz4compressor'} , crc_check_chance = 1.0 , dclocal_read_repair_chance = 0.1 , default_time_to_live = 0 , gc_grace_seconds = 0 , max_index_interval = 2048 , memtable_flush_period_in_ms = 0 , min_index_interval = 128 , read_repair_chance = 0.25 , speculative_retry = '99percentile'; create table "opscenter".events ( key text primary key, action bigint, api_source_ip text, column_family text, event_source text, "keyspace" text, level bigint, message text, source_node text, success boolean, target_node text, time bigint, user text ) compact storage , bloom_filter_fp_chance = 0.01 , caching = {'keys': 'all', 'rows_per_partition': 'none'} , comment = '{"info": "opscenter management data.", "version": [5, 2, 1]}' , compaction = {'class': 'org.apache.cassandra.db.compaction.sizetieredcompactionstrategy', 'max_threshold': '12', 'min_threshold': '8'} , compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.lz4compressor'} , crc_check_chance = 1.0 , dclocal_read_repair_chance = 0.1 , default_time_to_live = 0 , gc_grace_seconds = 864000 , max_index_interval = 2048 , memtable_flush_period_in_ms = 0 , min_index_interval = 128 , read_repair_chance = 0.25 , speculative_retry = '99percentile'; create table "opscenter".rollups300 ( key text, timestamp varint, value blob, primary key (key, timestamp) ) compact storage , clustering order (timestamp asc) , bloom_filter_fp_chance = 0.01 , caching = {'keys': 'all', 'rows_per_partition': 'none'} , comment = '{"info": "opscenter management data.", "version": [5, 2, 1]}' , compaction = {'class': 'org.apache.cassandra.db.compaction.sizetieredcompactionstrategy', 'max_threshold': '16', 'min_threshold': '4'} , compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.lz4compressor'} , crc_check_chance = 1.0 , dclocal_read_repair_chance = 0.1 , default_time_to_live = 0 , gc_grace_seconds = 0 , max_index_interval = 2048 , memtable_flush_period_in_ms = 0 , min_index_interval = 128 , read_repair_chance = 0.25 , speculative_retry = '99percentile';
Comments
Post a Comment