Опция Infinispan tcp в кластере
У меня проблема с бесконечностью (v9.1.3). У меня есть 4 узла / кластера в двух центрах обработки данных (DC). На DC1 один узел является успешным, один - отказывает. То же самое с DC2. Ошибка ниже:
Это при запуске сервера все остальные серверы должны быть включены? Добавлен jgroup incase, если я пропустил какие-либо настройки, связанные с кластером.
"timestamp" : "2017-12-20T16:36:25.564+10:00",
"message" : "ISPN000136: Error executing command PutKeyValueCommand, writing keys [Server1-37630]",
"thread" : "jgroups-4,ISPN,Server1-37630",
"level" : "ERROR",
"stack_trace" : "org.infinispan.remoting.RemoteException: ISPN000217: Received exception from PRDDC1SJBF001-27362, see cause for remote stack trace\n\tat org.infinispan.remoting.transport.ResponseCollectors.wrapRemoteException(ResponseCollectors.java:27)\n\tat org.infinispan.remoting.transport.jgroups.SyncMapResponseCollector.addException(SyncMapResponseCollector.java:48)\n\tat org.infinispan.remoting.transport.jgroups.SyncMapResponseCollector.addException(SyncMapResponseCollector.java:26)\n\tat org.infinispan.remoting.transport.ValidResponseCollector.addResponse(ValidResponseCollector.java:29)\n\tat org.infinispan.remoting.transport.impl.MultiTargetRequest.onResponse(MultiTargetRequest.java:91)\n\tat org.infinispan.remoting.transport.impl.RequestRepository.addResponse(RequestRepository.java:53)\n\tat org.infinispan.remoting.transport.jgroups.JGroupsTransport.processResponse(JGroupsTransport.java:1328)\n\tat org.infinispan.remoting.transport.jgroups.JGroupsTransport.processMessage(JGroupsTransport.java:1238)\n\tat org.infinispan.remoting.transport.jgroups.JGroupsTransport.access$200(JGroupsTransport.java:121)\n\tat org.infinispan.remoting.transport.jgroups.JGroupsTransport$ChannelCallbacks.receive(JGroupsTransport.java:1366)\n\tat org.jgroups.JChannel.up(JChannel.java:819)\n\tat org.jgroups.stack.ProtocolStack.up(ProtocolStack.java:893)\n\tat org.jgroups.protocols.pbcast.STATE_TRANSFER.up(STATE_TRANSFER.java:128)\n\tat org.jgroups.protocols.FRAG2.up(FRAG2.java:177)\n\tat org.jgroups.protocols.FlowControl.up(FlowControl.java:343)\n\tat org.jgroups.protocols.pbcast.GMS.up(GMS.java:864)\n\tat org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:240)\n\tat org.jgroups.protocols.UNICAST3.deliverMessage(UNICAST3.java:1002)\n\tat org.jgroups.protocols.UNICAST3.handleDataReceived(UNICAST3.java:728)\n\tat org.jgroups.protocols.UNICAST3.up(UNICAST3.java:383)\n\tat org.jgroups.protocols.pbcast.NAKACK2.up(NAKACK2.java:600)\n\tat org.jgroups.protocols.BARRIER.up(BARRIER.java:170)\n\tat org.jgroups.protocols.VERIFY_SUSPECT.up(VERIFY_SUSPECT.java:119)\n\tat org.jgroups.protocols.FD.up(FD.java:212)\n\tat org.jgroups.protocols.FD_SOCK.up(FD_SOCK.java:252)\n\tat org.jgroups.protocols.MERGE3.up(MERGE3.java:276)\n\tat org.jgroups.protocols.Discovery.up(Discovery.java:262)\n\tat org.jgroups.protocols.TP.passMessageUp(TP.java:1229)\n\tat org.jgroups.util.SubmitToThreadPool$SingleMessageHandler.run(SubmitToThreadPool.java:87)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)\n\tat java.lang.Thread.run(Thread.java:745)\n\tSuppressed: org.infinispan.remoting.RemoteException: ISPN000217: Received exception from PRDDC1SJBF002-24204, see cause for remote stack trace\n\t\t... 32 common frames omitted\n\tCaused by: java.lang.NullPointerException: null\n\t\tat org.infinispan.marshall.core.GlobalMarshaller.readWithExternalizer(GlobalMarshaller.java:694)\n\t\tat org.infinispan.marshall.core.GlobalMarshaller.readNonNullableObject(GlobalMarshaller.java:679)\n\t\tat org.infinispan.marshall.core.GlobalMarshaller.readNullableObject(GlobalMarshaller.java:361)\n\t\tat org.infinispan.marshall.core.BytesObjectInput.readObject(BytesObjectInput.java:40)\n\t\tat org.infinispan.commands.write.PutKeyValueCommand.readFrom(PutKeyValueCommand.java:121)\n\t\tat org.infinispan.marshall.exts.ReplicableCommandExternalizer.readCommandParameters(ReplicableCommandExternalizer.java:103)\n\t\tat org.infinispan.marshall.exts.ReplicableCommandExternalizer.readObject(ReplicableCommandExternalizer.java:92)\n\t\tat org.infinispan.marshall.exts.ReplicableCommandExternalizer.readObject(ReplicableCommandExternalizer.java:54)\n\t\tat org.infinispan.marshall.core.GlobalMarshaller.readWithExternalizer(GlobalMarshaller.java:694)\n\t\tat org.infinispan.marshall.core.GlobalMarshaller.readNonNullableObject(GlobalMarshaller.java:677)\n\t\tat org.infinispan.marshall.core.GlobalMarshaller.readNullableObject(GlobalMarshaller.java:361)\n\t\tat org.infinispan.marshall.core.BytesObjectInput.readObject(BytesObjectInput.java:40)\n\t\tat org.infinispan.commands.remote.SingleRpcCommand.readFrom(SingleRpcCommand.java:47)\n\t\tat org.infinispan.marshall.exts.ReplicableCommandExternalizer.readCommandParameters(ReplicableCommandExternalizer.java:103)\n\t\tat org.infinispan.marshall.exts.CacheRpcCommandExternalizer.readObject(CacheRpcCommandExternalizer.java:116)\n\t\tat org.infinispan.marshall.exts.CacheRpcCommandExternalizer.readObject(CacheRpcCommandExternalizer.java:57)\n\t\tat org.infinispan.marshall.core.GlobalMarshaller.readWithExternalizer(GlobalMarshaller.java:694)\n\t\tat org.infinispan.marshall.core.GlobalMarshaller.readNonNullableObject(GlobalMarshaller.java:677)\n\t\tat org.infinispan.marshall.core.GlobalMarshaller.readNullableObject(GlobalMarshaller.java:361)\n\t\tat org.infinispan.marshall.core.GlobalMarshaller.objectFromObjectInput(GlobalMarshaller.java:199)\n\t\tat org.infinispan.marshall.core.GlobalMarshaller.objectFromByteBuffer(GlobalMarshaller.java:228)\n\t\tat org.infinispan.remoting.transport.jgroups.JGroupsTransport.processRequest(JGroupsTransport.java:1288)\n\t\tat org.infinispan.remoting.transport.jgroups.JGroupsTransport.processMessage(JGroupsTransport.java:1235)\n\t\tat org.infinispan.remoting.transport.jgroups.JGroupsTransport.access$200(JGroupsTransport.java:121)\n\t\tat org.infinispan.remoting.transport.jgroups.JGroupsTransport$ChannelCallbacks.receive(JGroupsTransport.java:1366)\n\t\tat org.jgroups.JChannel.up(JChannel.java:819)\n\t\tat org.jgroups.stack.ProtocolStack.up(ProtocolStack.java:893)\n\t\tat org.jgroups.protocols.pbcast.STATE_TRANSFER.up(STATE_TRANSFER.java:128)\n\t\tat org.jgroups.protocols.FRAG2.up(FRAG2.java:177)\n\t\tat org.jgroups.protocols.FlowControl.up(FlowControl.java:359)\n\t\tat org.jgroups.protocols.pbcast.GMS.up(GMS.java:867)\n\t\tat org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:240)\n\t\tat org.jgroups.protocols.UNICAST3.up(UNICAST3.java:369)\n\t\tat org.jgroups.protocols.pbcast.NAKACK2.deliver(NAKACK2.java:941)\n\t\tat org.jgroups.protocols.pbcast.NAKACK2.handleMessage(NAKACK2.java:830)\n\t\tat org.jgroups.protocols.pbcast.NAKACK2.up(NAKACK2.java:610)\n\t\tat org.jgroups.protocols.BARRIER.up(BARRIER.java:170)\n\t\tat org.jgroups.protocols.VERIFY_SUSPECT.up(VERIFY_SUSPECT.java:129)\n\t\t... 9 common frames omitted\n\tSuppressed: org.infinispan.remoting.RemoteException: ISPN000217: Received exception from Server1-20720, see cause for remote stack trace\n\t\t... 32 common frames omitted\n\tCaused by: java.lang.NullPointerException: null\n\t\tat org.infinispan.marshall.core.GlobalMarshaller.readWithExternalizer(GlobalMarshaller.java:694)\n\t\tat org.infinispan.marshall.core.GlobalMarshaller.readNonNullableObject(GlobalMarshaller.java:679)\n\t\tat org.infinispan.marshall.core.GlobalMarshaller.readNullableObject(GlobalMarshaller.java:361)\n\t\tat org.infinispan.marshall.core.BytesObjectInput.readObject(BytesObjectInput.java:40)\n\t\tat org.infinispan.commands.write.PutKeyValueCommand.readFrom(PutKeyValueCommand.java:121)\n\t\tat org.infinispan.marshall.exts.ReplicableCommandExternalizer.readCommandParameters(ReplicableCommandExternalizer.java:103)\n\t\tat org.infinispan.marshall.exts.ReplicableCommandExternalizer.readObject(ReplicableCommandExternalizer.java:92)\n\t\tat org.infinispan.marshall.exts.ReplicableCommandExternalizer.readObject(ReplicableCommandExternalizer.java:54)\n\t\tat org.infinispan.marshall.core.GlobalMarshaller.readWithExternalizer(GlobalMarshaller.java:694)\n\t\tat org.infinispan.marshall.core.GlobalMarshaller.readNonNullableObject(GlobalMarshaller.java:677)\n\t\tat org.infinispan.marshall.core.GlobalMarshaller.readNullableObject(GlobalMarshaller.java:361)\n\t\tat org.infinispan.marshall.core.BytesObjectInput.readObject(BytesObjectInput.java:40)\n\t\tat org.infinispan.commands.remote.SingleRpcCommand.readFrom(SingleRpcCommand.java:47)\n\t\tat org.infinispan.marshall.exts.ReplicableCommandExternalizer.readCommandParameters(ReplicableCommandExternalizer.java:103)\n\t\tat org.infinispan.marshall.exts.CacheRpcCommandExternalizer.readObject(CacheRpcCommandExternalizer.java:116)\n\t\tat org.infinispan.marshall.exts.CacheRpcCommandExternalizer.readObject(CacheRpcCommandExternalizer.java:57)\n\t\tat org.infinispan.marshall.core.GlobalMarshaller.readWithExternalizer(GlobalMarshaller.java:694)\n\t\tat org.infinispan.marshall.core.GlobalMarshaller.readNonNullableObject(GlobalMarshaller.java:677)\n\t\tat org.infinispan.marshall.core.GlobalMarshaller.readNullableObject(GlobalMarshaller.java:361)\n\t\tat org.infinispan.marshall.core.GlobalMarshaller.objectFromObjectInput(GlobalMarshaller.java:199)\n\t\tat org.infinispan.marshall.core.GlobalMarshaller.objectFromByteBuffer(GlobalMarshaller.java:228)\n\t\tat org.infinispan.remoting.transport.jgroups.JGroupsTransport.processRequest(JGroupsTransport.java:1288)\n\t\tat org.infinispan.remoting.transport.jgroups.JGroupsTransport.processMessage(JGroupsTransport.java:1235)\n\t\tat org.infinispan.remoting.transport.jgroups.JGroupsTransport.access$200(JGroupsTransport.java:121)\n\t\tat org.infinispan.remoting.transport.jgroups.JGroupsTransport$ChannelCallbacks.receive(JGroupsTransport.java:1366)\n\t\tat org.jgroups.JChannel.up(JChannel.java:819)\n\t\tat org.jgroups.stack.ProtocolStack.up(ProtocolStack.java:893)\n\t\tat org.jgroups.protocols.pbcast.STATE_TRANSFER.up(STATE_TRANSFER.java:128)\n\t\tat org.jgroups.protocols.FRAG2.up(FRAG2.java:177)\n\t\tat org.jgroups.protocols.FlowControl.up(FlowControl.java:359)\n\t\tat org.jgroups.protocols.pbcast.GMS.up(GMS.java:867)\n\t\tat org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:240)\n\t\tat org.jgroups.protocols.UNICAST3.up(UNICAST3.java:369)\n\t\tat org.jgroups.protocols.pbcast.NAKACK2.deliver(NAKACK2.java:941)\n\t\tat org.jgroups.protocols.pbcast.NAKACK2.handleMessage(NAKACK2.java:830)\n\t\tat org.jgroups.protocols.pbcast.NAKACK2.up(NAKACK2.java:610)\n\t\tat org.jgroups.protocols.BARRIER.up(BARRIER.java:170)\n\t\tat org.jgroups.protocols.VERIFY_SUSPECT.up(VERIFY_SUSPECT.java:129)\n\t\t... 9 common frames omitted\n\tSuppressed: org.infinispan.remoting.RemoteException: ISPN000217: Received exception from PRDDC2SJBF002-55727, see cause for remote stack trace\n\t\t... 32 common frames omitted\n\tCaused by: java.lang.NullPointerException: null\n\t\tat org.infinispan.marshall.core.GlobalMarshaller.readWithExternalizer(GlobalMarshaller.java:694)\n\t\tat org.infinispan.marshall.core.GlobalMarshaller.readNonNullableObject(GlobalMarshaller.java:679)\n\t\tat org.infinispan.marshall.core.GlobalMarshaller.readNullableObject(GlobalMarshaller.java:361)\n\t\tat org.infinispan.marshall.core.BytesObjectInput.readObject(BytesObjectInput.java:40)\n\t\tat org.infinispan.commands.write.PutKeyValueCommand.readFrom(PutKeyValueCommand.java:121)\n\t\tat org.infinispan.marshall.exts.ReplicableCommandExternalizer.readCommandParameters(ReplicableCommandExternalizer.java:103)\n\t\tat org.infinispan.marshall.exts.ReplicableCommandExternalizer.readObject(ReplicableCommandExternalizer.java:92)\n\t\tat org.infinispan.marshall.exts.ReplicableCommandExternalizer.readObject(ReplicableCommandExternalizer.java:54)\n\t\tat org.infinispan.marshall.core.GlobalMarshaller.readWithExternalizer(GlobalMarshaller.java:694)\n\t\tat org.infinispan.marshall.core.GlobalMarshaller.readNonNullableObject(GlobalMarshaller.java:677)\n\t\tat org.infinispan.marshall.core.GlobalMarshaller.readNullableObject(GlobalMarshaller.java:361)\n\t\tat org.infinispan.marshall.core.BytesObjectInput.readObject(BytesObjectInput.java:40)\n\t\tat org.infinispan.commands.remote.SingleRpcCommand.readFrom(SingleRpcCommand.java:47)\n\t\tat org.infinispan.marshall.exts.ReplicableCommandExternalizer.readCommandParameters(ReplicableCommandExternalizer.java:103)\n\t\tat org.infinispan.marshall.exts.CacheRpcCommandExternalizer.readObject(CacheRpcCommandExternalizer.java:116)\n\t\tat org.infinispan.marshall.exts.CacheRpcCommandExternalizer.readObject(CacheRpcCommandExternalizer.java:57)\n\t\tat org.infinispan.marshall.core.GlobalMarshaller.readWithExternalizer(GlobalMarshaller.java:694)\n\t\tat org.infinispan.marshall.core.GlobalMarshaller.readNonNullableObject(GlobalMarshaller.java:677)\n\t\tat org.infinispan.marshall.core.GlobalMarshaller.readNullableObject(GlobalMarshaller.java:361)\n\t\tat org.infinispan.marshall.core.GlobalMarshaller.objectFromObjectInput(GlobalMarshaller.java:199)\n\t\tat org.infinispan.marshall.core.GlobalMarshaller.objectFromByteBuffer(GlobalMarshaller.java:228)\n\t\tat org.infinispan.remoting.transport.jgroups.JGroupsTransport.processRequest(JGroupsTransport.java:1288)\n\t\tat org.infinispan.remoting.transport.jgroups.JGroupsTransport.processMessage(JGroupsTransport.java:1235)\n\t\tat org.infinispan.remoting.transport.jgroups.JGroupsTransport.access$200(JGroupsTransport.java:121)\n\t\tat org.infinispan.remoting.transport.jgroups.JGroupsTransport$ChannelCallbacks.receive(JGroupsTransport.java:1366)\n\t\tat org.jgroups.JChannel.up(JChannel.java:819)\n\t\tat org.jgroups.stack.ProtocolStack.up(ProtocolStack.java:893)\n\t\tat org.jgroups.protocols.pbcast.STATE_TRANSFER.up(STATE_TRANSFER.java:128)\n\t\tat org.jgroups.protocols.FRAG2.up(FRAG2.java:177)\n\t\tat org.jgroups.protocols.FlowControl.up(FlowControl.java:359)\n\t\tat org.jgroups.protocols.pbcast.GMS.up(GMS.java:867)\n\t\tat org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:240)\n\t\tat org.jgroups.protocols.UNICAST3.up(UNICAST3.java:369)\n\t\tat org.jgroups.protocols.pbcast.NAKACK2.deliver(NAKACK2.java:941)\n\t\tat org.jgroups.protocols.pbcast.NAKACK2.handleMessage(NAKACK2.java:830)\n\t\tat org.jgroups.protocols.pbcast.NAKACK2.up(NAKACK2.java:610)\n\t\tat org.jgroups.protocols.BARRIER.up(BARRIER.java:170)\n\t\tat org.jgroups.protocols.VERIFY_SUSPECT.up(VERIFY_SUSPECT.java:129)\n\t\t... 9 common frames omitted\nCaused by: java.lang.NullPointerException: null\n\tat org.infinispan.marshall.core.GlobalMarshaller.readWithExternalizer(GlobalMarshaller.java:694)\n\tat org.infinispan.marshall.core.GlobalMarshaller.readNonNullableObject(GlobalMarshaller.java:679)\n\tat org.infinispan.marshall.core.GlobalMarshaller.readNullableObject(GlobalMarshaller.java:361)\n\tat org.infinispan.marshall.core.BytesObjectInput.readObject(BytesObjectInput.java:40)\n\tat org.infinispan.commands.write.PutKeyValueCommand.readFrom(PutKeyValueCommand.java:121)\n\tat org.infinispan.marshall.exts.ReplicableCommandExternalizer.readCommandParameters(ReplicableCommandExternalizer.java:103)\n\tat org.infinispan.marshall.exts.ReplicableCommandExternalizer.readObject(ReplicableCommandExternalizer.java:92)\n\tat org.infinispan.marshall.exts.ReplicableCommandExternalizer.readObject(ReplicableCommandExternalizer.java:54)\n\tat org.infinispan.marshall.core.GlobalMarshaller.readWithExternalizer(GlobalMarshaller.java:694)\n\tat org.infinispan.marshall.core.GlobalMarshaller.readNonNullableObject(GlobalMarshaller.java:677)\n\tat org.infinispan.marshall.core.GlobalMarshaller.readNullableObject(GlobalMarshaller.java:361)\n\tat org.infinispan.marshall.core.BytesObjectInput.readObject(BytesObjectInput.java:40)\n\tat org.infinispan.commands.remote.SingleRpcCommand.readFrom(SingleRpcCommand.java:47)\n\tat org.infinispan.marshall.exts.ReplicableCommandExternalizer.readCommandParameters(ReplicableCommandExternalizer.java:103)\n\tat org.infinispan.marshall.exts.CacheRpcCommandExternalizer.readObject(CacheRpcCommandExternalizer.java:116)\n\tat org.infinispan.marshall.exts.CacheRpcCommandExternalizer.readObject(CacheRpcCommandExternalizer.java:57)\n\tat org.infinispan.marshall.core.GlobalMarshaller.readWithExternalizer(GlobalMarshaller.java:694)\n\tat org.infinispan.marshall.core.GlobalMarshaller.readNonNullableObject(GlobalMarshaller.java:677)\n\tat org.infinispan.marshall.core.GlobalMarshaller.readNullableObject(GlobalMarshaller.java:361)\n\tat org.infinispan.marshall.core.GlobalMarshaller.objectFromObjectInput(GlobalMarshaller.java:199)\n\tat org.infinispan.marshall.core.GlobalMarshaller.objectFromByteBuffer(GlobalMarshaller.java:228)\n\tat org.infinispan.remoting.transport.jgroups.JGroupsTransport.processRequest(JGroupsTransport.java:1288)\n\tat org.infinispan.remoting.transport.jgroups.JGroupsTransport.processMessage(JGroupsTransport.java:1235)\n\tat org.infinispan.remoting.transport.jgroups.JGroupsTransport.access$200(JGroupsTransport.java:121)\n\tat org.infinispan.remoting.transport.jgroups.JGroupsTransport$ChannelCallbacks.receive(JGroupsTransport.java:1366)\n\tat org.jgroups.JChannel.up(JChannel.java:819)\n\tat org.jgroups.stack.ProtocolStack.up(ProtocolStack.java:893)\n\tat org.jgroups.protocols.pbcast.STATE_TRANSFER.up(STATE_TRANSFER.java:128)\n\tat org.jgroups.protocols.FRAG2.up(FRAG2.java:177)\n\tat org.jgroups.protocols.FlowControl.up(FlowControl.java:359)\n\tat org.jgroups.protocols.pbcast.GMS.up(GMS.java:867)\n\tat org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:240)\n\tat org.jgroups.protocols.UNICAST3.up(UNICAST3.java:369)\n\tat org.jgroups.protocols.pbcast.NAKACK2.deliver(NAKACK2.java:941)\n\tat org.jgroups.protocols.pbcast.NAKACK2.handleMessage(NAKACK2.java:830)\n\tat org.jgroups.protocols.pbcast.NAKACK2.up(NAKACK2.java:610)\n\tat org.jgroups.protocols.BARRIER.up(BARRIER.java:170)\n\tat org.jgroups.protocols.VERIFY_SUSPECT.up(VERIFY_SUSPECT.java:129)\n\t... 9 common frames omitted\n",
"hostname" : "Server1.x.com",
"appName" : "fuse-shared-cache"
Jgroup
<TCP bind_port="7801"
bind_addr="SITE_LOCAL" />
<TCPPING async_discovery="true"
initial_hosts="${jgroups.tcpping.initial_hosts:Server1.x.com[7801],Server2.x.com[7801],Server3.x.com[7801],Server4.x.com[7801]}" port_range="0" />
<MERGE3 min_interval="10000"
max_interval="30000" />
<FD_SOCK />
<FD timeout="3000"
max_tries="3" />
<VERIFY_SUSPECT timeout="1500" />
<BARRIER />
<pbcast.NAKACK2 use_mcast_xmit="false"
discard_delivered_msgs="true" />
<UNICAST3 />
<pbcast.STABLE desired_avg_gossip="50000"
max_bytes="4M" />
<pbcast.GMS print_local_addr="true"
join_timeout="2000" />
<MFC max_credits="2M"
min_threshold="0.4" />
<FRAG2 frag_size="60K" />
<!--RSVP resend_interval="2000" timeout="10000"/ -->
<pbcast.STATE_TRANSFER />