UnknownHostException in OperatorOpenShiftInfinispanCountersIT

[fedinskiy]

Occasionally, during our OCP runs , test OperatorOpenShiftInfinispanCountersIT fails with UnknownHostException during stopping phase. It looks like some kind of data race in our FW/TS.

16:03:36,780 INFO  Running command: oc delete -f target/test-classes/infinispan_cluster_config.yaml
16:03:36,858 INFO  oc: infinispan.infinispan.org "ts-ukmgkhysfz-infinispan-cluster" deleted
16:03:37,907 INFO  [two] 16:03:37,154 ISPN004007: Exception encountered. Retry 0 out of 0: java.net.UnknownHostException: Failed to resolve 'ts-ukmgkhysfz-infinispan-cluster.datagrid-cluster.svc.cluster.local' [A(1)] after 2 queries 
16:03:37,907 INFO  [two] 	at io.netty.resolver.dns.DnsResolveContext.finishResolve(DnsResolveContext.java:1097)
16:03:37,907 INFO  [two] 	at io.netty.resolver.dns.DnsResolveContext.tryToFinishResolve(DnsResolveContext.java:1044)
16:03:37,907 INFO  [two] 	at io.netty.resolver.dns.DnsResolveContext.query(DnsResolveContext.java:432)
16:03:37,907 INFO  [two] 	at io.netty.resolver.dns.DnsResolveContext.tryToFinishResolve(DnsResolveContext.java:1015)
16:03:37,907 INFO  [two] 	at io.netty.resolver.dns.DnsResolveContext.access$800(DnsResolveContext.java:66)
16:03:37,907 INFO  [two] 	at io.netty.resolver.dns.DnsResolveContext$2.operationComplete(DnsResolveContext.java:504)
16:03:37,907 INFO  [two] 	at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
16:03:37,907 INFO  [two] 	at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
16:03:37,907 INFO  [two] 	at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
16:03:37,907 INFO  [two] 	at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
16:03:37,907 INFO  [two] 	at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
16:03:37,907 INFO  [two] 	at io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625)
16:03:37,907 INFO  [two] 	at io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105)
16:03:37,907 INFO  [two] 	at io.netty.resolver.dns.DnsQueryContext.trySuccess(DnsQueryContext.java:317)
16:03:37,907 INFO  [two] 	at io.netty.resolver.dns.DnsQueryContext.finishSuccess(DnsQueryContext.java:309)
16:03:37,907 INFO  [two] 	at io.netty.resolver.dns.DnsNameResolver$DnsResponseHandler.channelRead(DnsNameResolver.java:1390)
16:03:37,907 INFO  [two] 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
16:03:37,907 INFO  [two] 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
16:03:37,907 INFO  [two] 	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
16:03:37,907 INFO  [two] 	at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
16:03:37,907 INFO  [two] 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
16:03:37,907 INFO  [two] 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
16:03:37,907 INFO  [two] 	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
16:03:37,907 INFO  [two] 	at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
16:03:37,907 INFO  [two] 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
16:03:37,907 INFO  [two] 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
16:03:37,907 INFO  [two] 	at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
16:03:37,907 INFO  [two] 	at io.netty.channel.nio.AbstractNioMessageChannel$NioMessageUnsafe.read(AbstractNioMessageChannel.java:97)
16:03:37,907 INFO  [two] 	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
16:03:37,907 INFO  [two] 	at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:689)
16:03:37,907 INFO  [two] 	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:652)
16:03:37,907 INFO  [two] 	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
16:03:37,907 INFO  [two] 	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
16:03:37,907 INFO  [two] 	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
16:03:37,907 INFO  [two] 	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
16:03:37,907 INFO  [two] 	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
16:03:37,907 INFO  [two] 	at java.base/java.lang.Thread.run(Thread.java:833)
16:03:37,907 INFO  [two] 16:03:37,157 ISPN004007: Exception encountered. Retry 0 out of 0: java.net.UnknownHostException: Failed to resolve 'ts-ukmgkhysfz-infinispan-cluster.datagrid-cluster.svc.cluster.local' [A(1)] after 2 queries 
16:03:37,908 INFO  [two] 	at io.netty.resolver.dns.DnsResolveContext.finishResolve(DnsResolveContext.java:1097)
16:03:37,908 INFO  [two] 	at io.netty.resolver.dns.DnsResolveContext.tryToFinishResolve(DnsResolveContext.java:1044)
16:03:37,908 INFO  [two] 	at io.netty.resolver.dns.DnsResolveContext.query(DnsResolveContext.java:432)
16:03:37,908 INFO  [two] 	at io.netty.resolver.dns.DnsResolveContext.tryToFinishResolve(DnsResolveContext.java:1015)
16:03:37,908 INFO  [two] 	at io.netty.resolver.dns.DnsResolveContext.access$800(DnsResolveContext.java:66)
16:03:37,908 INFO  [two] 	at io.netty.resolver.dns.DnsResolveContext$2.operationComplete(DnsResolveContext.java:504)
16:03:37,908 INFO  [two] 	at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
16:03:37,908 INFO  [two] 	at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
16:03:37,908 INFO  [two] 	at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
16:03:37,908 INFO  [two] 	at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
16:03:37,908 INFO  [two] 	at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
16:03:37,908 INFO  [two] 	at io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625)
16:03:37,908 INFO  [two] 	at io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105)
16:03:37,908 INFO  [two] 	at io.netty.resolver.dns.DnsQueryContext.trySuccess(DnsQueryContext.java:317)
16:03:37,908 INFO  [two] 	at io.netty.resolver.dns.DnsQueryContext.finishSuccess(DnsQueryContext.java:309)
16:03:37,908 INFO  [two] 	at io.netty.resolver.dns.DnsNameResolver$DnsResponseHandler.channelRead(DnsNameResolver.java:1390)
16:03:37,908 INFO  [two] 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
16:03:37,908 INFO  [two] 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
16:03:37,908 INFO  [two] 	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
16:03:37,908 INFO  [two] 	at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)
16:03:37,908 INFO  [two] 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
16:03:37,908 INFO  [two] 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
16:03:37,908 INFO  [two] 	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
16:03:37,908 INFO  [two] 	at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
16:03:37,908 INFO  [two] 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
16:03:37,908 INFO  [two] 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
16:03:37,908 INFO  [two] 	at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
16:03:37,908 INFO  [two] 	at io.netty.channel.nio.AbstractNioMessageChannel$NioMessageUnsafe.read(AbstractNioMessageChannel.java:97)
16:03:37,908 INFO  [two] 	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
16:03:37,908 INFO  [two] 	at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:689)
16:03:37,908 INFO  [two] 	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:652)
16:03:37,908 INFO  [two] 	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
16:03:37,908 INFO  [two] 	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
16:03:37,908 INFO  [two] 	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
16:03:37,908 INFO  [two] 	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
16:03:37,908 INFO  [two] 	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
16:03:37,908 INFO  [two] 	at java.base/java.lang.Thread.run(Thread.java:833)
16:03:38,952 INFO  Running command: oc scale dc/two --replicas=0 -n ts-ukmgkhysfz
16:03:39,051 INFO  oc: Warning: extensions/v1beta1 Scale is deprecated in v1.2+, unavailable in v1.16+