[2024-02-23T09:58:30,836][INFO ][o.e.m.j.JvmGcMonitorService] [t-cratedb-s-003] [gc][138534] overhead, spent [250ms] collecting in the last [1s]
[2024-02-23T10:02:09,927][INFO ][o.e.m.j.JvmGcMonitorService] [t-cratedb-s-003] [gc][138753] overhead, spent [458ms] collecting in the last [1s]
[2024-02-23T18:03:36,867][WARN ][o.e.i.e.Engine           ] [t-cratedb-s-003]  [.partitioned.metrics.04732dpg70r38dhk60o30c1g][9]failed engine [index id[Aw0xNzA4NjgwNzY5MjgxEDE2NzU0OWQwMjlmMzE5ZDANMTcwODY0NjQwMDAwMA==] origin[REPLICA] seq#[-2]]
org.elasticsearch.index.translog.TranslogException: Failed to write operation [Index{id='Aw0xNzA4NjgwNzY5MjgxEDE2NzU0OWQwMjlmMzE5ZDANMTcwODY0NjQwMDAwMA==', seqNo=-2, primaryTerm=0, version=-3, autoGeneratedIdTimestamp=-1}]
	at org.elasticsearch.index.translog.Translog.add(Translog.java:526) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:941) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:806) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:778) ~[crate-server-5.6.2.jar:?]
	at io.crate.execution.dml.upsert.TransportShardUpsertAction.processRequestItemsOnReplica(TransportShardUpsertAction.java:448) ~[crate-server-5.6.2.jar:?]
	at io.crate.execution.dml.upsert.TransportShardUpsertAction.processRequestItemsOnReplica(TransportShardUpsertAction.java:94) ~[crate-server-5.6.2.jar:?]
	at io.crate.execution.dml.TransportShardAction$2.call(TransportShardAction.java:119) ~[crate-server-5.6.2.jar:?]
	at io.crate.execution.dml.TransportShardAction$2.call(TransportShardAction.java:116) ~[crate-server-5.6.2.jar:?]
	at io.crate.execution.dml.TransportShardAction.wrapOperationInKillable(TransportShardAction.java:130) ~[crate-server-5.6.2.jar:?]
	at io.crate.execution.dml.TransportShardAction.shardOperationOnReplica(TransportShardAction.java:122) ~[crate-server-5.6.2.jar:?]
	at io.crate.execution.dml.TransportShardAction.shardOperationOnReplica(TransportShardAction.java:53) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.onResponse(TransportReplicationAction.java:557) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.onResponse(TransportReplicationAction.java:536) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.shard.IndexShard.lambda$innerAcquireReplicaOperationPermit$27(IndexShard.java:2925) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.action.ActionListener$3.onResponse(ActionListener.java:127) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.shard.IndexShardOperationPermits.acquire(IndexShardOperationPermits.java:292) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.shard.IndexShardOperationPermits.acquire(IndexShardOperationPermits.java:242) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.shard.IndexShard.lambda$acquireReplicaOperationPermit$25(IndexShard.java:2860) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.shard.IndexShard.innerAcquireReplicaOperationPermit(IndexShard.java:2964) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.shard.IndexShard.acquireReplicaOperationPermit(IndexShard.java:2859) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.action.support.replication.TransportReplicationAction.acquireReplicaOperationPermit(TransportReplicationAction.java:881) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.doRun(TransportReplicationAction.java:629) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.action.support.replication.TransportReplicationAction.handleReplicaRequest(TransportReplicationAction.java:521) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:59) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:331) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[crate-server-5.6.2.jar:?]
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) ~[?:?]
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) ~[?:?]
	at java.base/java.lang.Thread.run(Thread.java:1583) [?:?]
Caused by: java.lang.IllegalArgumentException: sequence number must be assigned
	at org.elasticsearch.index.seqno.SequenceNumbers.min(SequenceNumbers.java:90) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.translog.TranslogWriter.add(TranslogWriter.java:210) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.translog.Translog.add(Translog.java:519) ~[crate-server-5.6.2.jar:?]
	... 29 more
[2024-02-23T18:03:37,671][WARN ][o.e.i.e.Engine           ] [t-cratedb-s-003]  [.partitioned.metrics.04732dpg70rn.java:881) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.doRun(TransportReplicationAction.java:629) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.action.support.replication.TransportReplicationAction.handleReplicaRequest(TransportReplicationAction.java:521) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:59) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:331) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[crate-server-5.6.2.jar:?]
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) ~[?:?]
	at java.base/java.util.concurrent.ThreadPoolExecutor$Wo38dhk60o30c1g][0]failed engine [index id[Aw0xNzA4NjgwNzcyMzA3EGEyY2NkODhlYzE4OGRmYmINMTcwODY0NjQwMDAwMA==] origin[REPLICA] seq#[-2]]
org.elasticsearch.index.translog.TranslogException: Failed to write operation [Index{id='Aw0xNzA4NjgwNzcyMzA3EGEyY2NkODhlYzE4OGRmYmINMTcwODY0NjQwMDAwMA==', seqNo=-2, primaryTerm=0, version=-3, autoGeneratedIdTimestamp=-1}]
	at org.elasticsearch.index.translog.Translog.add(Translog.java:526) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:941) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:806) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:778) ~[crate-server-5.6.2.jar:?]
	at io.crate.execution.dml.upsert.TransportShardUpsertAction.processRequestItemsOnReplica(TransportShardUpsertAction.java:448) ~[crate-server-5.6.2.jar:?]
	at io.crate.execution.dml.upsert.TransportShardUpsertAction.processRequestItemsOnReplica(TransportShardUpssearch.index.shard.IndexShard.lambda$innerAcquireReplicaOperationPermit$27(IndexShard.java:2925) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.action.ActionListener$3.onResponse(ActionListener.java:127) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.shard.IndexShardOperationPermits.acquire(IndexShardOperationPermits.java:292) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.shard.IndexShardOperationPermits.acquire(IndexShardOperationPermits.java:242) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.shard.IndexShard.lambda$acquireReplicaOperationPermit$25(IndexShard.java:2860) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.shard.IndexShard.innerAcquireReplicaOperationPermit(IndexShard.java:2964) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.shard.IndexShard.acquireReplicaOperationPermit(IndexShard.java:2859) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.action.support.replication.TransportReplicationAction.acquireReplicaOperationPermit(TransportReplicationActioertAction.java:94) ~[crate-server-5.6.2.jar:?]
	at io.crate.execution.dml.TransportShardAction$2.call(TransportShardAction.java:119) ~[crate-server-5.6.2.jar:?]
	at io.crate.execution.dml.TransportShardAction$2.call(TransportShardAction.java:116) ~[crate-server-5.6.2.jar:?]
	at io.crate.execution.dml.TransportShardAction.wrapOperationInKillable(TransportShardAction.java:130) ~[crate-server-5.6.2.jar:?]
	at io.crate.execution.dml.TransportShardAction.shardOperationOnReplica(TransportShardAction.java:122) ~[crate-server-5.6.2.jar:?]
	at io.crate.execution.dml.TransportShardAction.shardOperationOnReplica(TransportShardAction.java:53) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.onResponse(TransportReplicationAction.java:557) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.onResponse(TransportReplicationAction.java:536) ~[crate-server-5.6.2.jar:?]
	at org.elasticrker.run(ThreadPoolExecutor.java:642) ~[?:?]
	at java.base/java.lang.Thread.run(Thread.java:1583) [?:?]
Caused by: java.lang.IllegalArgumentException: sequence number must be assigned
	at org.elasticsearch.index.seqno.SequenceNumbers.min(SequenceNumbers.java:90) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.translog.TranslogWriter.add(TranslogWriter.java:210) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.translog.Translog.add(Translog.java:519) ~[crate-server-5.6.2.jar:?]
	... 29 more
[2024-02-23T18:03:37,682][WARN ][o.e.i.c.IndicesClusterStateService] [t-cratedb-s-003] [.partitioned.metrics.04732dpg70r38dhk60o30c1g][0] marking and sending shard failed due to [shard failure, reason [index id[Aw0xNzA4NjgwNzcyMzA3EGEyY2NkODhlYzE4OGRmYmINMTcwODY0NjQwMDAwMA==] origin[REPLICA] seq#[-2]]]
org.elasticsearch.index.translog.TranslogException: Failed to write operation [Index{id='Aw0xNzA4NjgwNzcyMzA3EGEyY2NkODhlYzE4OGRmYmINMTcwODY0NjQwMDAwMA==', seqNo=-2, primaryTerm=0, version=-3, autoGeneratedIdTimestamp=-1}]
	at org.elasticsearch.index.translog.Translog.add(Translog.java:526) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:941) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:806) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:778) ~[crate-server-5.6.2.jar:?]
	at io.crate.execution.dml.upsert.TransportShardUpsertAction.processRequestItemsOnReplica(TransportShardUpsertAction.java:448) ~[crate-server-5.6.2.jar:?]
	at io.crate.execution.dml.upsert.TransportShardUpsertAction.processRequestItemsOnReplica(TransportShardUpsertAction.java:94) ~[crate-server-5.6.2.jar:?]
	at io.crate.execution.dml.TransportShardAction$2.call(TransportShardAction.java:119) ~[crate-server-5.6.2.jar:?]
	at io.crate.execution.dml.TransportShardAction$2.call(TransportShardAction.java:116) ~[crate-server-5.6.2.jar:?]
	at io.crate.execution.dml.TransportShardAction.wrapOperationInKillable(TransportShardAction.java:130) ~[crate-server-5.6.2.jar:?]
	at io.crate.execution.dml.TransportShardAction.shardOperationOnReplica(TransportShardAction.java:122) ~[crate-server-5.6.2.jar:?]
	at io.crate.execution.dml.TransportShardAction.shardOperationOnReplica(TransportShardAction.java:53) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.onResponse(TransportReplicationAction.java:557) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.onResponse(TransportReplicationAction.java:536) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.shard.IndexShard.lambda$innerAcquireReplicaOperationPermit$27(IndexShard.java:2925) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.action.ActionListener$3.onResponse(ActionListener.java:127) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.shard.IndexShardOperationPermits.acquire(IndexShardOperationPermits.java:292) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.shard.IndexShardOperationPermits.acquire(IndexShardOperationPermits.java:242) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.shard.IndexShard.lambda$acquireReplicaOperationPermit$25(IndexShard.java:2860) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.shard.IndexShard.innerAcquireReplicaOperationPermit(IndexShard.java:2964) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.shard.IndexShard.acquireReplicaOperationPermit(IndexShard.java:2859) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.action.support.replication.TransportReplicationAction.acquireReplicaOperationPermit(TransportReplicationAction.java:881) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.doRun(TransportReplicationAction.java:629) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.action.support.replication.TransportReplicationAction.handleReplicaRequest(TransportReplicationAction.java:521) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:59) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:331) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[crate-server-5.6.2.jar:?]
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) ~[?:?]
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) ~[?:?]
	at java.base/java.lang.Thread.run(Thread.java:1583) [?:?]
Caused by: java.lang.IllegalArgumentException: sequence number must be assigned
	at org.elasticsearch.index.seqno.SequenceNumbers.min(SequenceNumbers.java:90) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.translog.TranslogWriter.add(TranslogWriter.java:210) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.translog.Translog.add(Translog.java:519) ~[crate-server-5.6.2.jar:?]
	... 29 more
[2024-02-23T18:03:37,685][WARN ][o.e.i.c.IndicesClusterStateService] [t-cratedb-s-003] [.partitioned.metrics.04732dpg70r38dhk60o30c1g][9] marking and sending shard failed due to [shard failure, reason [index id[Aw0xNzA4NjgwNzY5MjgxEDE2NzU0OWQwMjlmMzE5ZDANMTcwODY0NjQwMDAwMA==] origin[REPLICA] seq#[-2]]]
org.elasticsearch.index.translog.TranslogException: Failed to write operation [Index{id='Aw0xNzA4NjgwNzY5MjgxEDE2NzU0OWQwMjlmMzE5ZDANMTcwODY0NjQwMDAwMA==', seqNo=-2, primaryTerm=0, version=-3, autoGeneratedIdTimestamp=-1}]
	at org.elasticsearch.index.translog.Translog.add(Translog.java:526) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:941) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:806) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:778) ~[crate-server-5.6.2.jar:?]
	at io.crate.execution.dml.upsert.TransportShardUpsertAction.processRequestItemsOnReplica(TransportShardUpsertAction.java:448) ~[crate-server-5.6.2.jar:?]
	at io.crate.execution.dml.upsert.TransportShardUpsertAction.processRequestItemsOnReplica(TransportShardUpsertAction.java:94) ~[crate-server-5.6.2.jar:?]
	at io.crate.execution.dml.TransportShardAction$2.call(TransportShardAction.java:119) ~[crate-server-5.6.2.jar:?]
	at io.crate.execution.dml.TransportShardAction$2.call(TransportShardAction.java:116) ~[crate-server-5.6.2.jar:?]
	at io.crate.execution.dml.TransportShardAction.wrapOperationInKillable(TransportShardAction.java:130) ~[crate-server-5.6.2.jar:?]
	at io.crate.execution.dml.TransportShardAction.shardOperationOnReplica(TransportShardAction.java:122) ~[crate-server-5.6.2.jar:?]
	at io.crate.execution.dml.TransportShardAction.shardOperationOnReplica(TransportShardAction.java:53) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.onResponse(TransportReplicationAction.java:557) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.onResponse(TransportReplicationAction.java:536) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.shard.IndexShard.lambda$innerAcquireReplicaOperationPermit$27(IndexShard.java:2925) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.action.ActionListener$3.onResponse(ActionListener.java:127) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.shard.IndexShardOperationPermits.acquire(IndexShardOperationPermits.java:292) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.shard.IndexShardOperationPermits.acquire(IndexShardOperationPermits.java:242) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.shard.IndexShard.lambda$acquireReplicaOperationPermit$25(IndexShard.java:2860) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.shard.IndexShard.innerAcquireReplicaOperationPermit(IndexShard.java:2964) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.shard.IndexShard.acquireReplicaOperationPermit(IndexShard.java:2859) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.action.support.replication.TransportReplicationAction.acquireReplicaOperationPermit(TransportReplicationAction.java:881) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.doRun(TransportReplicationAction.java:629) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.action.support.replication.TransportReplicationAction.handleReplicaRequest(TransportReplicationAction.java:521) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:59) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:331) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[crate-server-5.6.2.jar:?]
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) ~[?:?]
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) ~[?:?]
	at java.base/java.lang.Thread.run(Thread.java:1583) [?:?]
Caused by: java.lang.IllegalArgumentException: sequence number must be assigned
	at org.elasticsearch.index.seqno.SequenceNumbers.min(SequenceNumbers.java:90) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.translog.TranslogWriter.add(TranslogWriter.java:210) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.translog.Translog.add(Translog.java:519) ~[crate-server-5.6.2.jar:?]
	... 29 more
[2024-02-23T18:04:32,733][WARN ][o.e.i.e.Engine           ] [t-cratedb-s-003]  [.partitioned.metrics.04732dpg70r38dhk60o30c1g][4]failed engine [index id[Aw0xNzA4NjgwNzc5MTg4EGM3ZmUzMjdjZTVmNWJmM2MNMTcwODY0NjQwMDAwMA==] origin[REPLICA] seq#[-2]]
org.elasticsearch.index.translog.TranslogException: Failed to write operation [Index{id='Aw0xNzA4NjgwNzc5MTg4EGM3ZmUzMjdjZTVmNWJmM2MNMTcwODY0NjQwMDAwMA==', seqNo=-2, primaryTerm=0, version=-3, autoGeneratedIdTimestamp=-1}]
	at org.elasticsearch.index.translog.Translog.add(Translog.java:526) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:941) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:806) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:778) ~[crate-server-5.6.2.jar:?]
	at io.crate.execution.dml.upsert.TransportShardUpsertAction.processRequestItemsOnReplica(TransportShardUpsertAction.java:448) ~[crate-server-5.6.2.jar:?]
	at io.crate.execution.dml.upsert.TransportShardUpsertAction.processRequestItemsOnReplica(TransportShardUpsertAction.java:94) ~[crate-server-5.6.2.jar:?]
	at io.crate.execution.dml.TransportShardAction$2.call(TransportShardAction.java:119) ~[crate-server-5.6.2.jar:?]
	at io.crate.execution.dml.TransportShardAction$2.call(TransportShardAction.java:116) ~[crate-server-5.6.2.jar:?]
	at io.crate.execution.dml.TransportShardAction.wrapOperationInKillable(TransportShardAction.java:130) ~[crate-server-5.6.2.jar:?]
	at io.crate.execution.dml.TransportShardAction.shardOperationOnReplica(TransportShardAction.java:122) ~[crate-server-5.6.2.jar:?]
	at io.crate.execution.dml.TransportShardAction.shardOperationOnReplica(TransportShardAction.java:53) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.onResponse(TransportReplicationAction.java:557) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.onResponse(TransportReplicationAction.java:536) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.shard.IndexShard.lambda$innerAcquireReplicaOperationPermit$27(IndexShard.java:2925) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.action.ActionListener$3.onResponse(ActionListener.java:127) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.shard.IndexShardOperationPermits.acquire(IndexShardOperationPermits.java:292) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.shard.IndexShardOperationPermits.acquire(IndexShardOperationPermits.java:242) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.shard.IndexShard.lambda$acquireReplicaOperationPermit$25(IndexShard.java:2860) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.shard.IndexShard.innerAcquireReplicaOperationPermit(IndexShard.java:2964) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.shard.IndexShard.acquireReplicaOperationPermit(IndexShard.java:2859) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.action.support.replication.TransportReplicationAction.acquireReplicaOperationPermit(TransportReplicationAction.java:881) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.doRun(TransportReplicationAction.java:629) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.action.support.replication.TransportReplicationAction.handleReplicaRequest(TransportReplicationAction.java:521) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:59) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:331) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[crate-server-5.6.2.jar:?]
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) ~[?:?]
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) ~[?:?]
	at java.base/java.lang.Thread.run(Thread.java:1583) [?:?]
Caused by: java.lang.IllegalArgumentException: sequence number must be assigned
	at org.elasticsearch.index.seqno.SequenceNumbers.min(SequenceNumbers.java:90) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.translog.TranslogWriter.add(TranslogWriter.java:210) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.translog.Translog.add(Translog.java:519) ~[crate-server-5.6.2.jar:?]
	... 29 more
[2024-02-23T18:04:33,442][WARN ][o.e.i.e.Engine           ] [t-cratedb-s-003]  [.partitioned.metrics.04732dpg70r38dhk60o30c1g][9]failed engine [index id[Aw0xNzA4NjgwNzcxMzMzEDAyNTA5ZmMyZWY2MWJlMWINMTcwODY0NjQwMDAwMA==] origin[REPLICA] seq#[-2]]
org.elasticsearch.index.translog.TranslogException: Failed to write operation [Index{id='Aw0xNzA4NjgwNzcxMzMzEDAyNTA5ZmMyZWY2MWJlMWINMTcwODY0NjQwMDAwMA==', seqNo=-2, primaryTerm=0, version=-3, autoGeneratedIdTimestamp=-1}]
	at org.elasticsearch.index.translog.Translog.add(Translog.java:526) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:941) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:806) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:778) ~[crate-server-5.6.2.jar:?]
	at io.crate.execution.dml.upsert.TransportShardUpsertAction.processRequestItemsOnReplica(TransportShardUpsertAction.java:448) ~[crate-server-5.6.2.jar:?]
	at io.crate.execution.dml.upsert.TransportShardUpsertAction.processRequestItemsOnReplica(TransportShardUpsertAction.java:94) ~[crate-server-5.6.2.jar:?]
	at io.crate.execution.dml.TransportShardAction$2.call(TransportShardAction.java:119) ~[crate-server-5.6.2.jar:?]
	at io.crate.execution.dml.TransportShardAction$2.call(TransportShardAction.java:116) ~[crate-server-5.6.2.jar:?]
	at io.crate.execution.dml.TransportShardAction.wrapOperationInKillable(TransportShardAction.java:130) ~[crate-server-5.6.2.jar:?]
	at io.crate.execution.dml.TransportShardAction.shardOperationOnReplica(TransportShardAction.java:122) ~[crate-server-5.6.2.jar:?]
	at io.crate.execution.dml.TransportShardAction.shardOperationOnReplica(TransportShardAction.java:53) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.onResponse(TransportReplicationAction.java:557) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.onResponse(TransportReplicationAction.java:536) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.shard.IndexShard.lambda$innerAcquireReplicaOperationPermit$27(IndexShard.java:2925) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.action.ActionListener$3.onResponse(ActionListener.java:127) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.shard.IndexShardOperationPermits.acquire(IndexShardOperationPermits.java:292) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.shard.IndexShardOperationPermits.acquire(IndexShardOperationPermits.java:242) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.shard.IndexShard.lambda$acquireReplicaOperationPermit$25(IndexShard.java:2860) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.shard.IndexShard.innerAcquireReplicaOperationPermit(IndexShard.java:2964) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.shard.IndexShard.acquireReplicaOperationPermit(IndexShard.java:2859) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.action.support.replication.TransportReplicationAction.acquireReplicaOperationPermit(TransportReplicationAction.java:881) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.doRun(TransportReplicationAction.java:629) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.action.support.replication.TransportReplicationAction.handleReplicaRequest(TransportReplicationAction.java:521) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:59) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:331) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[crate-server-5.6.2.jar:?]
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) ~[?:?]
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) ~[?:?]
	at java.base/java.lang.Thread.run(Thread.java:1583) [?:?]
Caused by: java.lang.IllegalArgumentException: sequence number must be assigned
	at org.elasticsearch.index.seqno.SequenceNumbers.min(SequenceNumbers.java:90) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.translog.TranslogWriter.add(TranslogWriter.java:210) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.translog.Translog.add(Translog.java:519) ~[crate-server-5.6.2.jar:?]
	... 29 more
[2024-02-23T18:04:33,444][WARN ][o.e.i.c.IndicesClusterStateService] [t-cratedb-s-003] [.partitioned.metrics.04732dpg70r38dhk60o30c1g][4] marking and sending shard failed due to [shard failure, reason [index id[Aw0xNzA4NjgwNzc5MTg4EGM3ZmUzMjdjZTVmNWJmM2MNMTcwODY0NjQwMDAwMA==] origin[REPLICA] seq#[-2]]]
org.elasticsearch.index.translog.TranslogException: Failed to write operation [Index{id='Aw0xNzA4NjgwNzc5MTg4EGM3ZmUzMjdjZTVmNWJmM2MNMTcwODY0NjQwMDAwMA==', seqNo=-2, primaryTerm=0, version=-3, autoGeneratedIdTimestamp=-1}]
	at org.elasticsearch.index.translog.Translog.add(Translog.java:526) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:941) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:806) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:778) ~[crate-server-5.6.2.jar:?]
	at io.crate.execution.dml.upsert.TransportShardUpsertAction.processRequestItemsOnReplica(TransportShardUpsertAction.java:448) ~[crate-server-5.6.2.jar:?]
	at io.crate.execution.dml.upsert.TransportShardUpsertAction.processRequestItemsOnReplica(TransportShardUpsertAction.java:94) ~[crate-server-5.6.2.jar:?]
	at io.crate.execution.dml.TransportShardAction$2.call(TransportShardAction.java:119) ~[crate-server-5.6.2.jar:?]
	at io.crate.execution.dml.TransportShardAction$2.call(TransportShardAction.java:116) ~[crate-server-5.6.2.jar:?]
	at io.crate.execution.dml.TransportShardAction.wrapOperationInKillable(TransportShardAction.java:130) ~[crate-server-5.6.2.jar:?]
	at io.crate.execution.dml.TransportShardAction.shardOperationOnReplica(TransportShardAction.java:122) ~[crate-server-5.6.2.jar:?]
	at io.crate.execution.dml.TransportShardAction.shardOperationOnReplica(TransportShardAction.java:53) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.onResponse(TransportReplicationAction.java:557) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.onResponse(TransportReplicationAction.java:536) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.shard.IndexShard.lambda$innerAcquireReplicaOperationPermit$27(IndexShard.java:2925) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.action.ActionListener$3.onResponse(ActionListener.java:127) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.shard.IndexShardOperationPermits.acquire(IndexShardOperationPermits.java:292) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.shard.IndexShardOperationPermits.acquire(IndexShardOperationPermits.java:242) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.shard.IndexShard.lambda$acquireReplicaOperationPermit$25(IndexShard.java:2860) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.shard.IndexShard.innerAcquireReplicaOperationPermit(IndexShard.java:2964) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.shard.IndexShard.acquireReplicaOperationPermit(IndexShard.java:2859) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.action.support.replication.TransportReplicationAction.acquireReplicaOperationPermit(TransportReplicationAction.java:881) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.doRun(TransportReplicationAction.java:629) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.action.support.replication.TransportReplicationAction.handleReplicaRequest(TransportReplicationAction.java:521) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:59) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:331) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[crate-server-5.6.2.jar:?]
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) ~[?:?]
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) ~[?:?]
	at java.base/java.lang.Thread.run(Thread.java:1583) [?:?]
Caused by: java.lang.IllegalArgumentException: sequence number must be assigned
	at org.elasticsearch.index.seqno.SequenceNumbers.min(SequenceNumbers.java:90) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.translog.TranslogWriter.add(TranslogWriter.java:210) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.translog.Translog.add(Translog.java:519) ~[crate-server-5.6.2.jar:?]
	... 29 more
[2024-02-23T18:04:33,447][WARN ][o.e.i.c.IndicesClusterStateService] [t-cratedb-s-003] [.partitioned.metrics.04732dpg70r38dhk60o30c1g][9] marking and sending shard failed due to [shard failure, reason [index id[Aw0xNzA4NjgwNzcxMzMzEDAyNTA5ZmMyZWY2MWJlMWINMTcwODY0NjQwMDAwMA==] origin[REPLICA] seq#[-2]]]
org.elasticsearch.index.translog.TranslogException: Failed to write operation [Index{id='Aw0xNzA4NjgwNzcxMzMzEDAyNTA5ZmMyZWY2MWJlMWINMTcwODY0NjQwMDAwMA==', seqNo=-2, primaryTerm=0, version=-3, autoGeneratedIdTimestamp=-1}]
	at org.elasticsearch.index.translog.Translog.add(Translog.java:526) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:941) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:806) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:778) ~[crate-server-5.6.2.jar:?]
	at io.crate.execution.dml.upsert.TransportShardUpsertAction.processRequestItemsOnReplica(TransportShardUpsertAction.java:448) ~[crate-server-5.6.2.jar:?]
	at io.crate.execution.dml.upsert.TransportShardUpsertAction.processRequestItemsOnReplica(TransportShardUpsertAction.java:94) ~[crate-server-5.6.2.jar:?]
	at io.crate.execution.dml.TransportShardAction$2.call(TransportShardAction.java:119) ~[crate-server-5.6.2.jar:?]
	at io.crate.execution.dml.TransportShardAction$2.call(TransportShardAction.java:116) ~[crate-server-5.6.2.jar:?]
	at io.crate.execution.dml.TransportShardAction.wrapOperationInKillable(TransportShardAction.java:130) ~[crate-server-5.6.2.jar:?]
	at io.crate.execution.dml.TransportShardAction.shardOperationOnReplica(TransportShardAction.java:122) ~[crate-server-5.6.2.jar:?]
	at io.crate.execution.dml.TransportShardAction.shardOperationOnReplica(TransportShardAction.java:53) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.onResponse(TransportReplicationAction.java:557) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.onResponse(TransportReplicationAction.java:536) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.shard.IndexShard.lambda$innerAcquireReplicaOperationPermit$27(IndexShard.java:2925) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.action.ActionListener$3.onResponse(ActionListener.java:127) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.shard.IndexShardOperationPermits.acquire(IndexShardOperationPermits.java:292) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.shard.IndexShardOperationPermits.acquire(IndexShardOperationPermits.java:242) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.shard.IndexShard.lambda$acquireReplicaOperationPermit$25(IndexShard.java:2860) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.shard.IndexShard.innerAcquireReplicaOperationPermit(IndexShard.java:2964) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.shard.IndexShard.acquireReplicaOperationPermit(IndexShard.java:2859) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.action.support.replication.TransportReplicationAction.acquireReplicaOperationPermit(TransportReplicationAction.java:881) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.doRun(TransportReplicationAction.java:629) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.action.support.replication.TransportReplicationAction.handleReplicaRequest(TransportReplicationAction.java:521) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:59) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:331) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[crate-server-5.6.2.jar:?]
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) ~[?:?]
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) ~[?:?]
	at java.base/java.lang.Thread.run(Thread.java:1583) [?:?]
Caused by: java.lang.IllegalArgumentException: sequence number must be assigned
	at org.elasticsearch.index.seqno.SequenceNumbers.min(SequenceNumbers.java:90) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.translog.TranslogWriter.add(TranslogWriter.java:210) ~[crate-server-5.6.2.jar:?]
	at org.elasticsearch.index.translog.Translog.add(Translog.java:519) ~[crate-server-5.6.2.jar:?]
	... 29 more