Конфигурация темы Apache Kafka для производства
Я хочу развернуть кластер Kafka, и я хочу несколько предложений о конфигурации брокера.
Я действительно беспокоюсь о том, будет ли мой кластер высокодоступным, если каждый брокер отключится и если я попытаюсь перезапустить их, у меня будет потеря данных.
Кроме того, каков наилучший способ организации потребителей?
Это мои конфиги:
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# see kafka.server.KafkaConfig for additional details and defaults
############################# Server Basics #############################
broker.id=0
############################# Socket Server Settings #############################
port=9092
listeners=PLAINTEXT://localhost:9092
advertised.host.name=localhost
advertised.listeners=PLAINTEXT://localhost:9092
advertised.port=9092
host.name=localhost
num.network.threads=3
num.io.threads=8
socket.send.buffer.bytes=1048576
socket.receive.buffer.bytes=1048576
socket.request.max.bytes=104857600
queued.max.requests=16
fetch.purgatory.purge.interval.requests=100
producer.purgatory.purge.interval.requests=100
############################# Log Basics #############################
log.dirs=/tmp/kafka-logs
num.partitions=4
num.recovery.threads.per.data.dir=1
log.index.size.max.bytes=154624
log.index.interval.bytes=4096
message.max.bytes=1000000
auto.create.topics.enable=false
unclean.leader.election.enable=true
offsets.topic.replication.factor=3
leader.imbalance.check.interval.seconds=1
min.insync.replicas=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
max.poll.interval.ms=300000
max.poll.records=2
enable.auto.commit=true
auto.commit.interval.ms=1000
session.timeout.ms=30000
request.timeout.ms=50000
batch.size=16384
############################# Log Flush Policy #############################
default.replication.factor=3
log.flush.interval.messages=100000
log.flush.interval.ms=50000
log.flush.scheduler.interval.ms=2000
############################# Log Retention Policy #############################
log.retention.hours=168
#log.retention.bytes=1073741824
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
log.cleaner.enable=false
log.roll.hours=168
############################# Zookeeper #############################
zookeeper.connect=localhost:2181,localhost:2182,localhost:2183
zookeeper.connection.timeout.ms=6000
zk.sync.time.ms=2000
# Replication configurations
num.replica.fetchers=4
replica.fetch.max.bytes=1048576
replica.fetch.wait.max.ms=500
replica.high.watermark.checkpoint.interval.ms=5000
replica.socket.timeout.ms=30000
replica.socket.receive.buffer.bytes=65536
replica.lag.time.max.ms=10000
replica.lag.max.messages=4000
controller.socket.timeout.ms=30000
controller.message.queue.size=10
###################################
# Durability and hardening
###################################
retries=0
acks=all
auto.leader.rebalance.enable=true
leader.imbalance.per.broker.percentages=0
Любая помощь будет оценена.