Kafka


Java options Environment Variables PDS default values

JMX_PORT

9999

KAFKA_GC_LOG_OPTS

-verbose:gc

KAFKA_JMX_OPTS

  • -Dcom.sun.management.jmxremote=true

  • -Dcom.sun.management.jmxremote.authenticate=false

  • -Dcom.sun.management.jmxremote.ssl=false

KAFKA_HEAP_OPTS

  • -Xms4g

  • -Xmx4g

  • -XX:MetaspaceSize=96m

  • -XX:+UseG1GC

  • -XX:MaxGCPauseMillis=20

  • -XX:InitiatingHeapOccupancyPercent=35

  • -XX:G1HeapRegionSize=16M

  • -XX:MinMetaspaceFreeRatio=50

  • -XX:MaxMetaspaceFreeRatio=80

Metrics Exporter / Logging Configuration Environment Variables PDS default values

KAFKA_METRICS_MODE

default

LOG_LAYOUT

PATTERN

LOG_LEVEL_ROOTLOGGER

INFO

LOG_LEVEL_TOOLS_ROOTLOGGER

WARN

LOG4J_ADDITIVITY_KAFKA_AUTHORIZER_LOGGER

FALSE

LOG4J_ADDITIVITY_KAFKA_REQUEST_LOGGER

FALSE

LOG4J_ADDITIVITY_STATE_CHANGE_LOGGER

FALSE

LOG4J_LOGGER_KAFKA

INFO

LOG4J_LOGGER_KAFKA_AUTHORIZER_LOGGER

INFO

LOG4J_LOGGER_KAFKA_CONTROLLER

INFO

LOG4J_LOGGER_KAFKA_LOG_LOGCLEANER

INFO

LOG4J_LOGGER_KAFKA_NETWORK_PROCESSOR

FALSE

LOG4J_LOGGER_KAFKA_REQUEST_LOGGER

WARN

LOG4J_LOGGER_KAFKA_SERVER_KAFKAAPIS

FALSE

LOG4J_LOGGER_ORG_APACHE_KAFKA

INFO

LOG4J_LOGGER_ORG_APACHE_ZOOKEEPER

INFO

LOG4J_LOGGER_STATE_CHANGE_LOGGER

INFO

Broker Configurations Environment Variables PDS default values

alter.config.policy.class.name

ALTER_CONFIG_POLICY_CLASS_NAME

alter.log.dirs.replication.quota.window.num

ALTER_LOG_DIRS_REPLICATION_QUOTA_WINDOW_NUM

alter.log.dirs.replication.quota.window.size.seconds

ALTER_LOG_DIRS_REPLICATION_QUOTA_WINDOW_SIZE_SECONDS

authorizer.class.name

AUTHORIZER_CLASS_NAME

auto.create.topics.enable

AUTO_CREATE_TOPICS_ENABLEY

FALSE

auto.leader.rebalance.enable

AUTO_LEADER_REBALANCE_ENABLE

background.threads

BACKGROUND_THREADS

broker.id.generation.enable

BROKER_ID_GENERATION_ENABLE

broker.rack

BROKER_RACK

client.quota.callback.class

CLIENT_QUOTA_CALLBACK_CLASS

compression.type

COMPRESSION_TYPE

snappy

connection.failed.authentication.delay.ms

CONNECTION_FAILED_AUTHENTICATION_DELAY_MS

connections.max.idle.ms

CONNECTIONS_MAX_IDLE_MS

connections.max.reauth.ms

CONNECTIONS_MAX_REAUTH_MS

control.plane.listener.name

CONTROL_PLANE_LISTENER_NAME

controlled.shutdown.enable

CONTROLLED_SHUTDOWN_ENABLE

controlled.shutdown.max.retries

CONTROLLED_SHUTDOWN_MAX_RETRIES

controlled.shutdown.retry.backoff.ms

CONTROLLED_SHUTDOWN_RETRY_BACKOFF_MS

controller.listener.names

CONTROLLER_LISTENER_NAMES

controller.quorum.voters

CONTROLLER_QUORUM_VOTERS

controller.socket.timeout.ms

CONTROLLER_SOCKET_TIMEOUT_MS

create.topic.policy.class.name

CREATE_TOPIC_POLICY_CLASS_NAME

default.replication.factor

DEFAULT_REPLICATION_FACTOR

3

delegation.token.expiry.check.interval.ms

DELEGATION_TOKEN_EXPIRY_CHECK_INTERVAL_MS

delegation.token.expiry.time.ms

DELEGATION_TOKEN_EXPIRY_TIME_MS

delegation.token.master.key

DELEGATION_TOKEN_MASTER_KEY

delegation.token.max.lifetime.ms

DELEGATION_TOKEN_MAX_LIFETIME_MS

delete.records.purgatory.purge.interval.requests

DELETE_RECORDS_PURGATORY_PURGE_INTERVAL_REQUESTS

delete.topic.enable

DELETE_TOPIC_ENABLE

fetch.purgatory.purge.interval.requests

FETCH_PURGATORY_PURGE_INTERVAL_REQUESTS

100

group.initial.rebalance.delay.ms

GROUP_INITIAL_REBALANCE_DELAY_MS

Sets Kafka default - non overwritten parameters use Kafka defaults.

group.max.session.timeout.ms

GROUP_MAX_SESSION_TIMEOUT_MS

group.max.size

GROUP_MAX_SIZE

group.min.session.timeout.ms

GROUP_MIN_SESSION_TIMEOUT_MSY

inter.broker.protocol.version

INTER_BROKER_PROTOCOL_VERSION

kafka.metrics.polling.interval.secs

KAFKA_METRICS_POLLING_INTERVAL_SECS

kafka.metrics.reporters

KAFKA_METRICS_REPORTERS

leader.imbalance.check.interval.seconds

LEADER_IMBALANCE_CHECK_INTERVAL_SECONDS

leader.imbalance.per.broker.percentage

LEADER_IMBALANCE_PER_BROKER_PERCENTAGE

log.cleaner.backoff.ms

LOG_CLEANER_BACKOFF_MS

log.cleaner.dedupe.buffer.size

LOG_CLEANER_DEDUPE_BUFFER_SIZE

log.cleaner.delete.retention.ms

LOG_CLEANER_DELETE_RETENTION_MS

log.cleaner.enable

LOG_CLEANER_ENABLE

log.cleaner.io.buffer.load.factor

LOG_CLEANER_IO_BUFFER_LOAD_FACTOR

log.cleaner.io.buffer.size

LOG_CLEANER_IO_BUFFER_SIZE

log.cleaner.io.max.bytes.per.second

LOG_CLEANER_IO_MAX_BYTES_PER_SECOND

log.cleaner.max.compaction.lag.ms

LOG_CLEANER_MAX_COMPACTION_LAG_MS

log.cleaner.min.cleanable.ratio

LOG_CLEANER_MIN_CLEANABLE_RATIO

log.cleaner.min.compaction.lag.ms

LOG_CLEANER_MIN_COMPACTION_LAG_MS

log.cleaner.threads

LOG_CLEANER_THREADS

log.cleanup.policy

LOG_CLEANUP_POLICY

log.flush.interval.messages

LOG_FLUSH_INTERVAL_MESSAGES

20000

log.flush.interval.ms

LOG_FLUSH_INTERVAL_MS

10000

log.flush.offset.checkpoint.interval.ms

LOG_FLUSH_OFFSET_CHECKPOINT_INTERVAL_MS

log.flush.scheduler.interval.ms

LOG_FLUSH_SCHEDULER_INTERVAL_MS

2000

log.flush.start.offset.checkpoint.interval.ms

LOG_FLUSH_START_OFFSET_CHECKPOINT_INTERVAL_MS

log.index.interval.bytes

LOG_INDEX_INTERVAL_BYTES

log.index.size.max.bytes

LOG_INDEX_SIZE_MAX_BYTES

log.message.downconversion.enable

LOG_MESSAGE_DOWNCONVERSION_ENABLE

log.message.format.version

LOG_MESSAGE_FORMAT_VERSION

log.message.timestamp.difference.max.ms

LOG_MESSAGE_TIMESTAMP_DIFFERENCE_MAX_MS

log.message.timestamp.type

LOG_MESSAGE_TIMESTAMP_TYPE

LogAppendTime

log.preallocate

LOG_PREALLOCATE

log.retention.bytes

LOG_RETENTION_BYTES

log.retention.check.interval.ms

LOG_RETENTION_CHECK_INTERVAL_MS

log.retention.hours

LOG_RETENTION_HOURS

log.retention.minutes

LOG_RETENTION_MINUTES

log.retention.ms

LOG_RETENTION_MS

log.roll.hours

LOG_ROLL_HOURS

log.roll.jitter.hours

LOG_ROLL_JITTER_HOURS

log.roll.jitter.ms

LOG_ROLL_JITTER_MS

log.roll.ms

LOG_ROLL_MS

log.segment.bytes

LOG_SEGMENT_BYTES

log.segment.delete.delay.ms

LOG_SEGMENT_DELETE_DELAY_MS

max.connections

MAX_CONNECTIONS

max.connections.per.ip

MAX_CONNECTIONS_PER_IP

max.connections.per.ip.overrides

MAX_CONNECTIONS_PER_IP_OVERRIDES

max.incremental.fetch.session.cache.slots

MAX_INCREMENTAL_FETCH_SESSION_CACHE_SLOTS

message.max.bytes

MESSAGE_MAX_BYTES

metric.reporters

METRIC_REPORTERS

3

metrics.num.samples

METRICS_NUM_SAMPLES

metrics.recording.level

METRICS_RECORDING_LEVELS

metrics.sample.window.ms

METRICS_SAMPLE_WINDOW_MS

min.insync.replicas

MIN_INSYNC_REPLICAS

2

node.id

NODE_ID

num.io.threads

NUM_IO_THREADS

num.network.threads

NUM_NETWORK_THREADS

8

num.partitions

NUM_PARTITIONS

8

num.recovery.threads.per.data.dir

NUM_RECOVERY_THREADS_PER_DATA_DIR

num.replica.alter.log.dirs.threads

NUM_REPLICA_ALTER_LOG_DIRS_THREADS

num.replica.fetchers

NUM_REPLICA_FETCHERS

4

offset.metadata.max.bytes

OFFSET_METADATA_MAX_BYTES

offsets.commit.required.acks

OFFSETS_COMMIT_REQUIRED_ACKS

offsets.commit.timeout.ms

OFFSETS_COMMIT_TIMEOUT_MS

offsets.load.buffer.size

OFFSETS_LOAD_BUFFER_SIZE

offsets.retention.check.interval.ms

OFFSETS_RETENTION_CHECK_INTERVAL_MS

offsets.retention.minutes

OFFSETS_RETENTION_MINUTES

offsets.topic.compression.codec

OFFSETS_TOPIC_COMPRESSION_CODEC

offsets.topic.num.partitions

OFFSETS_TOPIC_NUM_PARTITIONS

offsets.topic.replication.factor

OFFSETS_TOPIC_SEGMENT_BYTES

password.encoder.cipher.algorithm

PASSWORD_ENCODER_CIPHER_ALGORITHM

password.encoder.iterations

PASSWORD_ENCODER_ITERATIONS

password.encoder.key.length

PASSWORD_ENCODER_KEY_LENGTH

password.encoder.keyfactory.algorithm

PASSWORD_ENCODER_KEYFACTORY_ALGORITHM

password.encoder.old.secret

PASSWORD_ENCODER_OLD_SECRET

password.encoder.secret

PASSWORD_ENCODER_SECRET

principal.builder.classs

PRINCIPAL_BUILDER_CLASS

process.roles

PROCESS_ROLES

producer.purgatory.purge.interval.requests

PRODUCER_PURGATORY_PURGE_INTERVAL_REQUESTS

100

queued.max.request.bytes

QUEUED_MAX_REQUEST_BYTES

queued.max.requests

QUEUED_MAX_REQUESTS

quota.window.num

QUOTA_WINDOW_NUM

quota.window.size.seconds

QUOTA_WINDOW_SIZE_SECONDS

replica.fetch.backoff.ms

REPLICA_FETCH_BACKOFF_MS

replica.fetch.max.bytes

REPLICA_FETCH_MAX_BYTES

replica.fetch.min.bytes

REPLICA_FETCH_MIN_BYTES

replica.fetch.response.max.bytes

REPLICA_FETCH_RESPONSE_MAX_BYTES

replica.fetch.wait.max.ms

REPLICA_FETCH_WAIT_MAX_MS

replica.high.watermark.checkpoint.interval.ms

REPLICA_HIGH_WATERMARK_CHECKPOINT_INTERVAL_MS

replica.lag.time.max.ms

REPLICA_LAG_TIME_MAX_MS

replica.selector.class

REPLICA_SELECTOR_CLASS

replica.socket.receive.buffer.bytes

REPLICA_SOCKET_RECEIVE_BUFFER_BYTES

1048576

replica.socket.timeout.ms

REPLICA_SOCKET_TIMEOUT_MS

replication.quota.window.num

REPLICATION_QUOTA_WINDOW_NUM

replication.quota.window.size.seconds

REPLICATION_QUOTA_WINDOW_SIZE_SECONDS

request.timeout.ms

REQUEST_TIMEOUT_MS

reserved.broker.max.id

RESERVED_BROKER_MAX_ID

sasl.client.callback.handler.class

SASL_CLIENT_CALLBACK_HANDLER_CLASS

sasl.enabled.mechanisms

SASL_ENABLED_MECHANISMS

sasl.jaas.config

SASL_JAAS_CONFIG

sasl.kerberos.kinit.cmd

SASL_KERBEROS_KINIT_CMD

sasl.kerberos.min.time.before.relogin

SASL_KERBEROS_MIN_TIME_BEFORE_RELOGIN

sasl.kerberos.principal.to.local.rules

SASL_KERBEROS_PRINCIPAL_TO_LOCAL_RULES

sasl.kerberos.service.name

SASL_KERBEROS_SERVICE_NAME

sasl.kerberos.ticket.renew.jitter

SASL_KERBEROS_TICKET_RENEW_JITTERS

sasl.kerberos.ticket.renew.window.factor

SASL_KERBEROS_TICKET_RENEW_WINDOW_FACTOR

sasl.login.callback.handler.class

SASL_LOGIN_CALLBACK_HANDLER_CLASS

sasl.login.class

SASL_LOGIN_CLASS

sasl.login.refresh.buffer.seconds

SASL_LOGIN_REFRESH_BUFFER_SECONDS

sasl.login.refresh.min.period.seconds

SASL_LOGIN_REFRESH_MIN_PERIOD_SECONDS

sasl.login.refresh.window.factor

SASL_LOGIN_REFRESH_WINDOW_FACTOR

sasl.login.refresh.window.jitter

SASL_LOGIN_REFRESH_WINDOW_JITTER

sasl.mechanism.inter.broker.protocol

SASL_MECHANISM_INTER_BROKER_PROTOCOL

sasl.server.callback.handler.class

SASL_SERVER_CALLBACK_HANDLER_CLASS

security.inter.broker.protocol

SECURITY_INTER_BROKER_PROTOCOL

security.providers

SECURITY_PROVIDERS

socket.receive.buffer.bytes

SOCKET_RECEIVE_BUFFER_BYTES

socket.request.max.bytes

SOCKET_REQUEST_MAX_BYTES

socket.send.buffer.bytes

SOCKET_SEND_BUFFER_BYTES

1048576

ssl.cipher.suites

SSL_CIPHER_SUITES

ssl.client.auth

SSL_CLIENT_AUTH

ssl.enabled.protocols

SSL_ENABLED_PROTOCOLS

ssl.endpoint.identification.algorithm

SSL_ENDPOINT_IDENTIFICATION_ALGORITHM

ssl.key.password

SSL_KEY_PASSWORD

ssl.keymanager.algorithm

SSL_KEYMANAGER_ALGORITHM

ssl.keystore.location

SSL_KEYSTORE_LOCATION

ssl.keystore.password

SSL_KEYSTORE_PASSWORD

ssl.keystore.type

SSL_KEYSTORE_TYPE

ssl.principal.mapping.rules

SSL_PRINCIPAL_MAPPING_RULES

ssl.protocol

SSL_PROTOCOL

ssl.provider

SSL_PROVIDER

ssl.secure.random.implementation

SSL_SECURE_RANDOM_IMPLEMENTATION

ssl.trustmanager.algorithm

SSL_TRUSTMANAGER_ALGORITHM

ssl.truststore.location

SSL_TRUSTSTORE_LOCATION

ssl.truststore.password

SSL_TRUSTSTORE_PASSWORD

ssl.truststore.type

SSL_TRUSTSTORE_TYPE

transaction.abort.timed.out.transaction.cleanup.interval.ms

TRANSACTION_ABORT_TIMED_OUT_TRANSACTION_CLEANUP_INTERVAL_MS

transaction.max.timeout.ms

TRANSACTION_MAX_TIMEOUT_MS

transaction.remove.expired.transaction.cleanup.interval.ms

TRANSACTION_REMOVE_EXPIRED_TRANSACTION_CLEANUP_INTERVAL_MS

transaction.state.log.load.buffer.size

TRANSACTION_STATE_LOG_LOAD_BUFFER_SIZE

transaction.state.log.min.isr

TRANSACTION_STATE_LOG_MIN_ISR

transaction.state.log.num.partitions

TRANSACTION_STATE_LOG_NUM_PARTITIONS

transaction.state.log.replication.factor

TRANSACTION_STATE_LOG_REPLICATION_FACTOR

transaction.state.log.segment.bytes

TRANSACTION_STATE_LOG_SEGMENT_BYTES

transactional.id.expiration.ms

TRANSACTIONAL_ID_EXPIRATION_MS

unclean.leader.election.enable

UNCLEAN_LEADER_ELECTION_ENABLE

FALSE

zookeeper.connection.string

ZOOKEEPER_CONNECTION_STRING

zookeeper.connection.timeout.ms

ZOOKEEPER_CONNECTION_TIMEOUT_MS

zookeeper.max.in.flight.requests

ZOOKEEPER_MAX_IN_FLIGHT_REQUESTS

zookeeper.session.timeout.ms

ZOOKEEPER_SESSION_TIMEOUT_MS

zookeeper.set.acl

ZOOKEEPER_SET_ACL

zookeeper.sync.time.ms

ZOOKEEPER_SYNC_TIME_MS



Last edited: Tuesday, Aug 30, 2022