kafka入门基础

1. Java安装

1.1 下载
https://download.oracle.com/java/18/archive/jdk-18.0.2.1_linu...
1.2 安装

tar zxvf jdk-18.0.2.1_linux-x64_bin.tar.gz -C /usr/local/share/
# vim /etc/profile                   // 添加以下内容

# The java envirment
JAVA_HOME=/usr/local/share/jdk-18.0.2.1/
CLASSPATH=.:$JAVA_HOME/lib/tools.jar
PATH=$JAVA_HOME/bin:$PATH
// 配置生效
# source /etc/profile

1.3 验证
java
java -version

2. kafka安装

2.1 下载
https://downloads.apache.org/kafka/3.8.0/kafka_2.13-3.8.0.tgz
2.2 安装
tar zxvf kafka_2.13-3.8.0.tgz -C /usr/local/share/
cd /usr/local/share/kafka_2.13-3.8.0/
vi config/zookeeper.properties

# the directory where the snapshot is stored.
dataDir=/usr/loacal/share/zookeerperLog
# the port at which the clients will connect
clientPort=2181
# disable the per-ip limit on the number of connections since this is a non-production config
maxClientCnxns=0
# Disable the adminserver by default to avoid port conflicts.
# Set the port to something non-conflicting if choosing to enable this
admin.enableServer=false

启动zookeeper
bin/zookeeper-server-start.sh config/zookeeper.properties

vi config/server.properties

broker.id=0
listeners=PLAINTEXT://localhost:9092
log.dirs=/usr/local/share/kafka_2.13-3.8.0/logs
zookeeper.connect=localhost:2181

启动kafka
bin/kafka-server-start.sh config/server.properties
2.3 验证
查看监听的端口

[root@localhost kafka_2.13-3.8.0]# netstat -lntup | grep -e 2181 -e 9092
tcp6       0      0 127.0.0.1:9092          :::*                    LISTEN      4250/java           
tcp6       0      0 :::2181                 :::*                    LISTEN      3237/java           
[root@localhost kafka_2.13-3.8.0]# 

3. topic

创建topic

[root@localhost kafka_2.13-3.8.0]# bin/kafka-topics.sh --create --topic testTopic --bootstrap-server localhost:9092
Created topic testTopic.
[root@localhost kafka_2.13-3.8.0]# 

查询topic

[root@localhost kafka_2.13-3.8.0]# bin/kafka-topics.sh --describe --topic testTopic --bootstrap-server localhost:9092
[2024-10-14 23:57:47,366] WARN [AdminClient clientId=adminclient-1] The DescribeTopicPartitions API is not supported, using Metadata API to describe topics. (org.apache.kafka.clients.admin.KafkaAdminClient)
Topic: testTopic    TopicId: 4THq5NDkQsyhYUI4ONoldg    PartitionCount: 1    ReplicationFactor: 1    Configs: 
    Topic: testTopic    Partition: 0    Leader: 0    Replicas: 0    Isr: 0    Elr: N/A    LastKnownElr: N/A
[root@localhost kafka_2.13-3.8.0]#

topic数据写入

[root@localhost kafka_2.13-3.8.0]# bin/kafka-console-producer.sh --topic testTopic --bootstrap-server localhost:9092
>This is a test message.
>This is a test messages.
>

说明：按ctrl+c停止写入
topic数据查询

[root@localhost kafka_2.13-3.8.0]# bin/kafka-console-consumer.sh --topic testTopic --from-beginning --bootstrap-server localhost:9092
This is a test message.
This is a test messages.

4. kafka systemctl开机启动

创建/etc/systemd/system/zookeeper.service

[Unit]
Description=Apache zookeeper server (broker)
After=network.target

[Service]
Type=simple
Environment="PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/share/jdk-18.0.2.1/bin"
User=root
Group=root
ExecStart=/usr/local/share/kafka_2.13-3.8.0/bin/zookeeper-server-start.sh  /usr/local/share/kafka_2.13-3.8.0/config/zookeeper.properties
ExecStop=/usr/local/share/kafka_2.13-3.8.0/bin/zookeeper-server-stop.sh

[Install]
WantedBy=multi-user.target

创建/etc/systemd/system/kafka.service

[Unit]
Description=Apache Kafka server (broker)
After=network.target  zookeeper.service

[Service]
Type=simple
Environment="PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/share/jdk-18.0.2.1/bin"
User=root
Group=root
ExecStart=/usr/local/share/kafka_2.13-3.8.0/bin/kafka-server-start.sh  /usr/local/share/kafka_2.13-3.8.0/config/server.properties
ExecStop=/usr/local/share/kafka_2.13-3.8.0/bin/kafka-server-stop.sh

[Install]
WantedBy=multi-user.target

systemctl daemon-reload
systemctl enable zookeeper
systemctl enable kafka

5. 使用filebeat发送nginx日志到kafka

安装filebeat
下载
https://artifacts.elastic.co/downloads/beats/filebeat/filebea...
安装
tar zxvf filebeat-8.15.2-linux-x86_64.tar.gz -C /usr/local/share/
[root@localhost filebeat-8.15.2-linux-x86_64]# ./filebeat modules list
[root@localhost filebeat-8.15.2-linux-x86_64]# ./filebeat modules enable nginx
[root@localhost filebeat-8.15.2-linux-x86_64]# cp filebeat.yml filebeat.yml.default
[root@localhost filebeat-8.15.2-linux-x86_64]# vim filebeat.yml

[root@localhost html]# cat /usr/local/share/filebeat-8.15.2-linux-x86_64/filebeat.yml
filebeat.inputs:
- type: log
  # Change to true to enable this input configuration.
  enabled: true
  # Paths that should be crawled and fetched. Glob based paths.
  paths:
    - /var/log/nginx/access.log 
#==========------------------------------kafka-----------------------------------
output.kafka:
  hosts: ["localhost:9092"]
  topic: nginxLog
  keep_alive: 10s
[root@localhost html]# 

[root@localhost filebeat-8.15.2-linux-x86_64]# ./filebeat -e

安装nginx
此处略

工具kafkatool

https://www.kafkatool.com/download3/offsetexplorer_64bit.exe

参考：

https://kafka.apache.org/quickstart