kafka发送及接收消息(demo)

1、producer

#-*- coding:utf-8 -*-
from kafka import KafkaProducer
from kafka import KafkaConsumer
from kafka.errors import KafkaError
from threading import Thread
import time


class kafkaProducer(object):
    def __init__(self, kafkatopic, kafkahost, kafkaport):
        self.kafkatopic = kafkatopic
        self.kafkahost = kafkahost
        self.kafkaport = kafkaport
        bootstrap_server = []
        for host in self.kafkahost:
            """auto_offset_reset='latest',# 消费kafka中最近的数据，如果设置为earliest则消费最早的数据，不管这些数据是否消费； consumer_timeout_ms:设置自动退出时间（毫秒）；"""
            bootstrap_server.append('{kafka_host}:{kafka_port}'.format(kafka_host=host, kafka_port=self.kafkaport))
        self.producer = KafkaProducer(bootstrap_servers=bootstrap_server)


    def sendMsg(self):
        try:
            for i in range(100):
                startTime = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
                if i % 2 == 0:
                    message_key = 'a'
                elif i % 3 == 0:
                    message_key = 'b'
                else:
                    message_key = 'c'
                # message = 'send_message_' + str(i+1) + ':' + startTime
                message = 'message_' + str(i)
                self.producer.send(self.kafkatopic, key=message_key.encode('utf-8'), value=message.encode('utf-8'),
                                   timestamp_ms=None)
                print(message_key, message, startTime)
                self.producer.flush()
                time.sleep(0.5)
            self.producer.close()
            print('****over****')
        except KafkaError as e:
            print('KafkaError:', e)


if __name__=='__main__':
    para_host = ['xxx', 'xxx', 'xxx']
    para_port = 9092
    producer = kafkaProducer("topic-1", para_host, para_port)
    producer.sendMsg()

2、consumer

from redis_try import *
from kafka import KafkaConsumer
from kafka.errors import KafkaError
from threading import Thread
import time


class kafkaConsumer(object):
    def __init__(self, kafkatopic, kafkahost, kafkaport):
        self.kafkatopic = kafkatopic
        self.kafkahost = kafkahost
        self.kafkaport = kafkaport
        bootstrap_server = []
        for host in self.kafkahost:
            bootstrap_server.append('{kafka_host}:{kafka_port}'.format(kafka_host=host, kafka_port=self.kafkaport))
        self.consumer = KafkaConsumer(self.kafkatopic, bootstrap_servers=bootstrap_server,
                                      auto_offset_reset='latest', consumer_timeout_ms=10000)

    def getMsg(self):
        try:
            for msg in self.consumer:
                # print('get_key_>>>>', msg.key.decode('utf-8'),
                #       ';get_message_>>>>', msg.value.decode('utf-8'))
                msg_value = msg.value.decode('utf-8')
                user, num, label = r.user_count(msg.key.decode('utf-8'))
                print("user:{} click:{} label:{} value:{}".format(user, num, label, msg_value))
            print('****done****')
        except KafkaError as e:
            print('KafkaError:', e)


if __name__ == '__main__':
    para_host = ['xxx', 'xxx', 'xxx']
    para_port = 9092
    r = redis_server(host='xxx', ports=[a, b, c], password='zzzz')# 调redis
    consumer = kafkaConsumer("topic-1", para_host, para_port)
    messages = consumer.getMsg()