python操作kafka写入json数据 对python操作kafka写入json数据的简单demo分享

安装kafka支持库pip install kafka-python

from kafka import KafkaProducer
import json
 
'''
 生产者demo
 向test_lyl2主题中循环写入10条json数据
 注意事项：要写入json数据需加上value_serializer参数，如下代码
'''
producer = KafkaProducer(
       value_serializer=lambda v: json.dumps(v).encode('utf-8'),
       bootstrap_servers=['192.168.12.101:6667','192.168.12.102:6667','192.168.12.103:6667']
       )
for i in range(10):
 data={
  "name":"李四",
  "age":23,
  "gender":"男",
  "id":i
 }
 producer.send('test_lyl2', data)
producer.close()

from kafka import KafkaConsumer
import json
 
'''
 消费者demo
 消费test_lyl2主题中的数据
 注意事项：如需以json格式读取数据需加上value_deserializer参数
'''
 
 
consumer = KafkaConsumer('test_lyl2',group_id="lyl-gid1",
       bootstrap_servers=['192.168.12.101:6667','192.168.12.102:6667','192.168.12.103:6667'],
       auto_offset_reset='earliest',value_deserializer=json.loads
       )
for message in consumer:
 print(message.value)