公网SASL_SSL方式生产和消费消息
更新时间:2024-06-18
在 Kafka 集群所在 VPC 网络外访问,使用 SASL_SSL 协议接入,接入点可以在 集群详情 页面查看。
环境准备
- 安装GCC
- 安装C++ 依赖库。
Shell
1yum install librdkafka-devel
2yum install cyrus-sasl
3yum install cyrus-sasl-scram
集群准备
1. 购买专享版消息服务for Kafka集群
开通消息服务 for Kafka服务后,在控制台页面点击『创建集群』,即可进行购买。
2. 为购买的集群创建主题
在控制台页面点击集群名称,进入集群详情页面。
在左侧的边栏中点击『主题管理』,进入主题管理页面。
在主题管理页面点击『创建主题』,进行主题的创建。
使用步骤:
步骤一:获取集群接入点
具体请参考:接入点查看。
步骤二:下载证书文件
下载证书文件:如何下载证书?
步骤三:编写测试代码
生产者代码示例
创建KafkaProducerDemo.c文件,具体代码示例如下:
Shell
1/*
2* librdkafka - Apache Kafka C library
3*
4* Copyright (c) 2017, Magnus Edenhill
5* All rights reserved.
6*
7* Redistribution and use in source and binary forms, with or without
8* modification, are permitted provided that the following conditions are met:
9*
10* 1. Redistributions of source code must retain the above copyright notice,
11* this list of conditions and the following disclaimer.
12* 2. Redistributions in binary form must reproduce the above copyright notice,
13* this list of conditions and the following disclaimer in the documentation
14* and/or other materials provided with the distribution.
15*
16* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
20* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26* POSSIBILITY OF SUCH DAMAGE.
27 */
28
29/**
30* Simple Apache Kafka producer
31* using the Kafka driver from librdkafka
32* (https://github.com/edenhill/librdkafka)
33 */
34
35#include <stdio.h>
36#include <signal.h>
37#include <string.h>
38
39
40/* Typical include path would be <librdkafka/rdkafka.h>, but this program
41* is builtin from within the librdkafka source tree and thus differs. */
42 #include "librdkafka/rdkafka.h"
43
44
45static volatile sig_atomic_t run = 1;
46
47/**
48* @brief Signal termination of program
49 */
50 static void stop(int sig) {
51 run = 0;
52 fclose(stdin); /* abort fgets() */
53 }
54
55
56/**
57* @brief Message delivery report callback.
58*
59* This callback is called exactly once per message, indicating if
60* the message was succesfully delivered
61* (rkmessage->err == RD_KAFKA_RESP_ERR_NO_ERROR) or permanently
62* failed delivery (rkmessage->err != RD_KAFKA_RESP_ERR_NO_ERROR).
63*
64* The callback is triggered from rd_kafka_poll() and executes on
65* the application's thread.
66 */
67 static void dr_msg_cb(rd_kafka_t *rk, const rd_kafka_message_t *rkmessage, void *opaque) {
68 if (rkmessage->err) {
69 fprintf(stderr, "%% Message delivery failed: %s\n", rd_kafka_err2str(rkmessage->err));
70 } else {
71 fprintf(stderr, "%% Message delivered (%zd bytes, partition %" PRId32 ")\n", rkmessage->len, rkmessage->partition);
72 }
73
74 /* The rkmessage is destroyed automatically by librdkafka */
75 }
76
77
78
79int main(int argc, char **argv) {
80rd_kafka_t *rk; /* Producer instance handle */
81rd_kafka_conf_t *conf; /* Temporary configuration object */
82
83 char errstr[512]; /* librdkafka API error reporting buffer */
84 char buf[512]; /* Message value temporary buffer */
85
86 const char *brokers; /* Argument: broker list */
87 const char *topic; /* Argument: topic to produce to */
88 const char *username; /* Argument: sasl username */
89 const char *password; /* Argument: sasl password */
90
91 /*
92 * Argument validation
93 */
94
95 //检查参数配置
96 if (argc != 5) {
97 fprintf(stderr, "%% Usage: %s <broker> <topic> <username> <password>\n", argv[0]);
98 return 1;
99 }
100
101 // 接入点信息
102 brokers = argv[1];
103 // 主题名称
104 topic = argv[2];
105 // 用户管理中创建的用户名称
106 username = argv[3];
107 // 用户管理中创建的用户密码
108 password = argv[4];
109
110
111 /*
112 * Create Kafka client configuration place-holder
113 */
114 conf = rd_kafka_conf_new();
115
116 /* Set bootstrap broker(s) as a comma-separated list of
117 * host or host:port (default port 9092).
118 * librdkafka will use the bootstrap brokers to acquire the full
119 * set of brokers from the cluster. */
120 if (rd_kafka_conf_set(conf, "bootstrap.servers", brokers, errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK) {
121 fprintf(stderr, "%s\n", errstr);
122 return 1;
123 }
124 // 认证机制支持PLAIN、SCRAM-SHA-512两种机制,根据集群所使用的认证方式进行选择
125 if (
126 rd_kafka_conf_set(conf, "ssl.ca.location", "ca.pem", errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK
127 || rd_kafka_conf_set(conf, "security.protocol", "sasl_ssl", errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK
128 || rd_kafka_conf_set(conf, "sasl.mechanism", "SCRAM-SHA-512", errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK
129 || rd_kafka_conf_set(conf, "sasl.username", username, errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK
130 || rd_kafka_conf_set(conf, "sasl.password", password, errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK
131 ) {
132 fprintf(stderr, "%s\n", errstr);
133 return -1;
134 }
135
136 /* Set the delivery report callback.
137 * This callback will be called once per message to inform
138 * the application if delivery succeeded or failed.
139 * See dr_msg_cb() above.
140 * The callback is only triggered from rd_kafka_poll() and
141 * rd_kafka_flush(). */
142 rd_kafka_conf_set_dr_msg_cb(conf, dr_msg_cb);
143
144 /*
145 * Create producer instance.
146 *
147 * NOTE: rd_kafka_new() takes ownership of the conf object
148 * and the application must not reference it again after
149 * this call.
150 */
151 rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf, errstr, sizeof(errstr));
152 if (!rk) {
153 fprintf(stderr, "%% Failed to create new producer: %s\n", errstr);
154 return 1;
155 }
156
157 /* Signal handler for clean shutdown */
158 signal(SIGINT, stop);
159
160 fprintf(stderr,
161 "%% Type some text and hit enter to produce message\n"
162 "%% Or just hit enter to only serve delivery reports\n"
163 "%% Press Ctrl-C or Ctrl-D to exit\n");
164
165 while (run && fgets(buf, sizeof(buf), stdin)) {
166 size_t len = strlen(buf);
167 rd_kafka_resp_err_t err;
168
169 /* Remove newline */
170 if (buf[len - 1] == '\n') {
171 buf[--len] = '\0';
172 }
173
174 /* Empty line: only serve delivery reports */
175 if (len == 0) {
176 rd_kafka_poll(rk, 0/*non-blocking */);
177 continue;
178 }
179
180 /*
181 * Send/Produce message.
182 * This is an asynchronous call, on success it will only
183 * enqueue the message on the internal producer queue.
184 * The actual delivery attempts to the broker are handled
185 * by background threads.
186 * The previously registered delivery report callback
187 * (dr_msg_cb) is used to signal back to the application
188 * when the message has been delivered (or failed).
189 */
190 retry:
191 err = rd_kafka_producev(
192 /* Producer handle */
193 rk,
194 /* Topic name */
195 RD_KAFKA_V_TOPIC(topic),
196 /* Make a copy of the payload. */
197 RD_KAFKA_V_MSGFLAGS(RD_KAFKA_MSG_F_COPY),
198 /* Message value and length */
199 RD_KAFKA_V_VALUE(buf, len),
200 /* Per-Message opaque, provided in
201 * delivery report callback as
202 * msg_opaque. */
203 RD_KAFKA_V_OPAQUE(NULL),
204 /* End sentinel */
205 RD_KAFKA_V_END
206 );
207
208 if (err) {
209 /*
210 * Failed to *enqueue* message for producing.
211 */
212 fprintf(stderr,
213 "%% Failed to produce to topic %s: %s\n", topic,
214 rd_kafka_err2str(err));
215
216 if (err == RD_KAFKA_RESP_ERR__QUEUE_FULL) {
217 /* If the internal queue is full, wait for
218 * messages to be delivered and then retry.
219 * The internal queue represents both
220 * messages to be sent and messages that have
221 * been sent or failed, awaiting their
222 * delivery report callback to be called.
223 *
224 * The internal queue is limited by the
225 * configuration property
226 * queue.buffering.max.messages */
227 rd_kafka_poll(rk, 1000 /*block for max 1000ms*/);
228 goto retry;
229 }
230 } else {
231 fprintf(stderr, "%% Enqueued message (%zd bytes) "
232 "for topic %s\n",
233 len, topic);
234 }
235
236
237 /* A producer application should continually serve
238 * the delivery report queue by calling rd_kafka_poll()
239 * at frequent intervals.
240 * Either put the poll call in your main loop, or in a
241 * dedicated thread, or call it after every
242 * rd_kafka_produce() call.
243 * Just make sure that rd_kafka_poll() is still called
244 * during periods where you are not producing any messages
245 * to make sure previously produced messages have their
246 * delivery report callback served (and any other callbacks
247 * you register). */
248 rd_kafka_poll(rk, 0 /*non-blocking*/);
249 }
250
251
252 /* Wait for final messages to be delivered or fail.
253 * rd_kafka_flush() is an abstraction over rd_kafka_poll() which
254 * waits for all messages to be delivered. */
255 fprintf(stderr, "%% Flushing final messages..\n");
256 rd_kafka_flush(rk, 10 * 1000 /* wait for max 10 seconds */);
257
258 /* If the output queue is still not empty there is an issue
259 * with producing messages to the clusters. */
260 if (rd_kafka_outq_len(rk) > 0) {
261 fprintf(stderr, "%% %d message(s) were not delivered\n", rd_kafka_outq_len(rk));
262 }
263
264 /* Destroy the producer instance */
265 rd_kafka_destroy(rk);
266
267 return 0;
268}
消费者代码示例
创建KafkaConsumerDemo.c文件,具体代码示例如下:
Shell
1/*
2* librdkafka - Apache Kafka C library
3*
4* Copyright (c) 2019, Magnus Edenhill
5* All rights reserved.
6*
7* Redistribution and use in source and binary forms, with or without
8* modification, are permitted provided that the following conditions are met:
9*
10* 1. Redistributions of source code must retain the above copyright notice,
11* this list of conditions and the following disclaimer.
12* 2. Redistributions in binary form must reproduce the above copyright notice,
13* this list of conditions and the following disclaimer in the documentation
14* and/or other materials provided with the distribution.
15*
16* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
20* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26* POSSIBILITY OF SUCH DAMAGE.
27 */
28
29/**
30* Simple high-level balanced Apache Kafka consumer
31* using the Kafka driver from librdkafka
32* (https://github.com/edenhill/librdkafka)
33 */
34
35#include <stdio.h>
36#include <signal.h>
37#include <string.h>
38#include <ctype.h>
39
40
41/* Typical include path would be <librdkafka/rdkafka.h>, but this program
42* is builtin from within the librdkafka source tree and thus differs. */
43 //#include <librdkafka/rdkafka.h>
44 #include "librdkafka/rdkafka.h"
45
46
47static volatile sig_atomic_t run = 1;
48
49/**
50* @brief Signal termination of program
51 */
52 static void stop(int sig) {
53 run = 0;
54 }
55
56
57
58/**
59* @returns 1 if all bytes are printable, else 0.
60 */
61 static int is_printable(const char *buf, size_t size) {
62 size_t i;
63
64 for (i = 0; i < size; i++) {
65 if (!isprint((int)buf[i])) {
66 return 0;
67 }
68 }
69
70 return 1;
71 }
72
73
74int main(int argc, char **argv) {
75rd_kafka_t *rk; /* Consumer instance handle */
76rd_kafka_conf_t *conf; /* Temporary configuration object */
77rd_kafka_resp_err_t err; /* librdkafka API error code */
78
79 char errstr[512]; /* librdkafka API error reporting buffer */
80
81 const char *brokers; /* Argument: broker list */
82 const char *groupid; /* Argument: Consumer group id */
83 const char *username; /* Argument: sasl username */
84 const char *password; /* Argument: sasl password */
85 char **topics; /* Argument: list of topics to subscribe to */
86
87 int topic_cnt; /* Number of topics to subscribe to */
88 rd_kafka_topic_partition_list_t *subscription; /* Subscribed topics */
89 int i;
90
91 /*
92 * Argument validation
93 */
94 if (argc < 6) {
95 fprintf(stderr, "%% Usage: %s <broker> <group.id> <username> <password> <topic1> <topic2>..\n", argv[0]);
96 return 1;
97 }
98
99 brokers = argv[1];
100 groupid = argv[2];
101 username = argv[3];
102 password = argv[4];
103 topics = &argv[5];
104
105 topic_cnt = argc - 5;
106
107
108 /*
109 * Create Kafka client configuration place-holder
110 */
111 conf = rd_kafka_conf_new();
112
113 /* Set bootstrap broker(s) as a comma-separated list of
114 * host or host:port (default port 9092).
115 * librdkafka will use the bootstrap brokers to acquire the full
116 * set of brokers from the cluster. */
117 if (rd_kafka_conf_set(conf, "bootstrap.servers", brokers, errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK) {
118 fprintf(stderr, "%s\n", errstr);
119 rd_kafka_conf_destroy(conf);
120 return 1;
121 }
122 // 认证机制支持PLAIN、SCRAM-SHA-512两种机制,根据集群所使用的认证方式进行选择
123 if (
124 rd_kafka_conf_set(conf, "ssl.ca.location", "ca.pem", errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK
125 || rd_kafka_conf_set(conf, "security.protocol", "sasl_ssl", errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK
126 || rd_kafka_conf_set(conf, "sasl.mechanism", "SCRAM-SHA-512", errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK
127 || rd_kafka_conf_set(conf, "sasl.username", username, errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK
128 || rd_kafka_conf_set(conf, "sasl.password", password, errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK
129 ) {
130 fprintf(stderr, "%s\n", errstr);
131 return -1;
132 }
133
134 /* Set the consumer group id.
135 * All consumers sharing the same group id will join the same
136 * group, and the subscribed topic' partitions will be assigned
137 * according to the partition.assignment.strategy
138 * (consumer config property) to the consumers in the group. */
139 if (rd_kafka_conf_set(conf, "group.id", groupid, errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK) {
140 fprintf(stderr, "%s\n", errstr);
141 rd_kafka_conf_destroy(conf);
142 return 1;
143 }
144
145 /* If there is no previously committed offset for a partition
146 * the auto.offset.reset strategy will be used to decide where
147 * in the partition to start fetching messages.
148 * By setting this to earliest the consumer will read all messages
149 * in the partition if there was no previously committed offset. */
150 if (rd_kafka_conf_set(conf, "auto.offset.reset", "earliest", errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK) {
151 fprintf(stderr, "%s\n", errstr);
152 rd_kafka_conf_destroy(conf);
153 return 1;
154 }
155
156 /*
157 * Create consumer instance.
158 *
159 * NOTE: rd_kafka_new() takes ownership of the conf object
160 * and the application must not reference it again after
161 * this call.
162 */
163 rk = rd_kafka_new(RD_KAFKA_CONSUMER, conf, errstr, sizeof(errstr));
164 if (!rk) {
165 fprintf(stderr, "%% Failed to create new consumer: %s\n", errstr);
166 return 1;
167 }
168
169 conf = NULL; /* Configuration object is now owned, and freed,
170 * by the rd_kafka_t instance. */
171
172
173 /* Redirect all messages from per-partition queues to
174 * the main queue so that messages can be consumed with one
175 * call from all assigned partitions.
176 *
177 * The alternative is to poll the main queue (for events)
178 * and each partition queue separately, which requires setting
179 * up a rebalance callback and keeping track of the assignment:
180 * but that is more complex and typically not recommended. */
181 rd_kafka_poll_set_consumer(rk);
182
183
184 /* Convert the list of topics to a format suitable for librdkafka */
185 subscription = rd_kafka_topic_partition_list_new(topic_cnt);
186 for (i = 0; i < topic_cnt; i++) {
187 rd_kafka_topic_partition_list_add(
188 subscription,
189 topics[i],
190 /* the partition is ignored
191 * by subscribe() */
192 RD_KAFKA_PARTITION_UA
193 );
194 }
195
196 /* Subscribe to the list of topics */
197 err = rd_kafka_subscribe(rk, subscription);
198 if (err) {
199 fprintf(stderr, "%% Failed to subscribe to %d topics: %s\n", subscription->cnt, rd_kafka_err2str(err));
200 rd_kafka_topic_partition_list_destroy(subscription);
201 rd_kafka_destroy(rk);
202 return 1;
203 }
204
205 fprintf(stderr,
206 "%% Subscribed to %d topic(s), "
207 "waiting for rebalance and messages...\n",
208 subscription->cnt);
209
210 rd_kafka_topic_partition_list_destroy(subscription);
211
212
213 /* Signal handler for clean shutdown */
214 signal(SIGINT, stop);
215
216 /* Subscribing to topics will trigger a group rebalance
217 * which may take some time to finish, but there is no need
218 * for the application to handle this idle period in a special way
219 * since a rebalance may happen at any time.
220 * Start polling for messages. */
221
222 while (run) {
223 rd_kafka_message_t *rkm;
224
225 rkm = rd_kafka_consumer_poll(rk, 100);
226 if (!rkm) {
227 continue; /* Timeout: no message within 100ms,
228 * try again. This short timeout allows
229 * checking for `run` at frequent intervals.
230 */
231 }
232
233 /* consumer_poll() will return either a proper message
234 * or a consumer error (rkm->err is set). */
235 if (rkm->err) {
236 /* Consumer errors are generally to be considered
237 * informational as the consumer will automatically
238 * try to recover from all types of errors. */
239 fprintf(stderr, "%% Consumer error: %s\n", rd_kafka_message_errstr(rkm));
240 rd_kafka_message_destroy(rkm);
241 continue;
242 }
243
244 /* Proper message. */
245 printf("Message on %s [%" PRId32 "] at offset %" PRId64 ":\n",
246 rd_kafka_topic_name(rkm->rkt), rkm->partition,
247 rkm->offset);
248
249 /* Print the message key. */
250 if (rkm->key && is_printable(rkm->key, rkm->key_len)) {
251 printf(" Key: %.*s\n", (int)rkm->key_len, (const char *)rkm->key);
252 } else if (rkm->key) {
253 printf(" Key: (%d bytes)\n", (int)rkm->key_len);
254 }
255
256 /* Print the message value/payload. */
257 if (rkm->payload && is_printable(rkm->payload, rkm->len)) {
258 printf(" Value: %.*s\n", (int)rkm->len, (const char *)rkm->payload);
259 } else if (rkm->payload) {
260 printf(" Value: (%d bytes)\n", (int)rkm->len);
261 }
262
263 rd_kafka_message_destroy(rkm);
264 }
265
266
267 /* Close the consumer: commit final offsets and leave the group. */
268 fprintf(stderr, "%% Closing consumer\n");
269 rd_kafka_consumer_close(rk);
270
271
272 /* Destroy the consumer */
273 rd_kafka_destroy(rk);
274
275 return 0;
276}
步骤四:编译并运行
编译并运行上述两个代码文件。
Bash
1# 启动消费者
2gcc -lrdkafka ./consumer.c -o consumer
3./consumer <broker> <group.id> <username> <password> <topic1> <topic2>..
4# 启动生产者
5gcc -lrdkafka ./producer.c -o producer
6./producer <broker> <topic> <username> <password>
步骤五:查看集群监控
查看消息是否发送成功或消费成功有两种方式:
- 在服务器端/控制台查看日志。
- 在专享版消息服务 for Kafka控制台查看集群监控,获取集群生产、消息情况。
推荐使用第二种方式,下面介绍如何查看集群监控。
(1)在专享版消息服务 for Kafka的控制台页面找到需要连接的集群,点击集群名称进入『集群详情』页面。
(2)页面跳转后,进入左侧边中的『集群详情』页面。
(3)点击左侧边栏中的『集群监控』,进入『集群监控』页面。
(4)通过查看『集群监控』页面,提供的不同纬度的监控信息(集群监控、节点监控、主题监控、消费组监控),即可获知集群的生产和消费情况。
集群监控的具体使用请参考:集群监控