using System.Net; using System.Text; using System.Text.Json; using Confluent.Kafka; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection.Extensions; using TrackCenter.Application.Helpers; using TrackCenter.Application.Services.Warning.Signals; using JsonSerializer = System.Text.Json.JsonSerializer; namespace TrackCenter.Application.Kafka; public static class WarningCalcServiceExtension { public static void AddWarningCalcServices(this IServiceCollection services, IConfiguration conf) { // Kafka 已按集群隔离,主题/消费组不再拼 Environment 后缀 // 预警信号批处理后台任务 services.AddHostedService(); // 预警信号 RegisterConsumer(services, conf, GlobalConst.WarningKafkaGroup, new[] { GlobalConst.WarningKafkaTopic }); RegisterProducer(services, conf, GlobalConst.WarningKafkaTopic); // 预警信号DLX RegisterConsumer(services, conf, GlobalConst.WarningDLXKafkaGroup, new[] { GlobalConst.WarningDLXKafkaTopic }); RegisterProducer(services, conf, GlobalConst.WarningDLXKafkaTopic); } private static void RegisterConsumer(IServiceCollection services, IConfiguration conf, string groupId, string[] subTopic) { services.AddJsonSerializer(); services.AddJsonSerializer(); services.AddSingleton>(sp => { DebounceHelper.DebounceWrapper? reportFuncWrapper = null; var consumerConfig = new ConsumerConfig { ClientId = Dns.GetHostName(), GroupId = groupId, BootstrapServers = conf.GetValue("Kafka:BootstrapServers"), AutoOffsetReset = AutoOffsetReset.Earliest, //偏移提交控制 EnablePartitionEof = false, EnableAutoOffsetStore = false, EnableAutoCommit = false, //链接控制 SessionTimeoutMs = conf.GetValue("Kafka:Consumer:SessionTimeoutMs"), MaxPollIntervalMs = conf.GetValue("Kafka:Consumer:MaxPollIntervalMs"), HeartbeatIntervalMs = conf.GetValue("Kafka:Consumer:HeartbeatIntervalMs"), //吞吐量 FetchMinBytes = conf.GetValue("Kafka:Consumer:FetchMinBytes"), FetchMaxBytes = conf.GetValue("Kafka:Consumer:FetchMaxBytes"), FetchWaitMaxMs = conf.GetValue("Kafka:Consumer:FetchWaitMaxMs"), CancellationDelayMaxMs = 100, // Debug = "all", // LogConnectionClose = false, // LogQueue = true }; ApplyOptionalSasl(consumerConfig, conf); var bb = new ConsumerBuilder(consumerConfig) .SetKeyDeserializer(sp.GetRequiredService>()) .SetValueDeserializer(sp.GetRequiredService>()) .SetErrorHandler((_, error) => { if (error.IsFatal) { Log.Error($"Kafka致命错误${error}"); return; } // tips: 防抖以防止高频上报错误 reportFuncWrapper ??= DebounceHelper.WrapMethod(TimeSpan.FromSeconds(5), () => { if (error.Code == ErrorCode.Local_Transport) Log.Error($"Kafka链接错误{error}"); else if (error.Code == ErrorCode.Local_AllBrokersDown) Log.Error($"Kafka所有Broker都挂了{error}"); }); reportFuncWrapper.Debounce(); }); var c = bb.Build(); c.Subscribe(subTopic); return c; }); } private static void RegisterProducer(IServiceCollection services, IConfiguration conf, string? defaultTopic = null) { services.AddJsonSerializer(); services.AddJsonSerializer(); services.AddSingleton>(sp => { DebounceHelper.DebounceWrapper? reportFuncWrapper = null; var producerConfig = new ProducerConfig { ClientId = Dns.GetHostName(), BootstrapServers = conf.GetValue("Kafka:BootstrapServers"), // 确保消息发送可靠 Acks = Acks.All, EnableIdempotence = true, MessageSendMaxRetries = conf.GetValue("Kafka:Producer:MessageSendMaxRetries"), RequestTimeoutMs = conf.GetValue("Kafka:Producer:RequestTimeoutMs"), // 消息体大小 CompressionType = conf.GetValue("Kafka:Producer:CompressionType"), CompressionLevel = conf.GetValue("Kafka:Producer:CompressionLevel"), // 消息发送速率控制(任意一条件先达到就发送) LingerMs = conf.GetValue("Kafka:Producer:LingerMs"), BatchSize = conf.GetValue("Kafka:Producer:BatchSize"), BatchNumMessages = conf.GetValue("Kafka:Producer:BatchNumMessages"), // 发送缓冲内存限制 QueueBufferingMaxMessages = conf.GetValue("Kafka:Producer:QueueBufferingMaxMessages"), QueueBufferingMaxKbytes = conf.GetValue("Kafka:Producer:QueueBufferingMaxKbytes"), // 分区器 一致性hash/随机 Partitioner = conf.GetValue("Kafka:Producer:Partitioner"), // Debug = "all", // LogConnectionClose = false, // LogQueue = true }; ApplyOptionalSasl(producerConfig, conf); var bb = new ProducerBuilder(producerConfig) .SetKeySerializer(sp.GetRequiredService>()) .SetValueSerializer(sp.GetRequiredService>()) .SetErrorHandler((_, error) => { if (error.IsFatal) { Log.Error($"Kafka致命错误${error}"); return; } // tips: 防抖以防止高频上报错误 reportFuncWrapper ??= DebounceHelper.WrapMethod(TimeSpan.FromSeconds(5), () => { if (error.Code == ErrorCode.Local_Transport) Log.Error($"Kafka链接错误{error}"); else if (error.Code == ErrorCode.Local_AllBrokersDown) Log.Error($"Kafka所有Broker都挂了{error}"); }); reportFuncWrapper.Debounce(); }); return bb.Build(); }); } /// /// 本地连外口 kafka-ext(SASL)时配置;集群内 PLAINTEXT 不配 SaslUsername 则跳过。 /// private static void ApplyOptionalSasl(ClientConfig cfg, IConfiguration conf) { var user = conf.GetValue("Kafka:SaslUsername"); if (string.IsNullOrWhiteSpace(user)) return; cfg.SecurityProtocol = conf.GetValue("Kafka:SecurityProtocol", SecurityProtocol.SaslPlaintext); cfg.SaslMechanism = conf.GetValue("Kafka:SaslMechanism", SaslMechanism.Plain); cfg.SaslUsername = user; cfg.SaslPassword = conf.GetValue("Kafka:SaslPassword"); } private static void AddJsonSerializer(this IServiceCollection services, Action? setup = null) { services.TryAddSingleton(_ => { var o = new JsonSerializerOptions(); setup?.Invoke(o); return new JsonCoreSerializer(o); }); } public static List> ConsumeBatch(this IConsumer consumer, int batchSize, TimeSpan timeout) { using var cts = new CancellationTokenSource(timeout); var list = new List>(); while (list.Count < batchSize && !cts.IsCancellationRequested) { var consumeResult = consumer.Consume(100); if (consumeResult != null) list.Add(consumeResult); } return list; } } internal sealed class JsonCoreSerializer : ISerializer, IDeserializer { private readonly Encoding _encoding; private readonly JsonSerializerOptions _options; public JsonCoreSerializer(JsonSerializerOptions options) { _options = options ?? throw new ArgumentNullException(nameof(options)); _encoding = Encoding.UTF8; } public byte[] Serialize(T data, SerializationContext context) { if (data is null) { return null; } var json = JsonSerializer.Serialize(data, data.GetType(), _options); var result = _encoding.GetBytes(json); return result; } public T Deserialize(ReadOnlySpan data, bool isNull, SerializationContext context) { if (isNull) { return default; } #if NETSTANDARD2_0 var json = _encoding.GetString(data.ToArray()); #else var json = _encoding.GetString(data); #endif var result = JsonSerializer.Deserialize(json, _options); return result; } }