diff --git a/src/SJZY.Track.Application/CAP/EventHandlers/TrackDeleteEventHandler.cs b/src/SJZY.Track.Application/CAP/EventHandlers/TrackDeleteEventHandler.cs index 33d65d4f..f1c7ad81 100644 --- a/src/SJZY.Track.Application/CAP/EventHandlers/TrackDeleteEventHandler.cs +++ b/src/SJZY.Track.Application/CAP/EventHandlers/TrackDeleteEventHandler.cs @@ -375,7 +375,7 @@ public class TrackDeleteEventHandler : ITransientDependency, ICapSubscribe { var scope = _serviceProvider.CreateScope(); var _warningSignalPublisher = scope.ServiceProvider.GetRequiredService(); - _warningSignalPublisher.PublishWarningSignal(WarningSignalType.轨迹删除, delTracks); + _warningSignalPublisher.PublishSignal(WarningSignalType.轨迹删除, delTracks); scope.Dispose(); return; } diff --git a/src/SJZY.Track.Application/CAP/EventHandlers/TrackUpsertEventHandler.cs b/src/SJZY.Track.Application/CAP/EventHandlers/TrackUpsertEventHandler.cs index 65070319..11cd7f91 100644 --- a/src/SJZY.Track.Application/CAP/EventHandlers/TrackUpsertEventHandler.cs +++ b/src/SJZY.Track.Application/CAP/EventHandlers/TrackUpsertEventHandler.cs @@ -522,7 +522,7 @@ public class TrackUpsertEventHandler : ITransientDependency, ICapSubscribe { var scope = _serviceProvider.CreateScope(); var _warningSignalPublisher = scope.ServiceProvider.GetRequiredService(); - _warningSignalPublisher.PublishWarningSignal(WarningSignalType.轨迹新增, tracks); + _warningSignalPublisher.PublishSignal(WarningSignalType.轨迹新增, tracks); scope.Dispose(); return; } @@ -559,7 +559,7 @@ public class TrackUpsertEventHandler : ITransientDependency, ICapSubscribe { var scope = _serviceProvider.CreateScope(); var _warningSignalPublisher = scope.ServiceProvider.GetRequiredService(); - _warningSignalPublisher.PublishWarningSignal(WarningSignalType.轨迹更新, tracks); + _warningSignalPublisher.PublishSignal(WarningSignalType.轨迹更新, tracks); scope.Dispose(); return; } diff --git a/src/SJZY.Track.Application/Kafka/WarningCalcBackgroundService.cs b/src/SJZY.Track.Application/Kafka/WarningCalcBackgroundService.cs index c1f0e04e..9aaaff06 100644 --- a/src/SJZY.Track.Application/Kafka/WarningCalcBackgroundService.cs +++ b/src/SJZY.Track.Application/Kafka/WarningCalcBackgroundService.cs @@ -1,7 +1,5 @@ using System.Diagnostics; using Confluent.Kafka; -using Confluent.Kafka.Core.Consumer; -using Confluent.Kafka.Core.Producer; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Options; using SJZY.Expand.ABP.Core.Logger; @@ -20,23 +18,20 @@ namespace SJZY.Track.Application.Kafka; public sealed class WarningCalcBackgroundService : BackgroundService { private readonly IServiceScopeFactory _serviceScopeFactory; - private readonly IKafkaConsumer _consumer; - private readonly IKafkaProducer _producer; - private readonly IKafkaProducer _dlxProducer; + private readonly IConsumer _consumer; + private readonly WarningSignalPublisher _warningSignalPublisher; private TrackWarningTaskOptions _options; public WarningCalcBackgroundService( IOptionsMonitor optionsMonitor, IServiceScopeFactory serviceScopeFactory, - IKafkaConsumer consumer, - IKafkaProducer producer, - IKafkaProducer dlxProducer + IConsumer consumer, + WarningSignalPublisher warningSignalPublisher ) { _serviceScopeFactory = serviceScopeFactory; _consumer = consumer; - _producer = producer; - _dlxProducer = dlxProducer; + _warningSignalPublisher = warningSignalPublisher; _options = optionsMonitor.CurrentValue; optionsMonitor.OnChange(x => _options = x); } @@ -175,7 +170,7 @@ public sealed class WarningCalcBackgroundService : BackgroundService var tasks = needRetryMetadata.Select(async metadata => { metadata.SetTrackSignalMessageHeader(); - await _producer.ProduceAsync(metadata.OwenMessage, CancellationToken.None); + await _warningSignalPublisher.PublishSignalAsync(metadata.OwenMessage, CancellationToken.None); }).ToArray(); await Task.WhenAll(tasks); @@ -193,7 +188,7 @@ public sealed class WarningCalcBackgroundService : BackgroundService { metadata.SetTrackSignalMessageHeader(); var msg = metadata.OwenMessage; - await _dlxProducer.ProduceAsync(new Message + await _warningSignalPublisher.PublishDlxSignalAsync(new Message { Key = msg.Value.OrderId, Value = msg.Value, @@ -282,6 +277,5 @@ public sealed class WarningCalcBackgroundService : BackgroundService base.Dispose(); _consumer.Close(); _consumer.Dispose(); - _producer.Dispose(); } } diff --git a/src/SJZY.Track.Application/Kafka/WarningCalcServiceExtension.cs b/src/SJZY.Track.Application/Kafka/WarningCalcServiceExtension.cs index 485e6ab8..1d6ec63c 100644 --- a/src/SJZY.Track.Application/Kafka/WarningCalcServiceExtension.cs +++ b/src/SJZY.Track.Application/Kafka/WarningCalcServiceExtension.cs @@ -1,20 +1,13 @@ -//#define ProtoBuf -#define NewtonsoftJson -using System.Globalization; using System.Net; +using System.Text; using System.Text.Json; -using System.Text.Json.Serialization; using Confluent.Kafka; -using Confluent.Kafka.Core.Consumer; -using Confluent.Kafka.Core.Producer; using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.Logging; -using Newtonsoft.Json; -using Newtonsoft.Json.Converters; -using Newtonsoft.Json.Serialization; +using Microsoft.Extensions.DependencyInjection.Extensions; using SJZY.Expand.ABP.Core.Logger; using SJZY.Track.Application.Helpers; using SJZY.Track.Application.Services.Warning.Signals; +using JsonSerializer = System.Text.Json.JsonSerializer; namespace SJZY.Track.Application.Kafka; @@ -37,140 +30,180 @@ public static class WarningCalcServiceExtension private static void RegisterConsumer(IServiceCollection services, IConfiguration conf, string groupId, string[] subTopic) { - services.AddSingleton>(sp => + services.AddJsonSerializer(); + services.AddJsonSerializer(); + services.AddSingleton>(sp => { - var bb = new KafkaConsumerBuilder() - .WithConsumerConfiguration(o => - { - o.WithClientId(Dns.GetHostName()) - .WithGroupId(groupId) - .WithBootstrapServers(conf.GetValue("Kafka:BootstrapServers")) - .WithAutoOffsetReset(AutoOffsetReset.Earliest) - //偏移提交控制 - .WithEnablePartitionEof(false) - .WithEnableAutoOffsetStore(false) - .WithEnableAutoCommit(false) - //链接控制 - .WithSessionTimeoutMs(conf.GetValue("Kafka:Consumer:SessionTimeoutMs")) - .WithMaxPollIntervalMs(conf.GetValue("Kafka:Consumer:MaxPollIntervalMs")) - .WithHeartbeatIntervalMs(conf.GetValue("Kafka:Consumer:HeartbeatIntervalMs")) - //吞吐量 - .WithFetchMinBytes(conf.GetValue("Kafka:Consumer:FetchMinBytes")) - .WithFetchMaxBytes(conf.GetValue("Kafka:Consumer:FetchMaxBytes")) - .WithFetchWaitMaxMs(conf.GetValue("Kafka:Consumer:FetchWaitMaxMs")) - //日志 - .WithDebug("consumer,cgrp,protocol") - .WithTopicSubscriptions(subTopic); - }); - - bb.WithLoggerFactory(sp.GetRequiredService()); DebounceHelper.DebounceWrapper? reportFuncWrapper = null; - bb.WithErrorHandler((consumer, error) => - { - if (error.IsFatal) + var bb = new ConsumerBuilder( + new ConsumerConfig + { + ClientId = Dns.GetHostName(), + GroupId = groupId, + BootstrapServers = conf.GetValue("Kafka:BootstrapServers"), + AutoOffsetReset = AutoOffsetReset.Earliest, + //偏移提交控制 + EnablePartitionEof = false, + EnableAutoOffsetStore = false, + EnableAutoCommit = false, + //链接控制 + SessionTimeoutMs = conf.GetValue("Kafka:Consumer:SessionTimeoutMs"), + MaxPollIntervalMs = conf.GetValue("Kafka:Consumer:MaxPollIntervalMs"), + HeartbeatIntervalMs = conf.GetValue("Kafka:Consumer:HeartbeatIntervalMs"), + //吞吐量 + FetchMinBytes = conf.GetValue("Kafka:Consumer:FetchMinBytes"), + FetchMaxBytes = conf.GetValue("Kafka:Consumer:FetchMaxBytes"), + FetchWaitMaxMs = conf.GetValue("Kafka:Consumer:FetchWaitMaxMs"), + //日志 + Debug = "consumer,cgrp,protocol", + CancellationDelayMaxMs = 100 + }) + .SetKeyDeserializer(sp.GetRequiredService>()) + .SetValueDeserializer(sp.GetRequiredService>()) + .SetErrorHandler((_, error) => { - LogHelper.Error($"Kafka致命错误${error}"); - return; - } + if (error.IsFatal) + { + LogHelper.Error($"Kafka致命错误${error}"); + return; + } - // tips: 防抖以防止高频上报错误 - reportFuncWrapper ??= DebounceHelper.WrapMethod(TimeSpan.FromSeconds(5), () => - { - if (error.Code == ErrorCode.Local_Transport) - LogHelper.Error($"Kafka链接错误{error}"); - else if (error.Code == ErrorCode.Local_AllBrokersDown) - LogHelper.Error($"Kafka所有Broker都挂了{error}"); + // tips: 防抖以防止高频上报错误 + reportFuncWrapper ??= DebounceHelper.WrapMethod(TimeSpan.FromSeconds(5), () => + { + if (error.Code == ErrorCode.Local_Transport) + LogHelper.Error($"Kafka链接错误{error}"); + else if (error.Code == ErrorCode.Local_AllBrokersDown) + LogHelper.Error($"Kafka所有Broker都挂了{error}"); + }); + reportFuncWrapper.Debounce(); }); - reportFuncWrapper.Debounce(); - }); -#if ProtoBuf - bb.WithProtobufNetDeserializer(); -#elif NewtonsoftJson - bb.WithNewtonsoftJsonDeserializer(builder => - builder.WithNullValueHandling(NullValueHandling.Ignore) // Ignores null values - .WithReferenceLoopHandling(ReferenceLoopHandling.Ignore) // Ignores reference loops - .WithContractResolver(new CamelCasePropertyNamesContractResolver()) // Camel case property names - ); -#else - bb.WithJsonCoreDeserializer(bc => - bc.WithReferenceHandler(ReferenceHandler.IgnoreCycles) // Handles circular references - .WithPropertyNamingPolicy(JsonNamingPolicy.CamelCase) // Uses camelCase naming - .WithDefaultIgnoreCondition(JsonIgnoreCondition.WhenWritingNull) - ); -#endif - return (bb.Build() as IKafkaConsumer)!; + var c = bb.Build(); + c.Subscribe(subTopic); + return c; }); } private static void RegisterProducer(IServiceCollection services, IConfiguration conf, string? defaultTopic = null) { - services.AddSingleton>(sp => + services.AddJsonSerializer(); + services.AddJsonSerializer(); + services.AddSingleton>(sp => { - var bb = new KafkaProducerBuilder() - .WithProducerConfiguration(o => - { - o.WithClientId(Dns.GetHostName()) - .WithBootstrapServers(conf.GetValue("Kafka:BootstrapServers")) - // 确保消息发送可靠 - .WithAcks(Acks.All) - .WithEnableIdempotence(true) // 幂等发送 - .WithMessageSendMaxRetries(conf.GetValue("Kafka:Producer:MessageSendMaxRetries")) - .WithRequestTimeoutMs(conf.GetValue("Kafka:Producer:RequestTimeoutMs")) - // 消息体大小 - .WithCompressionType(conf.GetValue("Kafka:Producer:CompressionType")) - .WithCompressionLevel(conf.GetValue("Kafka:Producer:CompressionLevel")) //Gzip 0-9 LZ4 0-12 数值越大压缩效果越好cpu占用越多 - // 消息发送速率控制(任意一条件先达到就发送) - .WithLingerMs(conf.GetValue("Kafka:Producer:LingerMs")) - .WithBatchSize(conf.GetValue("Kafka:Producer:BatchSize")) - .WithBatchNumMessages(conf.GetValue("Kafka:Producer:BatchNumMessages")) - // 发送缓冲内存限制 - .WithQueueBufferingMaxMessages(conf.GetValue("Kafka:Producer:QueueBufferingMaxMessages")) - .WithQueueBufferingMaxKbytes(conf.GetValue("Kafka:Producer:QueueBufferingMaxKbytes")) - // 分区器 一致性hash/随机 - .WithPartitioner(conf.GetValue("Kafka:Producer:Partitioner")) - .WithDefaultTopic(defaultTopic); - }); - - bb.WithLoggerFactory(sp.GetRequiredService()); DebounceHelper.DebounceWrapper? reportFuncWrapper = null; - bb.WithErrorHandler((producer, error) => - { - if (error.IsFatal) + var bb = new ProducerBuilder(new ProducerConfig() { - LogHelper.Error($"Kafka致命错误${error}"); - return; - } + ClientId = Dns.GetHostName(), + BootstrapServers = conf.GetValue("Kafka:BootstrapServers"), + // 确保消息发送可靠 + Acks = Acks.All, + EnableIdempotence = true, + MessageSendMaxRetries = conf.GetValue("Kafka:Producer:MessageSendMaxRetries"), + RequestTimeoutMs = conf.GetValue("Kafka:Producer:RequestTimeoutMs"), + // 消息体大小 + CompressionType = conf.GetValue("Kafka:Producer:CompressionType"), + CompressionLevel = conf.GetValue("Kafka:Producer:CompressionLevel"), + // 消息发送速率控制(任意一条件先达到就发送) + LingerMs = conf.GetValue("Kafka:Producer:LingerMs"), + BatchSize = conf.GetValue("Kafka:Producer:BatchSize"), + BatchNumMessages = conf.GetValue("Kafka:Producer:BatchNumMessages"), + // 发送缓冲内存限制 + QueueBufferingMaxMessages = conf.GetValue("Kafka:Producer:QueueBufferingMaxMessages"), + QueueBufferingMaxKbytes = conf.GetValue("Kafka:Producer:QueueBufferingMaxKbytes"), + // 分区器 一致性hash/随机 + Partitioner = conf.GetValue("Kafka:Producer:Partitioner"), + }) + .SetKeySerializer(sp.GetRequiredService>()) + .SetValueSerializer(sp.GetRequiredService>()) + .SetErrorHandler((_, error) => + { + if (error.IsFatal) + { + LogHelper.Error($"Kafka致命错误${error}"); + return; + } - // tips: 防抖以防止高频上报错误 - reportFuncWrapper ??= DebounceHelper.WrapMethod(TimeSpan.FromSeconds(5), () => - { - if (error.Code == ErrorCode.Local_Transport) - LogHelper.Error($"Kafka链接错误{error}"); - else if (error.Code == ErrorCode.Local_AllBrokersDown) - LogHelper.Error($"Kafka所有Broker都挂了{error}"); + // tips: 防抖以防止高频上报错误 + reportFuncWrapper ??= DebounceHelper.WrapMethod(TimeSpan.FromSeconds(5), () => + { + if (error.Code == ErrorCode.Local_Transport) + LogHelper.Error($"Kafka链接错误{error}"); + else if (error.Code == ErrorCode.Local_AllBrokersDown) + LogHelper.Error($"Kafka所有Broker都挂了{error}"); + }); + reportFuncWrapper.Debounce(); }); - reportFuncWrapper.Debounce(); - }); -#if ProtoBuf - // 2000w条消息数据包压缩从 2g -> 1g - bb.WithProtobufNetSerializer(); -#elif NewtonsoftJson - bb.WithNewtonsoftJsonSerializer(builder => - builder.WithNullValueHandling(NullValueHandling.Ignore) // Ignores null values - .WithReferenceLoopHandling(ReferenceLoopHandling.Ignore) // Ignores reference loops - .WithContractResolver(new CamelCasePropertyNamesContractResolver()) // Camel case property names - ); -#else - bb.WithJsonCoreSerializer(bc => - bc.WithReferenceHandler(ReferenceHandler.IgnoreCycles) // Handles circular references - .WithPropertyNamingPolicy(JsonNamingPolicy.CamelCase) // Uses camelCase naming - .WithDefaultIgnoreCondition(JsonIgnoreCondition.WhenWritingNull) - ); -#endif - return (bb.Build() as IKafkaProducer)!; + return bb.Build(); }); } + + private static void AddJsonSerializer(this IServiceCollection services, Action? setup = null) + { + services.TryAddSingleton(_ => + { + var o = new JsonSerializerOptions(); + setup?.Invoke(o); + return new JsonCoreSerializer(o); + }); + } + + public static List> ConsumeBatch(this IConsumer consumer, int batchSize, TimeSpan timeout) + { + using var cts = new CancellationTokenSource(timeout); + var list = new List>(); + while (list.Count < batchSize && !cts.IsCancellationRequested) + { + var consumeResult = consumer.Consume(100); + if (consumeResult != null) + list.Add(consumeResult); + } + + return list; + } +} + +internal sealed class JsonCoreSerializer : ISerializer, IDeserializer +{ + private readonly Encoding _encoding; + private readonly JsonSerializerOptions _options; + + public JsonCoreSerializer(JsonSerializerOptions options) + { + _options = options ?? throw new ArgumentNullException(nameof(options)); + _encoding = Encoding.UTF8; + } + + public byte[] Serialize(T data, SerializationContext context) + { + if (data is null) + { + return null; + } + + var json = JsonSerializer.Serialize(data, data.GetType(), _options); + + var result = _encoding.GetBytes(json); + + return result; + } + + public T Deserialize(ReadOnlySpan data, bool isNull, SerializationContext context) + { + if (isNull) + { + return default; + } + +#if NETSTANDARD2_0 + var json = _encoding.GetString(data.ToArray()); +#else + var json = _encoding.GetString(data); +#endif + var result = JsonSerializer.Deserialize(json, _options); + + return result; + } } diff --git a/src/SJZY.Track.Application/Kafka/WarningSignalPublisher.cs b/src/SJZY.Track.Application/Kafka/WarningSignalPublisher.cs index 95e0336f..313705af 100644 --- a/src/SJZY.Track.Application/Kafka/WarningSignalPublisher.cs +++ b/src/SJZY.Track.Application/Kafka/WarningSignalPublisher.cs @@ -1,5 +1,5 @@ using Confluent.Kafka; -using Confluent.Kafka.Core.Producer; +using Microsoft.Extensions.Configuration; using SJZY.Track.Application.Services.Warning.Signals; using SJZY.Track.Domain.Aggregates.TrackDetailsAggregate; using Volo.Abp.DependencyInjection; @@ -9,27 +9,48 @@ namespace SJZY.Track.Application.Kafka; /// /// 预警信号发布者 /// -public sealed class WarningSignalPublisher : ITransientDependency +public sealed class WarningSignalPublisher : IDisposable, ISingletonDependency { - private readonly IKafkaProducer _producer; + private readonly IProducer _producer; + private readonly IProducer _dlxProducer; + private readonly string env; - public WarningSignalPublisher(IKafkaProducer producer) + public WarningSignalPublisher(IProducer producer, IProducer dlxProducer, IConfiguration conf) { _producer = producer; + _dlxProducer = dlxProducer; + env = "-" + conf.GetValue("Environment")!.ToLower(); } - public void PublishWarningSignal(WarningSignalType type, List tracks) + public void PublishSignal(WarningSignalType type, List tracks) { foreach (var track in tracks) - PublishWarningSignal(type, track); + PublishSignal(type, track); } - public void PublishWarningSignal(WarningSignalType type, TrackDetailsIndex track) + public void PublishSignal(WarningSignalType type, TrackDetailsIndex track) { - _producer.Produce(new Message + _producer.Produce(GlobalConst.WarningKafkaTopic + env, new Message { Key = new WarningSignalKey(type, track.OrderId), Value = WarningSignal.FromTrack(track, type) }); } + + public Task PublishSignalAsync(Message msg, CancellationToken ct = default) + { + return _producer.ProduceAsync(GlobalConst.WarningKafkaTopic + env, msg, ct); + } + + + public Task PublishDlxSignalAsync(Message message, CancellationToken ct = default) + { + return _dlxProducer.ProduceAsync(GlobalConst.WarningDLXKafkaTopic + env, message, ct); + } + + public void Dispose() + { + _producer.Dispose(); + _dlxProducer.Dispose(); + } } diff --git a/src/SJZY.Track.Application/SJZY.Track.Application.csproj b/src/SJZY.Track.Application/SJZY.Track.Application.csproj index fcb4b392..34159b3e 100644 --- a/src/SJZY.Track.Application/SJZY.Track.Application.csproj +++ b/src/SJZY.Track.Application/SJZY.Track.Application.csproj @@ -9,10 +9,7 @@ - - - - +