change:控制es safely limit-10000

This commit is contained in:
xiaolipro 2024-08-13 10:12:14 +08:00
parent b4ae584e4d
commit 8f032a28ed
4 changed files with 14 additions and 37 deletions

View File

@ -28,23 +28,7 @@ public partial class TrackService
var res = new List<GetTrackOutput>(); var res = new List<GetTrackOutput>();
input.OrderNos = input.OrderNos.Where(x => !x.IsNullOrWhiteSpace()).Distinct().ToList(); // 单号去重、去空 input.OrderNos = input.OrderNos.Where(x => !x.IsNullOrWhiteSpace()).Distinct().ToList(); // 单号去重、去空
object[]? searchAfter = null; var details = await _trackDetailsRepository.GetListAsync(input.OrderType, input.OrderNos);
List<TrackDetailsIndex> details = new();
int limit = 1000, cnt, time = 10;
do
{
var (currentResults, nextSearchAfter) =
await _trackDetailsRepository.GetListAsync(input.OrderType, input.OrderNos, searchAfter, limit);
cnt = currentResults.Count;
details.AddRange(currentResults);
// 更新 searchAfter 以获取下一批数据
searchAfter = nextSearchAfter;
time--;
} while (cnt > 0 && time > 0);
if (details.Count == limit * time) throw new BusinessException(message: "查询轨迹量过大,请缩小查询范围");
foreach (var group in details.GroupBy(x => x.OrderId)) foreach (var group in details.GroupBy(x => x.OrderId))
{ {

View File

@ -9,8 +9,7 @@ public interface ITrackDetailsRepository : IElasticsearchClientService<TrackDeta
Task<List<TrackDetailsIndex>> GetLatestListAsync(int[] eventOrderIds, List<string> trackCodes); Task<List<TrackDetailsIndex>> GetLatestListAsync(int[] eventOrderIds, List<string> trackCodes);
Task<(List<TrackDetailsIndex> results, object[]? nextSearchAfter)> GetListAsync(int orderType, Task<List<TrackDetailsIndex>> GetListAsync(int orderType, List<string> orderNos);
List<string> orderNos, object[]? searchAfter = null, int limit = 1000);
Task<bool> AnyAsync(int orderId, string originTrackDataHash); Task<bool> AnyAsync(int orderId, string originTrackDataHash);

View File

@ -5,6 +5,7 @@ public partial class GlobalConst
public const string SystemUser = "系统"; public const string SystemUser = "系统";
public const string EmptyString = "--"; public const string EmptyString = "--";
public const int ChunkSize = 100; public const int ChunkSize = 100;
public const int SafelyLimit = 10000;
public const float NoneTrackTimeZone = 99999; public const float NoneTrackTimeZone = 99999;
} }

View File

@ -9,6 +9,7 @@ namespace SJZY.Track.Repository.Repositories;
internal class TrackDetailsRepository : ElasticSearchContextAbstract<TrackDetailsIndex>, ITrackDetailsRepository, internal class TrackDetailsRepository : ElasticSearchContextAbstract<TrackDetailsIndex>, ITrackDetailsRepository,
ITransientDependency ITransientDependency
{ {
const int SafelyLimit = GlobalConst.SafelyLimit;
public TrackDetailsRepository(ConnectionFactory connectionFactory) : base(connectionFactory) public TrackDetailsRepository(ConnectionFactory connectionFactory) : base(connectionFactory)
{ {
} }
@ -19,7 +20,7 @@ internal class TrackDetailsRepository : ElasticSearchContextAbstract<TrackDetail
var r = !forUpsert var r = !forUpsert
? await Context.SearchAsync<TrackDetailsIndex>(s => s ? await Context.SearchAsync<TrackDetailsIndex>(s => s
.Source() .Source()
.Size(1000) .Size(SafelyLimit)
.Query(q => q .Query(q => q
.Bool(b => b .Bool(b => b
.Must( .Must(
@ -36,7 +37,7 @@ internal class TrackDetailsRepository : ElasticSearchContextAbstract<TrackDetail
.Field(f => f.TrackTime) .Field(f => f.TrackTime)
.Field(f => f.TrackPlace) .Field(f => f.TrackPlace)
)) ))
.Size(1000) .Size(SafelyLimit)
.Query(q => q .Query(q => q
.Bool(b => b .Bool(b => b
.Must( .Must(
@ -83,7 +84,7 @@ internal class TrackDetailsRepository : ElasticSearchContextAbstract<TrackDetail
) )
) )
) )
.Size(orderIds.Length) .Size(SafelyLimit)
.Aggregations(a => a .Aggregations(a => a
.Terms("orders", t => t .Terms("orders", t => t
.Field(f => f.OrderId) .Field(f => f.OrderId)
@ -176,10 +177,9 @@ internal class TrackDetailsRepository : ElasticSearchContextAbstract<TrackDetail
} }
} }
public async Task<(List<TrackDetailsIndex> results, object[]? nextSearchAfter)> GetListAsync(int orderType, public async Task<List<TrackDetailsIndex>> GetListAsync(int orderType, List<string> orderNos)
List<string> orderNos, object[]? searchAfter = null, int limit = 1000)
{ {
if (orderNos.Count < 1) return (new List<TrackDetailsIndex>(), null); if (orderNos.Count < 1) return new List<TrackDetailsIndex>();
var r = await Context.SearchAsync<TrackDetailsIndex>(x => x var r = await Context.SearchAsync<TrackDetailsIndex>(x => x
.Source(s => s.Includes(fd => fd .Source(s => s.Includes(fd => fd
@ -202,17 +202,13 @@ internal class TrackDetailsRepository : ElasticSearchContextAbstract<TrackDetail
mu => mu mu => mu
.Term(f => f.OrderType, orderType) .Term(f => f.OrderType, orderType)
) )
// tips:对外不暴露服务商轨迹
.MustNot(mn => mn .MustNot(mn => mn
.Term(t => t.TrackCode.Suffix("keyword"), GlobalConst.EmptyString) .Term(t => t.TrackCode.Suffix("keyword"), GlobalConst.EmptyString)
) )
) )
) )
.Sort(s => s .Size(SafelyLimit)
.Ascending(f => f.TrackTime) // 需要根据某个字段排序
.Ascending(f => f.OrderNo) // 可以添加更多排序字段以确保唯一性
)
.Size(limit)
.SearchAfter(searchAfter)
); );
if (!r.IsValid) if (!r.IsValid)
@ -221,10 +217,7 @@ internal class TrackDetailsRepository : ElasticSearchContextAbstract<TrackDetail
$"轨迹查询失败,orderType:{orderType},orderNos:{orderNos},原因:{r.DebugInformation}。"); $"轨迹查询失败,orderType:{orderType},orderNos:{orderNos},原因:{r.DebugInformation}。");
} }
var results = r.Documents.ToList(); return r.Documents.ToList();
var nextSearchAfter = r.Hits.LastOrDefault()?.Sorts.ToArray();
return (results, nextSearchAfter);
} }
public async Task<List<TrackDetailsIndex>> GetListAsync(int orderId) public async Task<List<TrackDetailsIndex>> GetListAsync(int orderId)
@ -233,7 +226,7 @@ internal class TrackDetailsRepository : ElasticSearchContextAbstract<TrackDetail
.Query(q => .Query(q =>
q.Term(t => t.OrderId, orderId) q.Term(t => t.OrderId, orderId)
) )
.Size(1000) .Size(SafelyLimit)
.Sort(s => s.Field(f => f.ZeroTrackTime, SortOrder.Descending))); .Sort(s => s.Field(f => f.ZeroTrackTime, SortOrder.Descending)));
if (!r.IsValid) if (!r.IsValid)
@ -248,7 +241,7 @@ internal class TrackDetailsRepository : ElasticSearchContextAbstract<TrackDetail
{ {
var search = await Context.SearchAsync<TrackDetailsIndex>(x => x var search = await Context.SearchAsync<TrackDetailsIndex>(x => x
.Source(s => s.Includes(fd => fd.Field(f => f.IndexName))) .Source(s => s.Includes(fd => fd.Field(f => f.IndexName)))
.Size(orderIds.Length * trackCodes.Count * 2) .Size(SafelyLimit)
.Query(q => .Query(q =>
q.Terms(t => t.Field(f => f.OrderId).Terms(orderIds)) q.Terms(t => t.Field(f => f.OrderId).Terms(orderIds))
&& q.Terms(t => t.Field(f => f.TrackCode.Suffix("keyword")).Terms(trackCodes)) && q.Terms(t => t.Field(f => f.TrackCode.Suffix("keyword")).Terms(trackCodes))