ideship/track
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

change：控制es safely limit-10000

Browse Source
This commit is contained in:
xiaolipro 2024-08-13 10:12:14 +08:00
parent b4ae584e4d
commit 8f032a28ed
4 changed files with 14 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
src/SJZY.Track.Application/Services/ProvideServices/TrackService.Query.cs
View File

@ -28,23 +28,7 @@ public partial class TrackService
var res = new List<GetTrackOutput>();
input.OrderNos = input.OrderNos.Where(x => !x.IsNullOrWhiteSpace()).Distinct().ToList(); // 单号去重、去空
object[]? searchAfter = null;
List<TrackDetailsIndex> details = new();
int limit = 1000, cnt, time = 10;
do
{
var (currentResults, nextSearchAfter) =
await _trackDetailsRepository.GetListAsync(input.OrderType, input.OrderNos, searchAfter, limit);
cnt = currentResults.Count;
details.AddRange(currentResults);
// 更新 searchAfter 以获取下一批数据
searchAfter = nextSearchAfter;
time--;
} while (cnt > 0 && time > 0);
if (details.Count == limit * time) throw new BusinessException(message: "查询轨迹量过大，请缩小查询范围");
var details = await _trackDetailsRepository.GetListAsync(input.OrderType, input.OrderNos);
foreach (var group in details.GroupBy(x => x.OrderId))
{

3
src/SJZY.Track.Domain/Aggregates/TrackDetailsAggregate/ITrackDetailsRepository.cs
View File

@ -9,8 +9,7 @@ public interface ITrackDetailsRepository : IElasticsearchClientService<TrackDeta
Task<List<TrackDetailsIndex>> GetLatestListAsync(int[] eventOrderIds, List<string> trackCodes);
Task<(List<TrackDetailsIndex> results, object[]? nextSearchAfter)> GetListAsync(int orderType,
List<string> orderNos, object[]? searchAfter = null, int limit = 1000);
Task<List<TrackDetailsIndex>> GetListAsync(int orderType, List<string> orderNos);
Task<bool> AnyAsync(int orderId, string originTrackDataHash);

1
src/SJZY.Track.Domain/Consts/GlobalConst.cs
View File

@ -5,6 +5,7 @@ public partial class GlobalConst
public const string SystemUser = "系统";
public const string EmptyString = "--";
public const int ChunkSize = 100;
public const int SafelyLimit = 10000;
public const float NoneTrackTimeZone = 99999;
}

29
src/SJZY.Track.Repository/Repositories/TrackDetailsRepository.cs
View File

@ -9,6 +9,7 @@ namespace SJZY.Track.Repository.Repositories;
internal class TrackDetailsRepository : ElasticSearchContextAbstract<TrackDetailsIndex>, ITrackDetailsRepository,
ITransientDependency
{
const int SafelyLimit = GlobalConst.SafelyLimit;
public TrackDetailsRepository(ConnectionFactory connectionFactory) : base(connectionFactory)
{
}
@ -19,7 +20,7 @@ internal class TrackDetailsRepository : ElasticSearchContextAbstract<TrackDetail
var r = !forUpsert
? await Context.SearchAsync<TrackDetailsIndex>(s => s
.Source()
.Size(1000)
.Size(SafelyLimit)
.Query(q => q
.Bool(b => b
.Must(
@ -36,7 +37,7 @@ internal class TrackDetailsRepository : ElasticSearchContextAbstract<TrackDetail
.Field(f => f.TrackTime)
.Field(f => f.TrackPlace)
))
.Size(1000)
.Size(SafelyLimit)
.Query(q => q
.Bool(b => b
.Must(
@ -83,7 +84,7 @@ internal class TrackDetailsRepository : ElasticSearchContextAbstract<TrackDetail
)
)
)
.Size(orderIds.Length)
.Size(SafelyLimit)
.Aggregations(a => a
.Terms("orders", t => t
.Field(f => f.OrderId)
@ -176,10 +177,9 @@ internal class TrackDetailsRepository : ElasticSearchContextAbstract<TrackDetail
}
}
public async Task<(List<TrackDetailsIndex> results, object[]? nextSearchAfter)> GetListAsync(int orderType,
List<string> orderNos, object[]? searchAfter = null, int limit = 1000)
public async Task<List<TrackDetailsIndex>> GetListAsync(int orderType, List<string> orderNos)
{
if (orderNos.Count < 1) return (new List<TrackDetailsIndex>(), null);
if (orderNos.Count < 1) return new List<TrackDetailsIndex>();
var r = await Context.SearchAsync<TrackDetailsIndex>(x => x
.Source(s => s.Includes(fd => fd
@ -202,17 +202,13 @@ internal class TrackDetailsRepository : ElasticSearchContextAbstract<TrackDetail
mu => mu
.Term(f => f.OrderType, orderType)
)
// tips：对外不暴露服务商轨迹
.MustNot(mn => mn
.Term(t => t.TrackCode.Suffix("keyword"), GlobalConst.EmptyString)
)
)
)
.Sort(s => s
.Ascending(f => f.TrackTime) // 需要根据某个字段排序
.Ascending(f => f.OrderNo) // 可以添加更多排序字段以确保唯一性
)
.Size(limit)
.SearchAfter(searchAfter)
.Size(SafelyLimit)
);
if (!r.IsValid)
@ -221,10 +217,7 @@ internal class TrackDetailsRepository : ElasticSearchContextAbstract<TrackDetail
$"轨迹查询失败，orderType：{orderType}，orderNos：{orderNos}，原因：{r.DebugInformation}。");
}
var results = r.Documents.ToList();
var nextSearchAfter = r.Hits.LastOrDefault()?.Sorts.ToArray();
return (results, nextSearchAfter);
return r.Documents.ToList();
}
public async Task<List<TrackDetailsIndex>> GetListAsync(int orderId)
@ -233,7 +226,7 @@ internal class TrackDetailsRepository : ElasticSearchContextAbstract<TrackDetail
.Query(q =>
q.Term(t => t.OrderId, orderId)
)
.Size(1000)
.Size(SafelyLimit)
.Sort(s => s.Field(f => f.ZeroTrackTime, SortOrder.Descending)));
if (!r.IsValid)
@ -248,7 +241,7 @@ internal class TrackDetailsRepository : ElasticSearchContextAbstract<TrackDetail
{
var search = await Context.SearchAsync<TrackDetailsIndex>(x => x
.Source(s => s.Includes(fd => fd.Field(f => f.IndexName)))
.Size(orderIds.Length * trackCodes.Count * 2)
.Size(SafelyLimit)
.Query(q =>
q.Terms(t => t.Field(f => f.OrderId).Terms(orderIds))
&& q.Terms(t => t.Field(f => f.TrackCode.Suffix("keyword")).Terms(trackCodes))