Autotest
/
sharding
mirror of https://gitee.com/dotnetchina/sharding-core.git
1
0
Fork 0
Code Issues Releases Wiki Activity

优化group

This commit is contained in:
xuejiaming 2022-08-09 13:36:00 +08:00
parent a1cc753cd7
commit b1da6ee5b5
8 changed files with 87 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/ShardingCore/Sharding/Enumerators/StreamMergeAsync/InMemoryGroupByOrderStreamMergeAsyncEnumerator.cs
View File

@ -44,7 +44,7 @@ namespace ShardingCore.Sharding.Enumerators.StreamMergeAsync
list.Add(streamMergeAsyncEnumerator.GetCurrent());
_inMemoryReallyCount++;
}
return list.AsQueryable().OrderWithExpression(_streamMergeContext.Orders).GetEnumerator();
return list.AsQueryable().OrderWithExpression(_streamMergeContext.GroupByContext.PropertyOrders).GetEnumerator();
}
private IEnumerator<T> GetAllRows(IStreamMergeAsyncEnumerator<T> streamMergeAsyncEnumerator)
{
@ -60,7 +60,7 @@ namespace ShardingCore.Sharding.Enumerators.StreamMergeAsync
_inMemoryReallyCount++;
}
return list.AsQueryable().OrderWithExpression(_streamMergeContext.Orders).GetEnumerator();
return list.AsQueryable().OrderWithExpression(_streamMergeContext.GroupByContext.PropertyOrders).GetEnumerator();
}
public bool SkipFirst()

6
src/ShardingCore/Sharding/MergeContexts/GroupByContext.cs
View File

@ -1,3 +1,4 @@
using System.Collections.Generic;
using System.Linq.Expressions;
namespace ShardingCore.Sharding.MergeContexts
@ -18,6 +19,11 @@ namespace ShardingCore.Sharding.MergeContexts
/// 是否内存聚合
/// </summary>
public bool GroupMemoryMerge { get; set; }
public List<PropertyOrder> PropertyOrders { get; } = new List<PropertyOrder>();
public string GetOrderExpression()
{
return string.Join(",", PropertyOrders);
}
}
}

48
src/ShardingCore/Sharding/MergeContexts/QueryableRewriteEngine.cs
View File

@ -104,27 +104,39 @@ namespace ShardingCore.Sharding.MergeContexts
}
else
{
var groupKeys = selectGroupKeyProperties.Select(o=>o.PropertyName).ToHashSet();
var groupKeys = selectGroupKeyProperties.Select(o => o.PropertyName).ToHashSet();
bool groupMemoryMerge = false;
foreach (var propertyOrder in orders)
{
if (groupKeys.IsEmpty())
groupByContext.PropertyOrders.Add(propertyOrder);
if (!groupMemoryMerge && !groupKeys.IsEmpty())
{
break;
if (!groupKeys.Contains(propertyOrder.PropertyExpression))
{
groupMemoryMerge = true;
}
groupKeys.Remove(propertyOrder.PropertyExpression);
}
if (!groupKeys.Contains(propertyOrder.PropertyExpression))
{
groupMemoryMerge = true;
break;
}
groupKeys.Remove(propertyOrder.PropertyExpression);
}
//判断是否优先group key排序如果不是就是要内存聚合
groupByContext.GroupMemoryMerge = groupMemoryMerge;
var sort = string.Join(",", selectGroupKeyProperties.Select(o => $"{o.PropertyName} asc"));
reWriteQueryable = reWriteQueryable.RemoveAnyOrderBy().OrderWithExpression(sort, null);
if (groupByContext.GroupMemoryMerge)
{
if (groupByContext.GroupMemoryMerge)
{
var sort = string.Join(",", selectGroupKeyProperties.Select(o => $"{o.PropertyName} asc"));
reWriteQueryable = reWriteQueryable.RemoveAnyOrderBy().OrderWithExpression(sort, null);
}
orders.Clear();
foreach (var orderProperty in selectGroupKeyProperties)
{
orders.AddLast(new PropertyOrder(orderProperty.PropertyName, true,
orderProperty.OwnerType));
}
}
}
// else if (!mergeQueryCompilerContext.UseUnionAllMerge())
@ -188,11 +200,13 @@ namespace ShardingCore.Sharding.MergeContexts
//}
}
if (mergeQueryCompilerContext.UseUnionAllMerge() &
!mergeQueryCompilerContext.GetShardingDbContext().SupportUnionAllMerge())
if (mergeQueryCompilerContext.UseUnionAllMerge())
{
throw new ShardingCoreException(
$"if use {nameof(EntityFrameworkShardingQueryableExtension.UseUnionAllMerge)} plz rewrite {nameof(IQuerySqlGeneratorFactory)} with {nameof(IUnionAllMergeQuerySqlGeneratorFactory)} and {nameof(IQueryCompiler)} with {nameof(IUnionAllMergeQueryCompiler)}");
if (!mergeQueryCompilerContext.GetShardingDbContext().SupportUnionAllMerge())
{
throw new ShardingCoreException(
$"if use {nameof(EntityFrameworkShardingQueryableExtension.UseUnionAllMerge)} plz rewrite {nameof(IQuerySqlGeneratorFactory)} with {nameof(IUnionAllMergeQuerySqlGeneratorFactory)} and {nameof(IQueryCompiler)} with {nameof(IUnionAllMergeQueryCompiler)}");
}
}
return new RewriteResult(combineQueryable, reWriteQueryable);

23
src/ShardingCore/Sharding/MergeEngines/Executors/ShardingMergers/AbstractEnumerableShardingMerger.cs
View File

@ -23,6 +23,20 @@ namespace ShardingCore.Sharding.MergeEngines.Executors.ShardingMergers
}
public virtual IStreamMergeAsyncEnumerator<TEntity> StreamMerge(List<IStreamMergeAsyncEnumerator<TEntity>> parallelResults)
{
//如果是group in memory merger需要在内存中聚合好所有的 并且最后通过内存聚合在发挥
if (GetStreamMergeContext().GroupQueryMemoryMerge())
{
var multiAggregateOrderStreamMergeAsyncEnumerator = new MultiAggregateOrderStreamMergeAsyncEnumerator<TEntity>(_streamMergeContext, parallelResults);
//内存按key聚合好之后需要进行重排序按order
var inMemoryGroupByOrderStreamMergeAsyncEnumerator = new InMemoryGroupByOrderStreamMergeAsyncEnumerator<TEntity>(_streamMergeContext,multiAggregateOrderStreamMergeAsyncEnumerator, _async);
if (_streamMergeContext.IsPaginationQuery())
{
//分页的前提下还需要进行内存分页
return new PaginationStreamMergeAsyncEnumerator<TEntity>(_streamMergeContext,new[]{inMemoryGroupByOrderStreamMergeAsyncEnumerator});
}
return inMemoryGroupByOrderStreamMergeAsyncEnumerator;
}
if (_streamMergeContext.IsPaginationQuery())
return new PaginationStreamMergeAsyncEnumerator<TEntity>(_streamMergeContext, parallelResults);
if (_streamMergeContext.HasGroupQuery())
@ -32,6 +46,15 @@ namespace ShardingCore.Sharding.MergeEngines.Executors.ShardingMergers
protected virtual IStreamMergeAsyncEnumerator<TEntity> StreamInMemoryMerge(List<IStreamMergeAsyncEnumerator<TEntity>> parallelResults)
{
//如果是group in memory merger需要在内存中聚合好所有的 并且最后通过内存聚合在发挥
if (GetStreamMergeContext().GroupQueryMemoryMerge())
{
return new MultiAggregateOrderStreamMergeAsyncEnumerator<TEntity>(_streamMergeContext, parallelResults);
}
if (GetStreamMergeContext().IsPaginationQuery())
{
return new PaginationStreamMergeAsyncEnumerator<TEntity>(GetStreamMergeContext(), parallelResults, 0, GetStreamMergeContext().GetPaginationReWriteTake());//内存聚合分页不可以直接获取skip必须获取skip+take的数目
}
return StreamMerge(parallelResults);
}

10
src/ShardingCore/Sharding/MergeEngines/Executors/ShardingMergers/DefaultEnumerableShardingMerger.cs
View File

@ -9,13 +9,5 @@ namespace ShardingCore.Sharding.MergeEngines.Executors.ShardingMergers
public DefaultEnumerableShardingMerger(StreamMergeContext streamMergeContext,bool async) : base(streamMergeContext,async)
{
}
protected override IStreamMergeAsyncEnumerator<TEntity> StreamInMemoryMerge(List<IStreamMergeAsyncEnumerator<TEntity>> parallelResults)
{
if (GetStreamMergeContext().IsPaginationQuery())
return new PaginationStreamMergeAsyncEnumerator<TEntity>(GetStreamMergeContext(), parallelResults, 0, GetStreamMergeContext().GetPaginationReWriteTake());//内存聚合分页不可以直接获取skip必须获取skip+take的数目
return base.StreamInMemoryMerge(parallelResults);
}
}
}
}

36
src/ShardingCore/Sharding/MergeEngines/Executors/ShardingMergers/ReverseEnumerableShardingMerger.cs
View File

@ -11,23 +11,23 @@ namespace ShardingCore.Sharding.MergeEngines.Executors.ShardingMergers
{
}
protected override IStreamMergeAsyncEnumerator<TEntity> StreamInMemoryMerge(
List<IStreamMergeAsyncEnumerator<TEntity>> parallelResults)
{
if (GetStreamMergeContext().IsPaginationQuery() && GetStreamMergeContext().HasGroupQuery())
{
var multiAggregateOrderStreamMergeAsyncEnumerator =
new MultiAggregateOrderStreamMergeAsyncEnumerator<TEntity>(GetStreamMergeContext(),
parallelResults);
return new PaginationStreamMergeAsyncEnumerator<TEntity>(GetStreamMergeContext(),
new[] { multiAggregateOrderStreamMergeAsyncEnumerator }, 0,
GetStreamMergeContext().GetPaginationReWriteTake());
}
if (GetStreamMergeContext().IsPaginationQuery())
return new PaginationStreamMergeAsyncEnumerator<TEntity>(GetStreamMergeContext(), parallelResults, 0,
GetStreamMergeContext().GetPaginationReWriteTake());
return base.StreamInMemoryMerge(parallelResults);
}
// protected override IStreamMergeAsyncEnumerator<TEntity> StreamInMemoryMerge(
// List<IStreamMergeAsyncEnumerator<TEntity>> parallelResults)
// {
// // if (GetStreamMergeContext().IsPaginationQuery() && GetStreamMergeContext().HasGroupQuery())
// // {
// // var multiAggregateOrderStreamMergeAsyncEnumerator =
// // new MultiAggregateOrderStreamMergeAsyncEnumerator<TEntity>(GetStreamMergeContext(),
// // parallelResults);
// // return new PaginationStreamMergeAsyncEnumerator<TEntity>(GetStreamMergeContext(),
// // new[] { multiAggregateOrderStreamMergeAsyncEnumerator }, 0,
// // GetStreamMergeContext().GetPaginationReWriteTake());
// // }
//
// if (GetStreamMergeContext().IsPaginationQuery())
// return new PaginationStreamMergeAsyncEnumerator<TEntity>(GetStreamMergeContext(), parallelResults, 0,
// GetStreamMergeContext().GetPaginationReWriteTake());
// return base.StreamInMemoryMerge(parallelResults);
// }
}
}

5
src/ShardingCore/Sharding/MergeEngines/Executors/ShardingMergers/SequenceEnumerableShardingMerger.cs
View File

@ -22,5 +22,10 @@ namespace ShardingCore.Sharding.MergeEngines.Executors.ShardingMergers
return new MultiOrderStreamMergeAsyncEnumerator<TEntity>(GetStreamMergeContext(), parallelResults);
}
protected override IStreamMergeAsyncEnumerator<TEntity> StreamInMemoryMerge(List<IStreamMergeAsyncEnumerator<TEntity>> parallelResults)
{
return StreamMerge(parallelResults);
}
}
}

2
src/ShardingCore/Sharding/StreamMergeContext.cs
View File

@ -198,7 +198,7 @@ namespace ShardingCore.Sharding
/// <returns></returns>
public bool GroupQueryMemoryMerge()
{
return this.GroupByContext.GroupMemoryMerge;
return HasGroupQuery()&&this.GroupByContext.GroupMemoryMerge;
}
public bool IsMergeQuery()