性能优化顺序分表下的非迭代器方法支持自定义顺序查询熔断

This commit is contained in:
xuejiaming 2022-01-25 17:41:43 +08:00
parent 6bfcefac67
commit c05a2d8920
14 changed files with 271 additions and 65 deletions

View File

@ -113,7 +113,7 @@ namespace ShardingCore6x
}
[Params(10)]
[Params(1)]
public int N;

View File

@ -1,9 +1,9 @@
:start
::定义版本
set EFCORE2=2.4.1.01-preview1
set EFCORE3=3.4.1.01-preview1
set EFCORE5=5.4.1.01-preview1
set EFCORE6=6.4.1.01-preview1
set EFCORE2=2.4.1.01-rc1
set EFCORE3=3.4.1.01-rc1
set EFCORE5=5.4.1.01-rc1
set EFCORE6=6.4.1.01-rc1
::删除所有bin与obj下的文件
@echo off

View File

@ -221,8 +221,17 @@ namespace Sample.SqlServer.Controllers
[HttpGet]
public async Task<IActionResult> Get4()
{
var xxxaaa = await _defaultTableDbContext.Set<SysUserSalary>().FirstOrDefaultAsync();
Console.WriteLine("----0----");
var xxx = await _defaultTableDbContext.Set<SysUserSalary>().OrderByDescending(o=>o.DateOfMonth).FirstOrDefaultAsync();
return Ok(xxx);
Console.WriteLine("----1----");
var xxx1 = await _defaultTableDbContext.Set<SysUserSalary>().OrderByDescending(o=>o.DateOfMonth).LastOrDefaultAsync();
Console.WriteLine("----2----");
var xxx11 = await _defaultTableDbContext.Set<SysUserSalary>().OrderByDescending(o => o.DateOfMonth).FirstAsync();
Console.WriteLine("----3----");
var xxx21 = await _defaultTableDbContext.Set<SysUserSalary>().OrderByDescending(o => o.DateOfMonth).LastAsync();
Console.WriteLine("----4----");
return Ok(new{ xxx , xxx1});
}
}

View File

@ -8,8 +8,13 @@ namespace Sample.SqlServer.Shardings
{
public void Configure(EntityQueryBuilder<SysUserSalary> builder)
{
builder.AddOrder(o => o.DateOfMonth);
builder.AddConnectionsLimit(2, QueryableMethodNameEnum.First, QueryableMethodNameEnum.FirstOrDefault,QueryableMethodNameEnum.Any);
//当前表示按月分片,月份的排序字符串和int是一样的所以用某人的即可
builder.ShardingTailComparer(Comparer<string>.Default);
//DateOfMonth的排序和月份分片的后缀一致所以用true如果false,无果无关就不需要配置
builder.AddOrder(o => o.DateOfMonth,true);
builder.AddConnectionsLimit(2, QueryableMethodNameEnum.First, QueryableMethodNameEnum.FirstOrDefault,QueryableMethodNameEnum.Any,QueryableMethodNameEnum.LastOrDefault,QueryableMethodNameEnum.Last);
builder.AddDefaultSequenceQueryTrip(false, QueryableMethodNameEnum.FirstOrDefault);
}
}
}

View File

@ -28,7 +28,15 @@ namespace ShardingCore.Core.ShardingConfigurations.ConfigBuilders
{
ShardingCoreConfigBuilder = shardingCoreConfigBuilder;
}
/// <summary>
/// 添加一个分片配置 必填<code>ConfigId</code>和<code>AddDefaultDataSource(string dataSourceName, string connectionString)</code>
/// 如果全局未配置 必须配置<code>UseShardingQuery</code>和<code>UseShardingQuery</code>
/// </summary>
/// <param name="shardingGlobalConfigOptionsConfigure"></param>
/// <returns></returns>
/// <exception cref="ArgumentNullException"></exception>
/// <exception cref="ShardingCoreConfigException"></exception>
/// <exception cref="ArgumentException"></exception>
public ShardingConfigBuilder<TShardingDbContext> AddConfig(Action<ShardingConfigOptions<TShardingDbContext>> shardingGlobalConfigOptionsConfigure)
{
var shardingGlobalConfigOptions = new ShardingConfigOptions<TShardingDbContext>();

View File

@ -14,15 +14,20 @@ namespace ShardingCore.Core.ShardingConfigurations
public class ShardingConfigOptions<TShardingDbContext> where TShardingDbContext : DbContext, IShardingDbContext
{
/// <summary>
/// 配置id
/// 配置id,如果是单配置可以用guid代替,如果是多配置该属性表示每个配置的id
/// </summary>
public string ConfigId { get; set; }
/// <summary>
/// 优先级
/// 优先级多个配置之间的优先级
/// </summary>
public int Priority { get; set; }
/// <summary>
/// 全局配置最大的查询连接数限制,默认系统逻辑处理器<code>Environment.ProcessorCount</code>
/// </summary>
public int MaxQueryConnectionsLimit { get; set; } = Environment.ProcessorCount;
/// <summary>
/// 默认<code>ConnectionModeEnum.SYSTEM_AUTO</code>
/// </summary>
public ConnectionModeEnum ConnectionMode { get; set; } = ConnectionModeEnum.SYSTEM_AUTO;
/// <summary>
/// 读写分离配置
@ -61,10 +66,10 @@ namespace ShardingCore.Core.ShardingConfigurations
/// 添加读写分离配置
/// </summary>
/// <param name="readWriteSeparationConfigure"></param>
/// <param name="readStrategyEnum"></param>
/// <param name="defaultEnable"></param>
/// <param name="defaultPriority"></param>
/// <param name="readConnStringGetStrategy"></param>
/// <param name="readStrategyEnum">随机或者轮询</param>
/// <param name="defaultEnable">false表示哪怕您添加了读写分离也不会进行读写分离查询,只有需要的时候自行开启,true表示默认查询就是走的读写分离</param>
/// <param name="defaultPriority">默认优先级建议大于0</param>
/// <param name="readConnStringGetStrategy">LatestFirstTime:DbContext缓存,LatestEveryTime:每次都是最新</param>
/// <exception cref="ArgumentNullException"></exception>
public void AddReadWriteSeparation(
Func<IServiceProvider, IDictionary<string, IEnumerable<string>>> readWriteSeparationConfigure,
@ -81,21 +86,41 @@ namespace ShardingCore.Core.ShardingConfigurations
ShardingReadWriteSeparationOptions.ReadConnStringGetStrategy= readConnStringGetStrategy;
}
/// <summary>
/// 多个DbContext事务传播委托
/// </summary>
public Action<DbConnection, DbContextOptionsBuilder> ConnectionConfigure { get; private set; }
/// <summary>
/// 初始DbContext的创建委托
/// </summary>
public Action<string, DbContextOptionsBuilder> ConnectionStringConfigure { get; private set; }
/// <summary>
/// 仅内部DbContext生效的配置委托
/// </summary>
public Action<DbContextOptionsBuilder> InnerDbContextConfigure { get; private set; }
/// <summary>
/// 如何使用字符串创建DbContext
/// </summary>
/// <param name="queryConfigure"></param>
/// <exception cref="ArgumentNullException"></exception>
public void UseShardingQuery(Action<string, DbContextOptionsBuilder> queryConfigure)
{
ConnectionStringConfigure = queryConfigure ?? throw new ArgumentNullException(nameof(queryConfigure));
}
/// <summary>
/// 如何传递事务到其他DbContext
/// </summary>
/// <param name="transactionConfigure"></param>
/// <exception cref="ArgumentNullException"></exception>
public void UseShardingTransaction(Action<DbConnection, DbContextOptionsBuilder> transactionConfigure)
{
ConnectionConfigure = transactionConfigure ?? throw new ArgumentNullException(nameof(transactionConfigure));
}
public Action<DbContextOptionsBuilder> InnerDbContextConfigure { get; private set; }
/// <summary>
/// 仅内部DbContext生效,作为最外面的壳DbContext将不会生效
/// </summary>
/// <param name="innerDbContextConfigure"></param>
/// <exception cref="ArgumentNullException"></exception>
public void UseInnerDbContextConfigure(Action<DbContextOptionsBuilder> innerDbContextConfigure)
{
InnerDbContextConfigure= innerDbContextConfigure?? throw new ArgumentNullException(nameof(innerDbContextConfigure));

View File

@ -34,7 +34,13 @@ namespace ShardingCore.Core.ShardingConfigurations
/// 当查询遇到没有路由被命中时是否抛出错误
/// </summary>
public bool ThrowIfQueryRouteNotMatch { get; set; } = true;
/// <summary>
/// 全局启用分表路由表达式缓存,仅缓存单个表达式
/// </summary>
public bool? EnableTableRouteCompileCache { get; set; }
/// <summary>
/// 全局启用分库路由表达式缓存,仅缓存单个表达式
/// </summary>
public bool? EnableDataSourceRouteCompileCache { get; set; }
/// <summary>
/// 忽略建表时的错误
@ -142,32 +148,38 @@ namespace ShardingCore.Core.ShardingConfigurations
return _parallelTables;
}
/// <summary>
/// DbContext如何通过现有connection创建
/// 多个DbContext事务传播委托
/// </summary>
public Action<DbConnection, DbContextOptionsBuilder> ConnectionConfigure { get; private set; }
/// <summary>
/// DbContext如何通过连接字符串创建
/// 初始DbContext的创建委托
/// </summary>
public Action<string, DbContextOptionsBuilder> ConnectionStringConfigure { get; private set; }
/// <summary>
/// 仅内部DbContext生效的配置委托
/// </summary>
public Action<DbContextOptionsBuilder> InnerDbContextConfigure { get; private set; }
/// <summary>
/// DbContext如何通过连接字符串创建
/// 如何使用字符串创建DbContext
/// </summary>
public void UseShardingQuery(Action<string, DbContextOptionsBuilder> queryConfigure)
{
ConnectionStringConfigure = queryConfigure ?? throw new ArgumentNullException(nameof(queryConfigure));
}
/// <summary>
/// DbContext如何通过现有connection创建
/// 如何传递事务到其他DbContext
/// </summary>
public void UseShardingTransaction(Action<DbConnection, DbContextOptionsBuilder> transactionConfigure)
{
ConnectionConfigure = transactionConfigure ?? throw new ArgumentNullException(nameof(transactionConfigure));
}
/// <summary>
/// 仅内部DbContext配置的方法
/// </summary>
/// <param name="innerDbContextConfigure"></param>
/// <exception cref="ArgumentNullException"></exception>
public void UseInnerDbContextConfigure(Action<DbContextOptionsBuilder> innerDbContextConfigure)
{
InnerDbContextConfigure = innerDbContextConfigure ?? throw new ArgumentNullException(nameof(innerDbContextConfigure));

View File

@ -22,6 +22,9 @@ namespace ShardingCore.Core.VirtualDatabase.VirtualDataSources.Abstractions
/// 不能小于等于0 should greater than or equal zero
/// </summary>
int MaxQueryConnectionsLimit { get; }
/// <summary>
/// 连接模式,如果没有什么特殊情况请是用系统自动 默认<code>ConnectionModeEnum.SYSTEM_AUTO</code>
/// </summary>
ConnectionModeEnum ConnectionMode { get; }
/// <summary>
@ -52,6 +55,9 @@ namespace ShardingCore.Core.VirtualDatabase.VirtualDataSources.Abstractions
/// 不能为空 should not null
/// </summary>
IShardingComparer ShardingComparer { get; }
/// <summary>
/// 表确认管理者
/// </summary>
ITableEnsureManager TableEnsureManager { get; }
/// <summary>
/// 如何根据connectionString 配置 DbContextOptionsBuilder
@ -67,9 +73,15 @@ namespace ShardingCore.Core.VirtualDatabase.VirtualDataSources.Abstractions
/// <param name="dbContextOptionsBuilder"></param>
/// <returns></returns>
DbContextOptionsBuilder UseDbContextOptionsBuilder(DbConnection dbConnection, DbContextOptionsBuilder dbContextOptionsBuilder);
/// <summary>
/// 真实DbContextOptionBuilder的配置
/// </summary>
/// <param name="dbContextOptionsBuilder"></param>
void UseInnerDbContextOptionBuilder(DbContextOptionsBuilder dbContextOptionsBuilder);
/// <summary>
/// 使用读写分离
/// </summary>
/// <returns></returns>
bool UseReadWriteSeparation();
}

View File

@ -16,7 +16,6 @@ namespace ShardingCore.Helpers
public static Task<TResult[]> WhenAllFastFail<TResult>(params Task<TResult>[] tasks)
{
if (tasks is null || tasks.Length == 0) return Task.FromResult(Array.Empty<TResult>());
// defensive copy.
var defensive = tasks.Clone() as Task<TResult>[];

View File

@ -33,16 +33,24 @@ namespace ShardingCore.Sharding.EntityQueryConfigurations
return this;
}
/// <summary>
/// 使用当前属性order和comparer一样
/// 使用当前属性order和comparer有关联
/// </summary>
/// <typeparam name="TProperty"></typeparam>
/// <param name="primaryOrderPropertyExpression"></param>
/// <param name="isAsc">true:当前属性正序和comparer正序一样,false:当前属性倒序和comparer正序一样</param>
/// <returns></returns>
public EntityQueryBuilder<TEntity> AddOrder<TProperty>(Expression<Func<TEntity, TProperty>> primaryOrderPropertyExpression)
public EntityQueryBuilder<TEntity> AddOrder<TProperty>(Expression<Func<TEntity, TProperty>> primaryOrderPropertyExpression,bool isAsc=true)
{
_entityQueryMetadata.SeqQueryOrders.Add(primaryOrderPropertyExpression.GetPropertyAccess().Name);
_entityQueryMetadata.AddSeqComparerOrder(primaryOrderPropertyExpression.GetPropertyAccess().Name, isAsc);
return this;
}
/// <summary>
/// 添加链接限制,和程序启动配置的MaxQueryConnectionsLimit取最小值,非迭代器有效,说人话就是ToList不生效这个链接数限制
/// </summary>
/// <param name="connectionsLimit">连接数</param>
/// <param name="methodNames">查询方法</param>
/// <returns></returns>
/// <exception cref="ArgumentNullException"></exception>
public EntityQueryBuilder<TEntity> AddConnectionsLimit(int connectionsLimit,params QueryableMethodNameEnum[] methodNames)
{
if (connectionsLimit < 1)
@ -53,5 +61,21 @@ namespace ShardingCore.Sharding.EntityQueryConfigurations
}
return this;
}
/// <summary>
/// 配置默认方法不带排序的时候采用什么排序来触发熔断
/// </summary>
/// <param name="asc"></param>
/// <param name="methodNames"></param>
/// <returns></returns>
/// <exception cref="ArgumentNullException"></exception>
public EntityQueryBuilder<TEntity> AddDefaultSequenceQueryTrip(bool asc,params QueryableMethodNameEnum[] methodNames)
{
foreach (var methodName in methodNames)
{
_entityQueryMetadata.AddDefaultSequenceQueryTrip(asc,methodName);
}
return this;
}
}
}

View File

@ -23,18 +23,45 @@ namespace ShardingCore.Sharding.EntityQueryConfigurations
{ QueryableMethodNameEnum.Contains, nameof(Queryable.Contains) }
};
}
public ISet<string> SeqQueryOrders { get; }
public IComparer<string> DefaultTailComparer { get; set; }
public IDictionary<string,int> SeqConnectionsLimit { get; }
private readonly IDictionary<string,bool> _seqQueryOrders;
public IComparer<string> DefaultTailComparer { get; set; }
private readonly IDictionary<string, int> _seqConnectionsLimit;
private readonly IDictionary<string, bool> _seqQueryDefaults;
/// <summary>
///
/// </summary>
public EntityQueryMetadata()
{
SeqQueryOrders = new HashSet<string>();
DefaultTailComparer=Comparer<string>.Default;
SeqConnectionsLimit = new Dictionary<string, int>();
_seqQueryOrders = new Dictionary<string, bool>();
DefaultTailComparer =Comparer<string>.Default;
_seqConnectionsLimit = new Dictionary<string, int>();
_seqQueryDefaults = new Dictionary<string, bool>();
}
/// <summary>
/// 添加和默认数据库排序一样的排序
/// </summary>
/// <param name="propertyName"></param>
/// <param name="isAsc"></param>
public void AddSeqComparerOrder(string propertyName,bool isAsc)
{
if (_seqQueryOrders.ContainsKey(propertyName))
{
_seqQueryOrders[propertyName] = isAsc;
}
else
{
_seqQueryOrders.Add(propertyName, isAsc);
}
}
/// <summary>
/// 添加对应方法的连接数限制
/// </summary>
/// <param name="limit"></param>
/// <param name="methodNameEnum"></param>
/// <exception cref="ArgumentException"></exception>
public void AddConnectionsLimit(int limit, QueryableMethodNameEnum methodNameEnum)
{
if (!MethodNameSupports.TryGetValue(methodNameEnum, out var methodName))
@ -42,19 +69,24 @@ namespace ShardingCore.Sharding.EntityQueryConfigurations
throw new ArgumentException(methodNameEnum.ToString());
}
if (SeqConnectionsLimit.ContainsKey(methodName))
if (_seqConnectionsLimit.ContainsKey(methodName))
{
SeqConnectionsLimit[methodName]= limit;
_seqConnectionsLimit[methodName]= limit;
}
else
{
SeqConnectionsLimit.Add(methodName,limit);
_seqConnectionsLimit.Add(methodName,limit);
}
}
/// <summary>
/// 尝试获取当前查询方法配置的连接数限制
/// </summary>
/// <param name="methodName">First、FirstOrDefault...</param>
/// <param name="limit">连接数限制</param>
/// <returns></returns>
public bool TryGetConnectionsLimit(string methodName,out int limit)
{
if (SeqConnectionsLimit.TryGetValue(methodName, out var l))
if (_seqConnectionsLimit.TryGetValue(methodName, out var l))
{
limit = l;
return true;
@ -63,10 +95,60 @@ namespace ShardingCore.Sharding.EntityQueryConfigurations
limit = 0;
return false;
}
public bool ContainsComparerOrder(string propertyName)
/// <summary>
/// 是否包含当前排序字段
/// </summary>
/// <param name="propertyName"></param>
/// <param name="asc"></param>
/// <returns></returns>
public bool TryContainsComparerOrder(string propertyName,out bool asc)
{
return SeqQueryOrders.Contains(propertyName);
if (_seqQueryOrders.TryGetValue(propertyName, out var v))
{
asc = v;
return true;
}
asc = false;
return false;
}
/// <summary>
/// 默认顺序查询熔断
/// </summary>
/// <param name="asc"></param>
/// <param name="methodNameEnum"></param>
/// <exception cref="ArgumentException"></exception>
public void AddDefaultSequenceQueryTrip(bool asc,QueryableMethodNameEnum methodNameEnum)
{
if (!MethodNameSupports.TryGetValue(methodNameEnum, out var methodName))
{
throw new ArgumentException(methodNameEnum.ToString());
}
if (_seqQueryDefaults.ContainsKey(methodName))
{
_seqQueryDefaults[methodName] = asc;
}
else
{
_seqQueryDefaults.Add(methodName, asc);
}
}
/// <summary>
/// 当前方法是否配置了顺序排序查询熔断
/// </summary>
/// <param name="asc"></param>
/// <param name="methodName"></param>
/// <returns></returns>
public bool TryGetDefaultSequenceQueryTrip(string methodName,out bool asc)
{
if (_seqQueryDefaults.TryGetValue(methodName, out var v))
{
asc = v;
return true;
}
asc = false;
return false;
}
}

View File

@ -130,7 +130,7 @@ namespace ShardingCore.Sharding.MergeEngines.Abstractions
if (streamMergeContext.IsSeqQuery())
{
return sqlRouteUnits.OrderByAscDescIf(o => o.TableRouteResult.ReplaceTables.First().Tail,
(equalPropertyOrder ? streamMergeContext.PrimaryOrderIsAsc : !streamMergeContext.PrimaryOrderIsAsc), streamMergeContext.ShardingTailComparer);
(equalPropertyOrder ? streamMergeContext.TailComparerIsAsc : !streamMergeContext.TailComparerIsAsc), streamMergeContext.ShardingTailComparer);
}
return sqlRouteUnits;

View File

@ -20,7 +20,7 @@ namespace ShardingCore.Sharding.MergeEngines.Abstractions.InMemoryMerge
{
private readonly StreamMergeContext<TEntity> _mergeContext;
public AbstractInMemoryAsyncMergeEngine(StreamMergeContext<TEntity> streamMergeContext)
protected AbstractInMemoryAsyncMergeEngine(StreamMergeContext<TEntity> streamMergeContext)
{
_mergeContext = streamMergeContext;
}

View File

@ -22,6 +22,7 @@ using System.Linq.Expressions;
using System.Threading.Tasks;
using ShardingCore.Core.NotSupportShardingProviders;
using ShardingCore.Core.VirtualDatabase.VirtualTables;
using ShardingCore.Core.VirtualTables;
using ShardingCore.Sharding.EntityQueryConfigurations;
@ -79,9 +80,9 @@ namespace ShardingCore.Sharding
private readonly ConcurrentDictionary<DbContext, object> _parallelDbContexts;
/// <summary>
/// 主要排序是否是顺
/// 分表后缀比较是否重排正
/// </summary>
public bool PrimaryOrderIsAsc { get; } = true;
public bool TailComparerIsAsc { get; } = true;
private readonly bool _seqQuery=false;
@ -128,35 +129,64 @@ namespace ShardingCore.Sharding
if (virtualTable.EnableEntityQuery)
{
_seqQuery = true;
ShardingTailComparer =
virtualTable.EntityQueryMetadata.DefaultTailComparer ?? Comparer<string>.Default;
string methodName = null;
if (!MergeQueryCompilerContext.IsEnumerableQuery())
{
if (virtualTable.EntityQueryMetadata.TryGetConnectionsLimit(
((MethodCallExpression)MergeQueryCompilerContext.GetQueryExpression()).Method.Name,
out var limit))
methodName = ((MethodCallExpression)MergeQueryCompilerContext.GetQueryExpression()).Method.Name;
if (virtualTable.EntityQueryMetadata.TryGetConnectionsLimit(methodName,out var limit))
{
_maxParallelExecuteCount = Math.Min(limit,_maxParallelExecuteCount);
}
}
var propertyOrders = Orders as PropertyOrder[] ?? Orders.ToArray();
if (propertyOrders.IsNotEmpty())
if (TryGetSequenceQuery(propertyOrders, singleShardingEntityType, virtualTable, methodName,
out var tailComparerIsAsc))
{
var primaryOrder = propertyOrders[0];
//不是多级不能是匿名对象
if (primaryOrder.OwnerType == singleShardingEntityType && !primaryOrder.PropertyExpression.Contains("."))
{
if (virtualTable.EnableEntityQuery && virtualTable.EntityQueryMetadata.ContainsComparerOrder(primaryOrder.PropertyExpression))
{
PrimaryOrderIsAsc = primaryOrder.IsAsc;
}
}
_seqQuery = true;
TailComparerIsAsc = tailComparerIsAsc;
}
}
}
}
/// <summary>
/// 尝试获取当前方法是否采用顺序查询,如果有先判断排序没有的情况下判断默认
/// </summary>
/// <param name="propertyOrders"></param>
/// <param name="singleShardingEntityType"></param>
/// <param name="virtualTable"></param>
/// <param name="methodName"></param>
/// <param name="tailComparerIsAsc"></param>
/// <returns></returns>
private bool TryGetSequenceQuery(PropertyOrder[] propertyOrders, Type singleShardingEntityType,IVirtualTable virtualTable,string methodName, out bool tailComparerIsAsc)
{
if (propertyOrders.IsNotEmpty())
{
var primaryOrder = propertyOrders[0];
//不是多级不能是匿名对象
if (primaryOrder.OwnerType == singleShardingEntityType && !primaryOrder.PropertyExpression.Contains("."))
{
if (virtualTable.EnableEntityQuery && virtualTable.EntityQueryMetadata.TryContainsComparerOrder(primaryOrder.PropertyExpression, out var asc))
{
tailComparerIsAsc = asc ? primaryOrder.IsAsc : !primaryOrder.IsAsc;
return true;
}
}
}
if (virtualTable.EnableEntityQuery && methodName != null &&
virtualTable.EntityQueryMetadata.TryGetDefaultSequenceQueryTrip(methodName,out var defaultAsc))
{
tailComparerIsAsc = defaultAsc;
return true;
}
tailComparerIsAsc = true;
return false;
}
public void ReSetOrders(IEnumerable<PropertyOrder> orders)
{
Orders = orders;