[#266] 添加GetCompareValueByShardingKey方法优化当出现大量In时Expression的or函数或者and函数拼接导致stackoverflow异常

This commit is contained in:
xuejiaming 2024-04-20 08:39:15 +08:00
parent faeba5159d
commit b730c4e49a
10 changed files with 60 additions and 9 deletions

View File

@ -30,6 +30,10 @@ builder.Services.AddShardingDbContext<DefaultDbContext>()
}) })
.UseConfig(o => .UseConfig(o =>
{ {
// o.CacheEntrySize;
// o.CacheModelLockConcurrencyLevel
// o.CacheModelLockObjectSeconds
// o.CacheItemPriority
o.ThrowIfQueryRouteNotMatch = false; o.ThrowIfQueryRouteNotMatch = false;
o.AddDefaultDataSource("ds0", "server=127.0.0.1;port=3306;database=shardingTest;userid=root;password=root;"); o.AddDefaultDataSource("ds0", "server=127.0.0.1;port=3306;database=shardingTest;userid=root;password=root;");
o.UseShardingQuery((conn, b) => o.UseShardingQuery((conn, b) =>

View File

@ -131,6 +131,7 @@ namespace Sample.MySql.Controllers
[HttpGet] [HttpGet]
public async Task<IActionResult> Getxx() public async Task<IActionResult> Getxx()
{ {
var test = new Test(); var test = new Test();
test.UtcTime=DateTime.Now; test.UtcTime=DateTime.Now;
await _defaultTableDbContext.AddAsync(test); await _defaultTableDbContext.AddAsync(test);
@ -141,7 +142,7 @@ namespace Sample.MySql.Controllers
public async Task<IActionResult> Get() public async Task<IActionResult> Get()
{ {
var s = Guid.NewGuid().ToString(); var s = Guid.NewGuid().ToString();
var page =await _defaultTableDbContext.Set<SysUserLogByMonth>().Where(o=>o.Id==s).OrderByDescending(o=>o.Time).ToShardingPageAsync(1,2); // var page =await _defaultTableDbContext.Set<SysUserLogByMonth>().Include().ThenInclude().Where(o=>o.Id==s).OrderByDescending(o=>o.Time).ToShardingPageAsync(1,2);
// var virtualDataSource = _shardingRuntimeContext.GetVirtualDataSource(); // var virtualDataSource = _shardingRuntimeContext.GetVirtualDataSource();
// virtualDataSource.AddPhysicDataSource(new DefaultPhysicDataSource("2023", "xxxxxxxx", false)); // virtualDataSource.AddPhysicDataSource(new DefaultPhysicDataSource("2023", "xxxxxxxx", false));
// var dataSourceRouteManager = _shardingRuntimeContext.GetDataSourceRouteManager(); // var dataSourceRouteManager = _shardingRuntimeContext.GetDataSourceRouteManager();
@ -571,5 +572,15 @@ namespace Sample.MySql.Controllers
unShardingDbContext2.SaveChanges(); unShardingDbContext2.SaveChanges();
dbContextTransaction.Commit(); dbContextTransaction.Commit();
} }
[HttpGet]
public async Task<IActionResult> get131()
{
var list = new List<string>();
var idList = Enumerable.Range(1,50000).Select(o=>o.ToString()).ToList();
var sysUserMods = _defaultTableDbContext.Set<SysUserMod>()
.Where(o=>idList.Contains(o.Id)).ToList();
return Ok();
}
} }
} }

View File

@ -24,6 +24,15 @@ namespace Sample.MySql.Shardings
builder.ShardingProperty(o => o.Id); builder.ShardingProperty(o => o.Id);
} }
public override object GetCompareValueByShardingKey(object shardingKey, string shardingPropertyName)
{
if ("Id".Equals(shardingPropertyName))
{
return ShardingKeyToTail(shardingKey);
}
return base.GetCompareValueByShardingKey(shardingKey, shardingPropertyName);
}
// protected override List<TableRouteUnit> AfterShardingRouteUnitFilter(DataSourceRouteResult dataSourceRouteResult, List<TableRouteUnit> shardingRouteUnits) // protected override List<TableRouteUnit> AfterShardingRouteUnitFilter(DataSourceRouteResult dataSourceRouteResult, List<TableRouteUnit> shardingRouteUnits)
// { // {
// //拦截 // //拦截

View File

@ -226,7 +226,6 @@ namespace Sample.MySql
{ {
app.UseDeveloperExceptionPage(); app.UseDeveloperExceptionPage();
} }
app.ApplicationServices.UseAutoTryCompensateTable();
using (var scope = app.ApplicationServices.CreateScope()) using (var scope = app.ApplicationServices.CreateScope())
{ {
var unShardingDbContext = scope.ServiceProvider.GetService<UnShardingDbContext>(); var unShardingDbContext = scope.ServiceProvider.GetService<UnShardingDbContext>();
@ -236,6 +235,7 @@ namespace Sample.MySql
// var virtualTableRoute = (SysUserLogByMonthRoute)tableRouteManager.GetRoute(typeof(SysUserLogByMonth)); // var virtualTableRoute = (SysUserLogByMonthRoute)tableRouteManager.GetRoute(typeof(SysUserLogByMonth));
// virtualTableRoute.Append("2021"); // virtualTableRoute.Append("2021");
} }
app.ApplicationServices.UseAutoTryCompensateTable();
// var shardingRuntimeContext = app.ApplicationServices.GetRequiredService<IShardingRuntimeContext>(); // var shardingRuntimeContext = app.ApplicationServices.GetRequiredService<IShardingRuntimeContext>();
// var entityMetadataManager = shardingRuntimeContext.GetEntityMetadataManager(); // var entityMetadataManager = shardingRuntimeContext.GetEntityMetadataManager();
// var entityMetadata = entityMetadataManager.TryGet<SysUserMod>(); // var entityMetadata = entityMetadataManager.TryGet<SysUserMod>();

View File

@ -24,7 +24,7 @@ namespace ShardingCore.Core.VirtualRoutes.DataSourceRoutes.Abstractions
protected override List<string> DoRouteWithPredicate(List<string> allDataSourceNames, IQueryable queryable) protected override List<string> DoRouteWithPredicate(List<string> allDataSourceNames, IQueryable queryable)
{ {
//获取路由后缀表达式 //获取路由后缀表达式
var routeParseExpression = ShardingUtil.GetRouteParseExpression(queryable, EntityMetadata, GetRouteFilter, false); var routeParseExpression = ShardingUtil.GetRouteParseExpression(queryable, EntityMetadata, GetRouteFilter,GetCompareValueByShardingKey, false);
//表达式缓存编译 //表达式缓存编译
// var filter = CachingCompile(routeParseExpression); // var filter = CachingCompile(routeParseExpression);
var filter = routeParseExpression.GetRoutePredicate(); var filter = routeParseExpression.GetRoutePredicate();
@ -33,6 +33,11 @@ namespace ShardingCore.Core.VirtualRoutes.DataSourceRoutes.Abstractions
return dataSources; return dataSources;
} }
public virtual object GetCompareValueByShardingKey(object shardingKey, string shardingPropertyName)
{
return shardingKey;
}
/// <summary> /// <summary>
/// 如何路由到具体表 shardingKeyValue:分表的值, 返回结果:如果返回true表示返回该表 第一个参数 tail 第二参数是否返回该物理表 /// 如何路由到具体表 shardingKeyValue:分表的值, 返回结果:如果返回true表示返回该表 第一个参数 tail 第二参数是否返回该物理表

View File

@ -27,7 +27,7 @@ namespace ShardingCore.Core.VirtualRoutes.TableRoutes.Abstractions
protected override List<TableRouteUnit> DoRouteWithPredicate(DataSourceRouteResult dataSourceRouteResult, IQueryable queryable) protected override List<TableRouteUnit> DoRouteWithPredicate(DataSourceRouteResult dataSourceRouteResult, IQueryable queryable)
{ {
//获取路由后缀表达式 //获取路由后缀表达式
var routeParseExpression = ShardingUtil.GetRouteParseExpression(queryable, EntityMetadata, GetRouteFilter,true); var routeParseExpression = ShardingUtil.GetRouteParseExpression(queryable, EntityMetadata, GetRouteFilter,GetCompareValueByShardingKey,true);
//表达式缓存编译 //表达式缓存编译
// var filter =CachingCompile(routeParseExpression); // var filter =CachingCompile(routeParseExpression);
var filter =routeParseExpression.GetRoutePredicate(); var filter =routeParseExpression.GetRoutePredicate();
@ -40,6 +40,10 @@ namespace ShardingCore.Core.VirtualRoutes.TableRoutes.Abstractions
return sqlRouteUnits; return sqlRouteUnits;
} }
public virtual object GetCompareValueByShardingKey(object shardingKey, string shardingPropertyName)
{
return shardingKey;
}
/// <summary> /// <summary>
/// 如何路由到具体表 shardingKeyValue:分表的值, 返回结果:如果返回true表示返回该表 第一个参数 tail 第二参数是否返回该物理表 /// 如何路由到具体表 shardingKeyValue:分表的值, 返回结果:如果返回true表示返回该表 第一个参数 tail 第二参数是否返回该物理表

View File

@ -51,6 +51,7 @@ namespace ShardingCore.Core.Internal.Visitors
private readonly EntityMetadata _entityMetadata; private readonly EntityMetadata _entityMetadata;
private readonly Func<object, ShardingOperatorEnum, string, Func<string, bool>> _keyToTailWithFilter; private readonly Func<object, ShardingOperatorEnum, string, Func<string, bool>> _keyToTailWithFilter;
private readonly Func<object, string, object> _compareValueByKey;
/// <summary> /// <summary>
/// 是否是分表路由 /// 是否是分表路由
@ -63,10 +64,11 @@ namespace ShardingCore.Core.Internal.Visitors
private RoutePredicateExpression _where = RoutePredicateExpression.Default; private RoutePredicateExpression _where = RoutePredicateExpression.Default;
public QueryableRouteShardingTableDiscoverVisitor(EntityMetadata entityMetadata, public QueryableRouteShardingTableDiscoverVisitor(EntityMetadata entityMetadata,
Func<object, ShardingOperatorEnum, string, Func<string, bool>> keyToTailWithFilter, bool shardingTableRoute) Func<object, ShardingOperatorEnum, string, Func<string, bool>> keyToTailWithFilter,Func<object,string,object> compareValueByKey, bool shardingTableRoute)
{ {
_entityMetadata = entityMetadata; _entityMetadata = entityMetadata;
_keyToTailWithFilter = keyToTailWithFilter; _keyToTailWithFilter = keyToTailWithFilter;
_compareValueByKey = compareValueByKey;
_shardingTableRoute = shardingTableRoute; _shardingTableRoute = shardingTableRoute;
} }
@ -378,8 +380,15 @@ namespace ShardingCore.Core.Internal.Visitors
if (arrayObject is IEnumerable enumerableObj) if (arrayObject is IEnumerable enumerableObj)
{ {
var compareSet = new HashSet<object>();
foreach (var shardingValue in enumerableObj) foreach (var shardingValue in enumerableObj)
{ {
var compareValueByKey = _compareValueByKey(shardingValue,shardingPredicateResult.ShardingPropertyName);
if (!compareSet.Add(compareValueByKey))
{
continue;
}
var eq = _keyToTailWithFilter(shardingValue, var eq = _keyToTailWithFilter(shardingValue,
@in ? ShardingOperatorEnum.Equal : ShardingOperatorEnum.NotEqual, @in ? ShardingOperatorEnum.Equal : ShardingOperatorEnum.NotEqual,
shardingPredicateResult.ShardingPropertyName); shardingPredicateResult.ShardingPropertyName);

View File

@ -34,10 +34,10 @@ namespace ShardingCore.Utils
/// <param name="keyToTailExpression"></param> /// <param name="keyToTailExpression"></param>
/// <param name="shardingTableRoute">sharding table or data source</param> /// <param name="shardingTableRoute">sharding table or data source</param>
/// <returns></returns> /// <returns></returns>
public static RoutePredicateExpression GetRouteParseExpression(IQueryable queryable, EntityMetadata entityMetadata, Func<object, ShardingOperatorEnum,string, Func<string, bool>> keyToTailExpression,bool shardingTableRoute) public static RoutePredicateExpression GetRouteParseExpression(IQueryable queryable, EntityMetadata entityMetadata, Func<object, ShardingOperatorEnum,string, Func<string, bool>> keyToTailExpression,Func<object,string,object> compareValueByKey,bool shardingTableRoute)
{ {
QueryableRouteShardingTableDiscoverVisitor visitor = new QueryableRouteShardingTableDiscoverVisitor(entityMetadata, keyToTailExpression, shardingTableRoute); QueryableRouteShardingTableDiscoverVisitor visitor = new QueryableRouteShardingTableDiscoverVisitor(entityMetadata, keyToTailExpression,compareValueByKey, shardingTableRoute);
visitor.Visit(queryable.Expression); visitor.Visit(queryable.Expression);

View File

@ -24,6 +24,10 @@ namespace ShardingCore.CommonTest
_allDataSources = Enumerable.Range(0, 10).Select(o => o.ToString()).ToList(); _allDataSources = Enumerable.Range(0, 10).Select(o => o.ToString()).ToList();
} }
public static object GetCompareValueByShardingKey(object shardingKey, string shardingPropertyName)
{
return shardingKey;
}
public static Func<string, bool> GetRouteFilter(object shardingValue, ShardingOperatorEnum shardingOperator, public static Func<string, bool> GetRouteFilter(object shardingValue, ShardingOperatorEnum shardingOperator,
string propertyName) string propertyName)
{ {
@ -47,7 +51,7 @@ namespace ShardingCore.CommonTest
private void TestId(IQueryable<TestEntity> queryable, string[] dataSourceNames) private void TestId(IQueryable<TestEntity> queryable, string[] dataSourceNames)
{ {
var routePredicateExpression = var routePredicateExpression =
ShardingUtil.GetRouteParseExpression(queryable, _testEntityMetadata, GetRouteFilter, false); ShardingUtil.GetRouteParseExpression(queryable, _testEntityMetadata, GetRouteFilter,GetCompareValueByShardingKey, false);
Assert.NotNull(routePredicateExpression); Assert.NotNull(routePredicateExpression);
var routePredicate = routePredicateExpression.GetRoutePredicate(); var routePredicate = routePredicateExpression.GetRoutePredicate();

View File

@ -25,6 +25,11 @@ namespace ShardingCore.CommonTest
//[20220101....20220120] //[20220101....20220120]
_allTables = Enumerable.Range(0,20).Select(o=>dateTime.AddDays(o).ToString("yyyyMMdd")).ToList(); _allTables = Enumerable.Range(0,20).Select(o=>dateTime.AddDays(o).ToString("yyyyMMdd")).ToList();
} }
public static object GetCompareValueByShardingKey(object shardingKey, string shardingPropertyName)
{
return shardingKey;
}
public static Func<string, bool> GetRouteFilter(object shardingValue, ShardingOperatorEnum shardingOperator, public static Func<string, bool> GetRouteFilter(object shardingValue, ShardingOperatorEnum shardingOperator,
string propertyName) string propertyName)
{ {
@ -60,7 +65,7 @@ namespace ShardingCore.CommonTest
private void TestId(IQueryable<TestTimeEntity> queryable, string[] tables) private void TestId(IQueryable<TestTimeEntity> queryable, string[] tables)
{ {
var routePredicateExpression = ShardingUtil.GetRouteParseExpression(queryable,_testEntityMetadata,GetRouteFilter,true); var routePredicateExpression = ShardingUtil.GetRouteParseExpression(queryable,_testEntityMetadata,GetRouteFilter,GetCompareValueByShardingKey,true);
Assert.NotNull(routePredicateExpression); Assert.NotNull(routePredicateExpression);
var routePredicate = routePredicateExpression.GetRoutePredicate(); var routePredicate = routePredicateExpression.GetRoutePredicate();