Autotest
/
sharding
mirror of https://gitee.com/dotnetchina/sharding-core.git
1
0
Fork 0
Code Issues Releases Wiki Activity

[#266] 添加GetCompareValueByShardingKey方法优化当出现大量In时Expression的or函数或者and函数拼接导致stackoverflow异常

This commit is contained in:
xuejiaming 2024-04-20 08:39:15 +08:00
parent faeba5159d
commit b730c4e49a
10 changed files with 60 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
samples/Sample.AutoCreateIfPresent/Program.cs
View File

@ -30,6 +30,10 @@ builder.Services.AddShardingDbContext<DefaultDbContext>()
})
.UseConfig(o =>
{
// o.CacheEntrySize;
// o.CacheModelLockConcurrencyLevel
// o.CacheModelLockObjectSeconds
// o.CacheItemPriority
o.ThrowIfQueryRouteNotMatch = false;
o.AddDefaultDataSource("ds0", "server=127.0.0.1;port=3306;database=shardingTest;userid=root;password=root;");
o.UseShardingQuery((conn, b) =>

13
samples/Sample.MySql/Controllers/WeatherForecastController.cs
View File

@ -131,6 +131,7 @@ namespace Sample.MySql.Controllers
[HttpGet]
public async Task<IActionResult> Getxx()
{
var test = new Test();
test.UtcTime=DateTime.Now;
await _defaultTableDbContext.AddAsync(test);
@ -141,7 +142,7 @@ namespace Sample.MySql.Controllers
public async Task<IActionResult> Get()
{
var s = Guid.NewGuid().ToString();
var page =await _defaultTableDbContext.Set<SysUserLogByMonth>().Where(o=>o.Id==s).OrderByDescending(o=>o.Time).ToShardingPageAsync(1,2);
// var page =await _defaultTableDbContext.Set<SysUserLogByMonth>().Include().ThenInclude().Where(o=>o.Id==s).OrderByDescending(o=>o.Time).ToShardingPageAsync(1,2);
// var virtualDataSource = _shardingRuntimeContext.GetVirtualDataSource();
// virtualDataSource.AddPhysicDataSource(new DefaultPhysicDataSource("2023", "xxxxxxxx", false));
// var dataSourceRouteManager = _shardingRuntimeContext.GetDataSourceRouteManager();
@ -571,5 +572,15 @@ namespace Sample.MySql.Controllers
unShardingDbContext2.SaveChanges();
dbContextTransaction.Commit();
}
[HttpGet]
public async Task<IActionResult> get131()
{
var list = new List<string>();
var idList = Enumerable.Range(1,50000).Select(o=>o.ToString()).ToList();
var sysUserMods = _defaultTableDbContext.Set<SysUserMod>()
.Where(o=>idList.Contains(o.Id)).ToList();
return Ok();
}
}
}

9
samples/Sample.MySql/Shardings/SysUserModVirtualTableRoute.cs
View File

@ -24,6 +24,15 @@ namespace Sample.MySql.Shardings
builder.ShardingProperty(o => o.Id);
}
public override object GetCompareValueByShardingKey(object shardingKey, string shardingPropertyName)
{
if ("Id".Equals(shardingPropertyName))
{
return ShardingKeyToTail(shardingKey);
}
return base.GetCompareValueByShardingKey(shardingKey, shardingPropertyName);
}
// protected override List<TableRouteUnit> AfterShardingRouteUnitFilter(DataSourceRouteResult dataSourceRouteResult, List<TableRouteUnit> shardingRouteUnits)
// {
// //拦截

2
samples/Sample.MySql/Startup.cs
View File

@ -226,7 +226,6 @@ namespace Sample.MySql
{
app.UseDeveloperExceptionPage();
}
app.ApplicationServices.UseAutoTryCompensateTable();
using (var scope = app.ApplicationServices.CreateScope())
{
var unShardingDbContext = scope.ServiceProvider.GetService<UnShardingDbContext>();
@ -236,6 +235,7 @@ namespace Sample.MySql
// var virtualTableRoute = (SysUserLogByMonthRoute)tableRouteManager.GetRoute(typeof(SysUserLogByMonth));
// virtualTableRoute.Append("2021");
}
app.ApplicationServices.UseAutoTryCompensateTable();
// var shardingRuntimeContext = app.ApplicationServices.GetRequiredService<IShardingRuntimeContext>();
// var entityMetadataManager = shardingRuntimeContext.GetEntityMetadataManager();
// var entityMetadata = entityMetadataManager.TryGet<SysUserMod>();

7
src/ShardingCore/Core/VirtualRoutes/DataSourceRoutes/Abstractions/AbstractShardingOperatorVirtualDataSourceRoute.cs
View File

@ -24,7 +24,7 @@ namespace ShardingCore.Core.VirtualRoutes.DataSourceRoutes.Abstractions
protected override List<string> DoRouteWithPredicate(List<string> allDataSourceNames, IQueryable queryable)
{
//获取路由后缀表达式
var routeParseExpression = ShardingUtil.GetRouteParseExpression(queryable, EntityMetadata, GetRouteFilter, false);
var routeParseExpression = ShardingUtil.GetRouteParseExpression(queryable, EntityMetadata, GetRouteFilter,GetCompareValueByShardingKey, false);
//表达式缓存编译
// var filter = CachingCompile(routeParseExpression);
var filter = routeParseExpression.GetRoutePredicate();
@ -33,6 +33,11 @@ namespace ShardingCore.Core.VirtualRoutes.DataSourceRoutes.Abstractions
return dataSources;
}
public virtual object GetCompareValueByShardingKey(object shardingKey, string shardingPropertyName)
{
return shardingKey;
}
/// <summary>
/// 如何路由到具体表 shardingKeyValue:分表的值, 返回结果:如果返回true表示返回该表 第一个参数 tail 第二参数是否返回该物理表

6
src/ShardingCore/Core/VirtualRoutes/TableRoutes/Abstractions/AbstractShardingOperatorVirtualTableRoute.cs
View File

@ -27,7 +27,7 @@ namespace ShardingCore.Core.VirtualRoutes.TableRoutes.Abstractions
protected override List<TableRouteUnit> DoRouteWithPredicate(DataSourceRouteResult dataSourceRouteResult, IQueryable queryable)
{
//获取路由后缀表达式
var routeParseExpression = ShardingUtil.GetRouteParseExpression(queryable, EntityMetadata, GetRouteFilter,true);
var routeParseExpression = ShardingUtil.GetRouteParseExpression(queryable, EntityMetadata, GetRouteFilter,GetCompareValueByShardingKey,true);
//表达式缓存编译
// var filter =CachingCompile(routeParseExpression);
var filter =routeParseExpression.GetRoutePredicate();
@ -40,6 +40,10 @@ namespace ShardingCore.Core.VirtualRoutes.TableRoutes.Abstractions
return sqlRouteUnits;
}
public virtual object GetCompareValueByShardingKey(object shardingKey, string shardingPropertyName)
{
return shardingKey;
}
/// <summary>
/// 如何路由到具体表 shardingKeyValue:分表的值, 返回结果:如果返回true表示返回该表 第一个参数 tail 第二参数是否返回该物理表

11
src/ShardingCore/Sharding/Visitors/QueryableRouteDiscoverVisitor.cs
View File

@ -51,6 +51,7 @@ namespace ShardingCore.Core.Internal.Visitors
private readonly EntityMetadata _entityMetadata;
private readonly Func<object, ShardingOperatorEnum, string, Func<string, bool>> _keyToTailWithFilter;
private readonly Func<object, string, object> _compareValueByKey;
/// <summary>
/// 是否是分表路由
@ -63,10 +64,11 @@ namespace ShardingCore.Core.Internal.Visitors
private RoutePredicateExpression _where = RoutePredicateExpression.Default;
public QueryableRouteShardingTableDiscoverVisitor(EntityMetadata entityMetadata,
Func<object, ShardingOperatorEnum, string, Func<string, bool>> keyToTailWithFilter, bool shardingTableRoute)
Func<object, ShardingOperatorEnum, string, Func<string, bool>> keyToTailWithFilter,Func<object,string,object> compareValueByKey, bool shardingTableRoute)
{
_entityMetadata = entityMetadata;
_keyToTailWithFilter = keyToTailWithFilter;
_compareValueByKey = compareValueByKey;
_shardingTableRoute = shardingTableRoute;
}
@ -378,8 +380,15 @@ namespace ShardingCore.Core.Internal.Visitors
if (arrayObject is IEnumerable enumerableObj)
{
var compareSet = new HashSet<object>();
foreach (var shardingValue in enumerableObj)
{
var compareValueByKey = _compareValueByKey(shardingValue,shardingPredicateResult.ShardingPropertyName);
if (!compareSet.Add(compareValueByKey))
{
continue;
}
var eq = _keyToTailWithFilter(shardingValue,
@in ? ShardingOperatorEnum.Equal : ShardingOperatorEnum.NotEqual,
shardingPredicateResult.ShardingPropertyName);

4
src/ShardingCore/Utils/ShardingUtil.cs
View File

@ -34,10 +34,10 @@ namespace ShardingCore.Utils
/// <param name="keyToTailExpression"></param>
/// <param name="shardingTableRoute">sharding table or data source</param>
/// <returns></returns>
public static RoutePredicateExpression GetRouteParseExpression(IQueryable queryable, EntityMetadata entityMetadata, Func<object, ShardingOperatorEnum,string, Func<string, bool>> keyToTailExpression,bool shardingTableRoute)
public static RoutePredicateExpression GetRouteParseExpression(IQueryable queryable, EntityMetadata entityMetadata, Func<object, ShardingOperatorEnum,string, Func<string, bool>> keyToTailExpression,Func<object,string,object> compareValueByKey,bool shardingTableRoute)
{
QueryableRouteShardingTableDiscoverVisitor visitor = new QueryableRouteShardingTableDiscoverVisitor(entityMetadata, keyToTailExpression, shardingTableRoute);
QueryableRouteShardingTableDiscoverVisitor visitor = new QueryableRouteShardingTableDiscoverVisitor(entityMetadata, keyToTailExpression,compareValueByKey, shardingTableRoute);
visitor.Visit(queryable.Expression);

6
test/ShardingCore.CommonTest/ShardingDataSourceMod.cs
View File

@ -24,6 +24,10 @@ namespace ShardingCore.CommonTest
_allDataSources = Enumerable.Range(0, 10).Select(o => o.ToString()).ToList();
}
public static object GetCompareValueByShardingKey(object shardingKey, string shardingPropertyName)
{
return shardingKey;
}
public static Func<string, bool> GetRouteFilter(object shardingValue, ShardingOperatorEnum shardingOperator,
string propertyName)
{
@ -47,7 +51,7 @@ namespace ShardingCore.CommonTest
private void TestId(IQueryable<TestEntity> queryable, string[] dataSourceNames)
{
var routePredicateExpression =
ShardingUtil.GetRouteParseExpression(queryable, _testEntityMetadata, GetRouteFilter, false);
ShardingUtil.GetRouteParseExpression(queryable, _testEntityMetadata, GetRouteFilter,GetCompareValueByShardingKey, false);
Assert.NotNull(routePredicateExpression);
var routePredicate = routePredicateExpression.GetRoutePredicate();

7
test/ShardingCore.CommonTest/ShardingTableTime.cs
View File

@ -25,6 +25,11 @@ namespace ShardingCore.CommonTest
//[20220101....20220120]
_allTables = Enumerable.Range(0,20).Select(o=>dateTime.AddDays(o).ToString("yyyyMMdd")).ToList();
}
public static object GetCompareValueByShardingKey(object shardingKey, string shardingPropertyName)
{
return shardingKey;
}
public static Func<string, bool> GetRouteFilter(object shardingValue, ShardingOperatorEnum shardingOperator,
string propertyName)
{
@ -60,7 +65,7 @@ namespace ShardingCore.CommonTest
private void TestId(IQueryable<TestTimeEntity> queryable, string[] tables)
{
var routePredicateExpression = ShardingUtil.GetRouteParseExpression(queryable,_testEntityMetadata,GetRouteFilter,true);
var routePredicateExpression = ShardingUtil.GetRouteParseExpression(queryable,_testEntityMetadata,GetRouteFilter,GetCompareValueByShardingKey,true);
Assert.NotNull(routePredicateExpression);
var routePredicate = routePredicateExpression.GetRoutePredicate();