Autotest
/
sharding
mirror of https://gitee.com/dotnetchina/sharding-core.git
1
0
Fork 0
Code Issues Releases Wiki Activity

修复分片属性继承导致的不走高性能分页的bug

This commit is contained in:
xuejiaming 2023-09-13 13:54:39 +08:00
parent 87defaef06
commit e487d5e21c
7 changed files with 181 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
samples/Sample.MySql/Controllers/AsyncBatchService.cs Normal file
View File

@ -0,0 +1,20 @@
using Sample.MySql.DbContexts;
namespace Sample.MySql.Controllers;
public class AsyncBatchService
{
private readonly DefaultShardingDbContext _dbContext;
public AsyncBatchService(DefaultShardingDbContext dbContext)
{
_dbContext = dbContext;
}
public void SaveInOtherThread(List<object> list)
{
_dbContext.AddRangeAsync(list);
_dbContext.SaveChanges();
}
}

27
samples/Sample.MySql/Controllers/WeatherForecastController.cs
View File

@ -61,13 +61,15 @@ namespace Sample.MySql.Controllers
[Route("[controller]/[action]")]
public class WeatherForecastController : ControllerBase
{
private readonly IServiceProvider _serviceProvider;
private readonly UnShardingDbContext _unShardingDbContext;
private readonly DefaultShardingDbContext _defaultTableDbContext;
private readonly IShardingRuntimeContext _shardingRuntimeContext;
private readonly ABC _abc;
public WeatherForecastController(UnShardingDbContext unShardingDbContext,DefaultShardingDbContext defaultTableDbContext,IShardingRuntimeContext shardingRuntimeContext)
public WeatherForecastController(IServiceProvider serviceProvider,UnShardingDbContext unShardingDbContext,DefaultShardingDbContext defaultTableDbContext,IShardingRuntimeContext shardingRuntimeContext)
{
_serviceProvider = serviceProvider;
_unShardingDbContext = unShardingDbContext;
_defaultTableDbContext = defaultTableDbContext;
_shardingRuntimeContext = shardingRuntimeContext;
@ -126,7 +128,7 @@ namespace Sample.MySql.Controllers
//
// }
var dateTime = new DateTime(2021,1,1);
var x211 = await (from ut in _defaultTableDbContext.Set<SysTest>()
var x211 = await (from ut in _defaultTableDbContext.Set<SysTest>().UseUnionAllMerge()
join uu in _defaultTableDbContext.Set<SysUserLogByMonth>()
on ut.Id equals uu.Id
where uu.Time > dateTime
@ -218,6 +220,10 @@ namespace Sample.MySql.Controllers
.Select(o => new ssss(){ Id = o.Id, C = GetAll().Count(x => x.Id == o.Id) }).ToList();
var sysTests = GetAll();
var sysUserMods3 = _defaultTableDbContext.Set<SysTest>()
// .AsRoute(op=>
// {
// op.TryCreateOrAddMustTail<SysTest>(new[] { "00" });
// })
.Select(o => new ssss(){ Id = o.Id, C = sysTests.Count(x => x.Id == o.Id) }).ToList();
var resultX = await _defaultTableDbContext.Set<SysUserMod>()
.Where(o => o.Id == "2" || o.Id == "3").FirstOrDefaultAsync();
@ -446,5 +452,22 @@ namespace Sample.MySql.Controllers
// var sysUserMods2 = await _defaultTableDbContext.Set<SysTest>().FromSqlRaw("select * from SysTest where id='2'").ToListAsync();
return Ok();
}
// public void batachSave()
// {
// var objects = new List<List<object>>();
// foreach (List<object> o in objects)
// {
// Task.Run(()=>
// {
// using (var serviceScope = _serviceProvider.CreateScope())
// {
// var asyncBatchService = serviceScope.ServiceProvider.GetService<AsyncBatchService>();
// asyncBatchService.SaveInOtherThread(o);
// }
// });
// }
// }
}
}

9
samples/Sample.MySql/DbContexts/DefaultShardingDbContext.cs
View File

@ -23,6 +23,15 @@ namespace Sample.MySql.DbContexts
public DefaultShardingDbContext(DbContextOptions<DefaultShardingDbContext> options) : base(options)
{
var key = options.Extensions
.OrderBy(e => e.GetType().Name)
.Select(o =>
{
Console.WriteLine(o.GetType().Name);
return o;
})
.Aggregate(0L, (t, e) => (t * 397) ^ ((long)e.GetType().GetHashCode() * 397) ^ e.Info.GetServiceProviderHashCode());
Console.WriteLine("key:"+key);
//切记不要在构造函数中使用会让模型提前创建的方法
//ChangeTracker.QueryTrackingBehavior = QueryTrackingBehavior.NoTracking;
//Database.SetCommandTimeout(30000);

39
samples/Sample.MySql/MyJob.cs Normal file
View File

@ -0,0 +1,39 @@
using Microsoft.EntityFrameworkCore;
using Sample.MySql.DbContexts;
using ShardingCore;
using ShardingCore.Core.RuntimeContexts;
namespace Sample.MySql;
public class MyJob :IHostedService
{
private readonly IServiceProvider _serviceProvider;
private readonly IShardingRuntimeContext _shardingRuntimeContext;
public MyJob(IServiceProvider serviceProvider,IShardingRuntimeContext shardingRuntimeContext)
{
_serviceProvider = serviceProvider;
_shardingRuntimeContext = shardingRuntimeContext;
}
public Task StartAsync(CancellationToken cancellationToken)
{
// using (var serviceScope = _serviceProvider.CreateScope())
// {
// var defaultShardingDbContext = serviceScope.ServiceProvider.GetService<DefaultShardingDbContext>();
// }
//
// var dbContextOptionsBuilder = new DbContextOptionsBuilder<DefaultShardingDbContext>();
// dbContextOptionsBuilder.UseSharding(_shardingRuntimeContext);
// using (var dbcontext = new DefaultShardingDbContext(dbContextOptionsBuilder.Options))
// {
//
// }
throw new NotImplementedException();
}
public Task StopAsync(CancellationToken cancellationToken)
{
throw new NotImplementedException();
}
}

12
samples/Sample.MySql/Shardings/SysUserModVirtualTableRoute.cs
View File

@ -1,5 +1,7 @@
using Sample.MySql.Domain.Entities;
using ShardingCore.Core.EntityMetadatas;
using ShardingCore.Core.VirtualRoutes;
using ShardingCore.Core.VirtualRoutes.DataSourceRoutes.RouteRuleEngine;
using ShardingCore.VirtualRoutes.Mods;
namespace Sample.MySql.Shardings
@ -16,10 +18,20 @@ namespace Sample.MySql.Shardings
{
}
public override void Configure(EntityMetadataTableBuilder<SysUserMod> builder)
{
builder.ShardingProperty(o => o.Id);
}
// protected override List<TableRouteUnit> AfterShardingRouteUnitFilter(DataSourceRouteResult dataSourceRouteResult, List<TableRouteUnit> shardingRouteUnits)
// {
// //拦截
// if (shardingRouteUnits.Count > 10)
// {
// return shardingRouteUnits.Take(10).ToList();
// }
// return base.AfterShardingRouteUnitFilter(dataSourceRouteResult, shardingRouteUnits);
// }
}
}

23
samples/Sample.MySql/Startup.cs
View File

@ -130,24 +130,27 @@ namespace Sample.MySql
// o.UseEntityFrameworkCoreProxies = true;
o.ThrowIfQueryRouteNotMatch = false;
o.AutoUseWriteConnectionStringAfterWriteDb = true;
o.UseExecutorDbContextConfigure(op =>
{
});
o.UseShardingQuery((conStr, builder) =>
{
var logger = sp.ApplicationServiceProvider.GetService<ILogger<Startup>>();
logger.LogInformation(conStr);
builder.UseMySql(conStr, new MySqlServerVersion(new Version()))
// var logger = sp.ApplicationServiceProvider.GetService<ILogger<Startup>>();
// logger.LogInformation(conStr);
builder.UseMySql(conStr, new MySqlServerVersion(new Version()));
// .UseQueryTrackingBehavior(QueryTrackingBehavior.NoTracking)
.UseLoggerFactory(loggerFactory1)
.EnableSensitiveDataLogging();
// .UseLoggerFactory(loggerFactory1)
// .EnableSensitiveDataLogging();
//.UseQueryTrackingBehavior(QueryTrackingBehavior.NoTracking);
});
o.UseShardingTransaction((connection, builder) =>
{
builder
.UseMySql(connection, new MySqlServerVersion(new Version()))
// .UseQueryTrackingBehavior(QueryTrackingBehavior.NoTracking)
.UseLoggerFactory(loggerFactory1)
.EnableSensitiveDataLogging();
.UseMySql(connection, new MySqlServerVersion(new Version()));
// .UseQueryTrackingBehavior(QueryTrackingBehavior.NoTracking)
// .UseLoggerFactory(loggerFactory1)
// .EnableSensitiveDataLogging();
//.UseQueryTrackingBehavior(QueryTrackingBehavior.NoTracking);
});
o.AddDefaultDataSource("ds0",

92
src/ShardingCore/Sharding/MergeEngines/EnumeratorStreamMergeEngines/EnumeratorStreamMergeEngineFactory.cs
View File

@ -34,11 +34,12 @@ namespace ShardingCore.Sharding.MergeEngines.EnumeratorStreamMergeEngines
private readonly ITableRouteManager _tableRouteManager;
private readonly IEntityMetadataManager _entityMetadataManager;
private readonly IDataSourceRouteManager _dataSourceRouteManager;
private EnumeratorStreamMergeEngineFactory(StreamMergeContext streamMergeContext)
{
_streamMergeContext = streamMergeContext;
_shardingPageManager = streamMergeContext.ShardingRuntimeContext.GetShardingPageManager();
_tableRouteManager =streamMergeContext.ShardingRuntimeContext.GetTableRouteManager();
_tableRouteManager = streamMergeContext.ShardingRuntimeContext.GetTableRouteManager();
_entityMetadataManager = streamMergeContext.ShardingRuntimeContext.GetEntityMetadataManager();
_dataSourceRouteManager = streamMergeContext.ShardingRuntimeContext.GetDataSourceRouteManager();
}
@ -52,6 +53,7 @@ namespace ShardingCore.Sharding.MergeEngines.EnumeratorStreamMergeEngines
{
return _dataSourceRouteManager.GetRoute(entityType);
}
public IStreamEnumerable<TEntity> GetStreamEnumerable()
{
if (_streamMergeContext.IsRouteNotMatch())
@ -84,7 +86,8 @@ namespace ShardingCore.Sharding.MergeEngines.EnumeratorStreamMergeEngines
}
//未开启系统分表或者本次查询涉及多张分表
if (_streamMergeContext.IsPaginationQuery() && _streamMergeContext.IsSingleShardingEntityQuery() && _shardingPageManager.Current != null)
if (_streamMergeContext.IsPaginationQuery() && _streamMergeContext.IsSingleShardingEntityQuery() &&
_shardingPageManager.Current != null)
{
//获取虚拟表判断是否启用了分页配置
var shardingEntityType = _streamMergeContext.GetSingleShardingEntityType();
@ -114,7 +117,6 @@ namespace ShardingCore.Sharding.MergeEngines.EnumeratorStreamMergeEngines
private IStreamEnumerable<TEntity> DoNoOrderAppendEnumeratorStreamMergeEngine(Type shardingEntityType)
{
var isShardingDataSource = _entityMetadataManager.IsShardingDataSource(shardingEntityType);
var isShardingTable = _entityMetadataManager.IsShardingTable(shardingEntityType);
PaginationSequenceConfig dataSourceSequenceOrderConfig = null;
@ -124,28 +126,36 @@ namespace ShardingCore.Sharding.MergeEngines.EnumeratorStreamMergeEngines
var virtualDataSourceRoute = GetRoute(shardingEntityType);
if (virtualDataSourceRoute.EnablePagination)
{
dataSourceSequenceOrderConfig = virtualDataSourceRoute.PaginationMetadata.PaginationConfigs.OrderByDescending(o => o.AppendOrder)
.FirstOrDefault(o => o.AppendIfOrderNone && typeof(TEntity).ContainPropertyName(o.PropertyName));
dataSourceSequenceOrderConfig = virtualDataSourceRoute.PaginationMetadata.PaginationConfigs
.OrderByDescending(o => o.AppendOrder)
.FirstOrDefault(o =>
o.AppendIfOrderNone && typeof(TEntity).ContainPropertyName(o.PropertyName));
}
}
if (isShardingTable)
{
var tableRoute = _tableRouteManager.GetRoute(shardingEntityType);
if (tableRoute.EnablePagination)
{
tableSequenceOrderConfig = tableRoute.PaginationMetadata.PaginationConfigs.OrderByDescending(o => o.AppendOrder)
.FirstOrDefault(o => o.AppendIfOrderNone && typeof(TEntity).ContainPropertyName(o.PropertyName));
tableSequenceOrderConfig = tableRoute.PaginationMetadata.PaginationConfigs
.OrderByDescending(o => o.AppendOrder)
.FirstOrDefault(o =>
o.AppendIfOrderNone && typeof(TEntity).ContainPropertyName(o.PropertyName));
}
}
var useSequenceEnumeratorMergeEngine = isShardingDataSource && (dataSourceSequenceOrderConfig != null ||
(isShardingTable &&
!_streamMergeContext.IsCrossDataSource)) || (!isShardingDataSource && isShardingTable && tableSequenceOrderConfig != null);
!_streamMergeContext.IsCrossDataSource)) ||
(!isShardingDataSource && isShardingTable &&
tableSequenceOrderConfig != null);
if (useSequenceEnumeratorMergeEngine)
{
return new AppendOrderSequenceShardingEnumerable<TEntity>(_streamMergeContext, dataSourceSequenceOrderConfig, tableSequenceOrderConfig, _shardingPageManager.Current.RouteQueryResults);
return new AppendOrderSequenceShardingEnumerable<TEntity>(_streamMergeContext,
dataSourceSequenceOrderConfig, tableSequenceOrderConfig,
_shardingPageManager.Current.RouteQueryResults);
}
@ -154,7 +164,6 @@ namespace ShardingCore.Sharding.MergeEngines.EnumeratorStreamMergeEngines
private IStreamEnumerable<TEntity> DoOrderSequencePaginationEnumeratorStreamMergeEngine(Type shardingEntityType)
{
var orderCount = _streamMergeContext.Orders.Count();
var primaryOrder = _streamMergeContext.Orders.First();
var isShardingDataSource = _entityMetadataManager.IsShardingDataSource(shardingEntityType);
@ -170,33 +179,44 @@ namespace ShardingCore.Sharding.MergeEngines.EnumeratorStreamMergeEngines
virtualDataSourceRoute = GetRoute(shardingEntityType);
if (virtualDataSourceRoute.EnablePagination)
{
dataSourceSequenceOrderConfig = orderCount == 1 ? GetPaginationFullMatch(virtualDataSourceRoute.PaginationMetadata.PaginationConfigs, primaryOrder) : GetPaginationPrimaryMatch(virtualDataSourceRoute.PaginationMetadata.PaginationConfigs, primaryOrder);
dataSourceSequenceOrderConfig = orderCount == 1
? GetPaginationFullMatch(virtualDataSourceRoute.PaginationMetadata.PaginationConfigs,
primaryOrder)
: GetPaginationPrimaryMatch(virtualDataSourceRoute.PaginationMetadata.PaginationConfigs,
primaryOrder);
}
}
if (isShardingTable)
{
tableRoute = _tableRouteManager.GetRoute(shardingEntityType);
if (tableRoute.EnablePagination)
{
tableSequenceOrderConfig = orderCount == 1 ? GetPaginationFullMatch(tableRoute.PaginationMetadata.PaginationConfigs, primaryOrder) : GetPaginationPrimaryMatch(tableRoute.PaginationMetadata.PaginationConfigs, primaryOrder);
tableSequenceOrderConfig = orderCount == 1
? GetPaginationFullMatch(tableRoute.PaginationMetadata.PaginationConfigs, primaryOrder)
: GetPaginationPrimaryMatch(tableRoute.PaginationMetadata.PaginationConfigs, primaryOrder);
}
}
var useSequenceEnumeratorMergeEngine = isShardingDataSource && (dataSourceSequenceOrderConfig != null ||
(isShardingTable &&
!_streamMergeContext.IsCrossDataSource)) || (!isShardingDataSource && isShardingTable && tableSequenceOrderConfig != null);
!_streamMergeContext.IsCrossDataSource)) ||
(!isShardingDataSource && isShardingTable &&
tableSequenceOrderConfig != null);
if (useSequenceEnumeratorMergeEngine)
{
return new SequenceShardingEnumerable<TEntity>(_streamMergeContext, dataSourceSequenceOrderConfig, tableSequenceOrderConfig, _shardingPageManager.Current.RouteQueryResults, primaryOrder.IsAsc);
return new SequenceShardingEnumerable<TEntity>(_streamMergeContext, dataSourceSequenceOrderConfig,
tableSequenceOrderConfig, _shardingPageManager.Current.RouteQueryResults, primaryOrder.IsAsc);
}
var total = _shardingPageManager.Current.RouteQueryResults.Sum(o => o.QueryResult);
if (isShardingDataSource)
{
dataSourceUseReverse =
virtualDataSourceRoute.EnablePagination && EntityDataSourceUseReverseShardingPage(virtualDataSourceRoute, total);
virtualDataSourceRoute.EnablePagination &&
EntityDataSourceUseReverseShardingPage(virtualDataSourceRoute, total);
}
if (isShardingTable)
{
tableUseReverse =
@ -211,51 +231,65 @@ namespace ShardingCore.Sharding.MergeEngines.EnumeratorStreamMergeEngines
}
return null;
}
private bool EntityDataSourceUseReverseShardingPage(IVirtualDataSourceRoute virtualDataSourceRoute, long total)
{
if (virtualDataSourceRoute.PaginationMetadata.EnableReverseShardingPage && _streamMergeContext.Take.GetValueOrDefault() > 0)
if (virtualDataSourceRoute.PaginationMetadata.EnableReverseShardingPage &&
_streamMergeContext.Take.GetValueOrDefault() > 0)
{
if (virtualDataSourceRoute.PaginationMetadata.IsUseReverse(_streamMergeContext.Skip.GetValueOrDefault(), total))
if (virtualDataSourceRoute.PaginationMetadata.IsUseReverse(_streamMergeContext.Skip.GetValueOrDefault(),
total))
{
return true;
}
}
return false;
}
private bool EntityTableReverseShardingPage(IVirtualTableRoute tableRoute, long total)
{
if (tableRoute.PaginationMetadata.EnableReverseShardingPage && _streamMergeContext.Take.GetValueOrDefault() > 0)
if (tableRoute.PaginationMetadata.EnableReverseShardingPage &&
_streamMergeContext.Take.GetValueOrDefault() > 0)
{
if (tableRoute.PaginationMetadata.IsUseReverse(_streamMergeContext.Skip.GetValueOrDefault(), total))
{
return true;
}
}
return false;
}
private PaginationSequenceConfig GetPaginationFullMatch(ISet<PaginationSequenceConfig> paginationSequenceConfigs, PropertyOrder primaryOrder)
private PaginationSequenceConfig GetPaginationFullMatch(
ISet<PaginationSequenceConfig> paginationSequenceConfigs, PropertyOrder primaryOrder)
{
return paginationSequenceConfigs.FirstOrDefault(o => PaginationPrimaryMatch(o, primaryOrder));
}
private PaginationSequenceConfig GetPaginationPrimaryMatch(ISet<PaginationSequenceConfig> paginationSequenceConfigs, PropertyOrder primaryOrder)
private PaginationSequenceConfig GetPaginationPrimaryMatch(
ISet<PaginationSequenceConfig> paginationSequenceConfigs, PropertyOrder primaryOrder)
{
return paginationSequenceConfigs.Where(o => o.PaginationMatchEnum.HasFlag(PaginationMatchEnum.PrimaryMatch)).FirstOrDefault(o => PaginationPrimaryMatch(o, primaryOrder));
return paginationSequenceConfigs.Where(o => o.PaginationMatchEnum.HasFlag(PaginationMatchEnum.PrimaryMatch))
.FirstOrDefault(o => PaginationPrimaryMatch(o, primaryOrder));
}
private bool PaginationPrimaryMatch(PaginationSequenceConfig paginationSequenceConfig, PropertyOrder propertyOrder)
private bool PaginationPrimaryMatch(PaginationSequenceConfig paginationSequenceConfig,
PropertyOrder propertyOrder)
{
if (propertyOrder.PropertyExpression != paginationSequenceConfig.PropertyName)
return false;
if (paginationSequenceConfig.PaginationMatchEnum.HasFlag(PaginationMatchEnum.Owner))
return _streamMergeContext.GetSingleShardingEntityType() == paginationSequenceConfig.OrderPropertyInfo.DeclaringType;
{
var singleShardingEntityType = _streamMergeContext.GetSingleShardingEntityType();
return paginationSequenceConfig.OrderPropertyInfo.DeclaringType != null &&
(paginationSequenceConfig.OrderPropertyInfo.DeclaringType == singleShardingEntityType ||
paginationSequenceConfig.OrderPropertyInfo.DeclaringType.IsAssignableFrom(
singleShardingEntityType));
}
if (paginationSequenceConfig.PaginationMatchEnum.HasFlag(PaginationMatchEnum.Named))
return propertyOrder.PropertyExpression == paginationSequenceConfig.PropertyName;
return false;