From 191b57aa18855d190bc9d091dfeb1286b4ac33a6 Mon Sep 17 00:00:00 2001 From: xuejiaming <326308290@qq.com> Date: Thu, 2 Dec 2021 21:40:15 +0800 Subject: [PATCH] =?UTF-8?q?=E6=9C=89=E8=8A=B1=E4=B8=80=E7=82=B9=E7=82=B9?= =?UTF-8?q?=E8=B7=AF=E7=94=B1=E8=A1=A8=E8=BE=BE=E5=BC=8F=E5=B9=B6=E4=B8=94?= =?UTF-8?q?=E6=B7=BB=E5=8A=A0readme=E8=A1=A8=E8=BE=BE=E5=BC=8F=E7=BC=93?= =?UTF-8?q?=E5=AD=98=E5=92=8C=E6=80=A7=E8=83=BD=E8=AF=84=E4=BC=B0=E6=B5=8B?= =?UTF-8?q?=E8=AF=95benchmark?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 122 ++++++++++++++++-- .../ShardingCoreBenchmark/EFCoreCrud.cs | 4 +- .../DataSourceRouteRuleEngine.cs | 7 +- .../RoutingRuleEngine/TableRouteRuleEngine.cs | 5 +- 4 files changed, 123 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 2fbeede1..fb0481f3 100644 --- a/README.md +++ b/README.md @@ -25,16 +25,75 @@ Release | EF Core | .NET | .NET (Core) [5.x.x.x](https://www.nuget.org/packages/ShardingCore) | 5.0.10 | Standard 2.1 | 5.0+ [3.x.x.x](https://www.nuget.org/packages/ShardingCore) | 3.1.18 | Standard 2.0 | 2.0+ [2.x.x.x](https://www.nuget.org/packages/ShardingCore) | 2.2.6 | Standard 2.0 | 2.0+ -### 数据库支持 -数据库 | 是否支持 | 支持情况 ---- | --- | --- -SqlServer | 支持 | 支持 -MySql |支持 | 支持 -PostgreSql | 支持 | 支持 -SQLite | 支持 | 未测试 -Oracle | 支持 | 未测试 -其他 | 支持(只要efcore支持) | 未测试 +### 性能测试 +以下所有数据均在开启了表达式编译缓存的情况下测试，并且电脑处于长时间未关机并且开着很多vs和idea的情况下仅供参考,所有测试都是基于ShardingCore x.3.1.63+ version + +以下所有数据均在[源码中有案例](https://github.com/xuejmnet/sharding-core/blob/main/benchmarks/ShardingCoreBenchmark/EFCoreCrud.cs) + +efcore版本均为6.0 表结构为string型id的订单取模分成5张表 + +N代表执行次数 + +1.sql server 2012,data rows 7734363 =773w + +// * Summary * + +BenchmarkDotNet=v0.13.1, OS=Windows 10.0.18363.1500 (1909/November2019Update/19H2) +AMD Ryzen 9 3900X, 1 CPU, 24 logical and 12 physical cores +.NET SDK=6.0.100 + [Host] : .NET 6.0.0 (6.0.21.52210), X64 RyuJIT + DefaultJob : .NET 6.0.0 (6.0.21.52210), X64 RyuJIT + + +| Method | N | Mean | Error | StdDev | Median | +|---------------------------------------- |--- |-------------:|------------:|------------:|-------------:| +| NoShardingFirstOrDefaultAsync | 10 | 2.305 ms | 0.0419 ms | 0.0587 ms | 2.310 ms | +| ShardingFirstOrDefaultAsync | 10 | 4.200 ms | 0.0793 ms | 0.0815 ms | 4.205 ms | +| NoShardingNoIndexFirstOrDefaultAsync | 10 | 1,521.727 ms | 11.7909 ms | 11.0292 ms | 1,519.390 ms | +| ShardingNoIndexFirstOrDefaultAsync | 10 | 1,841.243 ms | 36.1808 ms | 49.5247 ms | 1,826.228 ms | +| NoShardingNoIndexCountAsync | 10 | 1,602.127 ms | 31.2448 ms | 26.0908 ms | 1,592.494 ms | +| ShardingNoIndexCountAsync | 10 | 1,946.878 ms | 33.9453 ms | 31.7525 ms | 1,948.952 ms | +| NoShardingNoIndexFirstOrDefaultAsync0w | 10 | 703.570 ms | 10.4157 ms | 9.2332 ms | 705.236 ms | +| ShardingNoIndexFirstOrDefaultAsync0w | 10 | 857.718 ms | 16.4004 ms | 15.3409 ms | 858.675 ms | +| NoShardingNoIndexFirstOrDefaultAsync99w | 10 | 818.947 ms | 16.2501 ms | 24.8156 ms | 814.093 ms | +| ShardingNoIndexFirstOrDefaultAsync99w | 10 | 957.405 ms | 15.8800 ms | 16.9915 ms | 953.739 ms | +| NoShardingNoIndexLikeToListAsync | 10 | 7,247.554 ms | 140.2374 ms | 191.9586 ms | 7,292.292 ms | +| ShardingNoIndexLikeToListAsync | 10 | 7,232.702 ms | 106.7630 ms | 99.8662 ms | 7,184.900 ms | +| NoShardingNoIndexToListAsync | 10 | 815.207 ms | 14.6120 ms | 21.4181 ms | 804.195 ms | +| ShardingNoIndexToListAsync | 10 | 948.056 ms | 7.3526 ms | 6.8776 ms | 944.511 ms | + +2.mysql 5.7,data rows 7553790=755w innerdb_buffer_size=3G + + +// * Summary * + +BenchmarkDotNet=v0.13.1, OS=Windows 10.0.18363.1500 (1909/November2019Update/19H2) +AMD Ryzen 9 3900X, 1 CPU, 24 logical and 12 physical cores +.NET SDK=6.0.100 + [Host] : .NET 6.0.0 (6.0.21.52210), X64 RyuJIT + DefaultJob : .NET 6.0.0 (6.0.21.52210), X64 RyuJIT + + +| Method | N | Mean | Error | StdDev | Median | +|---------------------------------------- |--- |--------------:|------------:|------------:|--------------:| +| NoShardingFirstOrDefaultAsync | 10 | 10.092 ms | 1.6571 ms | 4.5082 ms | 8.677 ms | +| ShardingFirstOrDefaultAsync | 10 | 9.082 ms | 0.1810 ms | 0.3445 ms | 9.096 ms | +| NoShardingNoIndexFirstOrDefaultAsync | 10 | 6.586 ms | 0.0795 ms | 0.0705 ms | 6.565 ms | +| ShardingNoIndexFirstOrDefaultAsync | 10 | 17.617 ms | 0.3345 ms | 0.3129 ms | 17.481 ms | +| NoShardingNoIndexCountAsync | 10 | 6.498 ms | 0.1188 ms | 0.1415 ms | 6.454 ms | +| ShardingNoIndexCountAsync | 10 | 17.791 ms | 0.2928 ms | 0.2739 ms | 17.805 ms | +| NoShardingNoIndexFirstOrDefaultAsync0w | 10 | 3.239 ms | 0.0285 ms | 0.0267 ms | 3.231 ms | +| ShardingNoIndexFirstOrDefaultAsync0w | 10 | 8.826 ms | 0.1719 ms | 0.1688 ms | 8.806 ms | +| NoShardingNoIndexFirstOrDefaultAsync99w | 10 | 3.260 ms | 0.0208 ms | 0.0194 ms | 3.257 ms | +| ShardingNoIndexFirstOrDefaultAsync99w | 10 | 8.634 ms | 0.1062 ms | 0.0994 ms | 8.653 ms | +| NoShardingNoIndexLikeToListAsync | 10 | 26,941.543 ms | 138.5988 ms | 129.6454 ms | 26,920.578 ms | +| ShardingNoIndexLikeToListAsync | 10 | 5,840.364 ms | 112.0434 ms | 115.0604 ms | 5,797.137 ms | +| NoShardingNoIndexToListAsync | 10 | 25,865.136 ms | 115.6391 ms | 102.5111 ms | 25,847.258 ms | +| ShardingNoIndexToListAsync | 10 | 5,502.922 ms | 92.7201 ms | 86.7305 ms | 5,483.847 ms | + +具体可以通过first前两次结果来计算得出结论单次查询的的损耗为0.3-0.4毫秒之间,通过数据聚合和数据路由的损耗单次在0.3ms-0.4ms,其中创建dbcontext为0.1毫秒目前没有好的优化方案,0.2毫秒左右是路由表达式解析和编译,复杂表达式可能更加耗时。P +sqlserver的各项数据在分表和未分表的情况下都几乎差不多可以得出在770w数据集情况下数据库还并未是数据瓶颈的关键，但是mysql可以看到在分表和未分表的情况下如果涉及到没有索引的全表扫描那么性能的差距将是分表后的表数目之多，测试中为5-6倍，也就是分表数目 - [使用介绍](#使用介绍) @@ -57,6 +116,7 @@ Oracle | 支持 | 未测试 - [批量操作](#批量操作) - [读写分离](#读写分离) - [高性能分页](#高性能分页) + - [表达式缓存](#表达式缓存) - [注意事项](#注意事项) - [计划(Future)](#计划) - [最后](#最后) @@ -708,6 +768,50 @@ var shardingPageResultAsync = await _defaultTableDbContext.Set().Ord ``` ### 注意:如果你是按时间排序无论何种排序建议开启并且加上时间顺序排序,如果你是取模或者自定义分表,建议将Id作为顺序排序,如果没有特殊情况请使用id排序并且加上反向排序作为性能优化,如果entity同时支持分表分库并且两个路由都支持同一个属性的顺序排序优先级为先分库后分表 +## 表达式缓存 +可以通过路由开启表达式缓存针对单个tail的表达式进行缓存支持=,>,>=,<,<=,equal +```c# + + public class OrderCreateTimeVirtualTableRoute:AbstractSimpleShardingMonthKeyDateTimeVirtualTableRoute + { + //开启表达式缓存 + public override bool EnableRouteParseCompileCache => true; + } +``` +针对表达式缓存可以自行重写父类方法来自行实现，也可以仅实现多tail表达式`AbstractShardingRouteParseCompileCacheVirtualTableRoute`,`AbstractShardingRouteParseCompileCacheVirtualDataSourceRoute` +```c# + public virtual Func CachingCompile(Expression> parseWhere) + { + if (EnableRouteParseCompileCache) + { + var doCachingCompile = DoCachingCompile(parseWhere); + if (doCachingCompile != null) + return doCachingCompile; + doCachingCompile = CustomerCachingCompile(parseWhere); + if (doCachingCompile != null) + return doCachingCompile; + } + return parseWhere.Compile(); + } + /// + /// 系统默认永久单表达式缓存 + /// + /// + /// 返回null会走这个方法如果还是null就会调用方法 + protected virtual Func DoCachingCompile(Expression> parseWhere) + { + var shouldCache = ShouldCache(parseWhere); + if(shouldCache) + return _routeCompileCaches.GetOrAdd(parseWhere, key => parseWhere.Compile()); + return null; + } + protected virtual Func CustomerCachingCompile(Expression> parseWhere) + { + return null; + } +``` + +开启表达式缓存可以将路由模块的编译由原先的0.14ms升级到0.013ms提示约0.13ms将近10倍性能 # 注意事项 使用该框架需要注意两点如果你的shardingdbcontext重写了以下服务可能无法使用 如果还想使用需要自己重写扩展[请参考](https://github.com/xuejmnet/sharding-core/blob/main/src/ShardingCore/DIExtension.cs) diff --git a/benchmarks/ShardingCoreBenchmark/EFCoreCrud.cs b/benchmarks/ShardingCoreBenchmark/EFCoreCrud.cs index 0b0fc0d7..cf7217c8 100644 --- a/benchmarks/ShardingCoreBenchmark/EFCoreCrud.cs +++ b/benchmarks/ShardingCoreBenchmark/EFCoreCrud.cs @@ -41,7 +41,7 @@ namespace ShardingCore6x { var services = new ServiceCollection(); - services.AddDbContext(o => o.UseMySql("server=127.0.0.1;port=3306;database=db1;userid=root;password=;", new MySqlServerVersion(new Version())) + services.AddDbContext(o => o.UseMySql("server=127.0.0.1;port=3306;database=db1;userid=root;password=L6yBtV6qNENrwBy7;", new MySqlServerVersion(new Version())) //UseSqlServer("Data Source=localhost;Initial Catalog=db1;Integrated Security=True;") , ServiceLifetime.Transient, ServiceLifetime.Transient); services.AddLogging(); @@ -52,7 +52,7 @@ namespace ShardingCore6x o.EnsureCreatedWithOutShardingTable = true; o.AutoTrackEntity = false; }).AddShardingTransaction((connection, builder) => builder.UseMySql(connection, new MySqlServerVersion(new Version()))) - .AddDefaultDataSource("ds0", "server=127.0.0.1;port=3306;database=db2;userid=root;password=;")//"Data Source=localhost;Initial Catalog=db2;Integrated Security=True;") + .AddDefaultDataSource("ds0", "server=127.0.0.1;port=3306;database=db2;userid=root;password=L6yBtV6qNENrwBy7;")//"Data Source=localhost;Initial Catalog=db2;Integrated Security=True;") .AddShardingTableRoute(op => { op.AddShardingTableRoute(); diff --git a/src/ShardingCore/Core/VirtualRoutes/DataSourceRoutes/RouteRuleEngine/DataSourceRouteRuleEngine.cs b/src/ShardingCore/Core/VirtualRoutes/DataSourceRoutes/RouteRuleEngine/DataSourceRouteRuleEngine.cs index c0f17b9c..03f1f5c0 100644 --- a/src/ShardingCore/Core/VirtualRoutes/DataSourceRoutes/RouteRuleEngine/DataSourceRouteRuleEngine.cs +++ b/src/ShardingCore/Core/VirtualRoutes/DataSourceRoutes/RouteRuleEngine/DataSourceRouteRuleEngine.cs @@ -38,11 +38,14 @@ namespace ShardingCore.Core.VirtualRoutes.DataSourceRoutes.RouteRuleEngine dataSourceMaps.Add(notShardingDataSourceEntityType, new HashSet() { _virtualDataSource.DefaultDataSourceName }); - var queryEntities = routeRuleContext.QueryEntities.Where(o => _entityMetadataManager.IsShardingDataSource(o)).ToList(); //if (queryEntities.Count > 1) // throw new ShardingCoreNotSupportedException($"{routeRuleContext.Queryable.ShardingPrint()}"); - foreach (var queryEntity in queryEntities) + foreach (var queryEntity in routeRuleContext.QueryEntities) { + if (!_entityMetadataManager.IsShardingDataSource(queryEntity)) + { + continue; + } var dataSourceConfigs = _virtualDataSource.RouteTo(queryEntity,new ShardingDataSourceRouteConfig(routeRuleContext.Queryable)); if (!dataSourceMaps.ContainsKey(queryEntity)) { diff --git a/src/ShardingCore/Core/VirtualRoutes/TableRoutes/RoutingRuleEngine/TableRouteRuleEngine.cs b/src/ShardingCore/Core/VirtualRoutes/TableRoutes/RoutingRuleEngine/TableRouteRuleEngine.cs index 95c27aec..8e742754 100644 --- a/src/ShardingCore/Core/VirtualRoutes/TableRoutes/RoutingRuleEngine/TableRouteRuleEngine.cs +++ b/src/ShardingCore/Core/VirtualRoutes/TableRoutes/RoutingRuleEngine/TableRouteRuleEngine.cs @@ -36,9 +36,10 @@ namespace ShardingCore.Core.VirtualRoutes.TableRoutes.RoutingRuleEngine var queryEntities = tableRouteRuleContext.Queryable.ParseQueryableEntities(); - var shardingEntities = queryEntities.Where(o => _entityMetadataManager.IsShardingTable(o)); - foreach (var shardingEntity in shardingEntities) + foreach (var shardingEntity in queryEntities) { + if(!_entityMetadataManager.IsShardingTable(shardingEntity)) + continue; var virtualTable = _virtualTableManager.GetVirtualTable(shardingEntity); var physicTables = virtualTable.RouteTo(new ShardingTableRouteConfig(queryable: tableRouteRuleContext.Queryable));