feat(#IW43M): 操作日志后台使用分页查询提高性能

comment #IW43M
使用后台分页提高数据量大时程序性能

close https://gitee.com/LongbowEnterprise/dashboard/issues?id=IW43M

Bootstrap.Admin/Query/QueryLogOption.cs

@@ -1,7 +1,6 @@
 using Bootstrap.DataAccess;
 using Longbow.Web.Mvc;
 using System;
-using System.Linq;
 
 namespace Bootstrap.Admin.Query
 {
@@ -14,56 +13,27 @@ namespace Bootstrap.Admin.Query
         /// 
         /// </summary>
         public string OperateType { get; set; }
+
         /// <summary>
         /// 
         /// </summary>
         public DateTime? OperateTimeStart { get; set; }
+
         /// <summary>
         /// 
         /// </summary>
         public DateTime? OperateTimeEnd { get; set; }
+
         /// <summary>
         /// 
         /// </summary>
         /// <returns></returns>
         public QueryData<Log> RetrieveData()
         {
-            var data = LogHelper.Retrieves();
+            var data = LogHelper.Retrieves(this, OperateTimeStart, OperateTimeEnd, OperateType);
-            if (!string.IsNullOrEmpty(OperateType))
-            {
-                data = data.Where(t => t.CRUD.Contains(OperateType));
-            }
-
-            if (OperateTimeStart > DateTime.MinValue)
-            {
-                data = data.Where(t => t.LogTime > OperateTimeStart);
-            }
-            if (OperateTimeEnd > DateTime.MinValue)
-            {
-                data = data.Where(t => t.LogTime < OperateTimeEnd.Value.AddDays(1));
-            }
-
             var ret = new QueryData<Log>();
-            ret.total = data.Count();
+            ret.total = data.TotalItems;
-            switch (Sort)
+            ret.rows = data.Items;
-            {
-                case "CRUD":
-                    data = Order == "asc" ? data.OrderBy(t => t.CRUD) : data.OrderByDescending(t => t.CRUD);
-                    break;
-                case "UserName":
-                    data = Order == "asc" ? data.OrderBy(t => t.UserName) : data.OrderByDescending(t => t.UserName);
-                    break;
-                case "LogTime":
-                    data = Order == "asc" ? data.OrderBy(t => t.LogTime) : data.OrderByDescending(t => t.LogTime);
-                    break;
-                case "Ip":
-                    data = Order == "asc" ? data.OrderBy(t => t.Ip) : data.OrderByDescending(t => t.Ip);
-                    break;
-                case "RequestUrl":
-                    data = Order == "asc" ? data.OrderBy(t => t.RequestUrl) : data.OrderByDescending(t => t.RequestUrl);
-                    break;
-            }
-            ret.rows = data.Skip(Offset).Take(Limit);
             return ret;
         }
     }

Bootstrap.Admin/appsettings.json

@@ -213,13 +213,6 @@
         "SlidingExpiration": true,
         "Desc": "通过角色ID获得所有应用程序数据"
       },
-      {
-        "Enabled": true,
-        "Key": "LogHelper-RetrieveLogs",
-        "Interval": 600000,
-        "SlidingExpiration": true,
-        "Desc": "所有日志数据"
-      },
       {
         "Enabled": true,
         "Key": "DictHelper-RetrieveDictsCategory",

Bootstrap.DataAccess.MongoDB/Log.cs

@@ -1,6 +1,8 @@
-using MongoDB.Driver;
+using Longbow.Web.Mvc;
+using MongoDB.Driver;
+using PetaPoco;
 using System;
-using System.Collections.Generic;
+using System.Linq;
 
 namespace Bootstrap.DataAccess.MongoDB
 {
@@ -12,8 +14,54 @@ namespace Bootstrap.DataAccess.MongoDB
         /// <summary>
         /// 
         /// </summary>
+        /// <param name="po"></param>
+        /// <param name="startTime"></param>
+        /// <param name="endTime"></param>
+        /// <param name="opType"></param>
         /// <returns></returns>
-        public override IEnumerable<DataAccess.Log> Retrieves() => DbManager.Logs.Find(l => l.LogTime >= DateTime.Now.AddDays(-7)).ToList();
+        public override Page<DataAccess.Log> Retrieves(PaginationOption po, DateTime? startTime, DateTime? endTime, string opType)
+        {
+            var filterBuilder = Builders<DataAccess.Log>.Filter;
+            var filter = filterBuilder.Empty;
+            if (startTime.HasValue) filter = filterBuilder.Gte("LogTime", startTime.Value);
+            if (endTime.HasValue) filter = filterBuilder.Lt("LogTime", endTime.Value.AddDays(1).AddSeconds(-1));
+            if (startTime == null && endTime == null) filter = filterBuilder.Gt("LogTime", DateTime.Today.AddMonths(0 - DictHelper.RetrieveAccessLogPeriod()));
+            if (!string.IsNullOrEmpty(opType)) filter = filterBuilder.Eq("CRUD", opType);
+
+            // sort
+            var sortBuilder = Builders<DataAccess.Log>.Sort;
+            SortDefinition<DataAccess.Log> sort = null;
+            switch (po.Sort)
+            {
+                case "CRUD":
+                    sort = po.Order == "asc" ? sortBuilder.Ascending(t => t.CRUD) : sortBuilder.Descending(t => t.CRUD);
+                    break;
+                case "UserName":
+                    sort = po.Order == "asc" ? sortBuilder.Ascending(t => t.UserName) : sortBuilder.Descending(t => t.UserName);
+                    break;
+                case "LogTime":
+                    sort = po.Order == "asc" ? sortBuilder.Ascending(t => t.LogTime) : sortBuilder.Descending(t => t.LogTime);
+                    break;
+                case "Ip":
+                    sort = po.Order == "asc" ? sortBuilder.Ascending(t => t.Ip) : sortBuilder.Descending(t => t.Ip);
+                    break;
+                case "RequestUrl":
+                    sort = po.Order == "asc" ? sortBuilder.Ascending(t => t.RequestUrl) : sortBuilder.Descending(t => t.RequestUrl);
+                    break;
+            }
+
+            var logs = DbManager.Logs.Find(filter).Sort(sort).ToList();
+            return new Page<DataAccess.Log>()
+            {
+                Context = logs,
+                CurrentPage = po.PageIndex,
+                ItemsPerPage = po.Limit,
+                TotalItems = logs.Count,
+                TotalPages = (long)Math.Ceiling(logs.Count * 1.0 / po.Limit),
+                Items = logs.Skip(po.Offset).Take(po.Limit).ToList()
+            };
+        }
+
         /// <summary>
         /// 删除日志信息
         /// </summary>

Bootstrap.DataAccess/Helper/LogHelper.cs

@@ -1,6 +1,7 @@
-using Longbow.Cache;
+using Longbow.Data;
-using Longbow.Data;
+using Longbow.Web.Mvc;
-using System.Collections.Generic;
+using PetaPoco;
+using System;
 
 namespace Bootstrap.DataAccess
 {
@@ -9,27 +10,18 @@ namespace Bootstrap.DataAccess
     /// </summary>
     public static class LogHelper
     {
-        /// <summary>
-        /// 
-        /// </summary>
-        public const string RetrieveLogsDataKey = "LogHelper-RetrieveLogs";
-
         /// <summary>
         /// 查询所有日志信息
         /// </summary>
+        /// <param name="op"></param>
         /// <returns></returns>
-        public static IEnumerable<Log> Retrieves() => CacheManager.GetOrAdd(RetrieveLogsDataKey, key => DbContextManager.Create<Log>().Retrieves());
+        public static Page<Log> Retrieves(PaginationOption op, DateTime? startTime, DateTime? endTime, string opType) => DbContextManager.Create<Log>().Retrieves(op, startTime, endTime, opType);
 
         /// <summary>
         /// 保存新增的日志信息
         /// </summary>
         /// <param name="p"></param>
         /// <returns></returns>
-        public static bool Save(Log p)
+        public static bool Save(Log p) => DbContextManager.Create<Log>().Save(p);
-        {
-            var ret = DbContextManager.Create<Log>().Save(p);
-            if (ret) CacheManager.Clear(RetrieveLogsDataKey);
-            return ret;
-        }
     }
 }

Bootstrap.DataAccess/Log.cs

@@ -1,5 +1,6 @@
-using System;
+using Longbow.Web.Mvc;
-using System.Collections.Generic;
+using PetaPoco;
+using System;
 
 namespace Bootstrap.DataAccess
 {
@@ -19,10 +20,24 @@ namespace Bootstrap.DataAccess
         public string RequestData { get; set; }
 
         /// <summary>
-        /// 查询所有日志信息
+        /// 查询所有操作日志信息
         /// </summary>
+        /// <param name="po"></param>
+        /// <param name="startTime"></param>
+        /// <param name="endTime"></param>
+        /// <param name="opType"></param>
         /// <returns></returns>
-        public virtual IEnumerable<Log> Retrieves() => DbManager.Create().Fetch<Log>("select * from Logs where LogTime > @0 order by LogTime desc", DateTime.Now.AddDays(-7));
+        public virtual Page<Log> Retrieves(PaginationOption po, DateTime? startTime, DateTime? endTime, string opType)
+        {
+            var sql = new Sql("select CRUD, UserName, LogTime, Ip, Browser, OS, City, RequestUrl, RequestData from Logs");
+            if (startTime.HasValue) sql.Append("where LogTime >= @0", startTime.Value);
+            if (endTime.HasValue) sql.Append("where LogTime < @0", endTime.Value.AddDays(1).AddSeconds(-1));
+            if (startTime == null && endTime == null) sql.Append("where LogTime > @0", DateTime.Today.AddMonths(0 - DictHelper.RetrieveExceptionsLogPeriod()));
+            if (!string.IsNullOrEmpty(opType)) sql.Append("where CRUD = @0", opType);
+            sql.Append($"order by {po.Sort} {po.Order}");
+
+            return DbManager.Create().Page<Log>(po.PageIndex, po.Limit, sql);
+        }
 
         /// <summary>
         /// 删除日志信息

UnitTest/Bootstrap.DataAccess/LogsTest.cs

@@ -1,4 +1,5 @@
-using Xunit;
+using Longbow.Web.Mvc;
+using Xunit;
 
 namespace Bootstrap.DataAccess
 {
@@ -35,7 +36,7 @@ namespace Bootstrap.DataAccess
                 RequestUrl = "~/Home/Index"
             };
             log.Save(log);
-            Assert.NotEmpty(log.Retrieves());
+            Assert.NotNull(log.Retrieves(new PaginationOption() { Limit = 20 }, null, null));
         }
     }
 }