p90524618
/
answer
mirror of https://gitee.com/answerdev/answer.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

refactor(sitemap): save sitemap in cache

This commit is contained in:
LinkinStars 2023-06-28 10:52:36 +08:00
parent 95a57417c1
commit a23080df0a
10 changed files with 106 additions and 124 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/wire_gen.go
View File

@ -222,7 +222,7 @@ func initApplication(debug bool, serverConf *conf.Server, dbConf *data.Database,
authUserMiddleware := middleware.NewAuthUserMiddleware(authService, siteInfoCommonService)
avatarMiddleware := middleware.NewAvatarMiddleware(serviceConf, uploaderService)
shortIDMiddleware := middleware.NewShortIDMiddleware(siteInfoCommonService)
templateRenderController := templaterender.NewTemplateRenderController(questionService, userService, tagService, answerService, commentService, dataData, siteInfoCommonService)
templateRenderController := templaterender.NewTemplateRenderController(questionService, userService, tagService, answerService, commentService, siteInfoCommonService, questionRepo)
templateController := controller.NewTemplateController(templateRenderController, siteInfoCommonService)
templateRouter := router.NewTemplateRouter(templateController, templateRenderController, siteInfoController)
connectorController := controller.NewConnectorController(siteInfoCommonService, emailService, userExternalLoginService)

3
internal/base/constant/cache_key.go
View File

@ -16,4 +16,7 @@ const (
ConfigKEY2ContentCacheKeyPrefix = "answer:config:key:"
ConnectorUserExternalInfoCacheKey = "answer:connector:"
ConnectorUserExternalInfoCacheTime = 10 * time.Minute
SiteMapQuestionCacheKeyPrefix = "answer:sitemap:question:%d"
SiteMapQuestionCacheTime = time.Hour
SitemapMaxSize = 50000
)

9
internal/controller/template_render/controller.go
View File

@ -1,9 +1,9 @@
package templaterender
import (
questioncommon "github.com/answerdev/answer/internal/service/question_common"
"math"
"github.com/answerdev/answer/internal/base/data"
"github.com/answerdev/answer/internal/service/comment"
"github.com/answerdev/answer/internal/service/siteinfo_common"
"github.com/google/wire"
@ -24,8 +24,8 @@ type TemplateRenderController struct {
tagService *tag.TagService
answerService *service.AnswerService
commentService *comment.CommentService
data *data.Data
siteInfoService siteinfo_common.SiteInfoCommonService
questionRepo questioncommon.QuestionRepo
}
func NewTemplateRenderController(
@ -34,9 +34,8 @@ func NewTemplateRenderController(
tagService *tag.TagService,
answerService *service.AnswerService,
commentService *comment.CommentService,
data *data.Data,
siteInfoService siteinfo_common.SiteInfoCommonService,
questionRepo questioncommon.QuestionRepo,
) *TemplateRenderController {
return &TemplateRenderController{
questionService: questionService,
@ -44,7 +43,7 @@ func NewTemplateRenderController(
tagService: tagService,
answerService: answerService,
commentService: commentService,
data: data,
questionRepo: questionRepo,
siteInfoService: siteInfoService,
}
}

1
internal/controller/template_render/index.go
View File

@ -1 +0,0 @@
package templaterender

71
internal/controller/template_render/question.go
View File

@ -1,8 +1,6 @@
package templaterender
import (
"encoding/json"
"fmt"
"html/template"
"net/http"
@ -32,48 +30,45 @@ func (t *TemplateRenderController) Sitemap(ctx *gin.Context) {
return
}
sitemapInfo := &schema.SiteMapList{}
infoStr, err := t.data.Cache.GetString(ctx, schema.SitemapCachekey)
questions, err := t.questionRepo.SitemapQuestions(ctx, 0, constant.SitemapMaxSize)
if err != nil {
log.Errorf("get Cache failed: %s", err)
return
}
hasTitle := false
if siteInfo.PermaLink == constant.PermaLinkQuestionIDAndTitle || siteInfo.PermaLink == constant.PermaLinkQuestionIDAndTitleByShortID {
hasTitle = true
}
if err = json.Unmarshal([]byte(infoStr), sitemapInfo); err != nil {
log.Errorf("get sitemap info failed: %s", err)
log.Errorf("get sitemap questions failed: %s", err)
return
}
if len(sitemapInfo.QuestionIDs) > 0 {
//question url list
ctx.Header("Content-Type", "application/xml")
ctx.Header("Content-Type", "application/xml")
if len(questions) < constant.SitemapMaxSize {
ctx.HTML(
http.StatusOK, "sitemap.xml", gin.H{
"xmlHeader": template.HTML(`<?xml version="1.0" encoding="UTF-8"?>`),
"list": sitemapInfo.QuestionIDs,
"general": general,
"hastitle": hasTitle,
},
)
} else {
//question list page
ctx.Header("Content-Type", "application/xml")
ctx.HTML(
http.StatusOK, "sitemap-list.xml", gin.H{
"xmlHeader": template.HTML(`<?xml version="1.0" encoding="UTF-8"?>`),
"page": sitemapInfo.MaxPageNum,
"list": questions,
"general": general,
"hastitle": siteInfo.PermaLink == constant.PermaLinkQuestionIDAndTitle ||
siteInfo.PermaLink == constant.PermaLinkQuestionIDAndTitleByShortID,
},
)
return
}
questionNum, err := t.questionRepo.GetQuestionCount(ctx)
if err != nil {
log.Error("GetQuestionCount error", err)
return
}
var pageList []int64
for page := int64(1); page*constant.SitemapMaxSize < questionNum; page++ {
pageList = append(pageList, page)
}
ctx.HTML(
http.StatusOK, "sitemap-list.xml", gin.H{
"xmlHeader": template.HTML(`<?xml version="1.0" encoding="UTF-8"?>`),
"page": pageList,
"general": general,
},
)
}
func (t *TemplateRenderController) SitemapPage(ctx *gin.Context, page int) error {
sitemapInfo := &schema.SiteMapPageList{}
general, err := t.siteInfoService.GetSiteGeneral(ctx)
if err != nil {
log.Error("get site general failed:", err)
@ -84,28 +79,20 @@ func (t *TemplateRenderController) SitemapPage(ctx *gin.Context, page int) error
log.Error("get site GetSiteSeo failed:", err)
return err
}
hasTitle := false
if siteInfo.PermaLink == constant.PermaLinkQuestionIDAndTitle || siteInfo.PermaLink == constant.PermaLinkQuestionIDAndTitleByShortID {
hasTitle = true
}
cachekey := fmt.Sprintf(schema.SitemapPageCachekey, page)
infoStr, err := t.data.Cache.GetString(ctx, cachekey)
questions, err := t.questionRepo.SitemapQuestions(ctx, page, constant.SitemapMaxSize)
if err != nil {
//If there is no cache, return directly.
return nil
}
if err = json.Unmarshal([]byte(infoStr), sitemapInfo); err != nil {
log.Errorf("get sitemap info failed: %s", err)
log.Errorf("get sitemap questions failed: %s", err)
return err
}
ctx.Header("Content-Type", "application/xml")
ctx.HTML(
http.StatusOK, "sitemap.xml", gin.H{
"xmlHeader": template.HTML(`<?xml version="1.0" encoding="UTF-8"?>`),
"list": sitemapInfo.PageData,
"list": questions,
"general": general,
"hastitle": hasTitle,
"hastitle": siteInfo.PermaLink == constant.PermaLinkQuestionIDAndTitle ||
siteInfo.PermaLink == constant.PermaLinkQuestionIDAndTitleByShortID,
},
)
return nil

63
internal/repo/question/question_repo.go
View File

@ -2,7 +2,9 @@ package question
import (
"context"
"encoding/json"
"fmt"
"github.com/segmentfault/pacman/log"
"strings"
"time"
"unicode"
@ -222,13 +224,13 @@ func (qr *questionRepo) GetQuestionList(ctx context.Context, question *entity.Qu
}
func (qr *questionRepo) GetQuestionCount(ctx context.Context) (count int64, err error) {
questionList := make([]*entity.Question, 0)
count, err = qr.data.DB.Context(ctx).In("question.status", []int{entity.QuestionStatusAvailable, entity.QuestionStatusClosed}).FindAndCount(&questionList)
session := qr.data.DB.Context(ctx)
session.Where("status = ? OR status = ?", entity.QuestionStatusAvailable, entity.QuestionStatusClosed)
count, err = session.Count(&entity.Question{Show: entity.QuestionShow})
if err != nil {
return count, errors.InternalServer(reason.DatabaseError).WithError(err).WithStack()
return 0, errors.InternalServer(reason.DatabaseError).WithError(err).WithStack()
}
return
return count, nil
}
func (qr *questionRepo) GetUserQuestionCount(ctx context.Context, userID string) (count int64, err error) {
@ -249,40 +251,51 @@ func (qr *questionRepo) GetQuestionCountByIDs(ctx context.Context, ids []string)
return
}
func (qr *questionRepo) GetQuestionIDsPage(ctx context.Context, page, pageSize int) (questionIDList []*schema.SiteMapQuestionInfo, err error) {
func (qr *questionRepo) SitemapQuestions(ctx context.Context, page, pageSize int) (
questionIDList []*schema.SiteMapQuestionInfo, err error) {
questionIDList = make([]*schema.SiteMapQuestionInfo, 0)
// try to get sitemap data from cache
cacheKey := fmt.Sprintf(constant.SiteMapQuestionCacheKeyPrefix, page)
cacheData, err := qr.data.Cache.GetString(ctx, cacheKey)
if err == nil && len(cacheKey) > 0 {
_ = json.Unmarshal([]byte(cacheData), &questionIDList)
return questionIDList, nil
}
// get sitemap data from db
rows := make([]*entity.Question, 0)
if page > 0 {
page = page - 1
} else {
page = 0
}
if pageSize == 0 {
pageSize = constant.DefaultPageSize
}
offset := page * pageSize
session := qr.data.DB.Context(ctx).Table("question")
session = session.In("question.status", []int{entity.QuestionStatusAvailable, entity.QuestionStatusClosed})
session.And("question.show = ?", entity.QuestionShow)
session = session.Limit(pageSize, offset)
session = session.OrderBy("question.created_at asc")
err = session.Select("id,title,created_at,post_update_time").Find(&rows)
session := qr.data.DB.Context(ctx)
session.Select("id,title,created_at,post_update_time")
session.Where("`show` = ?", entity.QuestionShow)
session.Where("status = ? OR status = ?", entity.QuestionStatusAvailable, entity.QuestionStatusClosed)
session.Limit(pageSize, page*pageSize)
session.Asc("created_at")
err = session.Find(&rows)
if err != nil {
return questionIDList, err
}
// warp data
for _, question := range rows {
item := &schema.SiteMapQuestionInfo{}
if handler.GetEnableShortID(ctx) {
item.ID = uid.EnShortID(question.ID)
}
item.Title = htmltext.UrlTitle(question.Title)
updateTime := fmt.Sprintf("%v", question.PostUpdateTime.Format(time.RFC3339))
if question.PostUpdateTime.Unix() < 1 {
updateTime = fmt.Sprintf("%v", question.CreatedAt.Format(time.RFC3339))
if question.PostUpdateTime.IsZero() {
item.UpdateTime = question.CreatedAt.Format(time.RFC3339)
} else {
item.UpdateTime = question.PostUpdateTime.Format(time.RFC3339)
}
item.UpdateTime = updateTime
questionIDList = append(questionIDList, item)
}
// set sitemap data to cache
cacheDataByte, _ := json.Marshal(questionIDList)
if err := qr.data.Cache.SetString(ctx, cacheKey, string(cacheDataByte), constant.SiteMapQuestionCacheTime); err != nil {
log.Error(err)
}
return questionIDList, nil
}

4
internal/router/template_router.go
View File

@ -17,7 +17,6 @@ func NewTemplateRouter(
templateController *controller.TemplateController,
templateRenderController *templaterender.TemplateRenderController,
siteInfoController *controller_admin.SiteInfoController,
) *TemplateRouter {
return &TemplateRouter{
templateController: templateController,
@ -26,7 +25,7 @@ func NewTemplateRouter(
}
}
// TemplateRouter template router
// RegisterTemplateRouter template router
func (a *TemplateRouter) RegisterTemplateRouter(r *gin.RouterGroup) {
r.GET("/sitemap.xml", a.templateController.Sitemap)
r.GET("/sitemap/:page", a.templateController.SitemapPage)
@ -35,7 +34,6 @@ func (a *TemplateRouter) RegisterTemplateRouter(r *gin.RouterGroup) {
r.GET("/custom.css", a.siteInfoController.GetCss)
r.GET("/", a.templateController.Index)
r.GET("/index", a.templateController.Index)
r.GET("/questions", a.templateController.QuestionList)
r.GET("/questions/:id", a.templateController.QuestionInfo)

18
internal/schema/question_schema.go
View File

@ -11,9 +11,6 @@ import (
)
const (
SitemapMaxSize = 50000
SitemapCachekey = "answer@sitemap"
SitemapPageCachekey = "answer@sitemap@page%d"
QuestionOperationPin = "pin"
QuestionOperationUnPin = "unpin"
QuestionOperationHide = "hide"
@ -427,21 +424,6 @@ type AdminSetQuestionStatusRequest struct {
QuestionID string `json:"question_id" form:"question_id"`
}
type SiteMapList struct {
QuestionIDs []*SiteMapQuestionInfo `json:"question_ids"`
MaxPageNum []int `json:"max_page_num"`
}
type SiteMapPageList struct {
PageData []*SiteMapQuestionInfo `json:"page_data"`
}
type SiteMapQuestionInfo struct {
ID string `json:"id"`
Title string `json:"title"`
UpdateTime string `json:"time"`
}
type PersonalQuestionPageReq struct {
Page int `validate:"omitempty,min=1" form:"page"`
PageSize int `validate:"omitempty,min=1" form:"page_size"`

16
internal/schema/sitemap_schema.go Normal file
View File

@ -0,0 +1,16 @@
package schema
type SiteMapList struct {
QuestionIDs []*SiteMapQuestionInfo `json:"question_ids"`
MaxPageNum []int `json:"max_page_num"`
}
type SiteMapPageList struct {
PageData []*SiteMapQuestionInfo `json:"page_data"`
}
type SiteMapQuestionInfo struct {
ID string `json:"id"`
Title string `json:"title"`
UpdateTime string `json:"time"`
}

43
internal/service/question_common/question.go
View File

@ -3,7 +3,6 @@ package questioncommon
import (
"context"
"encoding/json"
"fmt"
"math"
"time"
@ -52,7 +51,7 @@ type QuestionRepo interface {
GetQuestionCount(ctx context.Context) (count int64, err error)
GetUserQuestionCount(ctx context.Context, userID string) (count int64, err error)
GetQuestionCountByIDs(ctx context.Context, ids []string) (count int64, err error)
GetQuestionIDsPage(ctx context.Context, page, pageSize int) (questionIDList []*schema.SiteMapQuestionInfo, err error)
SitemapQuestions(ctx context.Context, page, pageSize int) (questionIDList []*schema.SiteMapQuestionInfo, err error)
}
// QuestionCommon user service
@ -553,40 +552,26 @@ func (as *QuestionCommon) RemoveAnswer(ctx context.Context, id string) (err erro
}
func (qs *QuestionCommon) SitemapCron(ctx context.Context) {
data := &schema.SiteMapList{}
questionNum, err := qs.questionRepo.GetQuestionCount(ctx)
if err != nil {
log.Error("GetQuestionCount error", err)
log.Error(err)
return
}
if questionNum <= schema.SitemapMaxSize {
questionIDList, err := qs.questionRepo.GetQuestionIDsPage(ctx, 0, int(questionNum))
if questionNum <= constant.SitemapMaxSize {
_, err = qs.questionRepo.SitemapQuestions(ctx, 0, int(questionNum))
if err != nil {
log.Error("GetQuestionIDsPage error", err)
log.Errorf("get site map question error: %v", err)
}
return
}
totalPages := int(math.Ceil(float64(questionNum) / float64(constant.SitemapMaxSize)))
for i := 1; i <= totalPages; i++ {
_, err = qs.questionRepo.SitemapQuestions(ctx, i, constant.SitemapMaxSize)
if err != nil {
log.Errorf("get site map question error: %v", err)
return
}
data.QuestionIDs = questionIDList
} else {
nums := make([]int, 0)
totalpages := int(math.Ceil(float64(questionNum) / float64(schema.SitemapMaxSize)))
for i := 1; i <= totalpages; i++ {
siteMapPagedata := &schema.SiteMapPageList{}
nums = append(nums, i)
questionIDList, err := qs.questionRepo.GetQuestionIDsPage(ctx, i, int(schema.SitemapMaxSize))
if err != nil {
log.Error("GetQuestionIDsPage error", err)
return
}
siteMapPagedata.PageData = questionIDList
if setCacheErr := qs.SetCache(ctx, fmt.Sprintf(schema.SitemapPageCachekey, i), siteMapPagedata); setCacheErr != nil {
log.Errorf("set sitemap cron SetCache failed: %s", setCacheErr)
}
}
data.MaxPageNum = nums
}
if setCacheErr := qs.SetCache(ctx, schema.SitemapCachekey, data); setCacheErr != nil {
log.Errorf("set sitemap cron SetCache failed: %s", setCacheErr)
}
}