refactor(sitemap): save sitemap in cache

cmd/wire_gen.go

@@ -222,7 +222,7 @@ func initApplication(debug bool, serverConf *conf.Server, dbConf *data.Database,
 	authUserMiddleware := middleware.NewAuthUserMiddleware(authService, siteInfoCommonService)
 	avatarMiddleware := middleware.NewAvatarMiddleware(serviceConf, uploaderService)
 	shortIDMiddleware := middleware.NewShortIDMiddleware(siteInfoCommonService)
-	templateRenderController := templaterender.NewTemplateRenderController(questionService, userService, tagService, answerService, commentService, dataData, siteInfoCommonService)
+	templateRenderController := templaterender.NewTemplateRenderController(questionService, userService, tagService, answerService, commentService, siteInfoCommonService, questionRepo)
 	templateController := controller.NewTemplateController(templateRenderController, siteInfoCommonService)
 	templateRouter := router.NewTemplateRouter(templateController, templateRenderController, siteInfoController)
 	connectorController := controller.NewConnectorController(siteInfoCommonService, emailService, userExternalLoginService)

internal/base/constant/cache_key.go

@@ -16,4 +16,7 @@ const (
 	ConfigKEY2ContentCacheKeyPrefix    = "answer:config:key:"
 	ConnectorUserExternalInfoCacheKey  = "answer:connector:"
 	ConnectorUserExternalInfoCacheTime = 10 * time.Minute
+	SiteMapQuestionCacheKeyPrefix      = "answer:sitemap:question:%d"
+	SiteMapQuestionCacheTime           = time.Hour
+	SitemapMaxSize                     = 50000
 )

internal/controller/template_render/controller.go

@@ -1,9 +1,9 @@
 package templaterender
 
 import (
+	questioncommon "github.com/answerdev/answer/internal/service/question_common"
 	"math"
 
-	"github.com/answerdev/answer/internal/base/data"
 	"github.com/answerdev/answer/internal/service/comment"
 	"github.com/answerdev/answer/internal/service/siteinfo_common"
 	"github.com/google/wire"
@@ -24,8 +24,8 @@ type TemplateRenderController struct {
 	tagService      *tag.TagService
 	answerService   *service.AnswerService
 	commentService  *comment.CommentService
-	data            *data.Data
 	siteInfoService siteinfo_common.SiteInfoCommonService
+	questionRepo    questioncommon.QuestionRepo
 }
 
 func NewTemplateRenderController(
@@ -34,9 +34,8 @@ func NewTemplateRenderController(
 	tagService *tag.TagService,
 	answerService *service.AnswerService,
 	commentService *comment.CommentService,
-	data *data.Data,
 	siteInfoService siteinfo_common.SiteInfoCommonService,
-
+	questionRepo questioncommon.QuestionRepo,
 ) *TemplateRenderController {
 	return &TemplateRenderController{
 		questionService: questionService,
@@ -44,7 +43,7 @@ func NewTemplateRenderController(
 		tagService:      tagService,
 		answerService:   answerService,
 		commentService:  commentService,
-		data:            data,
+		questionRepo:    questionRepo,
 		siteInfoService: siteInfoService,
 	}
 }

internal/controller/template_render/index.go

@@ -1 +0,0 @@
-package templaterender

internal/controller/template_render/question.go

@@ -1,8 +1,6 @@
 package templaterender
 
 import (
-	"encoding/json"
-	"fmt"
 	"html/template"
 	"net/http"
 
@@ -32,48 +30,45 @@ func (t *TemplateRenderController) Sitemap(ctx *gin.Context) {
 		return
 	}
 
-	sitemapInfo := &schema.SiteMapList{}
+	questions, err := t.questionRepo.SitemapQuestions(ctx, 0, constant.SitemapMaxSize)
-	infoStr, err := t.data.Cache.GetString(ctx, schema.SitemapCachekey)
 	if err != nil {
-		log.Errorf("get Cache failed: %s", err)
+		log.Errorf("get sitemap questions failed: %s", err)
-		return
-	}
-	hasTitle := false
-	if siteInfo.PermaLink == constant.PermaLinkQuestionIDAndTitle || siteInfo.PermaLink == constant.PermaLinkQuestionIDAndTitleByShortID {
-		hasTitle = true
-	}
-	if err = json.Unmarshal([]byte(infoStr), sitemapInfo); err != nil {
-		log.Errorf("get sitemap info failed: %s", err)
 		return
 	}
 
-	if len(sitemapInfo.QuestionIDs) > 0 {
-		//question url list
 	ctx.Header("Content-Type", "application/xml")
+	if len(questions) < constant.SitemapMaxSize {
 		ctx.HTML(
 			http.StatusOK, "sitemap.xml", gin.H{
 				"xmlHeader": template.HTML(`<?xml version="1.0" encoding="UTF-8"?>`),
-				"list":      sitemapInfo.QuestionIDs,
+				"list":      questions,
-				"general":   general,
-				"hastitle":  hasTitle,
-			},
-		)
-	} else {
-		//question list page
-		ctx.Header("Content-Type", "application/xml")
-		ctx.HTML(
-			http.StatusOK, "sitemap-list.xml", gin.H{
-				"xmlHeader": template.HTML(`<?xml version="1.0" encoding="UTF-8"?>`),
-				"page":      sitemapInfo.MaxPageNum,
 				"general":   general,
+				"hastitle": siteInfo.PermaLink == constant.PermaLinkQuestionIDAndTitle ||
+					siteInfo.PermaLink == constant.PermaLinkQuestionIDAndTitleByShortID,
 			},
 		)
 		return
 	}
+
+	questionNum, err := t.questionRepo.GetQuestionCount(ctx)
+	if err != nil {
+		log.Error("GetQuestionCount error", err)
+		return
+	}
+	var pageList []int64
+	for page := int64(1); page*constant.SitemapMaxSize < questionNum; page++ {
+		pageList = append(pageList, page)
+	}
+	ctx.HTML(
+		http.StatusOK, "sitemap-list.xml", gin.H{
+			"xmlHeader": template.HTML(`<?xml version="1.0" encoding="UTF-8"?>`),
+			"page":      pageList,
+			"general":   general,
+		},
+	)
 }
 
 func (t *TemplateRenderController) SitemapPage(ctx *gin.Context, page int) error {
-	sitemapInfo := &schema.SiteMapPageList{}
 	general, err := t.siteInfoService.GetSiteGeneral(ctx)
 	if err != nil {
 		log.Error("get site general failed:", err)
@@ -84,28 +79,20 @@ func (t *TemplateRenderController) SitemapPage(ctx *gin.Context, page int) error
 		log.Error("get site GetSiteSeo failed:", err)
 		return err
 	}
-	hasTitle := false
-	if siteInfo.PermaLink == constant.PermaLinkQuestionIDAndTitle || siteInfo.PermaLink == constant.PermaLinkQuestionIDAndTitleByShortID {
-		hasTitle = true
-	}
 
-	cachekey := fmt.Sprintf(schema.SitemapPageCachekey, page)
+	questions, err := t.questionRepo.SitemapQuestions(ctx, page, constant.SitemapMaxSize)
-	infoStr, err := t.data.Cache.GetString(ctx, cachekey)
 	if err != nil {
-		//If there is no cache, return directly.
+		log.Errorf("get sitemap questions failed: %s", err)
-		return nil
-	}
-	if err = json.Unmarshal([]byte(infoStr), sitemapInfo); err != nil {
-		log.Errorf("get sitemap info failed: %s", err)
 		return err
 	}
 	ctx.Header("Content-Type", "application/xml")
 	ctx.HTML(
 		http.StatusOK, "sitemap.xml", gin.H{
 			"xmlHeader": template.HTML(`<?xml version="1.0" encoding="UTF-8"?>`),
-			"list":      sitemapInfo.PageData,
+			"list":      questions,
 			"general":   general,
-			"hastitle":  hasTitle,
+			"hastitle": siteInfo.PermaLink == constant.PermaLinkQuestionIDAndTitle ||
+				siteInfo.PermaLink == constant.PermaLinkQuestionIDAndTitleByShortID,
 		},
 	)
 	return nil

internal/repo/question/question_repo.go

@@ -2,7 +2,9 @@ package question
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
+	"github.com/segmentfault/pacman/log"
 	"strings"
 	"time"
 	"unicode"
@@ -222,13 +224,13 @@ func (qr *questionRepo) GetQuestionList(ctx context.Context, question *entity.Qu
 }
 
 func (qr *questionRepo) GetQuestionCount(ctx context.Context) (count int64, err error) {
-	questionList := make([]*entity.Question, 0)
+	session := qr.data.DB.Context(ctx)
-
+	session.Where("status = ? OR status = ?", entity.QuestionStatusAvailable, entity.QuestionStatusClosed)
-	count, err = qr.data.DB.Context(ctx).In("question.status", []int{entity.QuestionStatusAvailable, entity.QuestionStatusClosed}).FindAndCount(&questionList)
+	count, err = session.Count(&entity.Question{Show: entity.QuestionShow})
 	if err != nil {
-		return count, errors.InternalServer(reason.DatabaseError).WithError(err).WithStack()
+		return 0, errors.InternalServer(reason.DatabaseError).WithError(err).WithStack()
 	}
-	return
+	return count, nil
 }
 
 func (qr *questionRepo) GetUserQuestionCount(ctx context.Context, userID string) (count int64, err error) {
@@ -249,40 +251,51 @@ func (qr *questionRepo) GetQuestionCountByIDs(ctx context.Context, ids []string)
 	return
 }
 
-func (qr *questionRepo) GetQuestionIDsPage(ctx context.Context, page, pageSize int) (questionIDList []*schema.SiteMapQuestionInfo, err error) {
+func (qr *questionRepo) SitemapQuestions(ctx context.Context, page, pageSize int) (
+	questionIDList []*schema.SiteMapQuestionInfo, err error) {
 	questionIDList = make([]*schema.SiteMapQuestionInfo, 0)
+
+	// try to get sitemap data from cache
+	cacheKey := fmt.Sprintf(constant.SiteMapQuestionCacheKeyPrefix, page)
+	cacheData, err := qr.data.Cache.GetString(ctx, cacheKey)
+	if err == nil && len(cacheKey) > 0 {
+		_ = json.Unmarshal([]byte(cacheData), &questionIDList)
+		return questionIDList, nil
+	}
+
+	// get sitemap data from db
 	rows := make([]*entity.Question, 0)
-	if page > 0 {
+	session := qr.data.DB.Context(ctx)
-		page = page - 1
+	session.Select("id,title,created_at,post_update_time")
-	} else {
+	session.Where("`show` = ?", entity.QuestionShow)
-		page = 0
+	session.Where("status = ? OR status = ?", entity.QuestionStatusAvailable, entity.QuestionStatusClosed)
-	}
+	session.Limit(pageSize, page*pageSize)
-	if pageSize == 0 {
+	session.Asc("created_at")
-		pageSize = constant.DefaultPageSize
+	err = session.Find(&rows)
-	}
-	offset := page * pageSize
-	session := qr.data.DB.Context(ctx).Table("question")
-	session = session.In("question.status", []int{entity.QuestionStatusAvailable, entity.QuestionStatusClosed})
-	session.And("question.show = ?", entity.QuestionShow)
-	session = session.Limit(pageSize, offset)
-	session = session.OrderBy("question.created_at asc")
-	err = session.Select("id,title,created_at,post_update_time").Find(&rows)
 	if err != nil {
 		return questionIDList, err
 	}
+
+	// warp data
 	for _, question := range rows {
 		item := &schema.SiteMapQuestionInfo{}
 		if handler.GetEnableShortID(ctx) {
 			item.ID = uid.EnShortID(question.ID)
 		}
 		item.Title = htmltext.UrlTitle(question.Title)
-		updateTime := fmt.Sprintf("%v", question.PostUpdateTime.Format(time.RFC3339))
+		if question.PostUpdateTime.IsZero() {
-		if question.PostUpdateTime.Unix() < 1 {
+			item.UpdateTime = question.CreatedAt.Format(time.RFC3339)
-			updateTime = fmt.Sprintf("%v", question.CreatedAt.Format(time.RFC3339))
+		} else {
+			item.UpdateTime = question.PostUpdateTime.Format(time.RFC3339)
 		}
-		item.UpdateTime = updateTime
 		questionIDList = append(questionIDList, item)
 	}
+
+	// set sitemap data to cache
+	cacheDataByte, _ := json.Marshal(questionIDList)
+	if err := qr.data.Cache.SetString(ctx, cacheKey, string(cacheDataByte), constant.SiteMapQuestionCacheTime); err != nil {
+		log.Error(err)
+	}
 	return questionIDList, nil
 }
 

internal/router/template_router.go

@@ -17,7 +17,6 @@ func NewTemplateRouter(
 	templateController *controller.TemplateController,
 	templateRenderController *templaterender.TemplateRenderController,
 	siteInfoController *controller_admin.SiteInfoController,
-
 ) *TemplateRouter {
 	return &TemplateRouter{
 		templateController:       templateController,
@@ -26,7 +25,7 @@ func NewTemplateRouter(
 	}
 }
 
-// TemplateRouter template router
+// RegisterTemplateRouter template router
 func (a *TemplateRouter) RegisterTemplateRouter(r *gin.RouterGroup) {
 	r.GET("/sitemap.xml", a.templateController.Sitemap)
 	r.GET("/sitemap/:page", a.templateController.SitemapPage)
@@ -35,7 +34,6 @@ func (a *TemplateRouter) RegisterTemplateRouter(r *gin.RouterGroup) {
 	r.GET("/custom.css", a.siteInfoController.GetCss)
 
 	r.GET("/", a.templateController.Index)
-	r.GET("/index", a.templateController.Index)
 
 	r.GET("/questions", a.templateController.QuestionList)
 	r.GET("/questions/:id", a.templateController.QuestionInfo)

internal/schema/question_schema.go

@@ -11,9 +11,6 @@ import (
 )
 
 const (
-	SitemapMaxSize         = 50000
-	SitemapCachekey        = "answer@sitemap"
-	SitemapPageCachekey    = "answer@sitemap@page%d"
 	QuestionOperationPin   = "pin"
 	QuestionOperationUnPin = "unpin"
 	QuestionOperationHide  = "hide"
@@ -427,21 +424,6 @@ type AdminSetQuestionStatusRequest struct {
 	QuestionID string `json:"question_id" form:"question_id"`
 }
 
-type SiteMapList struct {
-	QuestionIDs []*SiteMapQuestionInfo `json:"question_ids"`
-	MaxPageNum  []int                  `json:"max_page_num"`
-}
-
-type SiteMapPageList struct {
-	PageData []*SiteMapQuestionInfo `json:"page_data"`
-}
-
-type SiteMapQuestionInfo struct {
-	ID         string `json:"id"`
-	Title      string `json:"title"`
-	UpdateTime string `json:"time"`
-}
-
 type PersonalQuestionPageReq struct {
 	Page        int    `validate:"omitempty,min=1" form:"page"`
 	PageSize    int    `validate:"omitempty,min=1" form:"page_size"`

internal/schema/sitemap_schema.go

@@ -0,0 +1,16 @@
+package schema
+
+type SiteMapList struct {
+	QuestionIDs []*SiteMapQuestionInfo `json:"question_ids"`
+	MaxPageNum  []int                  `json:"max_page_num"`
+}
+
+type SiteMapPageList struct {
+	PageData []*SiteMapQuestionInfo `json:"page_data"`
+}
+
+type SiteMapQuestionInfo struct {
+	ID         string `json:"id"`
+	Title      string `json:"title"`
+	UpdateTime string `json:"time"`
+}

internal/service/question_common/question.go

@@ -3,7 +3,6 @@ package questioncommon
 import (
 	"context"
 	"encoding/json"
-	"fmt"
 	"math"
 	"time"
 
@@ -52,7 +51,7 @@ type QuestionRepo interface {
 	GetQuestionCount(ctx context.Context) (count int64, err error)
 	GetUserQuestionCount(ctx context.Context, userID string) (count int64, err error)
 	GetQuestionCountByIDs(ctx context.Context, ids []string) (count int64, err error)
-	GetQuestionIDsPage(ctx context.Context, page, pageSize int) (questionIDList []*schema.SiteMapQuestionInfo, err error)
+	SitemapQuestions(ctx context.Context, page, pageSize int) (questionIDList []*schema.SiteMapQuestionInfo, err error)
 }
 
 // QuestionCommon user service
@@ -553,40 +552,26 @@ func (as *QuestionCommon) RemoveAnswer(ctx context.Context, id string) (err erro
 }
 
 func (qs *QuestionCommon) SitemapCron(ctx context.Context) {
-	data := &schema.SiteMapList{}
 	questionNum, err := qs.questionRepo.GetQuestionCount(ctx)
 	if err != nil {
-		log.Error("GetQuestionCount error", err)
+		log.Error(err)
 		return
 	}
-	if questionNum <= schema.SitemapMaxSize {
+	if questionNum <= constant.SitemapMaxSize {
-		questionIDList, err := qs.questionRepo.GetQuestionIDsPage(ctx, 0, int(questionNum))
+		_, err = qs.questionRepo.SitemapQuestions(ctx, 0, int(questionNum))
 		if err != nil {
-			log.Error("GetQuestionIDsPage error", err)
+			log.Errorf("get site map question error: %v", err)
+		}
 		return
 	}
-		data.QuestionIDs = questionIDList
 
-	} else {
+	totalPages := int(math.Ceil(float64(questionNum) / float64(constant.SitemapMaxSize)))
-		nums := make([]int, 0)
+	for i := 1; i <= totalPages; i++ {
-		totalpages := int(math.Ceil(float64(questionNum) / float64(schema.SitemapMaxSize)))
+		_, err = qs.questionRepo.SitemapQuestions(ctx, i, constant.SitemapMaxSize)
-		for i := 1; i <= totalpages; i++ {
-			siteMapPagedata := &schema.SiteMapPageList{}
-			nums = append(nums, i)
-			questionIDList, err := qs.questionRepo.GetQuestionIDsPage(ctx, i, int(schema.SitemapMaxSize))
 		if err != nil {
-				log.Error("GetQuestionIDsPage error", err)
+			log.Errorf("get site map question error: %v", err)
 			return
 		}
-			siteMapPagedata.PageData = questionIDList
-			if setCacheErr := qs.SetCache(ctx, fmt.Sprintf(schema.SitemapPageCachekey, i), siteMapPagedata); setCacheErr != nil {
-				log.Errorf("set sitemap cron SetCache failed: %s", setCacheErr)
-			}
-		}
-		data.MaxPageNum = nums
-	}
-	if setCacheErr := qs.SetCache(ctx, schema.SitemapCachekey, data); setCacheErr != nil {
-		log.Errorf("set sitemap cron SetCache failed: %s", setCacheErr)
 	}
 }