sitemap cache

cmd/answer/wire_gen.go

@@ -164,7 +164,7 @@ func initApplication(debug bool, serverConf *conf.Server, dbConf *data.Database,
 	answerActivityRepo := activity.NewAnswerActivityRepo(dataData, activityRepo, userRankRepo)
 	questionActivityRepo := activity.NewQuestionActivityRepo(dataData, activityRepo, userRankRepo)
 	answerActivityService := activity2.NewAnswerActivityService(answerActivityRepo, questionActivityRepo)
-	questionService := service.NewQuestionService(questionRepo, tagCommonService, questionCommon, userCommon, revisionService, metaService, collectionCommon, answerActivityService)
+	questionService := service.NewQuestionService(questionRepo, tagCommonService, questionCommon, userCommon, revisionService, metaService, collectionCommon, answerActivityService, dataData)
 	questionController := controller.NewQuestionController(questionService, rankService)
 	answerService := service.NewAnswerService(answerRepo, questionRepo, questionCommon, userCommon, collectionCommon, userRepo, revisionService, answerActivityService, answerCommon, voteRepo)
 	dashboardService := dashboard.NewDashboardService(questionRepo, answerRepo, commentCommonRepo, voteRepo, userRepo, reportRepo, configRepo, siteInfoCommonService, serviceConf, dataData)
@@ -207,11 +207,11 @@ func initApplication(debug bool, serverConf *conf.Server, dbConf *data.Database,
 	uiRouter := router.NewUIRouter(siteinfoController)
 	authUserMiddleware := middleware.NewAuthUserMiddleware(authService)
 	avatarMiddleware := middleware.NewAvatarMiddleware(serviceConf, uploaderService)
-	templateRenderController := templaterender.NewTemplateRenderController(questionService, userService, tagService, answerService, commentService)
+	templateRenderController := templaterender.NewTemplateRenderController(questionService, userService, tagService, answerService, commentService, dataData, siteInfoCommonService)
 	templateController := controller.NewTemplateController(templateRenderController, siteInfoCommonService)
 	templateRouter := router.NewTemplateRouter(templateController, templateRenderController, siteInfoController)
 	ginEngine := server.NewHTTPServer(debug, staticRouter, answerAPIRouter, swaggerRouter, uiRouter, authUserMiddleware, avatarMiddleware, templateRouter)
-	scheduledTaskManager := cron.NewScheduledTaskManager(siteInfoCommonService)
+	scheduledTaskManager := cron.NewScheduledTaskManager(siteInfoCommonService, questionService)
 	application := newApplication(serverConf, ginEngine, scheduledTaskManager)
 	return application, func() {
 		cleanup2()

internal/base/cron/cron.go

@@ -1,18 +1,31 @@
 package cron
 
-import "github.com/answerdev/answer/internal/service/siteinfo_common"
+import (
+	"context"
+
+	"github.com/answerdev/answer/internal/service"
+	"github.com/answerdev/answer/internal/service/siteinfo_common"
+)
 
 // ScheduledTaskManager scheduled task manager
 type ScheduledTaskManager struct {
 	siteInfoService *siteinfo_common.SiteInfoCommonService
+	questionService *service.QuestionService
 }
 
 // NewScheduledTaskManager new scheduled task manager
-func NewScheduledTaskManager(siteInfoService *siteinfo_common.SiteInfoCommonService) *ScheduledTaskManager {
+func NewScheduledTaskManager(
-	manager := &ScheduledTaskManager{siteInfoService: siteInfoService}
+	siteInfoService *siteinfo_common.SiteInfoCommonService,
+	questionService *service.QuestionService,
+) *ScheduledTaskManager {
+	manager := &ScheduledTaskManager{
+		siteInfoService: siteInfoService,
+		questionService: questionService,
+	}
 	return manager
 }
 
 func (s *ScheduledTaskManager) Run() {
-
+	ctx := context.Background()
+	s.questionService.SitemapCron(ctx)
 }

internal/controller/template_render/controller.go

@@ -1,9 +1,12 @@
 package templaterender
 
 import (
-	"github.com/answerdev/answer/internal/service/comment"
 	"math"
 
+	"github.com/answerdev/answer/internal/base/data"
+	"github.com/answerdev/answer/internal/service/comment"
+	"github.com/answerdev/answer/internal/service/siteinfo_common"
+
 	"github.com/answerdev/answer/internal/schema"
 	"github.com/answerdev/answer/internal/service"
 	"github.com/answerdev/answer/internal/service/tag"
@@ -21,6 +24,8 @@ type TemplateRenderController struct {
 	tagService      *tag.TagService
 	answerService   *service.AnswerService
 	commentService  *comment.CommentService
+	data            *data.Data
+	siteInfoService *siteinfo_common.SiteInfoCommonService
 }
 
 func NewTemplateRenderController(
@@ -29,6 +34,9 @@ func NewTemplateRenderController(
 	tagService *tag.TagService,
 	answerService *service.AnswerService,
 	commentService *comment.CommentService,
+	data *data.Data,
+	siteInfoService *siteinfo_common.SiteInfoCommonService,
+
 ) *TemplateRenderController {
 	return &TemplateRenderController{
 		questionService: questionService,
@@ -36,6 +44,8 @@ func NewTemplateRenderController(
 		tagService:      tagService,
 		answerService:   answerService,
 		commentService:  commentService,
+		data:            data,
+		siteInfoService: siteInfoService,
 	}
 }
 

internal/controller/template_render/question.go

@@ -1,11 +1,14 @@
 package templaterender
 
 import (
+	"encoding/json"
+	"fmt"
 	"html/template"
 	"net/http"
 
 	"github.com/answerdev/answer/internal/schema"
 	"github.com/gin-gonic/gin"
+	"github.com/segmentfault/pacman/log"
 )
 
 func (t *TemplateRenderController) Index(ctx *gin.Context, req *schema.QuestionSearch) ([]*schema.QuestionInfo, int64, error) {
@@ -17,33 +20,71 @@ func (t *TemplateRenderController) QuestionDetail(ctx *gin.Context, id string) (
 }
 
 func (t *TemplateRenderController) Sitemap(ctx *gin.Context) {
-	if 1 == 1 {
+	general, err := t.siteInfoService.GetSiteGeneral(ctx)
-		//question list page
+	if err != nil {
-		ctx.Header("Content-Type", "application/xml")
+		log.Error("get site general failed:", err)
-		ctx.HTML(
-			http.StatusOK, "sitemap-list.xml", gin.H{
-				"xmlHeader": template.HTML(`<?xml version="1.0" encoding="UTF-8"?>`),
-				"list":      "string",
-			},
-		)
 		return
 	}
+
+	sitemapInfo := &schema.SiteMapList{}
+	infoStr, err := t.data.Cache.GetString(ctx, schema.SitemapCachekey)
+	if err != nil {
+		log.Errorf("get Cache failed: %s", err)
+		return
+	}
+	if err = json.Unmarshal([]byte(infoStr), sitemapInfo); err != nil {
+		log.Errorf("get sitemap info failed: %s", err)
+		return
+	}
+
+	if len(sitemapInfo.QuestionIDs) > 0 {
 		//question url list
 		ctx.Header("Content-Type", "application/xml")
 		ctx.HTML(
 			http.StatusOK, "sitemap.xml", gin.H{
 				"xmlHeader": template.HTML(`<?xml version="1.0" encoding="UTF-8"?>`),
-			"list":      "string",
+				"list":      sitemapInfo.QuestionIDs,
+				"general":   general,
 			},
 		)
+	} else {
+		//question list page
+		ctx.Header("Content-Type", "application/xml")
+		ctx.HTML(
+			http.StatusOK, "sitemap-list.xml", gin.H{
+				"xmlHeader": template.HTML(`<?xml version="1.0" encoding="UTF-8"?>`),
+				"page":      sitemapInfo.MaxPageNum,
+				"general":   general,
+			},
+		)
+		return
+	}
 }
 
 func (t *TemplateRenderController) SitemapPage(ctx *gin.Context, page int) error {
+	sitemapInfo := &schema.SiteMapPageList{}
+	general, err := t.siteInfoService.GetSiteGeneral(ctx)
+	if err != nil {
+		log.Error("get site general failed:", err)
+		return err
+	}
+
+	cachekey := fmt.Sprintf(schema.SitemapPageCachekey, page)
+	infoStr, err := t.data.Cache.GetString(ctx, cachekey)
+	if err != nil {
+		log.Errorf("get Cache failed: %s", err)
+		return err
+	}
+	if err = json.Unmarshal([]byte(infoStr), sitemapInfo); err != nil {
+		log.Errorf("get sitemap info failed: %s", err)
+		return err
+	}
 	ctx.Header("Content-Type", "application/xml")
 	ctx.HTML(
 		http.StatusOK, "sitemap.xml", gin.H{
 			"xmlHeader": template.HTML(`<?xml version="1.0" encoding="UTF-8"?>`),
-			"list":      "string",
+			"list":      sitemapInfo.PageData,
+			"general":   general,
 		},
 	)
 	return nil

internal/repo/question/question_repo.go

@@ -16,6 +16,7 @@ import (
 	"github.com/answerdev/answer/internal/schema"
 	questioncommon "github.com/answerdev/answer/internal/service/question_common"
 	"github.com/answerdev/answer/internal/service/unique"
+	"github.com/answerdev/answer/pkg/htmltext"
 
 	"github.com/segmentfault/pacman/errors"
 )
@@ -173,6 +174,36 @@ func (qr *questionRepo) GetQuestionCount(ctx context.Context) (count int64, err
 	return
 }
 
+func (qr *questionRepo) GetQuestionIDsPage(ctx context.Context, page, pageSize int) (questionIDList []*schema.SiteMapQuestionInfo, err error) {
+	questionIDList = make([]*schema.SiteMapQuestionInfo, 0)
+	rows := make([]*entity.Question, 0)
+	if page > 0 {
+		page = page - 1
+	} else {
+		page = 0
+	}
+	if pageSize == 0 {
+		pageSize = constant.DefaultPageSize
+	}
+	offset := page * pageSize
+	session := qr.data.DB.Table("question")
+	session = session.In("question.status", []int{entity.QuestionStatusAvailable, entity.QuestionStatusClosed})
+	session = session.Limit(pageSize, offset)
+	session = session.OrderBy("question.created_at asc")
+	err = session.Select("id,title,post_update_time").Find(&rows)
+	if err != nil {
+		return questionIDList, err
+	}
+	for _, question := range rows {
+		item := &schema.SiteMapQuestionInfo{}
+		item.ID = question.ID
+		item.Title = htmltext.UrlTitle(question.Title)
+		item.UpdateTime = question.PostUpdateTime.Format("2006-01-02 15:04:05")
+		questionIDList = append(questionIDList, item)
+	}
+	return questionIDList, nil
+}
+
 // GetQuestionPage get question page
 func (qr *questionRepo) GetQuestionPage(ctx context.Context, page, pageSize int, question *entity.Question) (questionList []*entity.Question, total int64, err error) {
 	questionList = make([]*entity.Question, 0)

internal/schema/question_schema.go

@@ -1,5 +1,12 @@
 package schema
 
+const (
+	// SitemapMaxSize  = 50000
+	SitemapMaxSize      = 10
+	SitemapCachekey     = "answer@sitemap"
+	SitemapPageCachekey = "answer@sitemap@page%d"
+)
+
 // RemoveQuestionReq delete question request
 type RemoveQuestionReq struct {
 	// question id
@@ -218,3 +225,18 @@ type AdminSetQuestionStatusRequest struct {
 	StatusStr  string `json:"status" form:"status"`
 	QuestionID string `json:"question_id" form:"question_id"`
 }
+
+type SiteMapList struct {
+	QuestionIDs []*SiteMapQuestionInfo `json:"question_ids"`
+	MaxPageNum  []int                  `json:"max_page_num"`
+}
+
+type SiteMapPageList struct {
+	PageData []*SiteMapQuestionInfo `json:"page_data"`
+}
+
+type SiteMapQuestionInfo struct {
+	ID         string `json:"id"`
+	Title      string `json:"title"`
+	UpdateTime string `json:"time"`
+}

internal/service/question_common/question.go

@@ -41,6 +41,7 @@ type QuestionRepo interface {
 	FindByID(ctx context.Context, id []string) (questionList []*entity.Question, err error)
 	CmsSearchList(ctx context.Context, search *schema.CmsQuestionSearch) ([]*entity.Question, int64, error)
 	GetQuestionCount(ctx context.Context) (count int64, err error)
+	GetQuestionIDsPage(ctx context.Context, page, pageSize int) (questionIDList []*schema.SiteMapQuestionInfo, err error)
 }
 
 // QuestionCommon user service

internal/service/question_service.go

@@ -3,10 +3,12 @@ package service
 import (
 	"encoding/json"
 	"fmt"
+	"math"
 	"strings"
 	"time"
 
 	"github.com/answerdev/answer/internal/base/constant"
+	"github.com/answerdev/answer/internal/base/data"
 	"github.com/answerdev/answer/internal/base/handler"
 	"github.com/answerdev/answer/internal/base/reason"
 	"github.com/answerdev/answer/internal/base/translator"
@@ -43,6 +45,7 @@ type QuestionService struct {
 	metaService           *meta.MetaService
 	collectionCommon      *collectioncommon.CollectionCommon
 	answerActivityService *activity.AnswerActivityService
+	data                  *data.Data
 }
 
 func NewQuestionService(
@@ -54,6 +57,8 @@ func NewQuestionService(
 	metaService *meta.MetaService,
 	collectionCommon *collectioncommon.CollectionCommon,
 	answerActivityService *activity.AnswerActivityService,
+	data *data.Data,
+
 ) *QuestionService {
 	return &QuestionService{
 		questionRepo:          questionRepo,
@@ -64,6 +69,7 @@ func NewQuestionService(
 		metaService:           metaService,
 		collectionCommon:      collectionCommon,
 		answerActivityService: answerActivityService,
+		data:                  data,
 	}
 }
 
@@ -1001,3 +1007,57 @@ func (qs *QuestionService) changeQuestionToRevision(ctx context.Context, questio
 	}
 	return questionRevision, nil
 }
+
+func (qs *QuestionService) SitemapCron(ctx context.Context) {
+	data := &schema.SiteMapList{}
+	questionNum, err := qs.questionRepo.GetQuestionCount(ctx)
+	if err != nil {
+		log.Error("GetQuestionCount error", err)
+		return
+	}
+	if questionNum <= schema.SitemapMaxSize {
+		questionIDList, err := qs.questionRepo.GetQuestionIDsPage(ctx, 0, int(questionNum))
+		if err != nil {
+			log.Error("GetQuestionIDsPage error", err)
+			return
+		}
+		data.QuestionIDs = questionIDList
+
+	} else {
+		nums := make([]int, 0)
+		totalpages := int(math.Ceil(float64(questionNum) / float64(schema.SitemapMaxSize)))
+		for i := 1; i <= totalpages; i++ {
+			siteMapPagedata := &schema.SiteMapPageList{}
+			nums = append(nums, i)
+			questionIDList, err := qs.questionRepo.GetQuestionIDsPage(ctx, i, int(schema.SitemapMaxSize))
+			if err != nil {
+				log.Error("GetQuestionIDsPage error", err)
+				return
+			}
+			siteMapPagedata.PageData = questionIDList
+			if setCacheErr := qs.SetCache(ctx, fmt.Sprintf(schema.SitemapPageCachekey, i), siteMapPagedata); setCacheErr != nil {
+				log.Errorf("set sitemap cron SetCache failed: %s", setCacheErr)
+			}
+		}
+		data.MaxPageNum = nums
+	}
+	if setCacheErr := qs.SetCache(ctx, schema.SitemapCachekey, data); setCacheErr != nil {
+		log.Errorf("set sitemap cron SetCache failed: %s", setCacheErr)
+	}
+
+	return
+
+}
+
+func (qs *QuestionService) SetCache(ctx context.Context, cachekey string, info interface{}) error {
+	infoStr, err := json.Marshal(info)
+	if err != nil {
+		return errors.InternalServer(reason.UnknownError).WithError(err).WithStack()
+	}
+
+	err = qs.data.Cache.SetString(ctx, cachekey, string(infoStr), schema.DashBoardCacheTime)
+	if err != nil {
+		return errors.InternalServer(reason.UnknownError).WithError(err).WithStack()
+	}
+	return nil
+}

ui/template/sitemap-list.xml

@@ -1,6 +1,8 @@
 {{ .xmlHeader }}
 <sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+  {{ range .page }}
   <sitemap>
-    <loc>http://www.example.com/sitemap1.xml</loc>
+    <loc>{{$.general.SiteUrl}}/sitemap/question-{{.}}.xml</loc>
   </sitemap>
+   {{ end }}
 </sitemapindex>

ui/template/sitemap.xml

@@ -1,7 +1,9 @@
 {{ .xmlHeader }}
 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+  {{ range .list }}
   <url>
-    <loc>http://www.example.com/foo1</loc>
+    <loc>{{$.general.SiteUrl}}/questions/{{.ID}}/{{.Title}}</loc>
-    <lastmod>2018-06-04</lastmod>
+    <lastmod>{{.UpdateTime}}</lastmod>
   </url>
+   {{ end }}
 </urlset>