revise digest add set tag occupied number

This commit is contained in:
xing 2024-04-07 21:30:57 +08:00
parent 6209f2b5e5
commit a78815f3d3
3 changed files with 68 additions and 8 deletions

View File

@ -4,10 +4,14 @@ import (
"context"
"fmt"
"github.com/fthvgb1/wp-go/app/pkg/config"
"github.com/fthvgb1/wp-go/app/pkg/logs"
"github.com/fthvgb1/wp-go/app/pkg/models"
"github.com/fthvgb1/wp-go/cache/cachemanager"
"github.com/fthvgb1/wp-go/cache/reload"
"github.com/fthvgb1/wp-go/helper"
str "github.com/fthvgb1/wp-go/helper/strings"
"github.com/fthvgb1/wp-go/plugin/digest"
"github.com/fthvgb1/wp-go/safety"
"regexp"
"strings"
"time"
@ -18,10 +22,45 @@ var more = regexp.MustCompile("<!--more(.*?)?-->")
var removeWpBlock = regexp.MustCompile("<!-- /?wp:.*-->")
var digestConfig *safety.Var[DigestConfig]
func InitDigestCache() {
cachemanager.NewMemoryMapCache(nil, digestRaw, config.GetConfig().CacheTime.DigestCacheTime, "digestPlugin", func() time.Duration {
return config.GetConfig().CacheTime.DigestCacheTime
})
digestConfig = reload.VarsBy(func() DigestConfig {
c, err := config.GetCustomizedConfig[DigestConfig]()
if err != nil {
logs.Error(err, "get digest config fail")
c.DigestWordCount = config.GetConfig().DigestWordCount
c.DigestAllowTag = config.GetConfig().DigestAllowTag
return c
}
if len(c.DigestTagOccupyNum) > 0 {
c.tagNum = map[string]int{}
for _, item := range c.DigestTagOccupyNum {
tags := strings.Split(item.Tag, "<")
for _, tag := range tags {
if tag == "" {
continue
}
c.tagNum[str.Join("<", tag)] = item.Num
}
}
}
return c
}, "digestConfig")
}
type DigestConfig struct {
DigestWordCount int `yaml:"digestWordCount"`
DigestAllowTag string `yaml:"digestAllowTag"`
DigestTagOccupyNum []struct {
Tag string `yaml:"tag"`
Num int `yaml:"num"`
} `yaml:"digestTagOccupyNum"`
tagNum map[string]int
}
func RemoveWpBlock(s string) string {
@ -45,7 +84,8 @@ func digestRaw(ctx context.Context, id uint64, arg ...any) (string, error) {
func Digests(content string, id uint64, limit int, fn func(id uint64, content, closeTag string) string) string {
closeTag := ""
content = RemoveWpBlock(content)
tag := config.GetConfig().DigestAllowTag
c := digestConfig.Load()
tag := c.DigestAllowTag
if tag == "" {
tag = "<a><b><blockquote><br><cite><code><dd><del><div><dl><dt><em><h1><h2><h3><h4><h5><h6><i><img><li><ol><p><pre><span><strong><ul>"
}
@ -54,7 +94,7 @@ func Digests(content string, id uint64, limit int, fn func(id uint64, content, c
if length <= limit {
return content
}
content, closeTag = digest.Html(content, limit)
content, closeTag = digest.Html(content, limit, c.tagNum)
if fn == nil {
return PostsMore(id, content, closeTag)
}

View File

@ -66,6 +66,15 @@ cacheTime:
digestWordCount: 300
# 摘要允许的标签 默认为<a><b><blockquote><br><cite><code><dd><del><div><dl><dt><em><h1><h2><h3><h4><h5><h6><i><img><li><ol><p><pre><span><strong><ul>
digestTag: "<a><b><blockquote><br><cite><code><dd><del><div><dl><dt><em><h1><h2><h3><h4><h5><h6><i><img><li><ol><p><pre><span><strong><ul>"
# 可以设置每个标签占用的字数默认都为0 set tag occupied num, default every tag occupied 0
digestTagOccupyNum: [
{
tag: "<img><table>",
num: 2
},
]
# 到达指定并发请求数时随机sleep
maxRequestSleepNum: 100
# 全局最大请求数超过直接403

View File

@ -3,6 +3,7 @@ package digest
import (
"github.com/fthvgb1/wp-go/helper/html"
"github.com/fthvgb1/wp-go/helper/slice"
str "github.com/fthvgb1/wp-go/helper/strings"
"regexp"
"strings"
"unicode/utf8"
@ -17,7 +18,7 @@ func StripTags(content, allowTag string) string {
return content
}
func Html(content string, limit int) (string, string) {
func Html(content string, limit int, m map[string]int) (string, string) {
closeTag := ""
length := utf8.RuneCountInString(content) + 1
if length <= limit {
@ -38,6 +39,7 @@ func Html(content string, limit int) (string, string) {
total := len(ru)
l, r := '<', '>'
i := -1
var tag []rune
for {
i++
for len(runeIndex) > 0 && i >= runeIndex[0][0] {
@ -60,10 +62,19 @@ func Html(content string, limit int) (string, string) {
continue
} else if ru[i] == r {
tagIn = false
if len(m) > 0 {
tags := str.Join("<", strings.Split(string(tag), " ")[0], ">")
tag = tag[:0]
if n, ok := m[tags]; ok && n > 0 {
end += n
}
}
continue
}
if tagIn == false {
end++
} else if len(m) > 0 {
tag = append(tag, ru[i])
}
}
if i > total {