xing/newfetch
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

fix

Browse Source
This commit is contained in:
xing 2022-07-06 10:55:28 +08:00
parent 3eb54a8c25
commit 28f4c8f799
1 changed files with 25 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
main.go
View File

@ -9,6 +9,7 @@ import (
"regexp" "regexp"
"strconv" "strconv"
"strings" "strings"
"sync"
"time" "time"
) )
@ -53,6 +54,19 @@ type message struct {
Data interface{} Data interface{}
} }
func setMap[T string | time.Duration | int](obj *mapXS[T], key string, v T) {
obj.Lock()
(*obj.mapX)[key] = v
obj.Unlock()
}
type mapX[T string | int | time.Duration] map[string]T
type mapXS[T string | int | time.Duration] struct {
*mapX[T]
*sync.Mutex
}
func newFetchHandler(fetchUrl string) *fetchHandler { func newFetchHandler(fetchUrl string) *fetchHandler {
return &fetchHandler{ return &fetchHandler{
fetchUrl: fetchUrl, fetchUrl: fetchUrl,
@ -85,11 +99,6 @@ func (f *fetchHandler) receiveMsg() {
f.keyword[r.conn] = t.Keyword f.keyword[r.conn] = t.Keyword
f.handle(r.conn) f.handle(r.conn)
} }
case "timeStepSet":
if t, ok := r.msg.Data.(int); ok {
f.reloadCron[r.conn] <- t
}
} }
} }
} }
@ -145,6 +154,7 @@ func (f *fetchHandler) parsesDom(html *http.Response, conn string) {
} }
var newFetch []fetchData var newFetch []fetchData
ti := time.Now() ti := time.Now()
log.Println(ti.Format("2006-01-02 15:04:05"))
compile := regexp.MustCompile(`(\d+)`) compile := regexp.MustCompile(`(\d+)`)
doc.Find("div[class=\"result-op c-container xpath-log new-pmd\"]").Each(func(i int, selection *goquery.Selection) { doc.Find("div[class=\"result-op c-container xpath-log new-pmd\"]").Each(func(i int, selection *goquery.Selection) {
@ -155,13 +165,16 @@ func (f *fetchHandler) parsesDom(html *http.Response, conn string) {
data.Desc = selection.Find(".c-row .c-color-text").First().Text() data.Desc = selection.Find(".c-row .c-color-text").First().Text()
data.Date = selection.Find("span[class=\"c-color-gray2 c-font-normal c-gap-right-xsmall\"]").First().Text() data.Date = selection.Find("span[class=\"c-color-gray2 c-font-normal c-gap-right-xsmall\"]").First().Text()
n := compile.FindAllStringSubmatch(data.Date, -1) n := compile.FindAllStringSubmatch(data.Date, -1)
nn, _ := strconv.Atoi(n[0][0]) if nil != n {
if strings.Contains(data.Date, "小时") { nn, _ := strconv.Atoi(n[0][0])
data.Date = ti.Add(-time.Duration(nn) * time.Hour).Format("2006-01-02 15:04") if strings.Contains(data.Date, "小时") {
} data.Date = ti.Add(-time.Duration(nn) * time.Hour).Format("2006-01-02 15:04")
if strings.Contains(data.Date, "分钟") { }
data.Date = ti.Add(-time.Duration(nn) * time.Minute).Format("2006-01-02 15:04") if strings.Contains(data.Date, "分钟") {
data.Date = ti.Add(-time.Duration(nn) * time.Minute).Format("2006-01-02 15:04")
}
} }
k := conn + "_" + data.Url + "_" + data.Title k := conn + "_" + data.Url + "_" + data.Title
if _, ok := f.hadFetchedMap[k]; !ok { if _, ok := f.hadFetchedMap[k]; !ok {
f.hadFetchData = append(f.hadFetchData, data) f.hadFetchData = append(f.hadFetchData, data)
@ -267,7 +280,7 @@ func main() {
for { for {
err := conn.ReadJSON(&msg.msg) err := conn.ReadJSON(&msg.msg)
if err != nil { if err != nil {
if !websocket.IsUnexpectedCloseError(err, websocket.CloseGoingAway) { if _, ok := h.connMap[remote]; ok && !websocket.IsUnexpectedCloseError(err, websocket.CloseGoingAway) {
delete(h.connMap, remote) delete(h.connMap, remote)
cc <- 1 cc <- 1
return return