From f6394cf1b335da198bede5f4bb55908525cfaff9 Mon Sep 17 00:00:00 2001 From: xing Date: Wed, 6 Jul 2022 17:03:42 +0800 Subject: [PATCH] done --- main.go | 99 ++++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 62 insertions(+), 37 deletions(-) diff --git a/main.go b/main.go index 33e017b..7cfd0e5 100644 --- a/main.go +++ b/main.go @@ -14,10 +14,11 @@ import ( ) type fetchData struct { - Url string `json:"url"` - Title string `json:"title"` - Desc string `json:"desc"` - Date string `json:"date"` + Url string `json:"url"` + Title string `json:"title"` + Desc string `json:"desc"` + Date string `json:"date"` + CreatedTime string `json:"created_time"` } type connChan struct { @@ -32,14 +33,14 @@ type dataChan struct { type fetchHandler struct { fetchUrl string hadFetchData []fetchData - cronTime map[string]time.Duration - keyword map[string]string - hadFetchedMap map[string]int - reloadCron map[string]chan int + cronTime mapXS[time.Duration] + keyword mapXS[string] + hadFetchedMap mapXS[int] + reloadCron mapXS[chan int] isOff chan int rMsgChan chan connChan newFetchItem chan dataChan - connMap map[string]*websocket.Conn + connMap mapXS[*websocket.Conn] } type setting struct { @@ -54,36 +55,60 @@ type message struct { Data interface{} } -func setMap[T string | time.Duration | int](obj *mapXS[T], key string, v T) { +func setMap[T mapT](obj *mapXS[T], key string, v T) { obj.Lock() (*obj.mapX)[key] = v obj.Unlock() } -type mapX[T string | int | time.Duration] map[string]T +func delMap[T mapT](obj *mapXS[T], key string) { + obj.Lock() + delete(*obj.mapX, key) + obj.Unlock() +} -type mapXS[T string | int | time.Duration] struct { +type mapT interface { + string | int | time.Duration | *websocket.Conn | chan int +} +type mapX[T mapT] map[string]T + +type mapXS[T mapT] struct { *mapX[T] *sync.Mutex } func newFetchHandler(fetchUrl string) *fetchHandler { return &fetchHandler{ - fetchUrl: fetchUrl, - keyword: make(map[string]string), - hadFetchedMap: make(map[string]int), - cronTime: make(map[string]time.Duration), - reloadCron: make(map[string]chan int), - isOff: make(chan int), - rMsgChan: make(chan connChan, 10), - newFetchItem: make(chan dataChan, 10), - connMap: make(map[string]*websocket.Conn), + fetchUrl: fetchUrl, + keyword: mapXS[string]{ + &mapX[string]{}, + &sync.Mutex{}, + }, + hadFetchedMap: mapXS[int]{ + &mapX[int]{}, + &sync.Mutex{}, + }, + cronTime: mapXS[time.Duration]{ + &mapX[time.Duration]{}, + &sync.Mutex{}, + }, + reloadCron: mapXS[chan int]{ + &mapX[chan int]{}, + &sync.Mutex{}, + }, + isOff: make(chan int), + rMsgChan: make(chan connChan, 10), + newFetchItem: make(chan dataChan, 10), + connMap: mapXS[*websocket.Conn]{ + &mapX[*websocket.Conn]{}, + &sync.Mutex{}, + }, } } func (f *fetchHandler) handle(conn string) { key := "纪检" - if kk, ok := f.keyword[conn]; ok && kk != "" { + if kk, ok := (*f.keyword.mapX)[conn]; ok && kk != "" { key = kk } f.parsesDom(f.fetch(f.fetchUrl+key), conn) @@ -95,8 +120,8 @@ func (f *fetchHandler) receiveMsg() { switch r.msg.Action { case "search": if t, ok := r.msg.Data.(*setting); ok { - f.reloadCron[r.conn] <- t.TimeStep - f.keyword[r.conn] = t.Keyword + (*f.reloadCron.mapX)[r.conn] <- t.TimeStep + setMap[string](&f.keyword, r.conn, t.Keyword) f.handle(r.conn) } } @@ -154,7 +179,6 @@ func (f *fetchHandler) parsesDom(html *http.Response, conn string) { } var newFetch []fetchData ti := time.Now() - log.Println(ti.Format("2006-01-02 15:04:05")) compile := regexp.MustCompile(`(\d+)`) doc.Find("div[class=\"result-op c-container xpath-log new-pmd\"]").Each(func(i int, selection *goquery.Selection) { @@ -162,6 +186,7 @@ func (f *fetchHandler) parsesDom(html *http.Response, conn string) { data.Url, _ = selection.Attr("mu") t := selection.Find(".news-title-font_1xS-F").First() data.Title = t.Text() + data.CreatedTime = ti.Format("2006-01-02 15:04:05") data.Desc = selection.Find(".c-row .c-color-text").First().Text() data.Date = selection.Find("span[class=\"c-color-gray2 c-font-normal c-gap-right-xsmall\"]").First().Text() n := compile.FindAllStringSubmatch(data.Date, -1) @@ -176,9 +201,9 @@ func (f *fetchHandler) parsesDom(html *http.Response, conn string) { } k := conn + "_" + data.Url + "_" + data.Title - if _, ok := f.hadFetchedMap[k]; !ok { + if _, ok := (*f.hadFetchedMap.mapX)[k]; !ok { f.hadFetchData = append(f.hadFetchData, data) - f.hadFetchedMap[k] = 1 + setMap(&f.hadFetchedMap, k, 1) newFetch = append(newFetch, data) } }) @@ -198,7 +223,7 @@ func (f *fetchHandler) sendFetchData() { for { data := <-f.newFetchItem - err := f.connMap[data.conn].WriteJSON(message{ + err := (*f.connMap.mapX)[data.conn].WriteJSON(message{ Status: true, Action: "newData", Message: "", @@ -211,21 +236,21 @@ func (f *fetchHandler) sendFetchData() { } func (f *fetchHandler) cronFetch(conn string, c chan int) { - step, ok := f.cronTime[conn] + step, ok := (*f.cronTime.mapX)[conn] if !ok { step = time.Second * 60 } t := time.NewTicker(step) - if _, ok := f.cronTime[conn]; !ok { - f.reloadCron[conn] = make(chan int) + if _, ok := (*f.cronTime.mapX)[conn]; !ok { + setMap(&f.reloadCron, conn, make(chan int)) } defer t.Stop() for { select { case <-t.C: f.handle(conn) - case tt := <-f.reloadCron[conn]: - f.cronTime[conn] = time.Duration(tt) * time.Second + case tt := <-(*f.reloadCron.mapX)[conn]: + setMap(&f.cronTime, conn, time.Duration(tt)*time.Second) go f.cronFetch(conn, c) return case <-c: @@ -265,8 +290,8 @@ func main() { return } remote := conn.RemoteAddr().String() - if _, ok := h.connMap[remote]; !ok { - h.connMap[remote] = conn + if _, ok := (*h.connMap.mapX)[remote]; !ok { + setMap(&h.connMap, remote, conn) } cc := make(chan int) go h.cronFetch(remote, cc) @@ -280,8 +305,8 @@ func main() { for { err := conn.ReadJSON(&msg.msg) if err != nil { - if _, ok := h.connMap[remote]; ok && !websocket.IsUnexpectedCloseError(err, websocket.CloseGoingAway) { - delete(h.connMap, remote) + if _, ok := (*h.connMap.mapX)[remote]; ok && !websocket.IsUnexpectedCloseError(err, websocket.CloseGoingAway) { + delMap(&h.connMap, remote) cc <- 1 return }