done
This commit is contained in:
xing
parent
28f4c8f799
commit
f6394cf1b3
99
main.go
99
main.go
|
|
@ -14,10 +14,11 @@ import (
|
|||
)
|
||||
|
||||
type fetchData struct {
|
||||
Url string `json:"url"`
|
||||
Title string `json:"title"`
|
||||
Desc string `json:"desc"`
|
||||
Date string `json:"date"`
|
||||
Url string `json:"url"`
|
||||
Title string `json:"title"`
|
||||
Desc string `json:"desc"`
|
||||
Date string `json:"date"`
|
||||
CreatedTime string `json:"created_time"`
|
||||
}
|
||||
|
||||
type connChan struct {
|
||||
|
|
@ -32,14 +33,14 @@ type dataChan struct {
|
|||
type fetchHandler struct {
|
||||
fetchUrl string
|
||||
hadFetchData []fetchData
|
||||
cronTime map[string]time.Duration
|
||||
keyword map[string]string
|
||||
hadFetchedMap map[string]int
|
||||
reloadCron map[string]chan int
|
||||
cronTime mapXS[time.Duration]
|
||||
keyword mapXS[string]
|
||||
hadFetchedMap mapXS[int]
|
||||
reloadCron mapXS[chan int]
|
||||
isOff chan int
|
||||
rMsgChan chan connChan
|
||||
newFetchItem chan dataChan
|
||||
connMap map[string]*websocket.Conn
|
||||
connMap mapXS[*websocket.Conn]
|
||||
}
|
||||
|
||||
type setting struct {
|
||||
|
|
@ -54,36 +55,60 @@ type message struct {
|
|||
Data interface{}
|
||||
}
|
||||
|
||||
func setMap[T string | time.Duration | int](obj *mapXS[T], key string, v T) {
|
||||
func setMap[T mapT](obj *mapXS[T], key string, v T) {
|
||||
obj.Lock()
|
||||
(*obj.mapX)[key] = v
|
||||
obj.Unlock()
|
||||
}
|
||||
|
||||
type mapX[T string | int | time.Duration] map[string]T
|
||||
func delMap[T mapT](obj *mapXS[T], key string) {
|
||||
obj.Lock()
|
||||
delete(*obj.mapX, key)
|
||||
obj.Unlock()
|
||||
}
|
||||
|
||||
type mapXS[T string | int | time.Duration] struct {
|
||||
type mapT interface {
|
||||
string | int | time.Duration | *websocket.Conn | chan int
|
||||
}
|
||||
type mapX[T mapT] map[string]T
|
||||
|
||||
type mapXS[T mapT] struct {
|
||||
*mapX[T]
|
||||
*sync.Mutex
|
||||
}
|
||||
|
||||
func newFetchHandler(fetchUrl string) *fetchHandler {
|
||||
return &fetchHandler{
|
||||
fetchUrl: fetchUrl,
|
||||
keyword: make(map[string]string),
|
||||
hadFetchedMap: make(map[string]int),
|
||||
cronTime: make(map[string]time.Duration),
|
||||
reloadCron: make(map[string]chan int),
|
||||
isOff: make(chan int),
|
||||
rMsgChan: make(chan connChan, 10),
|
||||
newFetchItem: make(chan dataChan, 10),
|
||||
connMap: make(map[string]*websocket.Conn),
|
||||
fetchUrl: fetchUrl,
|
||||
keyword: mapXS[string]{
|
||||
&mapX[string]{},
|
||||
&sync.Mutex{},
|
||||
},
|
||||
hadFetchedMap: mapXS[int]{
|
||||
&mapX[int]{},
|
||||
&sync.Mutex{},
|
||||
},
|
||||
cronTime: mapXS[time.Duration]{
|
||||
&mapX[time.Duration]{},
|
||||
&sync.Mutex{},
|
||||
},
|
||||
reloadCron: mapXS[chan int]{
|
||||
&mapX[chan int]{},
|
||||
&sync.Mutex{},
|
||||
},
|
||||
isOff: make(chan int),
|
||||
rMsgChan: make(chan connChan, 10),
|
||||
newFetchItem: make(chan dataChan, 10),
|
||||
connMap: mapXS[*websocket.Conn]{
|
||||
&mapX[*websocket.Conn]{},
|
||||
&sync.Mutex{},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (f *fetchHandler) handle(conn string) {
|
||||
key := "纪检"
|
||||
if kk, ok := f.keyword[conn]; ok && kk != "" {
|
||||
if kk, ok := (*f.keyword.mapX)[conn]; ok && kk != "" {
|
||||
key = kk
|
||||
}
|
||||
f.parsesDom(f.fetch(f.fetchUrl+key), conn)
|
||||
|
|
@ -95,8 +120,8 @@ func (f *fetchHandler) receiveMsg() {
|
|||
switch r.msg.Action {
|
||||
case "search":
|
||||
if t, ok := r.msg.Data.(*setting); ok {
|
||||
f.reloadCron[r.conn] <- t.TimeStep
|
||||
f.keyword[r.conn] = t.Keyword
|
||||
(*f.reloadCron.mapX)[r.conn] <- t.TimeStep
|
||||
setMap[string](&f.keyword, r.conn, t.Keyword)
|
||||
f.handle(r.conn)
|
||||
}
|
||||
}
|
||||
|
|
@ -154,7 +179,6 @@ func (f *fetchHandler) parsesDom(html *http.Response, conn string) {
|
|||
}
|
||||
var newFetch []fetchData
|
||||
ti := time.Now()
|
||||
log.Println(ti.Format("2006-01-02 15:04:05"))
|
||||
compile := regexp.MustCompile(`(\d+)`)
|
||||
|
||||
doc.Find("div[class=\"result-op c-container xpath-log new-pmd\"]").Each(func(i int, selection *goquery.Selection) {
|
||||
|
|
@ -162,6 +186,7 @@ func (f *fetchHandler) parsesDom(html *http.Response, conn string) {
|
|||
data.Url, _ = selection.Attr("mu")
|
||||
t := selection.Find(".news-title-font_1xS-F").First()
|
||||
data.Title = t.Text()
|
||||
data.CreatedTime = ti.Format("2006-01-02 15:04:05")
|
||||
data.Desc = selection.Find(".c-row .c-color-text").First().Text()
|
||||
data.Date = selection.Find("span[class=\"c-color-gray2 c-font-normal c-gap-right-xsmall\"]").First().Text()
|
||||
n := compile.FindAllStringSubmatch(data.Date, -1)
|
||||
|
|
@ -176,9 +201,9 @@ func (f *fetchHandler) parsesDom(html *http.Response, conn string) {
|
|||
}
|
||||
|
||||
k := conn + "_" + data.Url + "_" + data.Title
|
||||
if _, ok := f.hadFetchedMap[k]; !ok {
|
||||
if _, ok := (*f.hadFetchedMap.mapX)[k]; !ok {
|
||||
f.hadFetchData = append(f.hadFetchData, data)
|
||||
f.hadFetchedMap[k] = 1
|
||||
setMap(&f.hadFetchedMap, k, 1)
|
||||
newFetch = append(newFetch, data)
|
||||
}
|
||||
})
|
||||
|
|
@ -198,7 +223,7 @@ func (f *fetchHandler) sendFetchData() {
|
|||
for {
|
||||
data := <-f.newFetchItem
|
||||
|
||||
err := f.connMap[data.conn].WriteJSON(message{
|
||||
err := (*f.connMap.mapX)[data.conn].WriteJSON(message{
|
||||
Status: true,
|
||||
Action: "newData",
|
||||
Message: "",
|
||||
|
|
@ -211,21 +236,21 @@ func (f *fetchHandler) sendFetchData() {
|
|||
}
|
||||
|
||||
func (f *fetchHandler) cronFetch(conn string, c chan int) {
|
||||
step, ok := f.cronTime[conn]
|
||||
step, ok := (*f.cronTime.mapX)[conn]
|
||||
if !ok {
|
||||
step = time.Second * 60
|
||||
}
|
||||
t := time.NewTicker(step)
|
||||
if _, ok := f.cronTime[conn]; !ok {
|
||||
f.reloadCron[conn] = make(chan int)
|
||||
if _, ok := (*f.cronTime.mapX)[conn]; !ok {
|
||||
setMap(&f.reloadCron, conn, make(chan int))
|
||||
}
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-t.C:
|
||||
f.handle(conn)
|
||||
case tt := <-f.reloadCron[conn]:
|
||||
f.cronTime[conn] = time.Duration(tt) * time.Second
|
||||
case tt := <-(*f.reloadCron.mapX)[conn]:
|
||||
setMap(&f.cronTime, conn, time.Duration(tt)*time.Second)
|
||||
go f.cronFetch(conn, c)
|
||||
return
|
||||
case <-c:
|
||||
|
|
@ -265,8 +290,8 @@ func main() {
|
|||
return
|
||||
}
|
||||
remote := conn.RemoteAddr().String()
|
||||
if _, ok := h.connMap[remote]; !ok {
|
||||
h.connMap[remote] = conn
|
||||
if _, ok := (*h.connMap.mapX)[remote]; !ok {
|
||||
setMap(&h.connMap, remote, conn)
|
||||
}
|
||||
cc := make(chan int)
|
||||
go h.cronFetch(remote, cc)
|
||||
|
|
@ -280,8 +305,8 @@ func main() {
|
|||
for {
|
||||
err := conn.ReadJSON(&msg.msg)
|
||||
if err != nil {
|
||||
if _, ok := h.connMap[remote]; ok && !websocket.IsUnexpectedCloseError(err, websocket.CloseGoingAway) {
|
||||
delete(h.connMap, remote)
|
||||
if _, ok := (*h.connMap.mapX)[remote]; ok && !websocket.IsUnexpectedCloseError(err, websocket.CloseGoingAway) {
|
||||
delMap(&h.connMap, remote)
|
||||
cc <- 1
|
||||
return
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user