package main import ( "github.com/PuerkitoBio/goquery" "github.com/gin-gonic/gin" "github.com/gorilla/websocket" "github/fthvgb1/newsfetch/dist" "io/ioutil" "log" "net/http" "os/exec" "regexp" "runtime" "strconv" "strings" "sync" "time" ) type fetchData struct { Url string `json:"url"` Title string `json:"title"` Desc string `json:"desc"` Date string `json:"date"` CreatedTime string `json:"created_time"` } type connChan struct { conn string msg message } type dataChan struct { conn string item []fetchData } type fetchHandler struct { fetchUrl string hadFetchData []fetchData cronTime mapXS[time.Duration] keyword mapXS[string] hadFetchedMap mapXS[int] reloadCron mapXS[chan int] isOff chan int rMsgChan chan connChan newFetchItem chan dataChan connMap mapXS[*websocket.Conn] } type setting struct { Keyword string `json:"keyword"` TimeStep int `json:"timeStep"` } type message struct { Status bool Action string Message string Data interface{} } func setMap[T mapT](obj *mapXS[T], key string, v T) { obj.Lock() (*obj.mapX)[key] = v obj.Unlock() } func delMap[T mapT](obj *mapXS[T], key string) { obj.Lock() delete(*obj.mapX, key) obj.Unlock() } type mapT interface { string | int | time.Duration | *websocket.Conn | chan int } type mapX[T mapT] map[string]T type mapXS[T mapT] struct { *mapX[T] *sync.Mutex } func newFetchHandler(fetchUrl string) *fetchHandler { return &fetchHandler{ fetchUrl: fetchUrl, keyword: mapXS[string]{ &mapX[string]{}, &sync.Mutex{}, }, hadFetchedMap: mapXS[int]{ &mapX[int]{}, &sync.Mutex{}, }, cronTime: mapXS[time.Duration]{ &mapX[time.Duration]{}, &sync.Mutex{}, }, reloadCron: mapXS[chan int]{ &mapX[chan int]{}, &sync.Mutex{}, }, isOff: make(chan int), rMsgChan: make(chan connChan, 10), newFetchItem: make(chan dataChan, 10), connMap: mapXS[*websocket.Conn]{ &mapX[*websocket.Conn]{}, &sync.Mutex{}, }, } } func (f *fetchHandler) handle(conn string) { key := "纪检" if kk, ok := (*f.keyword.mapX)[conn]; ok && kk != "" { key = kk } f.parsesDom(f.fetch(f.fetchUrl+key), conn) } func (f *fetchHandler) receiveMsg() { for { r := <-f.rMsgChan switch r.msg.Action { case "search": if t, ok := r.msg.Data.(*setting); ok { (*f.reloadCron.mapX)[r.conn] <- t.TimeStep setMap[string](&f.keyword, r.conn, t.Keyword) f.handle(r.conn) } } } } func (f *fetchHandler) fetch(url string) *http.Response { defer func() { if r := recover(); r != nil { log.Println(r) } }() client := http.Client{ Transport: nil, CheckRedirect: nil, Jar: nil, Timeout: 10 * time.Second, } req, _ := http.NewRequest("GET", url, nil) req.Header.Add("accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9") req.Header.Add("accept-language", "zh-CN,zh;q=0.9") req.Header.Add("cache-control", "no-cache") req.Header.Add("connection", "keep-alive") req.Header.Add("cookie", "BIDUPSID=844E3DCAA2EEBF5C872DC99B967B6B7B; PSTM=1655872163; BAIDUID=844E3DCAA2EEBF5CB3E1D79750162204:FG=1; BD_UPN=123353; ORIGIN=2; ISSW=1; ISSW=1; BAIDUID_BFESS=844E3DCAA2EEBF5CB3E1D79750162204:FG=1; ZFY=jWFAySgO:AoQfb6emY9vnmEdptVao:Anj0FFkp028wFws:C; BD_HOME=1; delPer=0; BD_CK_SAM=1; PSINO=3; COOKIE_SESSION=42_0_2_2_3_0_1_0_2_0_0_0_18_0_51_0_1655888428_0_1655888377%7C3%230_0_1655888377%7C1; BAIDU_WISE_UID=wapp_1655902298617_702; ZD_ENTRY=google; channel=baidusearch; baikeVisitId=b3b23509-9330-4d33-82ae-b8eb37895917; BA_HECTOR=8k2g2g218ga40181ak1hbgg1n14; BDRCVFR[C0p6oIjvx-c]=mbxnW11j9Dfmh7GuZR8mvqV; BDSVRTM=1011; H_PS_PSSID=36550_36459_36673_36455_36453_36692_36165_36695_36697_36569_36075_36467_36316_36651") req.Header.Add("referer", "http://news.baidu.com/") req.Header.Add("sec-fetch-dest", "document") req.Header.Add("sec-fetch-mode", "navigate") req.Header.Add("sec-fetch-site", "cross-site") req.Header.Add("sec-fetch-user", "?1") req.Header.Add("upgrade-insecure-requests", "1") req.Header.Add("user-agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36") req.Header.Add("#sec-ch-ua", "\".Not/A)Brand\";v=\"99\", \"Google Chrome\";v=\"103\", \"Chromium\";v=\"103\"") req.Header.Add("sec-ch-ua-mobile", "?0") req.Header.Add("sec-ch-ua-platform", "\"Linux\"") req.Header.Add("postman-token", "81407fbc-2b96-54a7-0193-f640156714ab") response, err := client.Do(req) if err != nil { panic(err) } return response } func (f *fetchHandler) parsesDom(html *http.Response, conn string) { defer func() { if r := recover(); r != nil { log.Println(r) } }() doc, err := goquery.NewDocumentFromReader(html.Body) if err != nil { panic(err) } var newFetch []fetchData ti := time.Now() compile := regexp.MustCompile(`(\d+)`) doc.Find("div[class=\"result-op c-container xpath-log new-pmd\"]").Each(func(i int, selection *goquery.Selection) { data := fetchData{} data.Url, _ = selection.Attr("mu") t := selection.Find(".news-title-font_1xS-F").First() data.Title = t.Text() data.CreatedTime = ti.Format("2006-01-02 15:04:05") data.Desc = selection.Find(".c-row .c-color-text").First().Text() data.Date = selection.Find("span[class=\"c-color-gray2 c-font-normal c-gap-right-xsmall\"]").First().Text() n := compile.FindAllStringSubmatch(data.Date, -1) if nil != n { nn, _ := strconv.Atoi(n[0][0]) if strings.Contains(data.Date, "小时") { data.Date = ti.Add(-time.Duration(nn) * time.Hour).Format("2006-01-02 15:04") } if strings.Contains(data.Date, "分钟") { data.Date = ti.Add(-time.Duration(nn) * time.Minute).Format("2006-01-02 15:04") } } k := conn + "_" + data.Url + "_" + data.Title if _, ok := (*f.hadFetchedMap.mapX)[k]; !ok { f.hadFetchData = append(f.hadFetchData, data) setMap(&f.hadFetchedMap, k, 1) newFetch = append(newFetch, data) } }) if len(newFetch) > 0 { f.newFetchItem <- dataChan{ conn: conn, item: newFetch, } } err = html.Body.Close() if err != nil { panic(err) } } func (f *fetchHandler) sendFetchData() { for { data := <-f.newFetchItem err := (*f.connMap.mapX)[data.conn].WriteJSON(message{ Status: true, Action: "newData", Message: "", Data: data.item, }) if err != nil { log.Println(err) } } } func (f *fetchHandler) cronFetch(conn string, c chan int) { step, ok := (*f.cronTime.mapX)[conn] if !ok { step = time.Second * 60 } t := time.NewTicker(step) if _, ok := (*f.cronTime.mapX)[conn]; !ok { setMap(&f.reloadCron, conn, make(chan int)) } defer t.Stop() for { select { case <-t.C: f.handle(conn) case tt := <-(*f.reloadCron.mapX)[conn]: setMap(&f.cronTime, conn, time.Duration(tt)*time.Second) go f.cronFetch(conn, c) return case <-c: close(c) return } } } func main() { h := newFetchHandler("https://www.baidu.com/s?rtt=1&bsst=1&cl=2&tn=news&rsv_dl=ns_pc&word=") router := gin.Default() static := dist.Dist{ FS: dist.Static, } //router.Static("/css", "./dist/css") //router.Static("/js", "./dist/js") router.StaticFS("/js", http.FS(static)) router.StaticFS("/css", http.FS(static)) //router.StaticFile("/favicon.ico", "dist/favicon.ico") var upgrader = websocket.Upgrader{ CheckOrigin: func(r *http.Request) bool { return true }, } go h.sendFetchData() go h.receiveMsg() router.GET("/", func(c *gin.Context) { file, err := static.Open("index.html") if err != nil { c.String(404, "%s", err) return } bytes, err := ioutil.ReadAll(file) if err != nil { c.String(404, "%s", err) return } c.Data(200, "text/html", bytes) }) router.GET("ws", func(c *gin.Context) { conn, err := upgrader.Upgrade(c.Writer, c.Request, nil) if err != nil { c.JSON(201, message{ Status: false, Message: err.Error(), Data: nil, Action: "upgradeWs", }) log.Println(err) return } remote := conn.RemoteAddr().String() if _, ok := (*h.connMap.mapX)[remote]; !ok { setMap(&h.connMap, remote, conn) } cc := make(chan int) go h.cronFetch(remote, cc) go func() { msg := connChan{ conn: remote, msg: message{ Data: &setting{}, }, } for { err := conn.ReadJSON(&msg.msg) if err != nil { if _, ok := (*h.connMap.mapX)[remote]; ok && !websocket.IsUnexpectedCloseError(err, websocket.CloseGoingAway) { delMap(&h.connMap, remote) cc <- 1 return } log.Println(err) } else { h.rMsgChan <- msg } } }() }) go func() { time.Sleep(2 * time.Second) url := "http://127.0.0.1:8080" switch runtime.GOOS { case "linux": exec.Command(`xdg-open`, url).Start() case "windows": exec.Command(`cmd`, `/c`, `start`, url).Start() } }() err := router.Run(":8080") if err != nil { panic(err) } }