From ab97f5c624da698c8dfb546a5a84228175dd5d39 Mon Sep 17 00:00:00 2001 From: xing Date: Mon, 25 Jul 2022 22:38:30 +0800 Subject: [PATCH] =?UTF-8?q?=E5=BF=AB=E5=AE=8C=E5=B7=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.go | 131 ++++++++++-------- newsource/bjnews.go | 43 ++++++ newsource/ccdipager.go | 11 -- newsource/ifeng.go | 14 ++ newsource/model.go | 266 ++++++++++++++++++++++++++++++++++--- newsource/people.go | 15 +++ newsource/zaobao.go | 19 +++ src/components/WelCome.vue | 4 + 8 files changed, 420 insertions(+), 83 deletions(-) create mode 100644 newsource/bjnews.go create mode 100644 newsource/ifeng.go create mode 100644 newsource/people.go create mode 100644 newsource/zaobao.go diff --git a/main.go b/main.go index 6f4ff23..000cf01 100644 --- a/main.go +++ b/main.go @@ -2,6 +2,7 @@ package main import ( "embed" + "encoding/json" "errors" "fmt" "github.com/PuerkitoBio/goquery" @@ -17,7 +18,9 @@ import ( "net/url" "os/exec" "path/filepath" + "reflect" "runtime" + "runtime/debug" "strings" "sync" "time" @@ -141,10 +144,12 @@ func (f *fetchHandler) handle(conn string) { } for _, source := range newsource.GetSource() { r := f.fetch2(source, key) - if strings.ToUpper(source.Type) == "HTML" { - f.parsesDom(r, conn, source) - } else { - f.parseAjax(r, source, conn) + if r != nil { + if strings.ToUpper(source.Type) == "HTML" { + f.parsesDom(r, conn, source) + } else { + f.parseAjax(r, source, conn) + } } } } @@ -164,6 +169,11 @@ func (f *fetchHandler) receiveMsg() { } func (f *fetchHandler) fetch2(source newsource.Source, key string) *http.Response { + defer func() { + if r := recover(); r != nil { + log.Printf("err:%s. stack:%s", r, debug.Stack()) + } + }() jar, _ := cookiejar.New(nil) client := http.Client{ Transport: nil, @@ -173,22 +183,40 @@ func (f *fetchHandler) fetch2(source newsource.Source, key string) *http.Respons } searchUrl := source.SearchUrl source.Method = strings.ToUpper(source.Method) - if source.Method == "GET" { + if source.Method == "GET" && source.KeywordField != "" { if !strings.Contains(searchUrl, "?") { searchUrl += "?" + source.KeywordField + "=" + url.QueryEscape(key) } else { searchUrl += "&" + source.KeywordField + "=" + url.QueryEscape(key) } + } else if source.Method == "GET" && source.KeywordField == "" { + if strings.Contains(searchUrl, "${keyword}") { + searchUrl = strings.Replace(searchUrl, "${keyword}", url.QueryEscape(key), -1) + } else { + searchUrl += url.QueryEscape(key) + } } var req *http.Request if source.Method == "POST" { - body := source.KeywordField + "=" + key + body := "" if nil != source.ExternParam { - body += "&" - for s, s2 := range source.ExternParam { - body += s + "=" + s2 + "&" + if source.IsJson { + t := source.ExternParam + t[source.KeywordField] = key + bytes, err := json.Marshal(t) + if err != nil { + log.Printf("build post json param err:[%s]", err) + return nil + } + body = string(bytes) + } else { + body = source.KeywordField + "=" + key + body += "&" + for s, s2 := range source.ExternParam { + body += s + "=" + s2 + "&" + } + body = strings.TrimRight(body, "&") } - body = strings.TrimRight(body, "&") } req, _ = http.NewRequest(source.Method, searchUrl, strings.NewReader(body)) req.Header.Set("Content-Type", "application/x-www-form-urlencoded") @@ -220,46 +248,8 @@ func (f *fetchHandler) fetch2(source newsource.Source, key string) *http.Respons response, err := client.Do(req) if err != nil { - panic(err) - } - return response -} - -func (f *fetchHandler) fetch(url string) *http.Response { - defer func() { - if r := recover(); r != nil { - log.Println(r) - } - }() - client := http.Client{ - Transport: nil, - CheckRedirect: nil, - Jar: nil, - Timeout: 10 * time.Second, - } - req, _ := http.NewRequest("GET", url, nil) - - req.Header.Add("accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9") - req.Header.Add("accept-language", "zh-CN,zh;q=0.9") - req.Header.Add("cache-control", "no-cache") - req.Header.Add("connection", "keep-alive") - req.Header.Add("cookie", "BIDUPSID=844E3DCAA2EEBF5C872DC99B967B6B7B; PSTM=1655872163; BAIDUID=844E3DCAA2EEBF5CB3E1D79750162204:FG=1; BD_UPN=123353; ORIGIN=2; ISSW=1; ISSW=1; BAIDUID_BFESS=844E3DCAA2EEBF5CB3E1D79750162204:FG=1; ZFY=jWFAySgO:AoQfb6emY9vnmEdptVao:Anj0FFkp028wFws:C; BD_HOME=1; delPer=0; BD_CK_SAM=1; PSINO=3; COOKIE_SESSION=42_0_2_2_3_0_1_0_2_0_0_0_18_0_51_0_1655888428_0_1655888377%7C3%230_0_1655888377%7C1; BAIDU_WISE_UID=wapp_1655902298617_702; ZD_ENTRY=google; channel=baidusearch; baikeVisitId=b3b23509-9330-4d33-82ae-b8eb37895917; BA_HECTOR=8k2g2g218ga40181ak1hbgg1n14; BDRCVFR[C0p6oIjvx-c]=mbxnW11j9Dfmh7GuZR8mvqV; BDSVRTM=1011; H_PS_PSSID=36550_36459_36673_36455_36453_36692_36165_36695_36697_36569_36075_36467_36316_36651") - req.Header.Add("referer", "http://news.baidu.com/") - req.Header.Add("sec-fetch-dest", "document") - req.Header.Add("sec-fetch-mode", "navigate") - req.Header.Add("sec-fetch-site", "cross-site") - req.Header.Add("sec-fetch-user", "?1") - req.Header.Add("upgrade-insecure-requests", "1") - req.Header.Add("user-agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36") - req.Header.Add("#sec-ch-ua", "\".Not/A)Brand\";v=\"99\", \"Google Chrome\";v=\"103\", \"Chromium\";v=\"103\"") - req.Header.Add("sec-ch-ua-mobile", "?0") - req.Header.Add("sec-ch-ua-platform", "\"Linux\"") - req.Header.Add("postman-token", "81407fbc-2b96-54a7-0193-f640156714ab") - - response, err := client.Do(req) - - if err != nil { - panic(err) + log.Printf("request %s err: %s", req.URL, err) + return nil } return response } @@ -267,15 +257,41 @@ func (f *fetchHandler) fetch(url string) *http.Response { func (f *fetchHandler) parseAjax(response *http.Response, source newsource.Source, conn string) { defer func() { if r := recover(); r != nil { - log.Println(r) + log.Printf("parse ajax response err[%s]. stack:[%s]", r, debug.Stack()) } }() + nowDate := time.Now().Format("2006-01-02 15:04:06") var newFetch []data.FetchData - source.AjaxDealFun(&newFetch, response) - if len(newFetch) > 0 { - for i, fetchData := range newFetch { + if source.AjaxSimpleDeal != nil && source.AjaxDealFun == nil { + bytes, err := ioutil.ReadAll(response.Body) + if err != nil { + log.Printf("read response body err:[%s]", err) + return + } + if source.AjaxSimpleDeal != nil && source.Target != nil { + dst := reflect.New(source.Target).Elem() + err = json.Unmarshal(bytes, dst.Addr().Interface()) + if err != nil { + log.Printf("jsondecode err:[%s]", err) + return + } + source.AjaxSimpleDeal(dst.Interface(), &newFetch) + } + } else if source.AjaxDealFun != nil && source.AjaxSimpleDeal == nil { + source.AjaxDealFun(&newFetch, response) + } + + if len(newFetch) > 0 { + for i := 0; i < len(newFetch); i++ { + fetchData := newFetch[i] k := conn + "_" + fetchData.Url + "_" + fetchData.Title + if newFetch[i].CreatedTime == "" { + newFetch[i].CreatedTime = nowDate + } + if newFetch[i].Source == "" { + newFetch[i].Source = source.Name + } if _, ok := (*f.hadFetchedMap.mapX)[k]; !ok { f.hadFetchData = append(f.hadFetchData, fetchData) setMap(&f.hadFetchedMap, k, 1) @@ -297,7 +313,7 @@ func (f *fetchHandler) parseAjax(response *http.Response, source newsource.Sourc func (f *fetchHandler) parsesDom(html *http.Response, conn string, source newsource.Source) { defer func() { if r := recover(); r != nil { - log.Println(r) + log.Printf("parse html err:[%s]. stack:[%s]", r, debug.Stack()) } }() doc, err := goquery.NewDocumentFromReader(html.Body) @@ -309,6 +325,7 @@ func (f *fetchHandler) parsesDom(html *http.Response, conn string, source newsou doc.Find(source.ListQuery).Each(func(i int, selection *goquery.Selection) { fetchData := data.FetchData{ CreatedTime: nowDate, + Source: source.Name, } source.QueryHandler(i, selection, &fetchData) k := conn + "_" + fetchData.Url + "_" + fetchData.Title @@ -341,7 +358,7 @@ func (f *fetchHandler) sendFetchData() { Data: dataFetch.item, }) if err != nil { - log.Println(err) + log.Printf("send new fetch data err:[%s]", err) } } } @@ -434,7 +451,7 @@ func main() { cc <- 1 return } - log.Println(err) + log.Printf("websocket read client msg err:[%s]", err) } else { h.rMsgChan <- msg } diff --git a/newsource/bjnews.go b/newsource/bjnews.go new file mode 100644 index 0000000..30e2dbc --- /dev/null +++ b/newsource/bjnews.go @@ -0,0 +1,43 @@ +package newsource + +type BjNewsResponse struct { + Data struct { + Data []BjNewsItem `json:"data"` + } `json:"data"` +} + +type BjNewsDetailURL struct { + MURL string `json:"m_url"` + PcURL string `json:"pc_url"` +} + +type BjNewsItem struct { + Source BjNewsSource `json:"_source"` + Highlight BjNewsHighlight `json:"highlight"` +} + +type BjNewsSource struct { + SpecialType string `json:"special_type"` + PublishTime string `json:"publish_time"` + Title string `json:"title"` + Type int `json:"type"` + Keyword []interface{} `json:"keyword"` + UUID string `json:"uuid"` + CoverList string `json:"cover_list"` + HasVideo int `json:"has_video"` + ImgList []string `json:"img_list"` + Cover string `json:"cover"` + BigCover string `json:"big_cover"` + CommentNum int `json:"comment_num"` + ZanNum int `json:"zan_num"` + ShareNum string `json:"share_num"` + LiveStatus string `json:"live_status"` + XjhID string `json:"xjh_id"` + DetailURL BjNewsDetailURL `json:"detail_url"` +} + +type BjNewsHighlight struct { + Title string `json:"title"` + Desc string `json:"desc"` + Content string `json:"content"` +} diff --git a/newsource/ccdipager.go b/newsource/ccdipager.go index 587e8ff..5b8909a 100644 --- a/newsource/ccdipager.go +++ b/newsource/ccdipager.go @@ -1,10 +1,5 @@ package newsource -import ( - "regexp" - "strings" -) - type CCDIPAGERInfo struct { BM string `json:"BM"` BC string `json:"BC"` @@ -27,9 +22,3 @@ type CCDIPAGERData struct { Info []CCDIPAGERInfo `json:"info"` NextPage int `json:"nextPage"` } - -func stripTags(content string) string { - content = strings.Replace(content, " ", "", -1) - re := regexp.MustCompile(`<(.|\n)*?>`) - return re.ReplaceAllString(content, "") -} diff --git a/newsource/ifeng.go b/newsource/ifeng.go new file mode 100644 index 0000000..30c008c --- /dev/null +++ b/newsource/ifeng.go @@ -0,0 +1,14 @@ +package newsource + +type IFENGResponse struct { + Data struct { + Items []IFENGItem `json:"items"` + } `json:"data"` +} + +type IFENGItem struct { + ID string `json:"id"` + Source string `json:"source"` + Title string `json:"title"` + URL string `json:"url"` +} diff --git a/newsource/model.go b/newsource/model.go index 16cff21..03fc69d 100644 --- a/newsource/model.go +++ b/newsource/model.go @@ -8,6 +8,7 @@ import ( "io/ioutil" "log" "net/http" + "reflect" "regexp" "strconv" "strings" @@ -16,18 +17,27 @@ import ( ) type Source struct { - Name string - SearchUrl string - Method string - Type string - KeywordField string - ListQuery string - QueryHandler func(i int, selection *goquery.Selection, fetchData *data.FetchData) - AjaxHandler func(fetchData *data.FetchData) - Header map[string]string - HeaderFun func(r *http.Request) - ExternParam map[string]string - AjaxDealFun func(*[]data.FetchData, *http.Response) + Name string + SearchUrl string + Method string + Type string + KeywordField string + ListQuery string + QueryHandler func(i int, selection *goquery.Selection, fetchData *data.FetchData) + AjaxHandler func(fetchData *data.FetchData) + Header map[string]string + HeaderFun func(r *http.Request) + ExternParam map[string]string + AjaxDealFun func(*[]data.FetchData, *http.Response) + Target reflect.Type + UnmarshalerFun func([]byte) interface{} + AjaxSimpleDeal func(interface{}, *[]data.FetchData) + IsJson bool +} + +func StripTags(content string) string { + re := regexp.MustCompile(`<(.|\n)*?>`) + return re.ReplaceAllString(content, "") } func GetSource() []Source { @@ -36,7 +46,7 @@ func GetSource() []Source { nowDate := ti.Format("2006-01-02 15:04:05") return []Source{ { - Name: "中央纪委监察部", + Name: "中央纪委国家监委网站", SearchUrl: "https://www.ccdi.gov.cn/was5/web/search", Method: "post", ListQuery: ".center_box0 li", @@ -80,6 +90,7 @@ func GetSource() []Source { t := selection.Find(".news-title-font_1xS-F").First() data.Title = t.Text() data.Desc = selection.Find(".c-row .c-color-text").First().Text() + data.Source = selection.Find("span[aria-label*=\"新闻来源\"]").First().Text() data.Date = selection.Find("span[class=\"c-color-gray2 c-font-normal c-gap-right-xsmall\"]").First().Text() n := compile.FindAllStringSubmatch(data.Date, -1) if nil != n { @@ -151,7 +162,7 @@ func GetSource() []Source { if "" == v.YT { continue } - desc := stripTags(v.IRCONTENT) + desc := StripTags(v.IRCONTENT) l := utf8.RuneCountInString(desc) if l > 30 { l = 30 @@ -159,7 +170,7 @@ func GetSource() []Source { desc = string([]rune(desc)[:30]) d := data.FetchData{ Url: fmt.Sprintf("https://jjjcb.ccdi.gov.cn/epaper/index.html?guid=%s", v.ZBGUID), - Title: v.YT, + Title: v.DOCTITLE, Desc: desc, Date: v.DOCPUBTIME, CreatedTime: nowDate, @@ -169,5 +180,230 @@ func GetSource() []Source { } }, }, + { + Name: "中新网搜索", + SearchUrl: "https://sou.chinanews.com.cn/search.do", + KeywordField: "q", + Method: "get", + Type: "html", + ListQuery: "#news_list table", + QueryHandler: func(i int, selection *goquery.Selection, fetchData *data.FetchData) { + t := selection.Find(".news_item a").First() + fetchData.Title = t.Text() + fetchData.Url, _ = t.Attr("href") + fetchData.Desc = selection.Find(".news_content").First().Text() + tt := selection.Find(".news_other").First().Text() + fet := strings.Split(tt, "html") + fetchData.Date = fet[len(fet)-1] + }, + }, + { + Name: "新浪新闻搜索", + SearchUrl: "https://search.sina.com.cn/?range=title&c=news&time=&ie=utf-8", + Method: "get", + Type: "html", + KeywordField: "q", + ListQuery: "#wrap .box-result", + Header: map[string]string{ + "user-agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36", + "authority": "search.sina.com.cn", + "referer": "https://news.sina.com.cn/rol", + }, + QueryHandler: func(i int, selection *goquery.Selection, fetchData *data.FetchData) { + t := selection.Find("h2>a").First() + fetchData.Title = t.Text() + fetchData.Url, _ = t.Attr("href") + fetchData.Desc = selection.Find(".r-info .content").Text() + s := selection.Find("h2 >.fgray_time").First().Text() + ll := strings.Fields(s) + if len(ll) > 2 { + fetchData.Date = ll[1] + " " + ll[2] + fetchData.Source = ll[0] + } else { + fetchData.Date = ll[1] + fetchData.Source = ll[0] + } + + }, + }, + { + Name: "联合早报", + SearchUrl: "https://www.zaobao.com/search?pageNo=1&pageSize=10", + Method: "get", + Type: "ajax", + Header: map[string]string{ + "authority": "www.zaobao.com", + "user-agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36", + }, + KeywordField: "keywords", + AjaxDealFun: func(i *[]data.FetchData, response *http.Response) { + bytes, err := ioutil.ReadAll(response.Body) + if err != nil { + log.Println(err) + return + } + var r ZaoBaoResponse + err = json.Unmarshal(bytes, &r) + if err != nil { + log.Println(err) + return + } + for _, datum := range r.Result.Data { + t := time.UnixMilli(datum.PublicationDate) + v := data.FetchData{ + Url: "https://www.zaobao.com" + datum.URL, + Title: datum.Title, + Desc: datum.ContentPreview, + Date: t.Format("2006-01-02 15:04:05"), + } + *i = append(*i, v) + } + }, + }, + { + Name: "新京报", + SearchUrl: "https://s.bjnews.com.cn/bjnews/getlist?from=bw&page=1&orderby=0", + KeywordField: "bwsk", + Type: "ajax", + Method: "get", + Header: map[string]string{ + "user-agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36", + }, + Target: reflect.TypeOf(BjNewsResponse{}), + AjaxSimpleDeal: func(rr interface{}, i *[]data.FetchData) { + r := rr.(BjNewsResponse) + for _, v := range r.Data.Data { + item := data.FetchData{ + Url: v.Source.DetailURL.PcURL, + Title: v.Source.Title, + Desc: v.Highlight.Desc, + Date: v.Source.PublishTime, + } + *i = append(*i, item) + } + }, + }, + { + Name: "环球网", + SearchUrl: "https://www.baidu.com/s?wd=site:huanqiu.com%20", + Method: "get", + ListQuery: "div[class=\"result c-container xpath-log new-pmd\"]", + QueryHandler: func(i int, selection *goquery.Selection, data *data.FetchData) { + data.Url, _ = selection.Attr("mu") + t := selection.Find("h3[class='c-title t t tts-title'] a").First() + data.Title = t.Text() + data.Desc = selection.Find(".content-right_8Zs40").First().Text() + data.Source = selection.Find(".source_1Vdff .c-color-gray").First().Text() + data.Date = selection.Find("span[class=\"c-color-gray2\"]").First().Text() + n := compile.FindAllStringSubmatch(data.Date, -1) + if nil != n { + nn, _ := strconv.Atoi(n[0][0]) + if strings.Contains(data.Date, "小时") { + data.Date = ti.Add(-time.Duration(nn) * time.Hour).Format("2006-01-02 15:04") + } + if strings.Contains(data.Date, "分钟") { + data.Date = ti.Add(-time.Duration(nn) * time.Minute).Format("2006-01-02 15:04") + } + if strings.Contains(data.Date, "天") { + data.Date = ti.Add(-time.Duration(nn) * time.Hour * 24).Format("2006-01-02 15:04") + } + } + if strings.Contains(data.Date, "昨天") { + data.Date = ti.Add(-time.Duration(24) * time.Hour).Format("2006-01-02 15:04") + } + if strings.Contains(data.Date, "前天") { + data.Date = ti.Add(-time.Duration(48) * time.Hour).Format("2006-01-02 15:04") + } + }, + Type: "html", + HeaderFun: func(req *http.Request) { + req.Header.Add("accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9") + req.Header.Add("accept-language", "zh-CN,zh;q=0.9") + req.Header.Add("cache-control", "no-cache") + req.Header.Add("connection", "keep-alive") + req.Header.Add("cookie", "BIDUPSID=844E3DCAA2EEBF5C872DC99B967B6B7B; PSTM=1655872163; BAIDUID=844E3DCAA2EEBF5CB3E1D79750162204:FG=1; BD_UPN=123353; ORIGIN=2; ISSW=1; ISSW=1; BAIDUID_BFESS=844E3DCAA2EEBF5CB3E1D79750162204:FG=1; ZFY=jWFAySgO:AoQfb6emY9vnmEdptVao:Anj0FFkp028wFws:C; BD_HOME=1; delPer=0; BD_CK_SAM=1; PSINO=3; COOKIE_SESSION=42_0_2_2_3_0_1_0_2_0_0_0_18_0_51_0_1655888428_0_1655888377%7C3%230_0_1655888377%7C1; BAIDU_WISE_UID=wapp_1655902298617_702; ZD_ENTRY=google; channel=baidusearch; baikeVisitId=b3b23509-9330-4d33-82ae-b8eb37895917; BA_HECTOR=8k2g2g218ga40181ak1hbgg1n14; BDRCVFR[C0p6oIjvx-c]=mbxnW11j9Dfmh7GuZR8mvqV; BDSVRTM=1011; H_PS_PSSID=36550_36459_36673_36455_36453_36692_36165_36695_36697_36569_36075_36467_36316_36651") + req.Header.Add("referer", "http://news.baidu.com/") + req.Header.Add("sec-fetch-dest", "document") + req.Header.Add("sec-fetch-mode", "navigate") + req.Header.Add("sec-fetch-site", "cross-site") + req.Header.Add("sec-fetch-user", "?1") + req.Header.Add("upgrade-insecure-requests", "1") + req.Header.Add("user-agent", "Mozilla/5.0 (X11; Windows x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36") + req.Header.Add("#sec-ch-ua", "\".Not/A)Brand\";v=\"99\", \"Google Chrome\";v=\"103\", \"Chromium\";v=\"103\"") + req.Header.Add("sec-ch-ua-mobile", "?0") + req.Header.Add("sec-ch-ua-platform", "\"Windows\"") + req.Header.Add("postman-token", "81407fbc-2b96-54a7-0193-f640156714ab") + + }, + }, + { + Name: "凤凰新闻", + SearchUrl: "https://shankapi.ifeng.com/season/getSoFengData/all/${keyword}/1/getSoFengDataCallback?callback=getSoFengDataCallback2", + Type: "ajax", + Method: "get", + AjaxDealFun: func(i *[]data.FetchData, response *http.Response) { + bytes, err := ioutil.ReadAll(response.Body) + if err != nil { + log.Println(err) + return + } + r := string(bytes) + r = strings.Replace(r, "getSoFengDataCallback(", "", 1) + r = strings.TrimRight(r, ")") + var res IFENGResponse + err = json.Unmarshal([]byte(r), &res) + if err != nil { + log.Println(err) + return + } + for _, v := range res.Data.Items { + *i = append(*i, data.FetchData{ + Url: "https:" + v.URL, + Title: StripTags(v.Title), + Source: v.Source, + }) + } + }, + }, + { + Name: "人民网", + SearchUrl: "http://search.people.cn/search-platform/front/search", + Type: "ajax", + Method: "post", + IsJson: true, + KeywordField: "key", + Target: reflect.TypeOf(PeopleResponse{}), + ExternParam: map[string]string{ + "page": "1", + "limit": "10", + "hasTitle": "true", + "hasContent": "true", + "isFuzzy": "false", + "type": "0", + "sortType": "0", + "startTime": "0", + "endTime": "0", + }, + AjaxSimpleDeal: func(i interface{}, v *[]data.FetchData) { + r := i.(PeopleResponse) + for _, record := range r.Data.Records { + tt := time.UnixMilli(int64(record.InputTime)) + *v = append(*v, data.FetchData{ + Url: record.Url, + Title: StripTags(record.Title), + Desc: StripTags(record.Content), + Date: tt.Format("2006-01-02 15:04:05"), + Source: record.OriginName, + }) + } + }, + HeaderFun: func(req *http.Request) { + req.Header.Set("Content-Type", "application/json") + req.Header.Add("Cookie", "__jsluid_h=a1b7d0d8dad3604c9393bbcaf36ced1f; sso_c=0; sfr=1; __jsluid_h=fbf7d0abc29ec349c0c0c89c779c268c") + req.Header.Add("Origin", "http://search.people.cn") + req.Header.Add("Referer", "http://search.people.cn/s/?keyword=%E7%BA%AA%E6%A3%80&st=0") + req.Header.Add("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36") + }, + }, } } diff --git a/newsource/people.go b/newsource/people.go new file mode 100644 index 0000000..90d0782 --- /dev/null +++ b/newsource/people.go @@ -0,0 +1,15 @@ +package newsource + +type PeopleResponse struct { + Data struct { + Records []PeopleRecord + } +} + +type PeopleRecord struct { + Title string `json:"title"` + Url string `json:"url"` + OriginName string `json:"originName"` + Content string `json:"content"` + InputTime int `json:"inputTime"` +} diff --git a/newsource/zaobao.go b/newsource/zaobao.go new file mode 100644 index 0000000..8eb5528 --- /dev/null +++ b/newsource/zaobao.go @@ -0,0 +1,19 @@ +package newsource + +type ZaoBaoResponse struct { + Result struct { + Data []ZaoBaoItem `json:"data"` + } `json:"result"` +} + +type ZaoBaoItem struct { + ID string `json:"id"` + Title string `json:"title"` + Digest interface{} `json:"digest"` + ContentPreview string `json:"contentPreview"` + MainNode string `json:"mainNode"` + PublicationDate int64 `json:"publicationDate"` + URL string `json:"url"` + Pictures interface{} `json:"pictures"` + IsSticky int `json:"isSticky"` +} diff --git a/src/components/WelCome.vue b/src/components/WelCome.vue index 7a505ea..4fa5f67 100644 --- a/src/components/WelCome.vue +++ b/src/components/WelCome.vue @@ -31,6 +31,10 @@ + +