完善
This commit is contained in:
xing
parent
ab97f5c624
commit
eceeae2444
57
main.go
57
main.go
|
|
@ -36,21 +36,24 @@ type dataChan struct {
|
|||
}
|
||||
|
||||
type fetchHandler struct {
|
||||
fetchUrl string
|
||||
hadFetchData []data.FetchData
|
||||
cronTime mapXS[time.Duration]
|
||||
keyword mapXS[string]
|
||||
searchSource mapXS[[]string]
|
||||
hadFetchedMap mapXS[int]
|
||||
reloadCron mapXS[chan int]
|
||||
isOff chan int
|
||||
rMsgChan chan connChan
|
||||
newFetchItem chan dataChan
|
||||
connMap mapXS[*websocket.Conn]
|
||||
sourceMap map[string]newsource.Source
|
||||
sourceArr []string
|
||||
}
|
||||
|
||||
type setting struct {
|
||||
Keyword string `json:"keyword"`
|
||||
TimeStep int `json:"timeStep"`
|
||||
SearchSource []string `json:"searchSource"`
|
||||
}
|
||||
|
||||
type message struct {
|
||||
|
|
@ -86,6 +89,15 @@ func (r dist) Open(name string) (fs.File, error) {
|
|||
return file, err
|
||||
}
|
||||
|
||||
func isContain[T comparable](i T, arr []T) bool {
|
||||
for _, t := range arr {
|
||||
if i == t {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func setMap[T mapT](obj *mapXS[T], key string, v T) {
|
||||
obj.Lock()
|
||||
(*obj.mapX)[key] = v
|
||||
|
|
@ -99,7 +111,7 @@ func delMap[T mapT](obj *mapXS[T], key string) {
|
|||
}
|
||||
|
||||
type mapT interface {
|
||||
string | int | time.Duration | *websocket.Conn | chan int
|
||||
string | []string | int | time.Duration | *websocket.Conn | chan int
|
||||
}
|
||||
type mapX[T mapT] map[string]T
|
||||
|
||||
|
|
@ -108,9 +120,16 @@ type mapXS[T mapT] struct {
|
|||
*sync.Mutex
|
||||
}
|
||||
|
||||
func newFetchHandler(fetchUrl string) *fetchHandler {
|
||||
func newFetchHandler() *fetchHandler {
|
||||
var arr = make(map[string]newsource.Source)
|
||||
var x []string
|
||||
for _, source := range newsource.GetSource() {
|
||||
arr[source.Name] = source
|
||||
x = append(x, source.Name)
|
||||
}
|
||||
return &fetchHandler{
|
||||
fetchUrl: fetchUrl,
|
||||
sourceMap: arr,
|
||||
sourceArr: x,
|
||||
keyword: mapXS[string]{
|
||||
&mapX[string]{},
|
||||
&sync.Mutex{},
|
||||
|
|
@ -123,6 +142,10 @@ func newFetchHandler(fetchUrl string) *fetchHandler {
|
|||
&mapX[time.Duration]{},
|
||||
&sync.Mutex{},
|
||||
},
|
||||
searchSource: mapXS[[]string]{
|
||||
&mapX[[]string]{},
|
||||
&sync.Mutex{},
|
||||
},
|
||||
reloadCron: mapXS[chan int]{
|
||||
&mapX[chan int]{},
|
||||
&sync.Mutex{},
|
||||
|
|
@ -142,7 +165,8 @@ func (f *fetchHandler) handle(conn string) {
|
|||
if kk, ok := (*f.keyword.mapX)[conn]; ok && kk != "" {
|
||||
key = kk
|
||||
}
|
||||
for _, source := range newsource.GetSource() {
|
||||
for _, sourceName := range (*f.searchSource.mapX)[conn] {
|
||||
source := f.sourceMap[sourceName]
|
||||
r := f.fetch2(source, key)
|
||||
if r != nil {
|
||||
if strings.ToUpper(source.Type) == "HTML" {
|
||||
|
|
@ -160,9 +184,10 @@ func (f *fetchHandler) receiveMsg() {
|
|||
switch r.msg.Action {
|
||||
case "search":
|
||||
if t, ok := r.msg.Data.(*setting); ok {
|
||||
(*f.reloadCron.mapX)[r.conn] <- t.TimeStep
|
||||
setMap[string](&f.keyword, r.conn, t.Keyword)
|
||||
setMap[[]string](&f.searchSource, r.conn, t.SearchSource)
|
||||
f.handle(r.conn)
|
||||
(*f.reloadCron.mapX)[r.conn] <- t.TimeStep
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -283,6 +308,7 @@ func (f *fetchHandler) parseAjax(response *http.Response, source newsource.Sourc
|
|||
}
|
||||
|
||||
if len(newFetch) > 0 {
|
||||
var newF []data.FetchData
|
||||
for i := 0; i < len(newFetch); i++ {
|
||||
fetchData := newFetch[i]
|
||||
k := conn + "_" + fetchData.Url + "_" + fetchData.Title
|
||||
|
|
@ -295,13 +321,12 @@ func (f *fetchHandler) parseAjax(response *http.Response, source newsource.Sourc
|
|||
if _, ok := (*f.hadFetchedMap.mapX)[k]; !ok {
|
||||
f.hadFetchData = append(f.hadFetchData, fetchData)
|
||||
setMap(&f.hadFetchedMap, k, 1)
|
||||
} else {
|
||||
newFetch = newFetch[:i+copy(newFetch[i:], newFetch[i+1:])] // 删除中间1个元素
|
||||
newF = append(newF, newFetch[i])
|
||||
}
|
||||
}
|
||||
f.newFetchItem <- dataChan{
|
||||
conn: conn,
|
||||
item: newFetch,
|
||||
item: newF,
|
||||
}
|
||||
}
|
||||
err := response.Body.Close()
|
||||
|
|
@ -389,7 +414,7 @@ func (f *fetchHandler) cronFetch(conn string, c chan int) {
|
|||
}
|
||||
|
||||
func main() {
|
||||
h := newFetchHandler("https://www.baidu.com/s?rtt=1&bsst=1&cl=2&tn=news&rsv_dl=ns_pc&word=")
|
||||
h := newFetchHandler()
|
||||
router := gin.Default()
|
||||
static := dist{
|
||||
FS: st,
|
||||
|
|
@ -430,6 +455,12 @@ func main() {
|
|||
log.Println(err)
|
||||
return
|
||||
}
|
||||
_ = conn.WriteJSON(message{
|
||||
Status: true,
|
||||
Action: "sourceList",
|
||||
Message: "",
|
||||
Data: h.sourceArr,
|
||||
})
|
||||
remote := conn.RemoteAddr().String()
|
||||
if _, ok := (*h.connMap.mapX)[remote]; !ok {
|
||||
setMap(&h.connMap, remote, conn)
|
||||
|
|
@ -460,12 +491,12 @@ func main() {
|
|||
})
|
||||
go func() {
|
||||
time.Sleep(2 * time.Second)
|
||||
url := "http://127.0.0.1:8080"
|
||||
u := "http://127.0.0.1:8080"
|
||||
switch runtime.GOOS {
|
||||
case "linux":
|
||||
exec.Command(`xdg-open`, url).Start()
|
||||
exec.Command(`xdg-open`, u).Start()
|
||||
case "windows":
|
||||
exec.Command(`cmd`, `/c`, `start`, url).Start()
|
||||
exec.Command(`cmd`, `/c`, `start`, u).Start()
|
||||
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -8,12 +8,29 @@
|
|||
<el-form-item prop="password" label="搜索间隔时间，单位为秒">
|
||||
<el-input v-model="form.timeStep" type="number" step="1" ></el-input>
|
||||
</el-form-item>
|
||||
<el-form-item prop="searchSource" label="要抓取的新闻源">
|
||||
<el-select
|
||||
v-model="form.searchSource"
|
||||
multiple
|
||||
clearable filterable
|
||||
placeholder="可多选"
|
||||
style="width: 240px"
|
||||
>
|
||||
<el-option
|
||||
v-for="item in allSource"
|
||||
:key="item.k"
|
||||
:label="item.k"
|
||||
:value="item.v"
|
||||
/>
|
||||
</el-select>
|
||||
</el-form-item>
|
||||
|
||||
<el-form-item class="btns">
|
||||
<el-button @click="submit" type="primary">查询</el-button>
|
||||
</el-form-item>
|
||||
</el-form>
|
||||
<el-card class="box-card">
|
||||
<el-table
|
||||
<el-table :table-layout="tableLayout"
|
||||
:data="rowss" border stripe
|
||||
style="width: 100%">
|
||||
<el-table-column
|
||||
|
|
@ -32,12 +49,14 @@
|
|||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column
|
||||
align="center"
|
||||
prop="source"
|
||||
label="来源">
|
||||
</el-table-column>
|
||||
<el-table-column
|
||||
prop="desc"
|
||||
label="描述"
|
||||
:show-overflow-tooltip="true"
|
||||
>
|
||||
</el-table-column>
|
||||
<el-table-column
|
||||
|
|
@ -64,14 +83,20 @@
|
|||
</template>
|
||||
|
||||
<script>
|
||||
import {ref} from 'vue'
|
||||
|
||||
const tableLayout = ref('auto')
|
||||
export default {
|
||||
name: 'WelCome',
|
||||
data () {
|
||||
return {
|
||||
tableLayout: tableLayout,
|
||||
form: {
|
||||
keyword: '纪检',
|
||||
timeStep: 60
|
||||
timeStep: 60,
|
||||
searchSource: [],
|
||||
},
|
||||
allSource: [],
|
||||
ws: null,
|
||||
formRules: {
|
||||
keyword: [
|
||||
|
|
@ -81,6 +106,9 @@ export default {
|
|||
timeStep: [
|
||||
{ required: true, message: '请输入时间间隔，单位为秒', trigger: 'blur' },
|
||||
{ min: 1, max: 10000000, message: '长度在 1 到 10000000 秒', trigger: 'blur' }
|
||||
],
|
||||
searchSource: [
|
||||
{ required: true, message: '请选择新闻源', trigger: 'change' },
|
||||
]
|
||||
},
|
||||
rows: [],
|
||||
|
|
@ -135,6 +163,14 @@ export default {
|
|||
this.ws = new WebSocket('ws://127.0.0.1:8080/ws')
|
||||
this.ws.addEventListener('message', e => {
|
||||
const msg = JSON.parse(e.data)
|
||||
if(msg.Action==='sourceList'){
|
||||
this.allSource = msg.Data.map(value => {
|
||||
return {
|
||||
k:value,
|
||||
v:value
|
||||
}
|
||||
})
|
||||
}else if (msg.Action==='newData'){
|
||||
msg.Data.forEach(v => {
|
||||
v.isNew = true
|
||||
})
|
||||
|
|
@ -143,6 +179,8 @@ export default {
|
|||
// eslint-disable-next-line no-unused-vars
|
||||
const n = new Notification('有新抓取文章', { body: msg.Data[0].title + '等总共' + msg.Data.length + '条' })
|
||||
})
|
||||
}
|
||||
|
||||
})
|
||||
this.ws.onclose = () => {
|
||||
const a = setInterval(() => {
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user