From 1286338af0c4be6229c302601a14077612446fd2 Mon Sep 17 00:00:00 2001 From: xing Date: Sat, 14 Oct 2023 14:57:57 +0800 Subject: [PATCH] http tool, fix digest, reload add sort --- app/cmd/reload/reload.go | 84 ++++++--- app/plugins/digest.go | 5 + helper/httptool/http.go | 349 +++++++++++++++++++++++++++++++---- helper/httptool/http_test.go | 210 ++++++++++++++++++++- helper/slice/sort.go | 2 +- 5 files changed, 582 insertions(+), 68 deletions(-) diff --git a/app/cmd/reload/reload.go b/app/cmd/reload/reload.go index 692679e..ecb7016 100644 --- a/app/cmd/reload/reload.go +++ b/app/cmd/reload/reload.go @@ -2,11 +2,17 @@ package reload import ( "github.com/fthvgb1/wp-go/helper/number" + "github.com/fthvgb1/wp-go/helper/slice" "github.com/fthvgb1/wp-go/safety" "sync" ) -var calls []func() +type queue struct { + fn func() + order float64 +} + +var calls []queue var anyMap = safety.NewMap[string, any]() @@ -47,7 +53,7 @@ func GetAnyMapFnBys[K comparable, V, A any](namespace string, fn func(A) V) func } } -func safetyMapFn[K comparable, V, A any](namespace string) *safetyMap[K, V, A] { +func safetyMapFn[K comparable, V, A any](namespace string, order ...float64) *safetyMap[K, V, A] { vv, ok := safetyMaps.Load(namespace) var m *safetyMap[K, V, A] if ok { @@ -61,7 +67,7 @@ func safetyMapFn[K comparable, V, A any](namespace string) *safetyMap[K, V, A] { m = &safetyMap[K, V, A]{safety.NewMap[K, V](), sync.Mutex{}} Push(func() { m.val.Flush() - }) + }, order...) safetyMaps.Store(namespace, m) } safetyMapLock.Unlock() @@ -69,8 +75,8 @@ func safetyMapFn[K comparable, V, A any](namespace string) *safetyMap[K, V, A] { return m } -func GetAnyValMapBy[K comparable, V, A any](namespace string, key K, a A, fn func(A) V) V { - m := safetyMapFn[K, V, A](namespace) +func GetAnyValMapBy[K comparable, V, A any](namespace string, key K, a A, fn func(A) V, order ...float64) V { + m := safetyMapFn[K, V, A](namespace, order...) v, ok := m.val.Load(key) if ok { return v @@ -87,7 +93,7 @@ func GetAnyValMapBy[K comparable, V, A any](namespace string, key K, a A, fn fun return v } -func anyVal[T, A any](namespace string, counter bool) *safetyVar[T, A] { +func anyVal[T, A any](namespace string, counter bool, order ...float64) *safetyVar[T, A] { var vv *safetyVar[T, A] vvv, ok := safetyMaps.Load(namespace) if ok { @@ -105,7 +111,7 @@ func anyVal[T, A any](namespace string, counter bool) *safetyVar[T, A] { vv = &safetyVar[T, A]{safety.NewVar(v), sync.Mutex{}} Push(func() { vv.Val.Flush() - }) + }, getOrder(order...)) safetyMaps.Store(namespace, vv) } safetyMapLock.Unlock() @@ -113,8 +119,8 @@ func anyVal[T, A any](namespace string, counter bool) *safetyVar[T, A] { return vv } -func GetAnyValBy[T, A any](namespace string, tryTimes int, a A, fn func(A) (T, bool)) T { - var vv = anyVal[T, A](namespace, true) +func GetAnyValBy[T, A any](namespace string, tryTimes int, a A, fn func(A) (T, bool), order ...float64) T { + var vv = anyVal[T, A](namespace, true, order...) var ok bool v := vv.Val.Load() if v.ok { @@ -136,8 +142,8 @@ func GetAnyValBy[T, A any](namespace string, tryTimes int, a A, fn func(A) (T, b return v.v } -func GetAnyValBys[T, A any](namespace string, a A, fn func(A) T) T { - var vv = anyVal[T, A](namespace, false) +func GetAnyValBys[T, A any](namespace string, a A, fn func(A) T, order ...float64) T { + var vv = anyVal[T, A](namespace, false, order...) v := vv.Val.Load() if v.ok { return v.v @@ -155,49 +161,69 @@ func GetAnyValBys[T, A any](namespace string, a A, fn func(A) T) T { return v.v } -func Vars[T any](defaults T) *safety.Var[T] { +func Vars[T any](defaults T, order ...float64) *safety.Var[T] { ss := safety.NewVar(defaults) - calls = append(calls, func() { + ord := getOrder(order...) + calls = append(calls, queue{func() { ss.Store(defaults) - }) + }, ord}) return ss } -func VarsBy[T any](fn func() T) *safety.Var[T] { + +func getOrder(order ...float64) float64 { + var ord float64 + if len(order) > 0 { + ord = order[0] + } + return ord +} +func VarsBy[T any](fn func() T, order ...float64) *safety.Var[T] { ss := safety.NewVar(fn()) - calls = append(calls, func() { - ss.Store(fn()) + ord := getOrder(order...) + calls = append(calls, queue{ + func() { + ss.Store(fn()) + }, ord, }) return ss } -func MapBy[K comparable, T any](fn func(*safety.Map[K, T])) *safety.Map[K, T] { +func MapBy[K comparable, T any](fn func(*safety.Map[K, T]), order ...float64) *safety.Map[K, T] { m := safety.NewMap[K, T]() if fn != nil { fn(m) } - calls = append(calls, func() { - m.Flush() - if fn != nil { - fn(m) - } + ord := getOrder(order...) + calls = append(calls, queue{ + func() { + m.Flush() + if fn != nil { + fn(m) + } + }, ord, }) return m } -func SafeMap[K comparable, T any]() *safety.Map[K, T] { +func SafeMap[K comparable, T any](order ...float64) *safety.Map[K, T] { m := safety.NewMap[K, T]() - calls = append(calls, func() { + ord := getOrder(order...) + calls = append(calls, queue{func() { m.Flush() - }) + }, ord}) return m } -func Push(fn ...func()) { - calls = append(calls, fn...) +func Push(fn func(), order ...float64) { + ord := getOrder(order...) + calls = append(calls, queue{fn, ord}) } func Reload() { + slice.Sort(calls, func(i, j queue) bool { + return i.order > j.order + }) for _, call := range calls { - call() + call.fn() } anyMap.Flush() safetyMaps.Flush() diff --git a/app/plugins/digest.go b/app/plugins/digest.go index 276c699..b2147cd 100644 --- a/app/plugins/digest.go +++ b/app/plugins/digest.go @@ -12,6 +12,7 @@ import ( "regexp" "strings" "time" + "unicode/utf8" ) var digestCache *cache.MapCache[uint64, string] @@ -52,6 +53,10 @@ func Digests(content string, id uint64, limit int, fn func(id uint64, content, c tag = "

      • " } content = digest.StripTags(content, tag) + length := utf8.RuneCountInString(content) + 1 + if length <= limit { + return content + } content, closeTag = digest.Html(content, limit) if fn == nil { return PostsMore(id, content, closeTag) diff --git a/helper/httptool/http.go b/helper/httptool/http.go index 467ba09..bbd2efd 100644 --- a/helper/httptool/http.go +++ b/helper/httptool/http.go @@ -1,19 +1,29 @@ package httptool import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "github.com/fthvgb1/wp-go/helper/maps" + "github.com/fthvgb1/wp-go/helper/number" + str "github.com/fthvgb1/wp-go/helper/strings" + "golang.org/x/exp/constraints" "io" + "mime/multipart" "net/http" "net/url" + "os" + "strings" "time" ) -func GetString(u string, q map[string]string, timeout int64, a ...any) (r string, code int, err error) { - res, err := Get(u, q, timeout, a...) +func GetString(u string, q map[string]any, a ...any) (r string, code int, err error) { + res, err := Get(u, q, a...) if res != nil { code = res.StatusCode } if err != nil { - return "", code, err } defer res.Body.Close() @@ -25,45 +35,314 @@ func GetString(u string, q map[string]string, timeout int64, a ...any) (r string return } -func Get(u string, q map[string]string, timeout int64, a ...any) (res *http.Response, err error) { +func Get(u string, q map[string]any, a ...any) (res *http.Response, err error) { + cli, req, err := GetClient(u, q, a...) + res, err = cli.Do(req) + return +} + +func GetToJsonAny[T any](u string, q map[string]any, a ...any) (r T, code int, err error) { + rr, err := Get(u, q, a...) + if err != nil { + return + } + code = rr.StatusCode + b, err := io.ReadAll(rr.Body) + if err != nil { + return + } + rr.Body.Close() + err = json.Unmarshal(b, &r) + return +} + +func PostWwwString(u string, form map[string]any, a ...any) (r string, code int, err error) { + rr, err := Post(u, 1, form, a...) + if err != nil { + return "", 0, err + } + code = rr.StatusCode + rs, err := io.ReadAll(rr.Body) + if err != nil { + return "", code, err + } + rr.Body.Close() + r = string(rs) + return +} +func PostFormDataString(u string, form map[string]any, a ...any) (r string, code int, err error) { + rr, err := Post(u, 2, form, a...) + if err != nil { + return "", 0, err + } + code = rr.StatusCode + rs, err := io.ReadAll(rr.Body) + if err != nil { + return "", code, err + } + rr.Body.Close() + r = string(rs) + return +} + +func GetClient(u string, q map[string]any, a ...any) (res *http.Client, req *http.Request, err error) { parse, err := url.Parse(u) if err != nil { - return nil, err - } - cli := http.Client{ - Timeout: time.Duration(timeout) * time.Second, + return nil, nil, err } + cli := http.Client{} values := parse.Query() - for k, v := range q { - values.Add(k, v) - } + setValue(q, values) parse.RawQuery = values.Encode() - req := http.Request{ + req = &http.Request{ Method: "GET", URL: parse, } - if len(a) > 0 { - for _, arg := range a { - h, ok := arg.(map[string]string) - if ok && len(h) > 0 { - for k, v := range h { - req.Header.Add(k, v) - } - } - t, ok := arg.(time.Duration) - if ok { - cli.Timeout = t - } - checkRedirect, ok := arg.(func(req *http.Request, via []*http.Request) error) - if ok { - cli.CheckRedirect = checkRedirect - } - jar, ok := arg.(http.CookieJar) - if ok { - cli.Jar = jar - } - } - } - res, err = cli.Do(&req) + setArgs(&cli, req, a...) + return &cli, req, err +} + +// Post request +// +// u url +// +// types 1 x-www-form-urlencoded, 2 form-data, 3 json, 4 binary +func Post(u string, types int, form map[string]any, a ...any) (res *http.Response, err error) { + cli, req, err := PostClient(u, types, form, a...) + res, err = cli.Do(req) return } + +func PostClient(u string, types int, form map[string]any, a ...any) (cli *http.Client, req *http.Request, err error) { + parse, err := url.Parse(u) + if err != nil { + return + } + cli = &http.Client{} + req = &http.Request{ + Method: "POST", + URL: parse, + Header: http.Header{}, + } + switch types { + case 1: + values := url.Values{} + setValue(form, values) + req.Header.Add("Content-Type", "application/x-www-form-urlencoded") + b := strings.NewReader(values.Encode()) + req.Body = io.NopCloser(b) + case 2: + payload := &bytes.Buffer{} + writer := multipart.NewWriter(payload) + err = setFormData(form, writer) + if err != nil { + return + } + err = writer.Close() + if err != nil { + return + } + req.Body = io.NopCloser(payload) + req.Header.Add("Content-Type", writer.FormDataContentType()) + case 3: + fo, err := json.Marshal(form) + if err != nil { + return nil, nil, err + } + b := bytes.NewReader(fo) + req.Body = io.NopCloser(b) + req.Header.Add("Content-Type", "application/json") + case 4: + b, ok := maps.GetStrAnyVal[[]byte](form, "binary") + if !ok { + return nil, nil, errors.New("no binary value") + } + req.Body = io.NopCloser(bytes.NewReader(b)) + req.Header.Add("Content-Type", "application/octet-stream") + } + setArgs(cli, req, a...) + return +} + +func PostToJsonAny[T any](u string, types int, form map[string]any, a ...any) (r T, code int, err error) { + rr, err := Post(u, types, form, a...) + if err != nil { + return + } + code = rr.StatusCode + b, err := io.ReadAll(rr.Body) + if err != nil { + return + } + rr.Body.Close() + err = json.Unmarshal(b, &r) + return +} + +func setArgs(cli *http.Client, req *http.Request, a ...any) { + if len(a) < 1 { + return + } + for _, arg := range a { + h, ok := arg.(map[string]string) + if ok && len(h) > 0 { + for k, v := range h { + req.Header.Add(k, v) + } + } + hh, ok := arg.(http.Header) + if ok { + req.Header = hh + } + t, ok := arg.(time.Duration) + if ok { + cli.Timeout = t + } + checkRedirect, ok := arg.(func(req *http.Request, via []*http.Request) error) + if ok { + cli.CheckRedirect = checkRedirect + } + jar, ok := arg.(http.CookieJar) + if ok { + cli.Jar = jar + } + c, ok := arg.(string) + if ok && c != "" { + req.Header.Add("cookie", c) + } + } +} + +func set[T constraints.Integer | constraints.Float](a []T, k string, values url.Values) { + if !strings.Contains(k, "[]") { + k = str.Join(k, "[]") + } + for _, vv := range a { + values.Add(k, number.ToString(vv)) + } +} + +func setFormData(m map[string]any, values *multipart.Writer) (err error) { + for k, v := range m { + switch v.(type) { + case *os.File: + f := v.(*os.File) + if f == nil { + continue + } + ff, err := values.CreateFormFile(k, f.Name()) + if err != nil { + return err + } + _, err = io.Copy(ff, f) + if err != nil { + return err + } + case string: + err = values.WriteField(k, v.(string)) + case int64, int, int8, int32, int16, uint64, uint, uint8, uint32, uint16, float32, float64: + err = values.WriteField(k, fmt.Sprintf("%v", v)) + case []string: + if !strings.Contains(k, "[]") { + k = str.Join(k, "[]") + } + for _, vv := range v.([]string) { + err = values.WriteField(k, vv) + } + case *[]string: + if !strings.Contains(k, "[]") { + k = str.Join(k, "[]") + } + for _, vv := range *(v.(*[]string)) { + err = values.WriteField(k, vv) + } + case []int64: + err = sets(v.([]int64), k, values) + case []int: + err = sets(v.([]int), k, values) + case []int8: + err = sets(v.([]int8), k, values) + case []int16: + err = sets(v.([]int16), k, values) + case []int32: + err = sets(v.([]int32), k, values) + case []uint64: + err = sets(v.([]uint64), k, values) + case []uint: + err = sets(v.([]uint), k, values) + case []uint8: + err = sets(v.([]uint8), k, values) + case []uint16: + err = sets(v.([]uint16), k, values) + case []uint32: + err = sets(v.([]uint32), k, values) + case []float32: + err = sets(v.([]float32), k, values) + case []float64: + err = sets(v.([]float64), k, values) + } + } + return +} + +func sets[T constraints.Integer | constraints.Float](a []T, k string, values *multipart.Writer) error { + if !strings.Contains(k, "[]") { + k = str.Join(k, "[]") + } + for _, vv := range a { + err := values.WriteField(k, number.ToString(vv)) + if err != nil { + return err + } + } + return nil +} + +func setValue(m map[string]any, values url.Values) { + for k, v := range m { + switch v.(type) { + case string: + values.Add(k, v.(string)) + case int64, int, int8, int32, int16, uint64, uint, uint8, uint32, uint16, float32, float64: + values.Add(k, fmt.Sprintf("%v", v)) + case []string: + if !strings.Contains(k, "[]") { + k = str.Join(k, "[]") + } + for _, vv := range v.([]string) { + values.Add(k, vv) + } + case *[]string: + if !strings.Contains(k, "[]") { + k = str.Join(k, "[]") + } + for _, vv := range *(v.(*[]string)) { + values.Add(k, vv) + } + case []int64: + set(v.([]int64), k, values) + case []int: + set(v.([]int), k, values) + case []int8: + set(v.([]int8), k, values) + case []int16: + set(v.([]int16), k, values) + case []int32: + set(v.([]int32), k, values) + case []uint64: + set(v.([]uint64), k, values) + case []uint: + set(v.([]uint), k, values) + case []uint8: + set(v.([]uint8), k, values) + case []uint16: + set(v.([]uint16), k, values) + case []uint32: + set(v.([]uint32), k, values) + case []float32: + set(v.([]float32), k, values) + case []float64: + set(v.([]float64), k, values) + } + } +} diff --git a/helper/httptool/http_test.go b/helper/httptool/http_test.go index 867749f..0faa9c8 100644 --- a/helper/httptool/http_test.go +++ b/helper/httptool/http_test.go @@ -1,13 +1,16 @@ package httptool import ( + "net/http" + "reflect" "testing" + "time" ) func TestGetString(t *testing.T) { type args struct { u string - q map[string]string + q map[string]any timeout int64 a []any } @@ -22,9 +25,10 @@ func TestGetString(t *testing.T) { name: "wp.test", args: args{ u: "http://wp.test", - q: map[string]string{ + q: map[string]any{ "p": "2", "XDEBUG_SESSION_START": "34343", + "a": []int{2, 3}, }, timeout: 3, }, @@ -35,7 +39,7 @@ func TestGetString(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - gotR, gotCode, err := GetString(tt.args.u, tt.args.q, tt.args.timeout, tt.args.a...) + gotR, gotCode, err := GetString(tt.args.u, tt.args.q, tt.args.a...) if (err != nil) != tt.wantErr { t.Errorf("GetString() error = %v, wantErr %v", err, tt.wantErr) return @@ -49,3 +53,203 @@ func TestGetString(t *testing.T) { }) } } + +func TestPostWwwString(t *testing.T) { + type args struct { + u string + form map[string]any + timeout int64 + a []any + } + tests := []struct { + name string + args args + wantRes string + wantErr bool + }{ + { + name: "t1", + args: args{ + u: "http://wp.test?XDEBUG_SESSION_START=34244", + form: map[string]any{ + "aa": "bb", + "bb[]": []int{1, 2}, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotRes, _, err := PostWwwString(tt.args.u, tt.args.form, tt.args.a...) + if (err != nil) != tt.wantErr { + t.Errorf("Post() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(gotRes, tt.wantRes) { + t.Errorf("Post() gotRes = %v, want %v", gotRes, tt.wantRes) + } + }) + } +} + +func TestPost(t *testing.T) { + type args struct { + u string + types int + form map[string]any + timeout int64 + a []any + } + tests := []struct { + name string + args args + wantRes *http.Response + wantErr bool + }{ + { + name: "form-data", + args: args{ + u: "http://wp.test?XDEBUG_SESSION_START=3424", + types: 3, + form: map[string]any{ + "ff": "xxxff", + }, + timeout: 0, + a: nil, + }, + }, + { + name: "raw-json", + args: args{ + u: "http://wp.test?XDEBUG_SESSION_START=3424", + types: 3, + form: map[string]any{ + "ff": "xxxff", + "kk": 1, + }, + timeout: 0, + a: nil, + }, + }, + { + name: "binary", + args: args{ + u: "http://wp.test?XDEBUG_SESSION_START=3424", + types: 4, + form: map[string]any{ + "binary": []byte("ssssskkkkkk"), + }, + a: nil, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotRes, err := Post(tt.args.u, tt.args.types, tt.args.form, tt.args.a...) + if (err != nil) != tt.wantErr { + t.Errorf("Post() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(gotRes, tt.wantRes) { + t.Errorf("Post() gotRes = %v, want %v", gotRes, tt.wantRes) + } + }) + } +} + +type res struct { + Code int `json:"Code,omitempty"` + Message string `json:"Message" json:"Message,omitempty"` +} + +func TestPostToJsonAny(t *testing.T) { + type args struct { + u string + types int + form map[string]any + a []any + } + type testCase[T any] struct { + name string + args args + wantR T + wantCode int + wantErr bool + } + + tests := []testCase[res]{ + { + name: "res", + args: args{ + u: "http://wp.test?XDEBUG_SESSION_START=3424", + types: 1, + a: []any{3 * time.Second, map[string]string{"user-agent": "httptool"}}, + }, + wantR: res{ + 200, "ok", + }, + wantCode: 200, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotR, gotCode, err := PostToJsonAny[res](tt.args.u, tt.args.types, tt.args.form, tt.args.a...) + if (err != nil) != tt.wantErr { + t.Errorf("PostToJsonAny() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(gotR, tt.wantR) { + t.Errorf("PostToJsonAny() gotR = %v, want %v", gotR, tt.wantR) + } + if gotCode != tt.wantCode { + t.Errorf("PostToJsonAny() gotCode = %v, want %v", gotCode, tt.wantCode) + } + }) + } +} + +func TestGetToJsonAny(t *testing.T) { + type args struct { + u string + q map[string]any + a []any + } + type testCase[T any] struct { + name string + args args + wantR T + wantCode int + wantErr bool + } + tests := []testCase[res]{ + { + name: "t1", + args: args{ + u: "http://wp.test?XDEBUG_SESSION_START=3424", + q: map[string]any{ + "jjj": "ssss", + "fff": []int{1, 2, 3}, + }, + }, + wantR: res{ + 200, "ok", + }, + wantCode: 200, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotR, gotCode, err := GetToJsonAny[res](tt.args.u, tt.args.q, tt.args.a...) + if (err != nil) != tt.wantErr { + t.Errorf("GetToJsonAny() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(gotR, tt.wantR) { + t.Errorf("GetToJsonAny() gotR = %v, want %v", gotR, tt.wantR) + } + if gotCode != tt.wantCode { + t.Errorf("GetToJsonAny() gotCode = %v, want %v", gotCode, tt.wantCode) + } + }) + } +} diff --git a/helper/slice/sort.go b/helper/slice/sort.go index 834b2b8..d9af2ca 100644 --- a/helper/slice/sort.go +++ b/helper/slice/sort.go @@ -27,7 +27,7 @@ func (r anyArr[T]) Less(i, j int) bool { return r.fn(r.data[i], r.data[j]) } -// Sort fn 中i>j 为降序,反之为升序 +// Sort fn i>j 为降序 desc,反之为升序 asc func Sort[T any](arr []T, fn func(i, j T) bool) { slice := anyArr[T]{ data: arr,