From: qydysky Date: Thu, 5 Nov 2020 20:01:50 +0000 (+0800) Subject: 92 X-Git-Tag: v0.2.0~1 X-Git-Url: http://127.0.0.1:8081/?a=commitdiff_plain;h=a48c7a44c35a1afd09b73087be4e7c555f820b3a;p=part%2F.git 92 --- diff --git a/Flate.go b/Flate.go deleted file mode 100644 index 84ba386..0000000 --- a/Flate.go +++ /dev/null @@ -1,158 +0,0 @@ -package part - -import ( - // "sync" - // "os" - "github.com/klauspost/compress/flate" - // "io" - // "path/filepath" - // "strings" - "bytes" - // "time" - // "errors" -) - -type lflate struct {} - -func Flate() *lflate{ - return &lflate{} -} - -func (this *lflate) InFlate(byteS []byte, level int) ([]byte,error) { - buf := bytes.NewBuffer(nil) - - // 创建一个flate.Write - flateWrite, err := flate.NewWriter(buf, level) - if err != nil { - Logf().E(err.Error()) - return buf.Bytes(),err - } - defer flateWrite.Close() - // 写入待压缩内容 - flateWrite.Write(byteS) - flateWrite.Flush() - return buf.Bytes(),nil -} - -// func (this *lflate) UnFlate(zipFile string, destDir string) error { -// this.Lock() -// defer this.Unlock() - -// r, err := zip.OpenReader(zipFile) -// if err != nil { -// return err -// } -// defer func() { -// if err := r.Close(); err != nil { -// panic(err) -// } -// }() - -// os.MkdirAll(destDir, 0755) - -// // Closure to address file descriptors issue with all the deferred .Close() methods -// extractAndWriteFile := func(f *zip.File) error { -// rc, err := f.Open() -// if err != nil { -// return err -// } -// defer func() { -// if err := rc.Close(); err != nil { -// panic(err) -// } -// }() - -// path := filepath.Join(destDir, f.Name) - -// if f.FileInfo().IsDir() { -// os.MkdirAll(path, f.Mode()) -// } else { -// os.MkdirAll(filepath.Dir(path), f.Mode()) -// f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode()) -// if err != nil { -// return err -// } -// defer func() { -// if err := f.Close(); err != nil { -// panic(err) -// } -// }() - -// _, err = io.Copy(f, rc) -// if err != nil { -// return err -// } -// } -// return nil -// } - -// for _, f := range r.File { -// err := extractAndWriteFile(f) -// if err != nil { -// return err -// } -// } - -// return nil -// } - -// type rZip struct { -// poit *zip.ReadCloser -// buf map[string]*zip.File -// sync.Mutex -// } - -// func RZip() *rZip {return &rZip{}} - -// func (t *rZip) New(zipFile string) (error) { -// t.Lock() -// defer t.Unlock() - -// t.buf = make(map[string]*zip.File) - -// var err error -// t.poit, err = zip.OpenReader(zipFile) -// if err != nil {return err} - -// for _, _f := range t.poit.File { -// if _f.FileInfo().IsDir() {continue} -// t.buf[_f.Name] = _f -// } - -// return nil -// } - -// func (t *rZip) Read(path string) (*bytes.Buffer,string,error) { -// t.Lock() -// defer t.Unlock() - -// var timeLayoutStr = "2006-01-02 15:04:05" -// var err error - -// if f,ok := t.buf[path];ok { -// if rc, err := f.Open();err == nil { -// defer rc.Close(); - -// buf := new(bytes.Buffer) -// buf.ReadFrom(rc) -// return buf,f.FileHeader.Modified.Format(timeLayoutStr),nil -// } -// return &bytes.Buffer{},time.Now().UTC().Format(timeLayoutStr),err -// } -// return &bytes.Buffer{},time.Now().UTC().Format(timeLayoutStr),errors.New("not found") -// } - -// func (t *rZip) List() []string { -// var list []string -// for k := range t.buf { -// list=append(list,k) -// } -// return list -// } - -// func (t *rZip) Close() { -// t.poit.Close() -// for k := range t.buf { -// delete(t.buf, k) -// } -// } \ No newline at end of file diff --git a/Reqf.go b/Reqf.go index f74e972..457b494 100644 --- a/Reqf.go +++ b/Reqf.go @@ -11,6 +11,7 @@ import ( "errors" "io/ioutil" "net/url" + compress "github.com/qydysky/part/compress" // "encoding/binary" ) @@ -28,8 +29,8 @@ type Rval struct { } type req struct { - ResponseCode int Respon []byte + Response *http.Response UsedTime time.Duration cancelOpen bool @@ -156,7 +157,7 @@ func (this *req) Reqf_1(val Rval) (error) { if err = os.Rename(filepath+".dtmp", filepath); err != nil {return err} return nil } - this.ResponseCode = resp.StatusCode + this.Response = resp if !JustResponseCode { defer resp.Body.Close() if SaveToPath != "" && resp.StatusCode == 200 { @@ -167,8 +168,19 @@ func (this *req) Reqf_1(val Rval) (error) { var err error this.Respon,err = ioutil.ReadAll(resp.Body) if err != nil {return err} + if compress_type := resp.Header[`Content-Encoding`];compress_type!=nil{ + switch compress_type[0]{ + case `br`: + if this.Respon,err = compress.UnBr(this.Respon);e != nil {return err} + case `gzip`: + if this.Respon,err = compress.UnGzip(this.Respon);e != nil {return err} + case `deflate`: + if this.Respon,err = compress.UnFlate(this.Respon);e != nil {return err} + default: + } + } } - } + } else {resp.Body.Close()} this.UsedTime=time.Since(beginTime) @@ -184,4 +196,31 @@ func (t *req) Close(){ close(t.cancel) t.cancelOpen = false } +} + +func Cookies_String_2_Map(Cookies string) (o map[string]string) { + o = make(map[string]string) + list := strings.Split(Cookies, `; `) + for _,v := range list { + s := strings.SplitN(v, "=", 2) + o[s[0]] = s[1] + } + return +} + +func Map_2_Cookies_String(Cookies map[string]string) (o string) { + for k,v := range Cookies { + o += k +`=`+ v + `; ` + } + t := []rune(o) + o = string(t[:len(t)-2]) + return +} + +func Cookies_List_2_Map(Cookies []*http.Cookie) (o map[string]string) { + o = make(map[string]string) + for _,v := range Cookies { + o[v.Name] = v.Value + } + return } \ No newline at end of file diff --git a/compress/Brotli.go b/compress/Brotli.go new file mode 100644 index 0000000..90fce95 --- /dev/null +++ b/compress/Brotli.go @@ -0,0 +1,30 @@ +package part + +import ( + br "github.com/andybalholm/brotli" + "bytes" + "io" + "io/ioutil" +) + +func InBr(byteS []byte, level int) ([]byte,error) { + buf := bytes.NewBuffer(nil) + Write := br.NewWriterLevel(buf, level) + defer Write.Close() + + // 写入待压缩内容 + if _,err := Write.Write(byteS); err != nil {return buf.Bytes(),err} + if err := Write.Flush(); err != nil {return buf.Bytes(),err} + return buf.Bytes(),nil +} + +func UnBr(byteS []byte) ([]byte,error) { + buf := bytes.NewBuffer(byteS) + Read := br.NewReader(buf) + + rb, err := ioutil.ReadAll(Read) + if err == io.EOF || err == io.ErrUnexpectedEOF { + return rb, nil + } + return rb, err +} \ No newline at end of file diff --git a/compress/Brotli_test.go b/compress/Brotli_test.go new file mode 100644 index 0000000..23c25df --- /dev/null +++ b/compress/Brotli_test.go @@ -0,0 +1,22 @@ +package part + +import ( + "testing" +) + +func Test_Br(t *testing.T) { + s := []byte(`abc`) + t.Log(string(s)) + + b,e := InBr(s, 6) + if e != nil {t.Error(e);return} + t.Log(string(b)) + + c,e := UnBr(b) + if e != nil {t.Error(e);return} + t.Log(string(c)) + + for k,v := range c { + if v != []byte("abc")[k] {t.Error(`no match`)} + } +} diff --git a/compress/Flate.go b/compress/Flate.go new file mode 100644 index 0000000..3e1822b --- /dev/null +++ b/compress/Flate.go @@ -0,0 +1,36 @@ +package part + +import ( + "github.com/klauspost/compress/flate" + "bytes" + "io" + "io/ioutil" +) + +func InFlate(byteS []byte, level int) ([]byte,error) { + buf := bytes.NewBuffer(nil) + + // 创建一个flate.Write + flateWrite, err := flate.NewWriter(buf, level) + if err != nil { + return buf.Bytes(),err + } + defer flateWrite.Close() + // 写入待压缩内容 + flateWrite.Write(byteS) + flateWrite.Flush() + return buf.Bytes(),nil +} + +func UnFlate(byteS []byte) ([]byte,error) { + buf := bytes.NewBuffer(byteS) + // 创建一个flate.Write + flateRead := flate.NewReader(buf) + defer flateRead.Close() + // 写入待压缩内容 + rb, err := ioutil.ReadAll(flateRead) + if err == io.EOF || err == io.ErrUnexpectedEOF { + return rb, nil + } + return rb, err +} \ No newline at end of file diff --git a/compress/Flate_test.go b/compress/Flate_test.go new file mode 100644 index 0000000..c28ff32 --- /dev/null +++ b/compress/Flate_test.go @@ -0,0 +1,22 @@ +package part + +import ( + "testing" +) + +func Test_Flate(t *testing.T) { + s := []byte(`abc`) + t.Log(string(s)) + + b,e := InFlate(s, -1) + if e != nil {t.Error(e);return} + t.Log(string(b)) + + c,e := UnFlate(b) + if e != nil {t.Error(e);return} + t.Log(string(c)) + + for k,v := range c { + if v != []byte("abc")[k] {t.Error(`no match`)} + } +} diff --git a/compress/Gzip.go b/compress/Gzip.go new file mode 100644 index 0000000..2e503b4 --- /dev/null +++ b/compress/Gzip.go @@ -0,0 +1,35 @@ +package part + +import ( + gzip "github.com/klauspost/pgzip" + "bytes" + "io" + "io/ioutil" +) + +func InGzip(byteS []byte, level int) ([]byte,error) { + buf := bytes.NewBuffer(nil) + Write,err := gzip.NewWriterLevel(buf, level) + if err != nil { + return buf.Bytes(),err + } + defer Write.Close() + // 写入待压缩内容 + if _,err := Write.Write(byteS); err != nil {return buf.Bytes(),err} + if err := Write.Flush(); err != nil {return buf.Bytes(),err} + return buf.Bytes(),nil +} + +func UnGzip(byteS []byte) ([]byte,error) { + buf := bytes.NewBuffer(byteS) + Read,err := gzip.NewReader(buf) + if err != nil { + return buf.Bytes(),err + } + defer Read.Close() + rb, err := ioutil.ReadAll(Read) + if err == io.EOF || err == io.ErrUnexpectedEOF { + return rb, nil + } + return rb, err +} \ No newline at end of file diff --git a/compress/Gzip_test.go b/compress/Gzip_test.go new file mode 100644 index 0000000..dc59385 --- /dev/null +++ b/compress/Gzip_test.go @@ -0,0 +1,22 @@ +package part + +import ( + "testing" +) + +func Test_Gzip(t *testing.T) { + s := []byte(`abc`) + t.Log(string(s)) + + b,e := InGzip(s, -1) + if e != nil {t.Error(e);return} + t.Log(string(b)) + + c,e := UnGzip(b) + if e != nil {t.Error(e);return} + t.Log(string(c)) + + for k,v := range c { + if v != []byte("abc")[k] {t.Error(`no match`)} + } +} diff --git a/go.mod b/go.mod index de42bb7..f552430 100644 --- a/go.mod +++ b/go.mod @@ -4,8 +4,10 @@ go 1.14 require ( github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect + github.com/andybalholm/brotli v1.0.1 github.com/go-ole/go-ole v1.2.4 // indirect github.com/klauspost/compress v1.10.10 + github.com/klauspost/pgzip v1.2.5 github.com/miekg/dns v1.1.31 github.com/shirou/gopsutil v2.20.7+incompatible github.com/thedevsaddam/gojsonq v2.3.0+incompatible diff --git a/go.sum b/go.sum index 332e682..6275551 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,13 @@ github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/andybalholm/brotli v1.0.1 h1:KqhlKozYbRtJvsPrrEeXcO+N2l6NYT5A2QAFmSULpEc= +github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI= github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= github.com/klauspost/compress v1.10.10 h1:a/y8CglcM7gLGYmlbP/stPE5sR3hbhFRUjCBfd/0B3I= github.com/klauspost/compress v1.10.10/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= +github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/miekg/dns v1.1.31 h1:sJFOl9BgwbYAWOGEwr61FU28pqsBNdpRBnhGXtO06Oo= github.com/miekg/dns v1.1.31/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/shirou/gopsutil v2.20.6+incompatible h1:P37G9YH8M4vqkKcwBosp+URN5O8Tay67D2MbR361ioY=