From ebe30f6c8bcfd4f72f4dfc07caa3e72503eb0211 Mon Sep 17 00:00:00 2001 From: qydysky <32743305+qydysky@users.noreply.github.com> Date: Sun, 16 Oct 2022 12:33:31 +0800 Subject: [PATCH] file support golang.org/x/text/encoding --- file/FileWR.go | 71 +++++++++++++++++++++++++++++++-------------- file/FileWR_test.go | 43 ++++++++++++++++++++++----- go.mod | 6 ++-- go.sum | 21 ++++++++++++++ 4 files changed, 108 insertions(+), 33 deletions(-) diff --git a/file/FileWR.go b/file/FileWR.go index 63aa99e..f021e48 100644 --- a/file/FileWR.go +++ b/file/FileWR.go @@ -9,6 +9,7 @@ import ( "sync" l "github.com/qydysky/part/limit" + encoder "golang.org/x/text/encoding" ) var ( @@ -21,6 +22,8 @@ var ( type File struct { Config Config file *os.File + wr io.Writer + rr io.Reader sync.RWMutex } @@ -28,6 +31,10 @@ type Config struct { FilePath string //文件路径 CurIndex int64 //初始化光标位置 AutoClose bool //自动关闭句柄 + + // wrap with encoder + //https://pkg.go.dev/golang.org/x/text/encoding#section-directories + Coder encoder.Encoding } func New(filePath string, curIndex int64, autoClose bool) *File { @@ -65,7 +72,7 @@ func (t *File) CopyTo(to *File, byteInSec int64, tryLock bool) error { } defer to.Unlock() - return transfer(t.file, to.file, byteInSec) + return transfer(t, to, byteInSec) } func (t *File) Write(data []byte, tryLock bool) (int, error) { @@ -83,7 +90,7 @@ func (t *File) Write(data []byte, tryLock bool) (int, error) { } defer t.Unlock() - return t.file.Write(data) + return t.write().Write(data) } func (t *File) Read(data []byte) (int, error) { @@ -97,7 +104,7 @@ func (t *File) Read(data []byte) (int, error) { } defer t.RUnlock() - return t.file.Read(data) + return t.read().Read(data) } func (t *File) ReadUntil(separation byte, perReadSize int, maxReadSize int) (data []byte, e error) { @@ -117,9 +124,9 @@ func (t *File) ReadUntil(separation byte, perReadSize int, maxReadSize int) (dat ) for maxReadSize > 0 { - n, e = t.file.Read(tmpArea) + n, e = t.read().Read(tmpArea) - if e != nil { + if n == 0 && e != nil { if errors.Is(e, io.EOF) { e = nil } @@ -197,6 +204,25 @@ func (t *File) Close() error { return nil } +func (t *File) IsExist() bool { + if len(t.Config.FilePath) > 4096 { + panic(ErrFilePathTooLong) + } + + _, err := os.Stat(t.Config.FilePath) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return false + } else { + if !strings.Contains(err.Error(), "file name too long") { + panic(ErrFilePathTooLong) + } + return false + } + } + return true +} + func (t *File) getRWCloser() { if t.Config.AutoClose || t.file == nil { if !t.IsExist() { @@ -235,16 +261,16 @@ func (t *File) getRWCloser() { } } -func transfer(r io.ReadCloser, w io.WriteCloser, byteInSec int64) (e error) { +func transfer(r *File, w *File, byteInSec int64) (e error) { if byteInSec > 0 { limit := l.New(1, 1000, -1) defer limit.Close() buf := make([]byte, byteInSec) for { - n, err := r.Read(buf) + n, err := r.read().Read(buf) if n != 0 { - w.Write(buf[:n]) + w.write().Write(buf[:n]) } else if err != nil { e = err break @@ -252,27 +278,28 @@ func transfer(r io.ReadCloser, w io.WriteCloser, byteInSec int64) (e error) { limit.TO() } } else { - _, e = io.Copy(w, r) + _, e = io.Copy(w.write(), r.read()) } return nil } -func (t *File) IsExist() bool { - if len(t.Config.FilePath) > 4096 { - panic(ErrFilePathTooLong) +func (t *File) write() io.Writer { + if t.wr == nil { + t.wr = io.Writer(t.file) + if t.Config.Coder != nil { + t.wr = t.Config.Coder.NewEncoder().Writer(t.wr) + } } + return t.wr +} - _, err := os.Stat(t.Config.FilePath) - if err != nil { - if errors.Is(err, os.ErrNotExist) { - return false - } else { - if !strings.Contains(err.Error(), "file name too long") { - panic(ErrFilePathTooLong) - } - return false +func (t *File) read() io.Reader { + if t.rr == nil { + t.rr = io.Reader(t.file) + if t.Config.Coder != nil { + t.rr = t.Config.Coder.NewDecoder().Reader(t.rr) } } - return true + return t.rr } diff --git a/file/FileWR_test.go b/file/FileWR_test.go index 74673b5..4753577 100644 --- a/file/FileWR_test.go +++ b/file/FileWR_test.go @@ -3,6 +3,9 @@ package part import ( "bytes" "testing" + + "golang.org/x/text/encoding/simplifiedchinese" + "golang.org/x/text/encoding/unicode" ) func TestWriteReadDelSync(t *testing.T) { @@ -39,7 +42,8 @@ func TestWriteReadDelSync(t *testing.T) { func TestWriteReadDel(t *testing.T) { f := New("rwd.txt", 0, false) - if i, e := f.Write([]byte("sss"), true); i == 0 || e != nil { + f.Config.Coder = unicode.UTF8 + if i, e := f.Write([]byte("sssaaa"), true); i == 0 || e != nil { t.Fatal(e) } @@ -50,12 +54,14 @@ func TestWriteReadDel(t *testing.T) { var buf = make([]byte, 3) if i, e := f.Read(buf); i == 0 || e != nil { t.Fatal(i, e) - } else { - for _, v := range buf { - if v != 's' { - t.Fatal(v) - } - } + } else if !bytes.Equal(buf, []byte("sss")) { + t.Fatal(string(buf)) + } + + if i, e := f.Read(buf); i == 0 || e != nil { + t.Fatal(i, e) + } else if !bytes.Equal(buf, []byte("aaa")) { + t.Fatal(string(buf)) } if e := f.Close(); e != nil { @@ -158,3 +164,26 @@ func TestReadUntil(t *testing.T) { t.Fatal(e) } } + +func TestEncoderDecoder(t *testing.T) { + sf := New("GBK.txt", 0, true) + sf.Config.Coder = simplifiedchinese.GBK + if i, e := sf.Write([]byte("测1试s啊是3大家看s法$和"), true); i == 0 || e != nil { + t.Fatal(e) + } + + tf := New("UTF8.txt", 0, true) + tf.Config.Coder = unicode.UTF8 + if e := sf.CopyTo(tf, 5, true); e != nil { + t.Fatal(e) + } + + if data, e := tf.ReadUntil('\n', 3, 100); e != nil { + t.Fatal(e) + } else if !bytes.Equal(data, []byte("测1试s啊是3大家看s法$和")) { + t.Fatal(string(data)) + } + + sf.Delete() + tf.Delete() +} diff --git a/go.mod b/go.mod index 6a79f3f..01a420a 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.14 require ( github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect - github.com/andybalholm/brotli v1.0.1 + github.com/andybalholm/brotli v1.0.4 github.com/go-ole/go-ole v1.2.5 // indirect github.com/gorilla/websocket v1.4.2 github.com/klauspost/compress v1.11.6 @@ -14,9 +14,7 @@ require ( github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 github.com/thedevsaddam/gojsonq v2.3.0+incompatible github.com/thedevsaddam/gojsonq/v2 v2.5.2 - golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad // indirect - golang.org/x/net v0.0.0-20201224014010-6772e930b67b // indirect - golang.org/x/sys v0.0.0-20210113000019-eaf3bda374d2 // indirect + golang.org/x/text v0.3.8 // indirect ) //replace github.com/thedevsaddam/gojsonq v2.3.0+incompatible => github.com/thedevsaddam/gojsonq/v2 v2.5.2 diff --git a/go.sum b/go.sum index f67c4b2..f0fd860 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,8 @@ github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUW github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/andybalholm/brotli v1.0.1 h1:KqhlKozYbRtJvsPrrEeXcO+N2l6NYT5A2QAFmSULpEc= github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= +github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= +github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI= github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= github.com/go-ole/go-ole v1.2.5 h1:t4MGB5xEDZvXI+0rMjjsfBsD7yAgp/s9ZDkL1JndXwY= @@ -31,6 +33,7 @@ github.com/thedevsaddam/gojsonq v2.3.0+incompatible h1:i2lFTvGY4LvoZ2VUzedsFlRiy github.com/thedevsaddam/gojsonq v2.3.0+incompatible/go.mod h1:RBcQaITThgJAAYKH7FNp2onYodRz8URfsuEGpAch0NA= github.com/thedevsaddam/gojsonq/v2 v2.5.2 h1:CoMVaYyKFsVj6TjU6APqAhAvC07hTI6IQen8PHzHYY0= github.com/thedevsaddam/gojsonq/v2 v2.5.2/go.mod h1:bv6Xa7kWy82uT0LnXPE2SzGqTj33TAEeR560MdJkiXs= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -39,7 +42,10 @@ golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de h1:ikNHVSjEfnvz6sxdSPCaPt golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad h1:DN0cp81fZ3njFcrLCytUHRSUkqBjfTo4Tx9RJTWs0EY= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478 h1:l5EDrHhldLYb3ZRHDUhXF7Om7MvYXnkV9/iQNo1lX6g= @@ -48,7 +54,11 @@ golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgN golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201224014010-6772e930b67b h1:iFwSg7t5GZmB/Q5TjiEAsdoLDrdJRC1RiF2WhuV29Qw= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b h1:PxfKdU9lEEDYjdIzOtC4qFWgkU2rGHdKlKowJSMN9h0= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -64,10 +74,21 @@ golang.org/x/sys v0.0.0-20200810151505-1b9f1253b3ed/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210113000019-eaf3bda374d2 h1:F9vNgpIiamoF+Q1/c78bikg/NScXEtbZSNEpnRelOzs= golang.org/x/sys v0.0.0-20210113000019-eaf3bda374d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f h1:v4INt8xihDGvnrfjMDVXGxw9wrfxYyCjk0KbXjhR55s= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8 h1:nAL+RVCQ9uMn3vJZbV+MRnydTJFPf8qqY42YiA6MrqY= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -- 2.39.2