From: qydysky Date: Tue, 6 Aug 2024 16:19:13 +0000 (+0800) Subject: 1 X-Git-Tag: v0.28.20240806162427 X-Git-Url: http://127.0.0.1:8081/?a=commitdiff_plain;h=79c658a35f297a8d8312274a1ea52d771f4b0439;p=part%2F.git 1 --- diff --git a/CheckFile.go b/CheckFile.go index dd985e2..90b6758 100644 --- a/CheckFile.go +++ b/CheckFile.go @@ -1,254 +1,254 @@ package part -import ( - "errors" - "fmt" - "os" - "runtime" - "strings" - "syscall" - "time" - - reqf "github.com/qydysky/part/reqf" -) - -type checkfile struct { - RV []interface{} -} - -func Checkfile() *checkfile { - return &checkfile{} -} - -func (t *checkfile) Build(checkFile, root, checkDir, SplitString string, usemd5 bool) { - - v, _, _ := t.GetAllFile(checkDir) - _checkFile := Filel{ - File: checkFile, - Loc: 0, - } - - Logf().I("checkFile Build:begin") - - if usemd5 { - _checkFile.Context = append(_checkFile.Context, "UseMd5") - } - - _checkFile.Context = append(_checkFile.Context, SplitString) - - for _, value := range v { - if usemd5 { - md5, e := Md5().Md5File(value) - if e != nil { - md5 = "00000000000000000000000000000000" - } - _checkFile.Context = append(_checkFile.Context, md5) - } - _checkFile.Context = append(_checkFile.Context, value[len(root):]) - _checkFile.Context = append(_checkFile.Context, SplitString) - } - - File().FileWR(_checkFile) - Logf().I("checkFile Build:ok") - -} - -func (t *checkfile) IsExist(f string) bool { - if len(f) > 4096 { - return false - } - - _, err := os.Stat(f) - if err != nil { - if errors.Is(err, os.ErrNotExist) { - t.RV = append(t.RV, false, nil) - return false - } else { - if !strings.Contains(err.Error(), "file name too long") { - Logf().E(err) - } - t.RV = append(t.RV, false, err) - return false - } - } - t.RV = append(t.RV, true, nil) - return true -} - -func (t *checkfile) IsOpen(f string) bool { - if !t.IsExist(f) { - return false - } - - fi, e := os.OpenFile(f, syscall.O_RDONLY|syscall.O_EXCL, 0) - if e != nil { - return true - } - fi.Close() - return false -} - -func (t *checkfile) Checkfile(src string) (string, error) { - - _, str, err := t.GetAllFile(src) - - if err != nil { - return "", errors.New("获取文件列表错误！") - } - - return Md5().Md5String(str), nil - -} - -func (t *checkfile) GetAllFile(pathname string) ([]string, string, error) { - - var ( - returnVal string = "" - list []string - ) - - rd, err := os.ReadDir(pathname) - - if err != nil { - return list, returnVal, err - } - - for _, fi := range rd { - if fi.IsDir() { - _list, _returnVal, _ := t.GetAllFile(pathname + fi.Name() + "/") - returnVal += _returnVal - list = append(list, _list...) - } else { - returnVal += pathname + "/" + fi.Name() + "\n" - list = append(list, pathname+fi.Name()) - } - } - return list, returnVal, err -} - -func (t *checkfile) GetFileSize(path string) int64 { - - if !t.IsExist(path) { - return 0 - } - fileInfo, err := os.Stat(path) - if err != nil { - return 0 - } - return fileInfo.Size() -} - -func (t *checkfile) CheckList(checkFile, root, SplitString string) bool { - - if checkFile == "" || SplitString == "" { - Logf().E("[err]checkFile or SplitString has null.") - return false - } - Logf().I("===checkFile Check===") - - var checkFileString string - var checkFileList []string - if strings.Contains(checkFile, "https://") { - Logf().I("[wait]checkFile: Getting checkfile...") - - var r = reqf.Rval{ - Url: checkFile, - Timeout: 6, - Retry: 2, - } - req := reqf.New() - if e := req.Reqf(r); e != nil { - Logf().E("[err]checkFile:", checkFile, e.Error()) - return false - } else { - Logf().I("[ok]checkFile: Get checkfile.") - checkFileString = string(req.Respon) - } - } else { - var _checkFile = Filel{ - File: checkFile, - Loc: 0, - } - - checkFileString = File().FileWR(_checkFile) - } - - checkFileList = strings.Split(checkFileString, SplitString) - - var ( - returnVal bool = true - UseMd5 bool = strings.Contains(checkFileList[0], "UseMd5") - _value string - ) - - for _, value := range checkFileList[1:] { - if value == "" { - continue - } - - if UseMd5 { - _value = root + value[32:] - } else { - _value = root + value - } - - if !t.IsExist(_value) { - Logf().E("[err]checkFile:", _value, "not exist!") - returnVal = false - } else { - if UseMd5 { - if md5, _ := Md5().Md5File(_value); value[:32] != md5 { - Logf().E("[err]checkFile:", _value, "no match!") - returnVal = false - } - } - - if runtime.GOOS != "windows" && strings.Contains(_value, ".run") { - var want os.FileMode = 0700 - if !t.CheckFilePerm(_value, want) { - Logf().E("[err]checkFile:", _value, "no permission!") - returnVal = false - } - } - // fmt.Println("[ok]checkFile:",checkDir+value) - } - - } - if returnVal { - Logf().I("[ok]checkFile: all file pass!") - } - Logf().I("===checkFile Check===") - - return returnVal -} - -func (t *checkfile) GetFileModTime(path string) (error, int64) { - - if !t.IsExist(path) { - return errors.New("not exist"), time.Now().Unix() - } - - f, err := os.Open(path) - if err != nil { - fmt.Println("open file error") - return err, time.Now().Unix() - } - defer f.Close() - - fi, err := f.Stat() - if err != nil { - fmt.Println("stat fileinfo error") - return err, time.Now().Unix() - } - - return nil, fi.ModTime().Unix() -} - -func (t *checkfile) CheckFilePerm(f string, want os.FileMode) bool { - fi, err := os.Lstat(f) - if err != nil { - Logf().E("[err]cant get permission : ", f) - return false - } - return fi.Mode().Perm() >= want -} +// import ( +// "errors" +// "fmt" +// "os" +// "runtime" +// "strings" +// "syscall" +// "time" + +// reqf "github.com/qydysky/part/reqf" +// ) + +// type checkfile struct { +// RV []interface{} +// } + +// func Checkfile() *checkfile { +// return &checkfile{} +// } + +// func (t *checkfile) Build(checkFile, root, checkDir, SplitString string, usemd5 bool) { + +// v, _, _ := t.GetAllFile(checkDir) +// _checkFile := Filel{ +// File: checkFile, +// Loc: 0, +// } + +// Logf().I("checkFile Build:begin") + +// if usemd5 { +// _checkFile.Context = append(_checkFile.Context, "UseMd5") +// } + +// _checkFile.Context = append(_checkFile.Context, SplitString) + +// for _, value := range v { +// if usemd5 { +// md5, e := Md5().Md5File(value) +// if e != nil { +// md5 = "00000000000000000000000000000000" +// } +// _checkFile.Context = append(_checkFile.Context, md5) +// } +// _checkFile.Context = append(_checkFile.Context, value[len(root):]) +// _checkFile.Context = append(_checkFile.Context, SplitString) +// } + +// File().FileWR(_checkFile) +// Logf().I("checkFile Build:ok") + +// } + +// func (t *checkfile) IsExist(f string) bool { +// if len(f) > 4096 { +// return false +// } + +// _, err := os.Stat(f) +// if err != nil { +// if errors.Is(err, os.ErrNotExist) { +// t.RV = append(t.RV, false, nil) +// return false +// } else { +// if !strings.Contains(err.Error(), "file name too long") { +// Logf().E(err) +// } +// t.RV = append(t.RV, false, err) +// return false +// } +// } +// t.RV = append(t.RV, true, nil) +// return true +// } + +// func (t *checkfile) IsOpen(f string) bool { +// if !t.IsExist(f) { +// return false +// } + +// fi, e := os.OpenFile(f, syscall.O_RDONLY|syscall.O_EXCL, 0) +// if e != nil { +// return true +// } +// fi.Close() +// return false +// } + +// func (t *checkfile) Checkfile(src string) (string, error) { + +// _, str, err := t.GetAllFile(src) + +// if err != nil { +// return "", errors.New("获取文件列表错误！") +// } + +// return Md5().Md5String(str), nil + +// } + +// func (t *checkfile) GetAllFile(pathname string) ([]string, string, error) { + +// var ( +// returnVal string = "" +// list []string +// ) + +// rd, err := os.ReadDir(pathname) + +// if err != nil { +// return list, returnVal, err +// } + +// for _, fi := range rd { +// if fi.IsDir() { +// _list, _returnVal, _ := t.GetAllFile(pathname + fi.Name() + "/") +// returnVal += _returnVal +// list = append(list, _list...) +// } else { +// returnVal += pathname + "/" + fi.Name() + "\n" +// list = append(list, pathname+fi.Name()) +// } +// } +// return list, returnVal, err +// } + +// func (t *checkfile) GetFileSize(path string) int64 { + +// if !t.IsExist(path) { +// return 0 +// } +// fileInfo, err := os.Stat(path) +// if err != nil { +// return 0 +// } +// return fileInfo.Size() +// } + +// func (t *checkfile) CheckList(checkFile, root, SplitString string) bool { + +// if checkFile == "" || SplitString == "" { +// Logf().E("[err]checkFile or SplitString has null.") +// return false +// } +// Logf().I("===checkFile Check===") + +// var checkFileString string +// var checkFileList []string +// if strings.Contains(checkFile, "https://") { +// Logf().I("[wait]checkFile: Getting checkfile...") + +// var r = reqf.Rval{ +// Url: checkFile, +// Timeout: 6, +// Retry: 2, +// } +// req := reqf.New() +// if e := req.Reqf(r); e != nil { +// Logf().E("[err]checkFile:", checkFile, e.Error()) +// return false +// } else { +// Logf().I("[ok]checkFile: Get checkfile.") +// checkFileString = string(req.Respon) +// } +// } else { +// var _checkFile = Filel{ +// File: checkFile, +// Loc: 0, +// } + +// checkFileString = File().FileWR(_checkFile) +// } + +// checkFileList = strings.Split(checkFileString, SplitString) + +// var ( +// returnVal bool = true +// UseMd5 bool = strings.Contains(checkFileList[0], "UseMd5") +// _value string +// ) + +// for _, value := range checkFileList[1:] { +// if value == "" { +// continue +// } + +// if UseMd5 { +// _value = root + value[32:] +// } else { +// _value = root + value +// } + +// if !t.IsExist(_value) { +// Logf().E("[err]checkFile:", _value, "not exist!") +// returnVal = false +// } else { +// if UseMd5 { +// if md5, _ := Md5().Md5File(_value); value[:32] != md5 { +// Logf().E("[err]checkFile:", _value, "no match!") +// returnVal = false +// } +// } + +// if runtime.GOOS != "windows" && strings.Contains(_value, ".run") { +// var want os.FileMode = 0700 +// if !t.CheckFilePerm(_value, want) { +// Logf().E("[err]checkFile:", _value, "no permission!") +// returnVal = false +// } +// } +// // fmt.Println("[ok]checkFile:",checkDir+value) +// } + +// } +// if returnVal { +// Logf().I("[ok]checkFile: all file pass!") +// } +// Logf().I("===checkFile Check===") + +// return returnVal +// } + +// func (t *checkfile) GetFileModTime(path string) (error, int64) { + +// if !t.IsExist(path) { +// return errors.New("not exist"), time.Now().Unix() +// } + +// f, err := os.Open(path) +// if err != nil { +// fmt.Println("open file error") +// return err, time.Now().Unix() +// } +// defer f.Close() + +// fi, err := f.Stat() +// if err != nil { +// fmt.Println("stat fileinfo error") +// return err, time.Now().Unix() +// } + +// return nil, fi.ModTime().Unix() +// } + +// func (t *checkfile) CheckFilePerm(f string, want os.FileMode) bool { +// fi, err := os.Lstat(f) +// if err != nil { +// Logf().E("[err]cant get permission : ", f) +// return false +// } +// return fi.Mode().Perm() >= want +// } diff --git a/Json.go b/Json.go index f49bd50..f4d2f19 100644 --- a/Json.go +++ b/Json.go @@ -1,99 +1,99 @@ package part -import ( - "io" - "os" - "errors" - "strings" - goej "encoding/json" - "github.com/thedevsaddam/gojsonq/v2" -) +// import ( +// "io" +// "os" +// "errors" +// "strings" +// goej "encoding/json" +// "github.com/thedevsaddam/gojsonq/v2" +// ) -type json struct {} +// type json struct {} -func Json() (*json) {return &json{}} +// func Json() (*json) {return &json{}} -func (*json) Check(Source interface{}) error { - var jq *goej.Decoder - switch Source.(type) { - case string: - if Checkfile().IsExist(Source.(string)) { - fileObj,err := os.OpenFile(Source.(string),os.O_RDONLY,0644) - if err != nil { - return err - } - defer fileObj.Close() - jq = goej.NewDecoder(fileObj) - }else{ - jq = goej.NewDecoder(strings.NewReader(Source.(string))) - } - default: - return errors.New("json type != string") - } - - for { - _, err := jq.Token() - if err == io.EOF { - break - } - if err != nil { - return err - } - } +// func (*json) Check(Source interface{}) error { +// var jq *goej.Decoder +// switch Source.(type) { +// case string: +// if Checkfile().IsExist(Source.(string)) { +// fileObj,err := os.OpenFile(Source.(string),os.O_RDONLY,0644) +// if err != nil { +// return err +// } +// defer fileObj.Close() +// jq = goej.NewDecoder(fileObj) +// }else{ +// jq = goej.NewDecoder(strings.NewReader(Source.(string))) +// } +// default: +// return errors.New("json type != string") +// } - return nil -} +// for { +// _, err := jq.Token() +// if err == io.EOF { +// break +// } +// if err != nil { +// return err +// } +// } -func (*json) GetValFrom(Source interface{},key string)interface {}{ - var jq *gojsonq.JSONQ - switch Source.(type) { - case string: - if Checkfile().IsExist(Source.(string)) { - jq = gojsonq.New().File(Source.(string)) - }else{ - jq = gojsonq.New().FromString(Source.(string)) - } - default: - jq = gojsonq.New().FromInterface(Source) - } - return jq.More().Find(key) -} +// return nil +// } -func (*json) GetValFromS(Source string,key string)interface {}{ - var jq *gojsonq.JSONQ - jq = gojsonq.New().FromString(Source) - return jq.More().Find(key) -} +// func (*json) GetValFrom(Source interface{},key string)interface {}{ +// var jq *gojsonq.JSONQ +// switch Source.(type) { +// case string: +// if Checkfile().IsExist(Source.(string)) { +// jq = gojsonq.New().File(Source.(string)) +// }else{ +// jq = gojsonq.New().FromString(Source.(string)) +// } +// default: +// jq = gojsonq.New().FromInterface(Source) +// } +// return jq.More().Find(key) +// } -func (*json) GetArrayFrom(Source interface{},key string)[]interface {}{ - var jq *gojsonq.JSONQ - switch Source.(type) { - case string: - jq = gojsonq.New().FromString(Source.(string)) - default: - jq = gojsonq.New().FromInterface(Source) - } - return jq.Pluck(key).([]interface{}) -} +// func (*json) GetValFromS(Source string,key string)interface {}{ +// var jq *gojsonq.JSONQ +// jq = gojsonq.New().FromString(Source) +// return jq.More().Find(key) +// } -func (this *json) GetMultiValFrom(Source interface{},key []string) []interface{}{ - var jq *gojsonq.JSONQ - switch Source.(type) { - case string: - if Checkfile().IsExist(Source.(string)) { - jq = gojsonq.New().File(Source.(string)) - }else{ - jq = gojsonq.New().FromString(Source.(string)) - } - default: - jq = gojsonq.New().FromInterface(Source) - } +// func (*json) GetArrayFrom(Source interface{},key string)[]interface {}{ +// var jq *gojsonq.JSONQ +// switch Source.(type) { +// case string: +// jq = gojsonq.New().FromString(Source.(string)) +// default: +// jq = gojsonq.New().FromInterface(Source) +// } +// return jq.Pluck(key).([]interface{}) +// } - var returnVal []interface{} - for _,i := range key { - jq.Reset() - returnVal = append(returnVal,jq.More().Find(i)) - } +// func (this *json) GetMultiValFrom(Source interface{},key []string) []interface{}{ +// var jq *gojsonq.JSONQ +// switch Source.(type) { +// case string: +// if Checkfile().IsExist(Source.(string)) { +// jq = gojsonq.New().File(Source.(string)) +// }else{ +// jq = gojsonq.New().FromString(Source.(string)) +// } +// default: +// jq = gojsonq.New().FromInterface(Source) +// } - return returnVal -} \ No newline at end of file +// var returnVal []interface{} +// for _,i := range key { +// jq.Reset() +// returnVal = append(returnVal,jq.More().Find(i)) +// } + +// return returnVal +// } diff --git a/Md5.go b/Md5.go deleted file mode 100644 index b9fb0ef..0000000 --- a/Md5.go +++ /dev/null @@ -1,65 +0,0 @@ -package part - -import ( - "sync" - "io" - "crypto/md5" - "fmt" - "os" - "time" -) - -type md5l struct{sync.Mutex} - -func Md5() *md5l{ - return &md5l{} -} - -func (this *md5l) Md5String(str string) string { - this.Lock() - defer this.Unlock() - - w := md5.New() - io.WriteString(w, str) - md5str := fmt.Sprintf("%x", w.Sum(nil)) - return md5str -} - -func (this *md5l) Md5File(path string) (string, error) { - this.Lock() - defer this.Unlock() - - file, err := os.Open(path) - defer file.Close() - if err != nil { - return "",err - } - - h := md5.New() - _, err = io.Copy(h,file) - if err != nil { - return "",err - } - - return fmt.Sprintf("%x",h.Sum(nil)), nil -} - -func (this *md5l) GetFileModTime(path string) (error,int64) { - this.Lock() - defer this.Unlock() - - f, err := os.Open(path) - if err != nil { - fmt.Println("open file error") - return err,time.Now().Unix() - } - defer f.Close() - - fi, err := f.Stat() - if err != nil { - fmt.Println("stat fileinfo error") - return err,time.Now().Unix() - } - - return nil,fi.ModTime().Unix() -} diff --git a/Net.go b/Net.go index 276df43..9a5b0de 100644 --- a/Net.go +++ b/Net.go @@ -13,6 +13,7 @@ import ( "time" "github.com/miekg/dns" + pfile "github.com/qydysky/part/file" pool "github.com/qydysky/part/pool" ) @@ -602,7 +603,7 @@ func (this *netl) GetLocalDns() error { this.Dns.Server = string(ip) return nil - } else if Checkfile().IsExist("/etc/resolv.conf") { + } else if pfile.New("/etc/resolv.conf", 0, true).IsExist() { cmd := exec.Command("cat", "/etc/resolv.conf") output, _ := cmd.CombinedOutput() var ip []byte @@ -637,5 +638,4 @@ func MasterDomain(url_s string) (string, error) { } return strings.Join(list[len(list)-2:], "."), nil } - return "", nil } diff --git a/decoder/mp4/0.mp4 b/decoder/mp4/0.mp4 new file mode 100644 index 0000000..50c5d1f Binary files /dev/null and b/decoder/mp4/0.mp4 differ diff --git a/decoder/mp4/README.md b/decoder/mp4/README.md new file mode 100644 index 0000000..6c76354 --- /dev/null +++ b/decoder/mp4/README.md @@ -0,0 +1 @@ +## mp4 decoder \ No newline at end of file diff --git a/decoder/mp4/main.go b/decoder/mp4/main.go new file mode 100644 index 0000000..5d5763c --- /dev/null +++ b/decoder/mp4/main.go @@ -0,0 +1,511 @@ +package mp4 + +import ( + "bytes" + "errors" + "fmt" + "io" + "math" + + "github.com/dustin/go-humanize" + . "github.com/qydysky/part/decoder/tool" + perrors "github.com/qydysky/part/errors" + slice "github.com/qydysky/part/slice" +) + +var boxs map[string]bool + +func init() { + boxs = make(map[string]bool) + //isPureBox? || need to skip? + boxs["ftyp"] = true + boxs["moov"] = false + boxs["mvhd"] = true + boxs["trak"] = false + boxs["tkhd"] = true + boxs["mdia"] = false + boxs["mdhd"] = true + boxs["hdlr"] = true + boxs["minf"] = false || true + boxs["mvex"] = false || true + boxs["moof"] = false + boxs["mfhd"] = true + boxs["traf"] = false + boxs["tfhd"] = true + boxs["tfdt"] = true + boxs["trun"] = true + boxs["mdat"] = true +} + +type ie struct { + n string // box name + i int // start index + e int // end index +} + +type trak struct { + // firstTimeStamp int + // lastTimeStamp int + timescale int + trackID int + handlerType byte +} + +type timeStamp struct { + // firstTimeStamp int + timeStamp int + timescale int + data []byte + handlerType byte +} + +// func (t *timeStamp) resetTs() { +// t.timeStamp -= t.firstTimeStamp +// switch len(t.data) { +// case 4: +// copy(t.data, F.Itob32(int32(t.timeStamp))) +// case 8: +// copy(t.data, F.Itob64(int64(t.timeStamp))) +// } +// } + +func (t *timeStamp) getT() float64 { + return float64(t.timeStamp) / float64(t.timescale) +} + +type Fmp4Decoder struct { + traks map[int]*trak + buf *slice.Buf[byte] + + AVTDiff float64 // 音视频时间戳容差 +} + +func NewFmp4Decoder() *Fmp4Decoder { + return &Fmp4Decoder{ + traks: make(map[int]*trak), + buf: slice.New[byte](), + } +} + +func (t *Fmp4Decoder) Init_fmp4(buf []byte) (b []byte, err error) { + var ftypI, ftypE, moovI, moovE int + + ies, e := decode(buf, "ftyp") + if e != nil { + return + } + + err = deal(ies, + []string{"ftyp", "moov"}, + func(m []ie) (bool, error) { + ftypI = m[0].i + ftypE = m[0].e + moovI = m[1].i + moovE = m[1].e + return true, nil + }) + + if err != nil { + return nil, err + } + + err = deal(ies, + []string{"tkhd", "mdia", "mdhd", "hdlr"}, + func(m []ie) (bool, error) { + tackId := int(Btoi(buf, m[0].i+20, 4)) + t.traks[tackId] = &trak{ + trackID: tackId, + // firstTimeStamp: -1, + // lastTimeStamp: -1, + timescale: int(Btoi(buf, m[2].i+20, 4)), + handlerType: buf[m[3].i+16], + } + return false, nil + }) + + if err != nil { + return nil, err + } + + if len(t.traks) == 0 { + return nil, errors.New("未找到任何trak包") + } + + b = make([]byte, ftypE-ftypI+moovE-moovI) + copy(b[:ftypE-ftypI], buf[ftypI:ftypE]) + copy(b[ftypE-ftypI:], buf[moovI:moovE]) + return b, nil +} + +var ( + ErrBufTooLarge = errors.New("ErrBufTooLarge") + ErrMisTraks = errors.New("ErrMisTraks") +) + +func (t *Fmp4Decoder) Search_stream_fmp4(buf []byte, keyframe *slice.Buf[byte]) (cu int, err error) { + if len(buf) > humanize.MByte*100 { + return 0, ErrBufTooLarge + } + if len(t.traks) == 0 { + return 0, ErrMisTraks + } + t.buf.Reset() + keyframe.Reset() + + defer func() { + if err != nil { + keyframe.Reset() + cu = 0 + } + }() + + var ( + haveKeyframe bool + bufModified = t.buf.GetModified() + // maxSequenceNumber int //有时并不是单调增加 + maxVT float64 + maxAT float64 + + //get timeStamp + get_timeStamp = func(tfdt int) (ts timeStamp) { + switch buf[tfdt+8] { + case 0: + ts.data = buf[tfdt+16 : tfdt+20] + ts.timeStamp = int(Btoi32(buf, tfdt+16)) + case 1: + ts.data = buf[tfdt+12 : tfdt+20] + ts.timeStamp = int(Btoi64(buf, tfdt+12)) + } + return + } + + //get track type + get_track_type = func(tfhd, tfdt int) (ts timeStamp, handlerType byte) { + track, ok := t.traks[int(Btoi(buf, tfhd+12, 4))] + if ok { + ts := get_timeStamp(tfdt) + // if track.firstTimeStamp == -1 { + // track.firstTimeStamp = ts.timeStamp + // } + + // ts.firstTimeStamp = track.firstTimeStamp + ts.handlerType = track.handlerType + ts.timescale = track.timescale + + // if ts.timeStamp > track.lastTimeStamp { + // track.lastTimeStamp = ts.timeStamp + // ts.resetTs() + // } + + return ts, track.handlerType + } + return + } + + //is SampleEntries error? + checkSampleEntries = func(trun, mdat int) error { + if buf[trun+11] == 'b' { + for i := trun + 24; i < mdat; i += 12 { + if Btoi(buf, i+4, 4) < 1000 { + return errors.New("find sample size less then 1000") + } + } + } + return nil + } + + //is t error? + check_set_maxT = func(ts timeStamp, equal func(ts timeStamp) error, larger func(ts timeStamp) error) (err error) { + switch ts.handlerType { + case 'v': + if maxVT == 0 { + maxVT = ts.getT() + } else if maxVT == ts.getT() && equal != nil { + err = equal(ts) + } else if maxVT > ts.getT() && larger != nil { + err = larger(ts) + } else { + maxVT = ts.getT() + } + case 'a': + if maxAT == 0 { + maxAT = ts.getT() + } else if maxAT == ts.getT() && equal != nil { + err = equal(ts) + } else if maxAT > ts.getT() && larger != nil { + err = larger(ts) + } else { + maxAT = ts.getT() + } + default: + } + return + } + ) + + ies, e := decode(buf, "moof") + if e != nil { + return 0, e + } + + err = deals(ies, + [][]string{ + {"moof", "mfhd", "traf", "tfhd", "tfdt", "trun", "mdat"}, + {"moof", "mfhd", "traf", "tfhd", "tfdt", "trun", "traf", "tfhd", "tfdt", "trun", "mdat"}}, + []func(m []ie) (bool, error){ + func(m []ie) (bool, error) { + var ( + keyframeMoof = buf[m[5].i+20] == byte(0x02) + // moofSN = int(F.Btoi(buf, m[1].i+12, 4)) + ) + + { + ts, _ := get_track_type(m[3].i, m[4].i) + if ts.handlerType == 'v' { + if e := checkSampleEntries(m[5].i, m[6].i); e != nil { + //skip + t.buf.Reset() + haveKeyframe = false + cu = m[0].i + return false, e + } + } + if e := check_set_maxT(ts, func(_ timeStamp) error { + return errors.New("skip") + }, func(_ timeStamp) error { + t.buf.Reset() + haveKeyframe = false + cu = m[0].i + return errors.New("skip") + }); e != nil { + return false, e + } + } + + // fmt.Println(ts.getT(), "frame0", keyframeMoof, t.buf.size(), m[0].i, m[6].n, m[6].e) + + //deal frame + if keyframeMoof { + if v, e := t.buf.HadModified(bufModified); e == nil && v && !t.buf.IsEmpty() { + if e := t.buf.AppendTo(keyframe); e != nil { + return false, e + } + cu = m[0].i + t.buf.Reset() + } + haveKeyframe = true + } else if !haveKeyframe { + cu = m[6].e + } + if haveKeyframe { + if e := t.buf.Append(buf[m[0].i:m[6].e]); e != nil { + return false, e + } + } + return false, nil + }, + func(m []ie) (bool, error) { + var ( + keyframeMoof = buf[m[5].i+20] == byte(0x02) || buf[m[9].i+20] == byte(0x02) + // moofSN = int(F.Btoi(buf, m[1].i+12, 4)) + video timeStamp + audio timeStamp + ) + + // fmt.Println(moofSN, "frame1", keyframeMoof, t.buf.size(), m[0].i, m[10].n, m[10].e) + + { + ts, handlerType := get_track_type(m[3].i, m[4].i) + if handlerType == 'v' { + if e := checkSampleEntries(m[5].i, m[6].i); e != nil { + //skip + t.buf.Reset() + haveKeyframe = false + cu = m[0].i + return false, e + } + } + switch handlerType { + case 'v': + video = ts + case 's': + audio = ts + } + if e := check_set_maxT(ts, func(_ timeStamp) error { + return errors.New("skip") + }, func(_ timeStamp) error { + t.buf.Reset() + haveKeyframe = false + cu = m[0].i + return errors.New("skip") + }); e != nil { + return false, e + } + } + { + ts, handlerType := get_track_type(m[7].i, m[8].i) + if handlerType == 'v' { + if e := checkSampleEntries(m[9].i, m[10].i); e != nil { + //skip + t.buf.Reset() + haveKeyframe = false + cu = m[0].i + return false, e + } + } + switch handlerType { + case 'v': + video = ts + case 's': + audio = ts + } + if e := check_set_maxT(ts, func(_ timeStamp) error { + return errors.New("skip") + }, func(_ timeStamp) error { + t.buf.Reset() + haveKeyframe = false + cu = m[0].i + return errors.New("skip") + }); e != nil { + return false, e + } + } + + //sync audio timeStamp + if t.AVTDiff <= 0.1 { + t.AVTDiff = 0.1 + } + if diff := math.Abs(video.getT() - audio.getT()); diff > t.AVTDiff { + return false, fmt.Errorf("时间戳不匹配 %v %v (或许应调整fmp4音视频时间戳容差s>%.2f)", video.timeStamp, audio.timeStamp, diff) + // copy(video.data, F.Itob64(int64(audio.getT()*float64(video.timescale)))) + } + + //deal frame + if keyframeMoof { + if v, e := t.buf.HadModified(bufModified); e == nil && v && !t.buf.IsEmpty() { + if e := t.buf.AppendTo(keyframe); e != nil { + return false, e + } + cu = m[0].i + t.buf.Reset() + } + haveKeyframe = true + } else if !haveKeyframe { + cu = m[10].e + } + if haveKeyframe { + if e := t.buf.Append(buf[m[0].i:m[10].e]); e != nil { + return false, e + } + } + return false, nil + }}) + return +} + +func deal(ies []ie, boxNames []string, fs func([]ie) (breakloop bool, err error)) (err error) { + return deals(ies, [][]string{boxNames}, []func([]ie) (breakloop bool, err error){fs}) +} + +func deals(ies []ie, boxNames [][]string, fs []func([]ie) (breakloop bool, e error)) (err error) { + if len(boxNames) != len(fs) { + panic("boxNames与fs数量不相等") + } + var matchCounts = make([]int, len(boxNames)) + for cu := 0; cu < len(ies) && len(boxNames) != 0; cu++ { + for i := 0; i < len(boxNames); i++ { + if ies[cu].n == boxNames[i][matchCounts[i]] { + matchCounts[i] += 1 + if matchCounts[i] == len(boxNames[i]) { + matchCounts[i] = 0 + if breakloop, e := fs[i](ies[cu-len(boxNames[i])+1 : cu+1]); e != nil { + return e + } else if breakloop { + boxNames = append(boxNames[:i], boxNames[i+1:]...) + fs = append(fs[:i], fs[i+1:]...) + matchCounts = append(matchCounts[:i], matchCounts[i+1:]...) + i -= 1 + } + } + } else { + matchCounts[i] = 0 + } + } + } + return +} + +var ( + ErrMisBox = perrors.New("decode", "ErrMisBox") + ErrCantResync = perrors.New("decode") +) + +func decode(buf []byte, reSyncboxName string) (m []ie, err error) { + var cu int + + for cu < len(buf)-3 { + boxName, i, e, E := searchBox(buf, &cu) + if E != nil { + if errors.Is(E, io.EOF) { + if len(m) == 0 { + err = ErrMisBox + } + return + } + err = E + if reSyncI := bytes.Index(buf[cu:], []byte(reSyncboxName)); reSyncI != -1 { + cu += reSyncI - 4 + m = m[:0] + continue + } + err = ErrCantResync.WithReason(E.Error() + "> 未能reSync") + return + } + + m = append(m, ie{ + n: boxName, + i: i, + e: e, + }) + } + + return +} + +var ( + ErrUnkownBox = perrors.New("ErrUnkownBox") +) + +func searchBox(buf []byte, cu *int) (boxName string, i int, e int, err error) { + i = *cu + e = i + int(Btoi(buf, *cu, 4)) + boxName = string(buf[*cu+4 : *cu+8]) + isPureBoxOrNeedSkip, ok := boxs[boxName] + if !ok { + err = ErrUnkownBox.WithReason("未知包: " + boxName) + } else if e > len(buf) { + err = io.EOF + } else if isPureBoxOrNeedSkip { + *cu = e + } else { + *cu += 8 + } + return +} + +// func testBox(buf []byte, no string) { +// fmt.Println("testBox", "===>") +// err := deal(buf, +// []string{"moof", "mfhd", +// "traf", "tfhd", "tfdt", "trun", +// "traf", "tfhd", "tfdt", "trun", +// "mdat"}, +// func(m []*ie) bool { +// moofSN := int(F.Btoi(buf, m[1].i+12, 4)) +// keyframeMoof := buf[m[5].i+20] == byte(0x02) || buf[m[9].i+20] == byte(0x02) +// fmt.Println(moofSN, "frame", keyframeMoof, m[0].i, m[10].n, m[10].e) +// return false +// }) +// fmt.Println("err", err) +// fmt.Println("testBox", "<===") +// } diff --git a/decoder/mp4/main_test.go b/decoder/mp4/main_test.go new file mode 100644 index 0000000..53f780b --- /dev/null +++ b/decoder/mp4/main_test.go @@ -0,0 +1,68 @@ +package mp4 + +import ( + "errors" + "io" + "testing" + + "github.com/dustin/go-humanize" + . "github.com/qydysky/part/decoder/tool" + pfile "github.com/qydysky/part/file" + pslice "github.com/qydysky/part/slice" +) + +func TestMain(t *testing.T) { + f := pfile.New("0.mp4", 0, false) + data, e := f.ReadAll(humanize.KByte, humanize.MByte*10) + if !errors.Is(e, io.EOF) { + t.Fatal(e) + } + + fmp4decoder := NewFmp4Decoder() + if _, e := fmp4decoder.Init_fmp4(data); e != nil { + t.Fatal(e) + } + + buf := pslice.New[byte]() + if _, e = fmp4decoder.Search_stream_fmp4(data, buf); e != nil && !errors.Is(e, io.EOF) { + t.Fatal(e) + } + + t.Log(buf.Size()) +} + +func TestMain1(t *testing.T) { + f := pfile.New("0.mp4", 0, false) + data, e := f.ReadAll(humanize.KByte, humanize.MByte*10) + if !errors.Is(e, io.EOF) { + t.Fatal(e) + } + + if boxs, e := decode(data, "ftyp"); e != nil { + t.Fatal(e) + } else { + for i := 0; i < len(boxs); i++ { + if boxs[i].n == "mdat" { + mdat := boxs[i] + t.Logf("box type: %s", data[mdat.i+4:mdat.i+8]) + t.Logf("box len: %d", Btoi32(data, mdat.i)) + t.Logf("box cont: %x ...", data[mdat.i+8:mdat.i+8+20]) + for i := mdat.i + 8; i < mdat.e; { + nvlL := Btoi32(data, i) + i += 4 + // t.Logf("nvl l: %d", nvlL) + t.Logf("nvl h: %x %x %d", data[i], data[i]&0x1f, nvlL) + if data[i]&0x1f == 5 || data[i]&0x1f == 1 { + t.Logf("%0.8b", data[i+1:i+20]) + // r := NewBitsReader(data[i+1 : i+int(nvlL)]) + // first_mb_in_slice := r.UE() + // t.Logf("first_mb_in_slice: %d", first_mb_in_slice) + // return + } + i += int(nvlL) + } + return + } + } + } +} diff --git a/decoder/tool/bitsReader.go b/decoder/tool/bitsReader.go new file mode 100644 index 0000000..99dbb03 --- /dev/null +++ b/decoder/tool/bitsReader.go @@ -0,0 +1,61 @@ +package tool + +type BitsReader struct { + Data []byte + len uint + readedByte uint + readedInByte uint8 +} + +func NewBitsReader(Data []byte) *BitsReader { + return &BitsReader{Data: Data, len: uint(len(Data))} +} + +func U[T uint8 | uint16 | uint32 | uint64](t *BitsReader, n uint8) (r T) { + r = T(t.Data[t.readedByte]) + r &= 0b11111111 >> t.readedInByte + if 8-t.readedInByte > n { + r = r >> (8 - t.readedInByte - n) + t.readedInByte += n + } else if 8-t.readedInByte == n { + t.readedInByte = 0 + t.readedByte += 1 + } else { + r = r << (n - (8 - t.readedInByte)) + t.readedInByte = 0 + t.readedByte += 1 + r |= U[T](t, (n - (8 - t.readedInByte))) + } + return +} + +// unsigned integer Exp-Golomb-coded syntax element with the left bit first +func UE[T uint8 | uint16 | uint32 | uint64](t *BitsReader) (r T) { + var move uint8 + for U[uint8](t, 1) == 0 { + move += 1 + } + return (U[T](t, move) | 1<<(move)) - 1 +} + +// signed integer Exp-Golomb-coded syntax element with the left bit first +func SE[T int8 | int16 | int32 | int64](t *BitsReader) (r T) { + var move uint8 + for U[uint8](t, 1) == 0 { + move += 1 + } + switch any(r).(type) { + case int8: + r = T(U[uint8](t, move-1) | 1<<(move-1)) + case int16: + r = T(U[uint16](t, move-1) | 1<<(move-1)) + case int32: + r = T(U[uint32](t, move-1) | 1<<(move-1)) + case int64: + r = T(U[uint64](t, move-1) | 1<<(move-1)) + } + if U[uint8](t, 1) == 0 { + r = -r + } + return +} diff --git a/decoder/tool/bitsReader_test.go b/decoder/tool/bitsReader_test.go new file mode 100644 index 0000000..91d52a7 --- /dev/null +++ b/decoder/tool/bitsReader_test.go @@ -0,0 +1,67 @@ +package tool + +import ( + "testing" +) + +func Benchmark_u(b *testing.B) { + r := NewBitsReader([]byte{0x03, 0x03}) + b.ResetTimer() + for i := 0; i < b.N; i++ { + r.readedByte = 0 + r.readedInByte = 0 + U[uint16](r, 16) + } +} + +func Benchmark_u1(b *testing.B) { + r := []byte{0x03, 0x03} + b.ResetTimer() + for i := 0; i < b.N; i++ { + Btoui16(r, 0) + } +} + +func Test_u(t *testing.T) { + r := NewBitsReader([]byte{0x03, 0x03}) + t.Logf("%d %d", U[uint16](r, 16), Btoui16([]byte{0x03, 0x03}, 0)) +} + +func Test_ue(t *testing.T) { + r := NewBitsReader([]byte{0b01101100}) + if r := UE[uint8](r); r != 2 { + t.Fatal(r) + } + if r := UE[uint8](r); r != 2 { + t.Fatal(r) + } +} + +func Test_se(t *testing.T) { + r := NewBitsReader([]byte{0b00101001, 0b00100000}) + if r := SE[int8](r); r != 2 { + t.Fatal() + } + if r := SE[int8](r); r != -2 { + t.Fatal() + } +} + +func Test_mix(t *testing.T) { + r := NewBitsReader([]byte{0b00101011, 0b00100101, 0b0}) + if r := SE[int8](r); r != 2 { + t.Fatal() + } + if r := UE[uint8](r); r != 2 { + t.Fatal(r) + } + if r := SE[int8](r); r != -2 { + t.Fatal(r) + } + if r := U[uint8](r, 1); r != 1 { + t.Fatal(r) + } + if r := UE[uint8](r); r != 1 { + t.Fatal(r) + } +} diff --git a/decoder/tool/sue.go b/decoder/tool/sue.go deleted file mode 100644 index 2abe55a..0000000 --- a/decoder/tool/sue.go +++ /dev/null @@ -1,30 +0,0 @@ -package tool - -// unsigned integer Exp-Golomb-coded syntax element with the left bit first -func ue(d int) (r, move int) { - for d&0b10000000 == 0b00000000 { - move += 1 - d = d << 1 - } - for i := move; i >= 0; i-- { - r |= d & 0b10000000 >> (7 - i) - d = d << 1 - } - return r - 1, 2*move + 1 -} - -// signed integer Exp-Golomb-coded syntax element with the left bit first -func se(d int) (r, move int) { - for d&0b10000000 == 0b00000000 { - move += 1 - d = d << 1 - } - for i := move; i > 0; i-- { - r |= d & 0b10000000 >> (8 - i) - d = d << 1 - } - if d&0b10000000 == 0b00000000 { - r = -r - } - return r, 2*move + 1 -} diff --git a/decoder/tool/sue_test.go b/decoder/tool/sue_test.go deleted file mode 100644 index 139cf07..0000000 --- a/decoder/tool/sue_test.go +++ /dev/null @@ -1,20 +0,0 @@ -package tool - -import ( - "testing" -) - -func Test_ue(t *testing.T) { - if r, m := ue(0b01101100); r != 2 || m != 3 { - t.Fatal() - } -} - -func Test_se(t *testing.T) { - if r, m := se(0b00101000); r != 2 || m != 5 { - t.Fatal() - } - if r, m := se(0b00100000); r != -2 || m != 5 { - t.Fatal() - } -} diff --git a/file/FileWR.go b/file/FileWR.go index ba86a36..57fb7ba 100644 --- a/file/FileWR.go +++ b/file/FileWR.go @@ -10,6 +10,7 @@ import ( "path/filepath" "strings" "sync" + "syscall" pio "github.com/qydysky/part/io" encoder "golang.org/x/text/encoding" @@ -566,6 +567,39 @@ func (t *File) Stat() (fs.FileInfo, error) { return info, nil } +func (t *File) GetFileModTime() (mod int64, err error) { + fi, err := t.Stat() + if err != nil { + return -1, err + } + return fi.ModTime().Unix(), nil +} + +func (t *File) GetFileSize() (int64, error) { + fi, err := t.Stat() + if err != nil { + return -1, err + } + return fi.Size(), nil +} + +func (t *File) HavePerm(want os.FileMode) (bool, error) { + fi, err := t.Stat() + if err != nil { + return false, err + } + return fi.Mode().Perm() >= want, nil +} + +func (t *File) IsOpen() bool { + if !t.IsExist() { + return false + } + fi, e := os.OpenFile(t.Config.FilePath, syscall.O_RDONLY|syscall.O_EXCL, 0) + defer fi.Close() + return e != nil +} + func (t *File) getRWCloser(mode ...fs.FileMode) { fmode := fs.ModePerm if len(mode) != 0 { diff --git a/hash/Md5.go b/hash/Md5.go new file mode 100644 index 0000000..7777b4a --- /dev/null +++ b/hash/Md5.go @@ -0,0 +1,45 @@ +package part + +import ( + "crypto/md5" + "fmt" + "io" + "os" +) + +func Md5String(str string) string { + return fmt.Sprintf("%x", md5.Sum([]byte(str))) +} + +func Md5File(path string) (string, error) { + file, err := os.Open(path) + defer file.Close() + if err != nil { + return "", err + } + + h := md5.New() + _, err = io.Copy(h, file) + if err != nil { + return "", err + } + + return fmt.Sprintf("%x", h.Sum(nil)), nil +} + +// func GetFileModTime(path string) (error, int64) { +// f, err := os.Open(path) +// if err != nil { +// fmt.Println("open file error") +// return err, time.Now().Unix() +// } +// defer f.Close() + +// fi, err := f.Stat() +// if err != nil { +// fmt.Println("stat fileinfo error") +// return err, time.Now().Unix() +// } + +// return nil, fi.ModTime().Unix() +// } diff --git a/hash/Md5_test.go b/hash/Md5_test.go new file mode 100644 index 0000000..cfe9445 --- /dev/null +++ b/hash/Md5_test.go @@ -0,0 +1,9 @@ +package part + +import ( + "testing" +) + +func TestMain(t *testing.T) { + t.Log(Md5String("sss")) +} diff --git a/reqf/Reqf.go b/reqf/Reqf.go index 3425a4c..04a3818 100644 --- a/reqf/Reqf.go +++ b/reqf/Reqf.go @@ -69,6 +69,7 @@ var ( ErrWriteRes = errors.New("ErrWriteRes") ErrReadRes = errors.New("ErrReadRes") ErrPostStrOrRawPipe = errors.New("ErrPostStrOrRawPipe") + ErrNoDate = errors.New("ErrNoDate") ) type Req struct { @@ -430,3 +431,11 @@ func ToForm(m map[string]string) (postStr string, contentType string) { buf.WriteString(`-----------------------------` + sign + `--`) return buf.String(), `multipart/form-data; boundary=---------------------------` + sign } + +func ResDate(res *http.Response) (time.Time, error) { + if date := res.Header.Get("date"); date != `` { + return time.Parse(time.RFC1123, date) + } else { + return time.Time{}, ErrNoDate + } +} diff --git a/reqf/Reqf_test.go b/reqf/Reqf_test.go index 5cf13a1..7c0e802 100644 --- a/reqf/Reqf_test.go +++ b/reqf/Reqf_test.go @@ -512,4 +512,7 @@ func Test_req5(t *testing.T) { t.Log(r.Respon, buf) t.Fatal() } + if _, e := ResDate(r.Response); e != nil { + t.Fatal() + } }