From: qydysky Date: Thu, 22 Jun 2023 04:23:18 +0000 (+0000) Subject: log support db X-Git-Tag: v0.28.0+202306223ef3143 X-Git-Url: http://127.0.0.1:8081/?a=commitdiff_plain;h=3ef3143a72355b6d85fa675b00fee6f4f2e8323d;p=part%2F.git log support db --- diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 48ad977..bab1dcd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -25,6 +25,7 @@ jobs: run: | go test -count 1 -timeout 30s -v . go test -count 1 -timeout 5s -v -race github.com/qydysky/part/signal + go test -count 1 -timeout 5s -v -race github.com/qydysky/part/log go test -count 1 -timeout 15s -v -race github.com/qydysky/part/reqf go test -count 1 -timeout 15s -v -race github.com/qydysky/part/limit go test -count 1 -timeout 20s -v -race github.com/qydysky/part/file @@ -35,7 +36,7 @@ jobs: go test -count 1 -timeout 5s -v -race github.com/qydysky/part/sync go test -count 1 -timeout 10s -v -race github.com/qydysky/part/web go test -count 1 -timeout 10s -v -run "Test_Client" -race github.com/qydysky/part/websocket - CC=gcc;CXX=g++;CGO_ENABLED=1;GOOS=linux go test -count 1 -timeout 10s -v -race -ldflags '-extldflags=-static -extldflags=-lm' github.com/qydysky/part/sql + go test -count 1 -timeout 10s -v -race github.com/qydysky/part/sql - name: Set Release Name run: | diff --git a/log/Log.go b/log/Log.go index b943f3e..ed2c058 100644 --- a/log/Log.go +++ b/log/Log.go @@ -1,13 +1,18 @@ package part import ( + "context" + "database/sql" + "fmt" "io" "log" "os" + "strings" "time" f "github.com/qydysky/part/file" m "github.com/qydysky/part/msgq" + psql "github.com/qydysky/part/sql" ) var ( @@ -22,15 +27,19 @@ type Log_interface struct { type Config struct { To time.Duration File string - Stdout bool + DBConn *sql.DB + + // $1:Prefix $2:Base $2:Msgs + DBInsert string + Stdout bool Prefix_string map[string]struct{} Base_string []any } type Msg_item struct { - Prefix string - Msg_obj []any + Prefix string + Msgs []any Config } @@ -43,7 +52,6 @@ func New(c Config) (o *Log_interface) { if c.File != `` { f.New(c.File, 0, true).Create() } - if o.To != 0 { o.MQ = m.NewTypeTo[Msg_item](o.To) } else { @@ -64,9 +72,20 @@ func New(c Config) (o *Log_interface) { log.Println(err) } } + if msg.DBConn != nil && msg.DBInsert != `` { + sqlTx := psql.BeginTx[any](msg.DBConn, context.Background()) + sqlTx.SimpleDo( + msg.DBInsert, + msg.Prefix, + strings.TrimSpace(fmt.Sprintln(msg.Base_string...)), + strings.TrimSpace(fmt.Sprintln(msg.Msgs...))) + if _, err := sqlTx.Fin(); err != nil { + log.Println(err) + } + } log.New(io.MultiWriter(showObj...), msg.Prefix, - log.Ldate|log.Ltime).Println(msg.Msg_obj...) + log.Ldate|log.Ltime).Println(append(msg.Base_string, msg.Msgs...)) return false }) //启动阻塞 @@ -111,7 +130,7 @@ func (I *Log_interface) Log_to_file(fileP string) (O *Log_interface) { O = I // O.Block(100) - if O.File != `` { + if O.File != `` && fileP != `` { O.File = fileP f.New(O.File, 0, true).Create() } else { @@ -120,6 +139,21 @@ func (I *Log_interface) Log_to_file(fileP string) (O *Log_interface) { return } +// Open 日志输出至DB +func (I *Log_interface) LDB(db *sql.DB, insert string) (O *Log_interface) { + O = I + // + O.Block(100) + if db != nil && insert != `` { + O.DBConn = db + O.DBInsert = insert + } else { + O.DBConn = nil + O.DBInsert = `` + } + return +} + func (I *Log_interface) LFile(fileP string) (O *Log_interface) { return I.Log_to_file(fileP) } @@ -133,6 +167,9 @@ func (I *Log_interface) Block(ms int) (O *Log_interface) { func (I *Log_interface) Close() { I.MQ.ClearAll() + if I.DBConn != nil { + (*I.DBConn).Close() + } } // 日志等级 @@ -154,9 +191,9 @@ func (I *Log_interface) L(prefix string, i ...any) (O *Log_interface) { } O.MQ.Push_tag(`L`, Msg_item{ - Prefix: prefix, - Msg_obj: append(O.Base_string, i), - Config: O.Config, + Prefix: prefix, + Msgs: i, + Config: O.Config, }) return } diff --git a/log/Log_test.go b/log/Log_test.go index 0c79679..ff03f16 100644 --- a/log/Log_test.go +++ b/log/Log_test.go @@ -3,11 +3,16 @@ package part import ( // "fmt" + "context" + "database/sql" + "errors" "testing" - "time" - "net/http" _ "net/http/pprof" + + _ "modernc.org/sqlite" + + psql "github.com/qydysky/part/sql" ) func Test_1(t *testing.T) { @@ -36,20 +41,56 @@ func Test_1(t *testing.T) { var n *Log_interface func Test_2(t *testing.T) { + db, err := sql.Open("sqlite", ":memory:") + if err != nil { + t.Fatal(err) + } + defer db.Close() + + { + tx := psql.BeginTx[any](db, context.Background(), &sql.TxOptions{}) + tx = tx.Do(psql.SqlFunc[any]{ + Query: "create table log (p test,base text,msg text)", + SkipSqlErr: true, + }) + if _, err := tx.Fin(); err != nil { + t.Fatal(err) + } + } + n = New(Config{ File: `1.log`, Stdout: true, Prefix_string: map[string]struct{}{`T:`: On, `I:`: On, `W:`: On, `E:`: On}, }) - go func() { - http.ListenAndServe("0.0.0.0:8899", nil) - }() - // n = nil - for { - n := n.Base_add(`>1`) - n.L(`T:`, `s`) - time.Sleep(time.Second * time.Duration(1)) - // n=nil + ndb := n.Base_add(`>1`) + ndb = ndb.LDB(db, `insert into log (p,base,msg) values (?,?,?)`) + ndb.L(`T:`, `s`) + n.L(`T:`, `p`) + + { + type logg struct { + P string `sql:"p"` + Base string + Msg string `sql:"s"` + } + tx := psql.BeginTx[any](db, context.Background(), &sql.TxOptions{}) + tx = tx.SimpleDo("select p,base,msg as s from log") + tx.AfterQF(func(_ *any, rows *sql.Rows, e *error) { + if ls, err := psql.DealRows[logg](rows, func() logg { return logg{} }); err == nil { + if len(ls) != 1 { + *e = errors.New("num wrong") + } + if ls[0].Msg != "s" { + *e = errors.New("msg wrong") + } + } else { + *e = err + } + }) + if _, err := tx.Fin(); err != nil { + t.Fatal(err) + } } } diff --git a/sql/Sql.go b/sql/Sql.go index 15b4784..82941fa 100644 --- a/sql/Sql.go +++ b/sql/Sql.go @@ -19,16 +19,15 @@ type CanTx interface { BeginTx(ctx context.Context, opts *sql.TxOptions) (*sql.Tx, error) } -type BeforeF[T any] func(dataP *T, sqlf *SqlFunc[T], txE error) (dataPR *T, stopErr error) -type AfterEF[T any] func(dataP *T, result sql.Result, txE error) (dataPR *T, stopErr error) -type AfterQF[T any] func(dataP *T, rows *sql.Rows, txE error) (dataPR *T, stopErr error) +type BeforeF[T any] func(ctxVP *T, sqlf *SqlFunc[T], e *error) +type AfterEF[T any] func(ctxVP *T, result sql.Result, e *error) +type AfterQF[T any] func(ctxVP *T, rows *sql.Rows, e *error) type SqlTx[T any] struct { canTx CanTx ctx context.Context opts *sql.TxOptions sqlFuncs []*SqlFunc[T] - dataP *T fin bool } @@ -54,11 +53,35 @@ func BeginTx[T any](canTx CanTx, ctx context.Context, opts ...*sql.TxOptions) *S return &tx } +func (t *SqlTx[T]) SimpleDo(query string, args ...any) *SqlTx[T] { + t.sqlFuncs = append(t.sqlFuncs, &SqlFunc[T]{ + Query: query, + Args: args, + }) + return t +} + func (t *SqlTx[T]) Do(sqlf SqlFunc[T]) *SqlTx[T] { t.sqlFuncs = append(t.sqlFuncs, &sqlf) return t } +// PlaceHolder will replaced by ? +func (t *SqlTx[T]) SimplePlaceHolderA(query string, ptr any) *SqlTx[T] { + return t.DoPlaceHolder(SqlFunc[T]{ + Query: query, + }, ptr) +} + +// PlaceHolder will replaced by $%d +func (t *SqlTx[T]) SimplePlaceHolderB(query string, ptr any) *SqlTx[T] { + return t.DoPlaceHolder(SqlFunc[T]{ + Query: query, + }, ptr, func(index int, holder string) (replaceTo string) { + return fmt.Sprintf("$%d", index+1) + }) +} + func (t *SqlTx[T]) DoPlaceHolder(sqlf SqlFunc[T], ptr any, replaceF ...func(index int, holder string) (replaceTo string)) *SqlTx[T] { dataR := reflect.ValueOf(ptr).Elem() index := 0 @@ -101,9 +124,10 @@ func (t *SqlTx[T]) AfterQF(f AfterQF[T]) *SqlTx[T] { return t } -func (t *SqlTx[T]) Fin() (dataP *T, e error) { +func (t *SqlTx[T]) Fin() (ctxVP T, e error) { if t.fin { - return nil, fmt.Errorf("BeginTx; [] >> fin") + e = fmt.Errorf("BeginTx; [] >> fin") + return } var hasErr bool @@ -117,11 +141,10 @@ func (t *SqlTx[T]) Fin() (dataP *T, e error) { sqlf := t.sqlFuncs[i] if sqlf.beforeF != nil { - if datap, err := sqlf.beforeF(t.dataP, sqlf, e); err != nil { + sqlf.beforeF(&ctxVP, sqlf, &e) + if e != nil { e = errors.Join(e, fmt.Errorf("%s; >> %s", sqlf.Query, err)) hasErr = true - } else { - t.dataP = datap } } @@ -148,11 +171,10 @@ func (t *SqlTx[T]) Fin() (dataP *T, e error) { e = errors.Join(e, fmt.Errorf("%s; %s >> %s", sqlf.Query, sqlf.Args, err)) } } else if sqlf.afterEF != nil { - if datap, err := sqlf.afterEF(t.dataP, res, e); err != nil { + sqlf.afterEF(&ctxVP, res, &e) + if e != nil { hasErr = true e = errors.Join(e, fmt.Errorf("%s; %s >> %s", sqlf.Query, sqlf.Args, err)) - } else { - t.dataP = datap } } case Queryf: @@ -162,11 +184,10 @@ func (t *SqlTx[T]) Fin() (dataP *T, e error) { e = errors.Join(e, fmt.Errorf("%s; %s >> %s", sqlf.Query, sqlf.Args, err)) } } else if sqlf.afterQF != nil { - if datap, err := sqlf.afterQF(t.dataP, res, e); err != nil { + sqlf.afterQF(&ctxVP, res, &e) + if e != nil { hasErr = true e = errors.Join(e, fmt.Errorf("%s; %s >> %s", sqlf.Query, sqlf.Args, err)) - } else { - t.dataP = datap } } } @@ -184,14 +205,14 @@ func (t *SqlTx[T]) Fin() (dataP *T, e error) { } } t.fin = true - return t.dataP, e + return } func IsFin[T any](t *SqlTx[T]) bool { return t == nil || t.fin } -func DealRows[T any](rows *sql.Rows, newT func() T) (*[]T, error) { +func DealRows[T any](rows *sql.Rows, newT func() T) ([]T, error) { rowNames, err := rows.Columns() if err != nil { return nil, err @@ -249,18 +270,5 @@ func DealRows[T any](rows *sql.Rows, newT func() T) (*[]T, error) { res = append(res, stu) } - return &res, nil -} - -// for mysql,oracle not postgresql -func SimpleQ[T any](canTx CanTx, query string, ptr *T) (*[]T, error) { - tx := BeginTx[[]T](canTx, context.Background()) - tx.DoPlaceHolder(SqlFunc[[]T]{Query: query}, ptr) - tx.AfterQF(func(_ *[]T, rows *sql.Rows, txE error) (dataPR *[]T, stopErr error) { - if txE != nil { - return nil, txE - } - return DealRows(rows, func() T { return *ptr }) - }) - return tx.Fin() + return res, nil } diff --git a/sql/Sql_test.go b/sql/Sql_test.go index 2c92aeb..1880c0e 100644 --- a/sql/Sql_test.go +++ b/sql/Sql_test.go @@ -57,51 +57,52 @@ func TestMain(t *testing.T) { Ty: Queryf, Ctx: ctx, Query: "select msg from log", - }).AfterQF(func(dataP *[]string, rows *sql.Rows, err error) (dataPR *[]string, stopErr error) { + }).AfterQF(func(dataP *[]string, rows *sql.Rows, err *error) { names := make([]string, 0) for rows.Next() { var name string - if err := rows.Scan(&name); err != nil { - return nil, err + if *err = rows.Scan(&name); *err != nil { + return } names = append(names, name) } rows.Close() if len(names) != 1 || dateTime != names[0] { - return nil, errors.New("no") + *err = errors.New("no") + return } - return &names, nil + *dataP = names }) tx = tx.Do(SqlFunc[[]string]{ Ty: Execf, Ctx: ctx, - }).BeforeF(func(dataP *[]string, sqlf *SqlFunc[[]string], txE error) (dataPR *[]string, stopErr error) { + }).BeforeF(func(dataP *[]string, sqlf *SqlFunc[[]string], txE *error) { sqlf.Query = "insert into log2 values (?)" sqlf.Args = append(sqlf.Args, (*dataP)[0]) - return dataP, nil }) tx = tx.Do(SqlFunc[[]string]{ Ty: Queryf, Ctx: ctx, Query: "select msg from log2", - }).AfterQF(func(dataP *[]string, rows *sql.Rows, err error) (dataPR *[]string, stopErr error) { + }).AfterQF(func(dataP *[]string, rows *sql.Rows, err *error) { names := make([]string, 0) for rows.Next() { var name string - if err := rows.Scan(&name); err != nil { - return nil, err + if *err = rows.Scan(&name); *err != nil { + return } names = append(names, name) } rows.Close() if len(names) != 1 || dateTime != names[0] { - return nil, errors.New("no2") + *err = errors.New("no2") + return } - return &names, nil + *dataP = names }) if _, e := tx.Fin(); e != nil { @@ -184,7 +185,8 @@ func TestMain3(t *testing.T) { if _, e := tx.Fin(); e != nil { t.Log(e) } - if _, err := SimpleQ(db, "insert into log123 values ({Msg},{Msg2})", &logg{Msg: 3, Msg2: "b"}); err != nil { + tx1 := BeginTx[any](db, context.Background()).SimplePlaceHolderA("insert into log123 values ({Msg},{Msg2})", &logg{Msg: 3, Msg2: "b"}) + if _, err := tx1.Fin(); err != nil { t.Fatal(err) } } @@ -192,25 +194,30 @@ func TestMain3(t *testing.T) { selectLog123 := SqlFunc[[]logg]{Query: "select msg as Msg, msg2 as Msg2 from log123 where msg = {Msg}"} tx := BeginTx[[]logg](db, context.Background()) tx.DoPlaceHolder(selectLog123, &logg{Msg: 2, Msg2: "b"}) - tx.AfterQF(func(_ *[]logg, rows *sql.Rows, txE error) (dataPR *[]logg, stopErr error) { - return DealRows(rows, func() logg { return logg{} }) + tx.AfterQF(func(ctxVP *[]logg, rows *sql.Rows, txE *error) { + *ctxVP, *txE = DealRows(rows, func() logg { return logg{} }) }) if v, e := tx.Fin(); e != nil { t.Fatal(e) } else { - if (*v)[0].Msg2 != "b" || (*v)[0].Msg != 2 { + if v[0].Msg2 != "b" || v[0].Msg != 2 { t.Fatal() } } } { - if v, err := SimpleQ(db, "select msg as Msg, msg2 as Msg2 from log123 where msg2 = {Msg2}", &logg{Msg2: "b"}); err != nil { + tx1 := BeginTx[[]logg](db, context.Background()). + SimplePlaceHolderA("select msg as Msg, msg2 as Msg2 from log123 where msg2 = {Msg2}", &logg{Msg2: "b"}). + AfterQF(func(ctxVP *[]logg, rows *sql.Rows, e *error) { + *ctxVP, *e = DealRows[logg](rows, func() logg { return logg{} }) + }) + if v, err := tx1.Fin(); err != nil { t.Fatal(err) } else { - if (*v)[0].Msg2 != "b" || (*v)[0].Msg != 2 { + if v[0].Msg2 != "b" || v[0].Msg != 2 { t.Fatal() } - if (*v)[1].Msg2 != "b" || (*v)[1].Msg != 3 { + if v[1].Msg2 != "b" || v[1].Msg != 3 { t.Fatal() } } @@ -280,18 +287,19 @@ func Local_TestPostgresql(t *testing.T) { if _, e := BeginTx[any](db, context.Background(), &sql.TxOptions{}).Do(SqlFunc[any]{ Query: "select created as sss from test", - afterQF: func(_ *any, rows *sql.Rows, txE error) (dataPR *any, stopErr error) { + afterQF: func(_ *any, rows *sql.Rows, txE *error) { if rowsP, e := DealRows[test1](rows, func() test1 { return test1{} }); e != nil { - return nil, e + *txE = e } else { - if len(*rowsP) != 1 { - return nil, errors.New("no match") + if len(rowsP) != 1 { + *txE = errors.New("no match") + return } - if (*rowsP)[0].Created != "1" { - return nil, errors.New("no match") + if rowsP[0].Created != "1" { + *txE = errors.New("no match") + return } } - return nil, nil }, }).Fin(); e != nil { t.Fatal(e)