Merge pull request #88 from taosdata/feat/TD-18301

fix: adapt to new raw_block
This commit is contained in:
Xuefeng Tan 2022-08-10 20:25:01 +08:00 committed by GitHub
commit 3d21433f78
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 159 additions and 61 deletions

View file

@ -604,13 +604,13 @@ func stmtParseTag(tags json.RawMessage, fields []*wrapper.StmtField) ([]driver.V
func blockConvert(block unsafe.Pointer, blockSize int, fields []*wrapper.StmtField) [][]driver.Value {
colCount := len(fields)
r := make([][]driver.Value, colCount)
payloadOffset := uintptr(4 * colCount)
nullBitMapOffset := uintptr(wrapper.BitmapLen(blockSize))
pHeader := uintptr(block) + payloadOffset + 12 + uintptr(6*colCount) // length i32, group u64
lengthOffset := wrapper.RawBlockGetColumnLengthOffset(colCount)
pHeader := uintptr(block) + wrapper.RawBlockGetColDataOffset(colCount)
pStart := pHeader
for column := 0; column < colCount; column++ {
r[column] = make([]driver.Value, blockSize)
colLength := *((*int32)(unsafe.Pointer(uintptr(block) + 12 + uintptr(6*colCount) + uintptr(column)*4)))
colLength := *((*int32)(unsafe.Pointer(uintptr(block) + lengthOffset + uintptr(column)*wrapper.Int32Size)))
if wrapper.IsVarDataType(uint8(fields[column].FieldType)) {
convertF := rawConvertVarDataMap[fields[column].FieldType]
pStart = pHeader + uintptr(4*blockSize)

View file

@ -296,7 +296,6 @@ func execute(c *gin.Context, logger *logrus.Entry, taosConnect unsafe.Pointer, s
}
precision := wrapper.TaosResultPrecision(res)
fetched := false
payloadOffset := uintptr(4 * fieldsCount)
pHeaderList := make([]uintptr, fieldsCount)
pStartList := make([]uintptr, fieldsCount)
for {
@ -331,10 +330,11 @@ func execute(c *gin.Context, logger *logrus.Entry, taosConnect unsafe.Pointer, s
thread.Unlock()
blockSize := result.N
nullBitMapOffset := uintptr(ctools.BitmapLen(blockSize))
tmpPHeader := uintptr(block) + payloadOffset + 12 + uintptr(6*fieldsCount) // length i32, group u64
lengthOffset := wrapper.RawBlockGetColumnLengthOffset(fieldsCount)
tmpPHeader := uintptr(block) + wrapper.RawBlockGetColDataOffset(fieldsCount) // length i32, group u64
tmpPStart := tmpPHeader
for column := 0; column < fieldsCount; column++ {
colLength := *((*int32)(unsafe.Pointer(uintptr(block) + 12 + uintptr(6*fieldsCount) + uintptr(column)*4)))
colLength := *((*int32)(unsafe.Pointer(uintptr(block) + lengthOffset + uintptr(column)*wrapper.Int32Size)))
if ctools.IsVarDataType(rowsHeader.ColTypes[column]) {
pHeaderList[column] = tmpPHeader
tmpPStart = tmpPHeader + uintptr(4*blockSize)

View file

@ -549,7 +549,7 @@ func (t *TaosStmt) close(ctx context.Context, session *melody.Session, req *Stmt
stmt.clean()
}
func (t *TaosStmt) setTagsBlock(ctx context.Context, session *melody.Session, reqID, stmtID, rows, columns uint64, block unsafe.Pointer) {
func (t *TaosStmt) setTagsBlock(ctx context.Context, session *melody.Session, reqID, stmtID uint64, rows, columns int, block unsafe.Pointer) {
if rows != 1 {
wsStmtErrorMsg(ctx, session, 0xffff, "rows not equal 1", STMTSetTags, reqID, &stmtID)
return
@ -587,7 +587,7 @@ func (t *TaosStmt) setTagsBlock(ctx context.Context, session *melody.Session, re
wsWriteJson(session, resp)
return
}
if int(columns) != tagNums {
if columns != tagNums {
wsStmtErrorMsg(ctx, session, 0xffff, "stmt tags count not match", STMTSetTags, reqID, &stmtID)
return
}
@ -617,7 +617,7 @@ func (t *TaosStmt) setTagsBlock(ctx context.Context, session *melody.Session, re
wsWriteJson(session, resp)
}
func (t *TaosStmt) bindBlock(ctx context.Context, session *melody.Session, reqID, stmtID, rows, columns uint64, block unsafe.Pointer) {
func (t *TaosStmt) bindBlock(ctx context.Context, session *melody.Session, reqID, stmtID uint64, rows, columns int, block unsafe.Pointer) {
if t.conn == nil {
wsStmtErrorMsg(ctx, session, 0xffff, "taos not connected", STMTBind, reqID, &stmtID)
return
@ -664,12 +664,12 @@ func (t *TaosStmt) bindBlock(ctx context.Context, session *melody.Session, reqID
return
}
}
if int(columns) != colNums {
if columns != colNums {
wsStmtErrorMsg(ctx, session, 0xffff, "stmt column count not match", STMTBind, reqID, &stmtID)
return
}
s = log.GetLogNow(isDebug)
data := blockConvert(block, int(rows), fields)
data := blockConvert(block, rows, fields)
logger.Debugln("block convert cost:", log.GetLogDuration(isDebug, s))
s = log.GetLogNow(isDebug)
thread.Lock()
@ -835,16 +835,14 @@ func (ctl *Restful) InitStmt() {
//p0 uin64 代表 req_id
//p0+8 uint64 代表 stmt_id
//p0+16 uint64 代表 操作类型(1 (set tag) 2 (bind))
//p0+24 uint64 代表 列数
//p0+32 uint64 代表 行数
//p0+40 raw block
//p0+24 raw block
p0 := *(*uintptr)(unsafe.Pointer(&data))
reqID := *(*uint64)(unsafe.Pointer(p0))
stmtID := *(*uint64)(unsafe.Pointer(p0 + uintptr(8)))
action := *(*uint64)(unsafe.Pointer(p0 + uintptr(16)))
columns := *(*uint64)(unsafe.Pointer(p0 + uintptr(24)))
counts := *(*uint64)(unsafe.Pointer(p0 + uintptr(32)))
block := unsafe.Pointer(p0 + uintptr(40))
block := unsafe.Pointer(p0 + uintptr(24))
columns := wrapper.RawBlockGetNumOfCols(block)
rows := wrapper.RawBlockGetNumOfRows(block)
if ctl.stmtM.IsClosed() {
return
}
@ -853,10 +851,10 @@ func (ctl *Restful) InitStmt() {
switch action {
case BindMessage:
t := session.MustGet(TaosStmtKey)
t.(*TaosStmt).bindBlock(ctx, session, reqID, stmtID, counts, columns, block)
t.(*TaosStmt).bindBlock(ctx, session, reqID, stmtID, int(rows), int(columns), block)
case SetTagsMessage:
t := session.MustGet(TaosStmtKey)
t.(*TaosStmt).setTagsBlock(ctx, session, reqID, stmtID, counts, columns, block)
t.(*TaosStmt).setTagsBlock(ctx, session, reqID, stmtID, int(rows), int(columns), block)
}
})

View file

@ -470,14 +470,39 @@ func TestBlock(t *testing.T) {
//p0 uin64 代表 req_id
//p0+8 uint64 代表 stmt_id
//p0+16 uint64 代表 类型(1 set tag 2 bind)
//p0+24 uint64 代表 列数
//p0+32 uint64 代表 行数
//p0+40 raw block
rawBlock := []byte{150, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 8, 0, 0, 0, 1, 0, 1, 0, 0, 0, 2, 0, 1, 0, 0, 0, 3, 0, 2, 0, 0, 0, 4, 0, 4, 0, 0, 0, 5, 0, 8, 0, 0, 0, 11, 0, 1, 0, 0, 0, 12, 0, 2, 0, 0, 0, 13, 0, 4, 0, 0, 0, 14, 0, 8, 0, 0, 0, 6, 0, 4, 0, 0, 0, 7, 0, 8, 0, 0, 0, 8, 0, 22, 0, 0, 0, 10, 0, 82, 0, 0, 0, 24, 0, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 6, 0, 0, 0, 12, 0, 0, 0, 24, 0, 0, 0, 3, 0, 0, 0, 6, 0, 0, 0, 12, 0, 0, 0, 24, 0, 0, 0, 12, 0, 0, 0, 24, 0, 0, 0, 17, 0, 0, 0, 48, 0, 0, 0, 0, 142, 23, 228, 90, 129, 1, 0, 0, 118, 27, 228, 90, 129, 1, 0, 0, 94, 31, 228, 90, 129, 1, 0, 0, 32, 1, 0, 0, 32, 2, 22, 0, 32, 3, 0, 33, 0, 0, 0, 32, 4, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 32, 5, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 6, 66, 0, 32, 7, 0, 77, 0, 0, 0, 32, 8, 0, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 32, 9, 0, 0, 0, 0, 0, 0, 0, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 32, 65, 0, 128, 124, 68, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 38, 64, 0, 0, 0, 0, 0, 92, 145, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 255, 255, 255, 255, 6, 0, 98, 105, 110, 97, 114, 121, 7, 0, 98, 105, 110, 97, 114, 121, 50, 0, 0, 0, 0, 22, 0, 0, 0, 255, 255, 255, 255, 20, 0, 110, 0, 0, 0, 99, 0, 0, 0, 104, 0, 0, 0, 97, 0, 0, 0, 114, 0, 0, 0, 24, 0, 110, 0, 0, 0, 99, 0, 0, 0, 104, 0, 0, 0, 97, 0, 0, 0, 114, 0, 0, 0, 50, 0, 0, 0}
//p0+24 raw block
rawBlock := []byte{
0x01, 0x00, 0x00, 0x00, 0x98, 0x01, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, 0x08, 0x00, 0x00,
0x00, 0x01, 0x01, 0x00, 0x00, 0x00, 0x02, 0x01, 0x00, 0x00, 0x00, 0x03, 0x02, 0x00, 0x00, 0x00,
0x04, 0x04, 0x00, 0x00, 0x00, 0x05, 0x08, 0x00, 0x00, 0x00, 0x0b, 0x01, 0x00, 0x00, 0x00, 0x0c,
0x02, 0x00, 0x00, 0x00, 0x0d, 0x04, 0x00, 0x00, 0x00, 0x0e, 0x08, 0x00, 0x00, 0x00, 0x06, 0x04,
0x00, 0x00, 0x00, 0x07, 0x08, 0x00, 0x00, 0x00, 0x08, 0x16, 0x00, 0x00, 0x00, 0x0a, 0x52, 0x00,
0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x06, 0x00,
0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x06, 0x00,
0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x18, 0x00,
0x00, 0x00, 0x11, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, 0x00, 0x2c, 0x5b, 0x70, 0x86, 0x82,
0x01, 0x00, 0x00, 0x14, 0x5f, 0x70, 0x86, 0x82, 0x01, 0x00, 0x00, 0xfc, 0x62, 0x70, 0x86, 0x82,
0x01, 0x00, 0x00, 0x20, 0x01, 0x00, 0x00, 0x20, 0x02, 0x16, 0x00, 0x20, 0x03, 0x00, 0x21, 0x00,
0x00, 0x00, 0x20, 0x04, 0x00, 0x00, 0x00, 0x2c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20,
0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x37, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x06, 0x42, 0x00, 0x20, 0x07, 0x00, 0x4d,
0x00, 0x00, 0x00, 0x20, 0x08, 0x00, 0x00, 0x00, 0x58, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x20, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x63, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x20, 0x41, 0x00, 0x80,
0x7c, 0x44, 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x26, 0x40, 0x00,
0x00, 0x00, 0x00, 0x00, 0x5c, 0x91, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x06, 0x00, 0x62, 0x69, 0x6e,
0x61, 0x72, 0x79, 0x07, 0x00, 0x62, 0x69, 0x6e, 0x61, 0x72, 0x79, 0x32, 0x00, 0x00, 0x00, 0x00,
0x16, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x14, 0x00, 0x6e, 0x00, 0x00, 0x00, 0x63, 0x00,
0x00, 0x00, 0x68, 0x00, 0x00, 0x00, 0x61, 0x00, 0x00, 0x00, 0x72, 0x00, 0x00, 0x00, 0x18, 0x00,
0x6e, 0x00, 0x00, 0x00, 0x63, 0x00, 0x00, 0x00, 0x68, 0x00, 0x00, 0x00, 0x61, 0x00, 0x00, 0x00,
0x72, 0x00, 0x00, 0x00, 0x32, 0x00, 0x00, 0x00,
}
now := time.Now()
binary.LittleEndian.PutUint64(rawBlock[153:], uint64(now.UnixNano()))
binary.LittleEndian.PutUint64(rawBlock[161:], uint64(now.Add(time.Second).UnixNano()))
binary.LittleEndian.PutUint64(rawBlock[169:], uint64(now.Add(time.Second*2).UnixNano()))
binary.LittleEndian.PutUint64(rawBlock[155:], uint64(now.UnixNano()))
binary.LittleEndian.PutUint64(rawBlock[163:], uint64(now.Add(time.Second).UnixNano()))
binary.LittleEndian.PutUint64(rawBlock[171:], uint64(now.Add(time.Second*2).UnixNano()))
s := httptest.NewServer(router)
defer s.Close()
ws, _, err := websocket.DefaultDialer.Dial("ws"+strings.TrimPrefix(s.URL, "http")+"/rest/stmt", nil)
@ -624,15 +649,11 @@ func TestBlock(t *testing.T) {
status = AfterBind
reqID := uint64(10)
action := uint64(2)
columns := uint64(14)
rows := uint64(3)
block := &bytes.Buffer{}
writeUint64(block, reqID)
writeUint64(block, stmtID)
writeUint64(block, action)
writeUint64(block, columns)
writeUint64(block, rows)
block.Write(rawBlock)
blockData := block.Bytes()
t.Log(blockData)

View file

@ -463,7 +463,7 @@ func (t *TMQ) fetch(ctx context.Context, session *melody.Session, req *TMQFetchR
} else {
message.buffer.Reset()
}
blockLength := int(*(*int32)(block))
blockLength := int(wrapper.RawBlockGetLength(block))
message.buffer.Grow(blockLength + 24)
writeUint64(message.buffer, 0)
writeUint64(message.buffer, req.ReqID)

View file

@ -413,7 +413,7 @@ func (t *Taos) fetchBlock(ctx context.Context, session *melody.Session, req *WSF
return
}
resultS.Lock()
blockLength := int(*(*int32)(resultS.Block))
blockLength := int(wrapper.RawBlockGetLength(resultS.Block))
if resultS.buffer == nil {
resultS.buffer = new(bytes.Buffer)
} else {

2
go.mod
View file

@ -37,6 +37,6 @@ require (
github.com/spf13/viper v1.9.0
github.com/stretchr/testify v1.7.1
github.com/swaggo/swag v1.7.6
github.com/taosdata/driver-go/v3 v3.0.0-20220803121203-b99dfb2af5d8
github.com/taosdata/driver-go/v3 v3.0.0-20220810121502-84ebab64fbd7
gonum.org/v1/gonum v0.8.2 // indirect
)

4
go.sum
View file

@ -1571,8 +1571,8 @@ github.com/swaggo/swag v1.7.6/go.mod h1:7vLqNYEtYoIsD14wXgy9oDS65MNiDANrPtbk9rnL
github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
github.com/taosdata/driver-go/v3 v3.0.0-20220803121203-b99dfb2af5d8 h1:+edStK2Qj2M71O+S1/6nnN1BcN53GVi47vUpJaJLtzg=
github.com/taosdata/driver-go/v3 v3.0.0-20220803121203-b99dfb2af5d8/go.mod h1:lT4lpI3wo3hXRwP3nzm7xDs/YgYbw5YU58XingVlfsY=
github.com/taosdata/driver-go/v3 v3.0.0-20220810121502-84ebab64fbd7 h1:hKpP9hWH18t4l9zWryOeBBC3IG2tSyKtAWAmHO2MwrQ=
github.com/taosdata/driver-go/v3 v3.0.0-20220810121502-84ebab64fbd7/go.mod h1:lT4lpI3wo3hXRwP3nzm7xDs/YgYbw5YU58XingVlfsY=
github.com/tbrandon/mbserver v0.0.0-20170611213546-993e1772cc62 h1:Oj2e7Sae4XrOsk3ij21QjjEgAcVSeo9nkp0dI//cD2o=
github.com/tbrandon/mbserver v0.0.0-20170611213546-993e1772cc62/go.mod h1:qUzPVlSj2UgxJkVbH0ZwuuiR46U8RBMDT5KLY78Ifpw=
github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I=

View file

@ -4,6 +4,7 @@ import (
"unsafe"
"github.com/taosdata/driver-go/v3/common"
"github.com/taosdata/driver-go/v3/wrapper"
"github.com/taosdata/taosadapter/v3/tools/jsonbuilder"
)
@ -47,47 +48,47 @@ func WriteRawJsonBool(builder *jsonbuilder.Stream, pStart uintptr, row int) {
}
func WriteRawJsonTinyint(builder *jsonbuilder.Stream, pStart uintptr, row int) {
builder.WriteInt8(*((*int8)(unsafe.Pointer(pStart + uintptr(row)*1))))
builder.WriteInt8(*((*int8)(unsafe.Pointer(pStart + uintptr(row)*wrapper.Int8Size))))
}
func WriteRawJsonSmallint(builder *jsonbuilder.Stream, pStart uintptr, row int) {
builder.WriteInt16(*((*int16)(unsafe.Pointer(pStart + uintptr(row)*2))))
builder.WriteInt16(*((*int16)(unsafe.Pointer(pStart + uintptr(row)*wrapper.Int16Size))))
}
func WriteRawJsonInt(builder *jsonbuilder.Stream, pStart uintptr, row int) {
builder.WriteInt32(*((*int32)(unsafe.Pointer(pStart + uintptr(row)*4))))
builder.WriteInt32(*((*int32)(unsafe.Pointer(pStart + uintptr(row)*wrapper.Int32Size))))
}
func WriteRawJsonBigint(builder *jsonbuilder.Stream, pStart uintptr, row int) {
builder.WriteInt64(*((*int64)(unsafe.Pointer(pStart + uintptr(row)*8))))
builder.WriteInt64(*((*int64)(unsafe.Pointer(pStart + uintptr(row)*wrapper.Int64Size))))
}
func WriteRawJsonUTinyint(builder *jsonbuilder.Stream, pStart uintptr, row int) {
builder.WriteUint8(*((*uint8)(unsafe.Pointer(pStart + uintptr(row)*1))))
builder.WriteUint8(*((*uint8)(unsafe.Pointer(pStart + uintptr(row)*wrapper.UInt8Size))))
}
func WriteRawJsonUSmallint(builder *jsonbuilder.Stream, pStart uintptr, row int) {
builder.WriteUint16(*((*uint16)(unsafe.Pointer(pStart + uintptr(row)*2))))
builder.WriteUint16(*((*uint16)(unsafe.Pointer(pStart + uintptr(row)*wrapper.UInt16Size))))
}
func WriteRawJsonUInt(builder *jsonbuilder.Stream, pStart uintptr, row int) {
builder.WriteUint32(*((*uint32)(unsafe.Pointer(pStart + uintptr(row)*4))))
builder.WriteUint32(*((*uint32)(unsafe.Pointer(pStart + uintptr(row)*wrapper.UInt32Size))))
}
func WriteRawJsonUBigint(builder *jsonbuilder.Stream, pStart uintptr, row int) {
builder.WriteUint64(*((*uint64)(unsafe.Pointer(pStart + uintptr(row)*8))))
builder.WriteUint64(*((*uint64)(unsafe.Pointer(pStart + uintptr(row)*wrapper.UInt64Size))))
}
func WriteRawJsonFloat(builder *jsonbuilder.Stream, pStart uintptr, row int) {
builder.WriteFloat32(*((*float32)(unsafe.Pointer(pStart + uintptr(row)*4))))
builder.WriteFloat32(*((*float32)(unsafe.Pointer(pStart + uintptr(row)*wrapper.Float32Size))))
}
func WriteRawJsonDouble(builder *jsonbuilder.Stream, pStart uintptr, row int) {
builder.WriteFloat64(*((*float64)(unsafe.Pointer(pStart + uintptr(row)*8))))
builder.WriteFloat64(*((*float64)(unsafe.Pointer(pStart + uintptr(row)*wrapper.Float64Size))))
}
func WriteRawJsonTime(builder *jsonbuilder.Stream, pStart uintptr, row int, precision int, timeFormat FormatTimeFunc) {
value := *((*int64)(unsafe.Pointer(pStart + uintptr(row)*8)))
value := *((*int64)(unsafe.Pointer(pStart + uintptr(row)*wrapper.Int64Size)))
timeFormat(builder, value, precision)
}

View file

@ -8,11 +8,37 @@ import (
"github.com/stretchr/testify/assert"
"github.com/taosdata/driver-go/v3/common"
"github.com/taosdata/driver-go/v3/wrapper"
"github.com/taosdata/taosadapter/v3/tools/jsonbuilder"
)
func TestJsonWriteRawBlock(t *testing.T) {
raw := []byte{143, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 8, 0, 0, 0, 1, 0, 1, 0, 0, 0, 2, 0, 1, 0, 0, 0, 3, 0, 2, 0, 0, 0, 4, 0, 4, 0, 0, 0, 5, 0, 8, 0, 0, 0, 11, 0, 1, 0, 0, 0, 12, 0, 2, 0, 0, 0, 13, 0, 4, 0, 0, 0, 14, 0, 8, 0, 0, 0, 6, 0, 4, 0, 0, 0, 7, 0, 8, 0, 0, 0, 8, 0, 22, 0, 0, 0, 10, 0, 82, 0, 0, 0, 15, 0, 0, 64, 0, 0, 16, 0, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0, 4, 0, 0, 0, 8, 0, 0, 0, 16, 0, 0, 0, 2, 0, 0, 0, 4, 0, 0, 0, 8, 0, 0, 0, 16, 0, 0, 0, 8, 0, 0, 0, 16, 0, 0, 0, 13, 0, 0, 0, 42, 0, 0, 0, 18, 0, 0, 0, 0, 214, 138, 67, 209, 129, 1, 0, 0, 190, 142, 67, 209, 129, 1, 0, 0, 64, 1, 0, 64, 1, 0, 64, 1, 0, 0, 0, 64, 1, 0, 0, 0, 0, 0, 0, 0, 64, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 1, 0, 64, 1, 0, 0, 0, 64, 1, 0, 0, 0, 0, 0, 0, 0, 64, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 128, 63, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 240, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 255, 255, 255, 11, 0, 116, 101, 115, 116, 95, 98, 105, 110, 97, 114, 121, 0, 0, 0, 0, 255, 255, 255, 255, 40, 0, 116, 0, 0, 0, 101, 0, 0, 0, 115, 0, 0, 0, 116, 0, 0, 0, 95, 0, 0, 0, 110, 0, 0, 0, 99, 0, 0, 0, 104, 0, 0, 0, 97, 0, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 7, 0, 123, 34, 97, 34, 58, 49, 125, 7, 0, 123, 34, 97, 34, 58, 49, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}
raw := []byte{
0x01, 0x00, 0x00, 0x00, 0x77, 0x01, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x0f, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, 0x08, 0x00, 0x00,
0x00, 0x01, 0x01, 0x00, 0x00, 0x00, 0x02, 0x01, 0x00, 0x00, 0x00, 0x03, 0x02, 0x00, 0x00, 0x00,
0x04, 0x04, 0x00, 0x00, 0x00, 0x05, 0x08, 0x00, 0x00, 0x00, 0x0b, 0x01, 0x00, 0x00, 0x00, 0x0c,
0x02, 0x00, 0x00, 0x00, 0x0d, 0x04, 0x00, 0x00, 0x00, 0x0e, 0x08, 0x00, 0x00, 0x00, 0x06, 0x04,
0x00, 0x00, 0x00, 0x07, 0x08, 0x00, 0x00, 0x00, 0x08, 0x16, 0x00, 0x00, 0x00, 0x0a, 0x52, 0x00,
0x00, 0x00, 0x0f, 0x00, 0x40, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x02,
0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x02,
0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x08,
0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x16, 0x00, 0x00, 0x00, 0x12,
0x00, 0x00, 0x00, 0x00, 0x74, 0x00, 0x90, 0x86, 0x82, 0x01, 0x00, 0x00, 0x5c, 0x04, 0x90, 0x86,
0x82, 0x01, 0x00, 0x00, 0x40, 0x01, 0x00, 0x40, 0x02, 0x00, 0x40, 0x03, 0x00, 0x00, 0x00, 0x40,
0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x06, 0x00, 0x40, 0x07, 0x00, 0x00,
0x00, 0x40, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x09, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x20, 0x41,
0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x26, 0x40, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x06, 0x00, 0x62,
0x69, 0x6e, 0x61, 0x72, 0x79, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x14, 0x00, 0x6e,
0x00, 0x00, 0x00, 0x63, 0x00, 0x00, 0x00, 0x68, 0x00, 0x00, 0x00, 0x61, 0x00, 0x00, 0x00, 0x72,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0x07, 0x00, 0x7b, 0x22, 0x61,
0x22, 0x3a, 0x31, 0x7d, 0x07, 0x00, 0x7b, 0x22, 0x61, 0x22, 0x3a, 0x31, 0x7d, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
}
w := &strings.Builder{}
builder := jsonbuilder.BorrowStream(w)
defer jsonbuilder.ReturnStream(builder)
@ -20,15 +46,15 @@ func TestJsonWriteRawBlock(t *testing.T) {
fieldTypes := []uint8{9, 1, 2, 3, 4, 5, 11, 12, 13, 14, 6, 7, 8, 10, 15}
blockSize := 2
precision := 0
payloadOffset := uintptr(4 * fieldsCount)
pHeaderList := make([]uintptr, fieldsCount)
pStartList := make([]uintptr, fieldsCount)
nullBitMapOffset := uintptr(BitmapLen(blockSize))
block := unsafe.Pointer(*(*uintptr)(unsafe.Pointer(&raw)))
tmpPHeader := uintptr(block) + payloadOffset + 12 + uintptr(6*fieldsCount) // length i32, group u64
lengthOffset := wrapper.RawBlockGetColumnLengthOffset(fieldsCount)
tmpPHeader := uintptr(block) + wrapper.RawBlockGetColDataOffset(fieldsCount)
tmpPStart := tmpPHeader
for column := 0; column < fieldsCount; column++ {
colLength := *((*int32)(unsafe.Pointer(uintptr(block) + 12 + uintptr(6*fieldsCount) + uintptr(column)*4)))
colLength := *((*int32)(unsafe.Pointer(uintptr(block) + lengthOffset + uintptr(column)*wrapper.Int32Size)))
if IsVarDataType(fieldTypes[column]) {
pHeaderList[column] = tmpPHeader
tmpPStart = tmpPHeader + uintptr(4*blockSize)
@ -73,5 +99,5 @@ func TestJsonWriteRawBlock(t *testing.T) {
builder.WriteObjectEnd()
err := builder.Flush()
assert.NoError(t, err)
assert.Equal(t, "{[\"2022-07-06T02:07:53.558Z\",true,1,1,1,1,1,1,1,1,1,1,\"test_binary\",\"test_nchar\",{\"a\":1}],[\"2022-07-06T02:07:54.558Z\",null,null,null,null,null,null,null,null,null,null,null,null,null,{\"a\":1}]}", w.String())
assert.Equal(t, "{[\"2022-08-10T07:02:40.5Z\",true,2,3,4,5,6,7,8,9,10,11,\"binary\",\"nchar\",{\"a\":1}],[\"2022-08-10T07:02:41.5Z\",null,null,null,null,null,null,null,null,null,null,null,null,null,{\"a\":1}]}", w.String())
}

View file

@ -4,6 +4,7 @@ import (
"database/sql/driver"
"reflect"
"testing"
"time"
"github.com/taosdata/driver-go/v3/common"
)
@ -22,16 +23,61 @@ func TestParseBlock(t *testing.T) {
want1 [][]driver.Value
}{
{
name: "",
name: "raw",
args: args{
data: []byte{2, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4, 0, 0, 0, 52, 0, 0, 0, 0, 0, 100, 0, 0, 0, 100, 0, 0, 0, 100, 0, 0, 0, 100, 0, 0, 0, 100, 0, 0, 0, 100, 0, 0, 0, 100, 0, 0, 0, 100, 0, 0, 0, 100, 0, 0, 0, 100, 0, 0, 0, 100, 0, 0, 0, 100, 0, 0, 0, 100, 0, 0, 0},
colTypes: []uint8{common.TSDB_DATA_TYPE_INT},
rows: 13,
data: []byte{
0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x01, 0x00, 0x00, 0x00, 0x98, 0x01, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, 0x08, 0x00, 0x00,
0x00, 0x01, 0x01, 0x00, 0x00, 0x00, 0x02, 0x01, 0x00, 0x00, 0x00, 0x03, 0x02, 0x00, 0x00, 0x00,
0x04, 0x04, 0x00, 0x00, 0x00, 0x05, 0x08, 0x00, 0x00, 0x00, 0x0b, 0x01, 0x00, 0x00, 0x00, 0x0c,
0x02, 0x00, 0x00, 0x00, 0x0d, 0x04, 0x00, 0x00, 0x00, 0x0e, 0x08, 0x00, 0x00, 0x00, 0x06, 0x04,
0x00, 0x00, 0x00, 0x07, 0x08, 0x00, 0x00, 0x00, 0x08, 0x16, 0x00, 0x00, 0x00, 0x0a, 0x52, 0x00,
0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x06, 0x00,
0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x06, 0x00,
0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x18, 0x00,
0x00, 0x00, 0x11, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, 0x00, 0x2c, 0x5b, 0x70, 0x86, 0x82,
0x01, 0x00, 0x00, 0x14, 0x5f, 0x70, 0x86, 0x82, 0x01, 0x00, 0x00, 0xfc, 0x62, 0x70, 0x86, 0x82,
0x01, 0x00, 0x00, 0x20, 0x01, 0x00, 0x00, 0x20, 0x02, 0x16, 0x00, 0x20, 0x03, 0x00, 0x21, 0x00,
0x00, 0x00, 0x20, 0x04, 0x00, 0x00, 0x00, 0x2c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20,
0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x37, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x06, 0x42, 0x00, 0x20, 0x07, 0x00, 0x4d,
0x00, 0x00, 0x00, 0x20, 0x08, 0x00, 0x00, 0x00, 0x58, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x20, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x63, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x20, 0x41, 0x00, 0x80,
0x7c, 0x44, 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x26, 0x40, 0x00,
0x00, 0x00, 0x00, 0x00, 0x5c, 0x91, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x06, 0x00, 0x62, 0x69, 0x6e,
0x61, 0x72, 0x79, 0x07, 0x00, 0x62, 0x69, 0x6e, 0x61, 0x72, 0x79, 0x32, 0x00, 0x00, 0x00, 0x00,
0x16, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x14, 0x00, 0x6e, 0x00, 0x00, 0x00, 0x63, 0x00,
0x00, 0x00, 0x68, 0x00, 0x00, 0x00, 0x61, 0x00, 0x00, 0x00, 0x72, 0x00, 0x00, 0x00, 0x18, 0x00,
0x6e, 0x00, 0x00, 0x00, 0x63, 0x00, 0x00, 0x00, 0x68, 0x00, 0x00, 0x00, 0x61, 0x00, 0x00, 0x00,
0x72, 0x00, 0x00, 0x00, 0x32, 0x00, 0x00, 0x00,
},
colTypes: []uint8{
common.TSDB_DATA_TYPE_TIMESTAMP,
common.TSDB_DATA_TYPE_BOOL,
common.TSDB_DATA_TYPE_TINYINT,
common.TSDB_DATA_TYPE_SMALLINT,
common.TSDB_DATA_TYPE_INT,
common.TSDB_DATA_TYPE_BIGINT,
common.TSDB_DATA_TYPE_UTINYINT,
common.TSDB_DATA_TYPE_USMALLINT,
common.TSDB_DATA_TYPE_UINT,
common.TSDB_DATA_TYPE_UBIGINT,
common.TSDB_DATA_TYPE_FLOAT,
common.TSDB_DATA_TYPE_DOUBLE,
common.TSDB_DATA_TYPE_BINARY,
common.TSDB_DATA_TYPE_NCHAR,
},
rows: 3,
precision: 0,
},
want: 2,
want: 1,
want1: [][]driver.Value{
{int32(100)}, {int32(100)}, {int32(100)}, {int32(100)}, {int32(100)}, {int32(100)}, {int32(100)}, {int32(100)}, {int32(100)}, {int32(100)}, {int32(100)}, {int32(100)}, {int32(100)},
{time.Unix(0, 1660112886572*1e6).Local(), true, int8(2), int16(3), int32(4), int64(5), uint8(6), uint16(7), uint32(8), uint64(9), float32(10), float64(11), "binary", "nchar"},
{time.Unix(0, 1660112887572*1e6).Local(), false, int8(22), int16(33), int32(44), int64(55), uint8(66), uint16(77), uint32(88), uint64(99), float32(1010), float64(1111), "binary2", "nchar2"},
{time.Unix(0, 1660112888572*1e6).Local(), nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil},
},
},
}
@ -65,15 +111,21 @@ func TestParseTmqBlock(t *testing.T) {
{
name: "",
args: args{
data: []byte{2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4, 0, 0, 0, 52, 0, 0, 0, 0, 0, 100, 0, 0, 0, 100, 0, 0, 0, 100, 0, 0, 0, 100, 0, 0, 0, 100, 0, 0, 0, 100, 0, 0, 0, 100, 0, 0, 0, 100, 0, 0, 0, 100, 0, 0, 0, 100, 0, 0, 0, 100, 0, 0, 0, 100, 0, 0, 0, 100, 0, 0, 0},
colTypes: []uint8{common.TSDB_DATA_TYPE_INT},
rows: 13,
data: []byte{
0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x01, 0x00, 0x00, 0x00, 0x3c, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, 0x08, 0x00, 0x00,
0x00, 0x04, 0x04, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0xe7,
0x43, 0x9e, 0x86, 0x82, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,
},
colTypes: []uint8{common.TSDB_DATA_TYPE_TIMESTAMP, common.TSDB_DATA_TYPE_INT},
rows: 1,
precision: 0,
},
id: 2,
messageID: 1,
want1: [][]driver.Value{
{int32(100)}, {int32(100)}, {int32(100)}, {int32(100)}, {int32(100)}, {int32(100)}, {int32(100)}, {int32(100)}, {int32(100)}, {int32(100)}, {int32(100)}, {int32(100)}, {int32(100)},
{time.Unix(0, 1660115895271*1e6).Local(), int32(1)},
},
},
}