diff --git a/include/common/tcol.h b/include/common/tcol.h index 1c0306c6728..e85d268d39d 100644 --- a/include/common/tcol.h +++ b/include/common/tcol.h @@ -81,6 +81,7 @@ uint8_t columnLevelVal(const char* level); uint8_t columnEncodeVal(const char* encode); uint16_t columnCompressVal(const char* compress); +bool withColCompress(uint8_t tableType); bool withExtSchema(uint8_t tableType); bool hasRefCol(uint8_t tableType); bool checkColumnEncode(char encode[TSDB_CL_COMPRESS_OPTION_LEN]); diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index f851522e247..08445da093b 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -28,9 +28,6 @@ extern "C" { #include "ttypes.h" #include "streamMsg.h" -#define TABLE_TOTAL_COL_NUM(pMeta) ((pMeta)->tableInfo.numOfColumns + (pMeta)->tableInfo.numOfTags) -#define TABLE_META_SIZE(pMeta) \ - (NULL == (pMeta) ? 0 : (sizeof(STableMeta) + TABLE_TOTAL_COL_NUM((pMeta)) * sizeof(SSchema) + (pMeta)->numOfColRefs * sizeof(SColRef))) #define VGROUPS_INFO_SIZE(pInfo) \ (NULL == (pInfo) ? 0 : (sizeof(SVgroupsInfo) + (pInfo)->numOfVgroups * sizeof(SVgroupInfo))) diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 6a2fb79f7d9..196ea8f0fbe 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -24,6 +24,7 @@ extern "C" { #include "systable.h" #include "tarray.h" #include "thash.h" +#include "tcol.h" #include "tlog.h" #include "tmsg.h" #include "tmsgcb.h" @@ -160,6 +161,45 @@ typedef struct STableMeta { } STableMeta; #pragma pack(pop) +#define TABLE_TOTAL_COL_NUM(pMeta) ((pMeta)->tableInfo.numOfColumns + (pMeta)->tableInfo.numOfTags) + +#define TABLE_META_BASE_SIZE(pMeta) \ + (NULL == (pMeta) ? 0 : (sizeof(STableMeta) + TABLE_TOTAL_COL_NUM((pMeta)) * sizeof(SSchema))) + +#define TABLE_META_SCHEMA_EXT_SIZE(pMeta) \ + ((withExtSchema((pMeta)->tableType) && NULL != (pMeta)->schemaExt) ? (pMeta)->tableInfo.numOfColumns * sizeof(SSchemaExt) : 0) + +#define TABLE_META_COL_REF_SIZE(pMeta) \ + ((hasRefCol((pMeta)->tableType) && NULL != (pMeta)->colRef) ? (pMeta)->numOfColRefs * sizeof(SColRef) : 0) + +#define TABLE_META_FULL_SIZE(pMeta) \ + (NULL == (pMeta) ? 0 : (TABLE_META_BASE_SIZE((pMeta)) + TABLE_META_SCHEMA_EXT_SIZE((pMeta)) + TABLE_META_COL_REF_SIZE((pMeta)))) + +static inline void tableMetaResetPointers(STableMeta *pMeta) { + if (NULL == pMeta) { + return; + } + + char *pCursor = (char *)pMeta + TABLE_META_BASE_SIZE(pMeta); + + if (withExtSchema(pMeta->tableType) && NULL != pMeta->schemaExt) { + pMeta->schemaExt = (SSchemaExt *)pCursor; + pCursor += pMeta->tableInfo.numOfColumns * sizeof(SSchemaExt); + } else { + pMeta->schemaExt = NULL; + } + + if (hasRefCol(pMeta->tableType) && NULL != pMeta->colRef) { + pMeta->colRef = (SColRef *)pCursor; + pCursor += pMeta->numOfColRefs * sizeof(SColRef); + } else { + pMeta->colRef = NULL; + } + + pMeta->tagRef = NULL; + pMeta->numOfTagRefs = 0; +} + typedef struct SViewMeta { uint64_t viewId; int64_t ownerId; @@ -425,6 +465,10 @@ void initQueryModuleMsgHandle(); const SSchema* tGetTbnameColumnSchema(); bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_t numOfTags, bool isVirtual); int32_t getAsofJoinReverseOp(EOperatorType op); +bool hasDecimalBytesTypeInfo(int32_t bytes); +void schemaToRefDataType(const SSchema* pSchema, STypeMod typeMod, SDataType* pType); +bool isSameRefDataType(const SDataType* pLeft, const SDataType* pRight); +int32_t getNormalColSchemaIndex(const STableMeta* pTableMeta, const char* pColName); int32_t queryCreateCTableMetaFromMsg(STableMetaRsp* msg, SCTableMeta* pMeta); int32_t queryCreateVCTableMetaFromMsg(STableMetaRsp *msg, SVCTableMeta **pMeta); diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index 89130acda39..09a74c41b2f 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -1234,7 +1234,8 @@ static STableMeta* getTableMeta(SCatalog* pCatalog, SRequestConnInfo* conn, char return pTableMeta; } -static int32_t checkColRef(STableMeta* pTableMeta, char* colName, uint8_t precision, const SSchema* pSchema) { +static int32_t checkColRef(STableMeta* pTableMeta, char* colName, uint8_t precision, const char* pSchemaName, + const SDataType* pType) { int32_t code = TSDB_CODE_SUCCESS; if (pTableMeta->tableInfo.precision != precision) { code = TSDB_CODE_PAR_INVALID_REF_COLUMN_TYPE; @@ -1258,14 +1259,22 @@ static int32_t checkColRef(STableMeta* pTableMeta, char* colName, uint8_t precis const SSchema* pRefCol = getNormalColSchema(pTableMeta, colName); if (NULL == pRefCol) { code = TSDB_CODE_PAR_INVALID_REF_COLUMN; - uError("virtual table's column:\"%s\"'s reference column:\"%s\" not exist", pSchema->name, colName); + uError("virtual table's column:\"%s\"'s reference column:\"%s\" not exist", pSchemaName, colName); goto end; } - if (pRefCol->type != pSchema->type || pRefCol->bytes != pSchema->bytes) { + int32_t refColIndex = getNormalColSchemaIndex(pTableMeta, colName); + const SSchemaExt* pRefSchemaExt = + (refColIndex >= 0 && pTableMeta->schemaExt && refColIndex < pTableMeta->tableInfo.numOfColumns) + ? pTableMeta->schemaExt + refColIndex + : NULL; + SDataType refType = {0}; + schemaToRefDataType(pRefCol, NULL != pRefSchemaExt ? pRefSchemaExt->typeMod : 0, &refType); + + if (!isSameRefDataType(pType, &refType)) { code = TSDB_CODE_PAR_INVALID_REF_COLUMN_TYPE; uError("virtual table's column:\"%s\"'s type and reference column:\"%s\"'s type not match, %d %d %d %d", - pSchema->name, colName, pSchema->type, pSchema->bytes, pRefCol->type, pRefCol->bytes); + pSchemaName, colName, pType->type, pType->bytes, refType.type, refType.bytes); goto end; } @@ -1273,18 +1282,21 @@ end: return code; } -static int32_t checkColRefForCreate(SCatalog* pCatalog, SRequestConnInfo* conn, SColRef* pColRef, int32_t acctId, uint8_t precision, SSchema* pSchema) { +static int32_t checkColRefForCreate(SCatalog* pCatalog, SRequestConnInfo* conn, SColRef* pColRef, int32_t acctId, + uint8_t precision, SSchema* pSchema, const SSchemaExt* pSchemaExt) { STableMeta* pTableMeta = getTableMeta(pCatalog, conn, pColRef->refDbName, pColRef->refTableName, acctId); if (pTableMeta == NULL) { return terrno; } - int32_t code = checkColRef(pTableMeta, pColRef->refColName, precision, pSchema); + SDataType colType = {0}; + schemaToRefDataType(pSchema, NULL != pSchemaExt ? pSchemaExt->typeMod : 0, &colType); + int32_t code = checkColRef(pTableMeta, pColRef->refColName, precision, pSchema->name, &colType); taosMemoryFreeClear(pTableMeta); return code; } static int32_t checkColRefForAdd(SCatalog* pCatalog, SRequestConnInfo* conn, int32_t acctId, char* dbName, char* tbName, char* colName, - char* dbNameSrc, char* tbNameSrc, char* colNameSrc, int8_t type, int32_t bytes) { + char* dbNameSrc, char* tbNameSrc, char* colNameSrc, int8_t type, int32_t bytes, STypeMod typeMod) { int32_t code = 0; STableMeta* pTableMeta = getTableMeta(pCatalog, conn, dbName, tbName, acctId); if (pTableMeta == NULL) { @@ -1297,9 +1309,9 @@ static int32_t checkColRefForAdd(SCatalog* pCatalog, SRequestConnInfo* conn, int goto end; } - SSchema pSchema = {.type = type, .bytes = bytes}; - tstrncpy(pSchema.name, colNameSrc, TSDB_COL_NAME_LEN); - code = checkColRef(pTableMeta, colName, pTableMetaSrc->tableInfo.precision, &pSchema); + SDataType colType = {.type = type, .bytes = bytes}; + fillTypeFromTypeMod(&colType, typeMod); + code = checkColRef(pTableMeta, colName, pTableMetaSrc->tableInfo.precision, colNameSrc, &colType); end: taosMemoryFreeClear(pTableMeta); @@ -1327,7 +1339,14 @@ static int32_t checkColRefForAlter(SCatalog* pCatalog, SRequestConnInfo* conn, i goto end; } - code = checkColRef(pTableMeta, colName, pTableMetaSrc->tableInfo.precision, pSchema); + int32_t schemaIdx = getNormalColSchemaIndex(pTableMetaSrc, colNameSrc); + const SSchemaExt* pSchemaExt = + (schemaIdx >= 0 && pTableMetaSrc->schemaExt && schemaIdx < pTableMetaSrc->tableInfo.numOfColumns) + ? pTableMetaSrc->schemaExt + schemaIdx + : NULL; + SDataType colType = {0}; + schemaToRefDataType(pSchema, NULL != pSchemaExt ? pSchemaExt->typeMod : 0, &colType); + code = checkColRef(pTableMeta, colName, pTableMetaSrc->tableInfo.precision, pSchema->name, &colType); end: taosMemoryFreeClear(pTableMeta); @@ -1420,7 +1439,11 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, uint32_t metaLen) { SColRef* pColRef = pCreateReq->colRef.pColRef + i; if (!pColRef || !pColRef->hasRef) continue; SSchema* pSchema = pTableMeta->schema + i; - RAW_RETURN_CHECK(checkColRefForCreate(pCatalog, &conn, pColRef, pTscObj->acctId, pTableMeta->tableInfo.precision, pSchema)); + const SSchemaExt* pSchemaExt = + (pTableMeta->schemaExt && i < pTableMeta->tableInfo.numOfColumns) ? pTableMeta->schemaExt + i : NULL; + RAW_RETURN_CHECK( + checkColRefForCreate(pCatalog, &conn, pColRef, pTscObj->acctId, pTableMeta->tableInfo.precision, pSchema, + pSchemaExt)); } SArray* pTagVals = NULL; @@ -1942,7 +1965,7 @@ static int32_t taosAlterTable(TAOS* taos, void* meta, uint32_t metaLen) { pRequest->pDb, req.tbName, req.colName)); }else if (req.action == TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COLUMN_REF && tmqWriteCheckRef) { RAW_RETURN_CHECK(checkColRefForAdd(pCatalog, &conn, pTscObj->acctId, req.refDbName, req.refTbName, req.refColName, - pRequest->pDb, req.tbName, req.colName, req.type, req.bytes)); + pRequest->pDb, req.tbName, req.colName, req.type, req.bytes, req.typeMod)); } tEncodeSize(tEncodeSVAlterTbReq, &req, tlen, code); diff --git a/source/common/src/msg/tmsg.c b/source/common/src/msg/tmsg.c index cbe796e8a97..6a1ca9564fd 100644 --- a/source/common/src/msg/tmsg.c +++ b/source/common/src/msg/tmsg.c @@ -15475,7 +15475,9 @@ int32_t tEncodeSVAlterTbReq(SEncoder *pEncoder, const SVAlterTbReq *pReq) { } TAOS_CHECK_EXIT(tEncodeI64(pEncoder, pReq->ctimeMs)); TAOS_CHECK_EXIT(tEncodeI8(pEncoder, pReq->source)); - if (pReq->action == TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION || pReq->action == TSDB_ALTER_TABLE_ADD_COLUMN) { + if (pReq->action == TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION || + pReq->action == TSDB_ALTER_TABLE_ADD_COLUMN || + pReq->action == TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COLUMN_REF) { TAOS_CHECK_EXIT(tEncodeI32(pEncoder, pReq->typeMod)); } @@ -15788,7 +15790,9 @@ int32_t tDecodeSVAlterTbReq(SDecoder *pDecoder, SVAlterTbReq *pReq) { if (!tDecodeIsEnd(pDecoder)) { TAOS_CHECK_EXIT(tDecodeI8(pDecoder, &pReq->source)); } - if (pReq->action == TSDB_ALTER_TABLE_ADD_COLUMN || pReq->action == TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION) { + if (pReq->action == TSDB_ALTER_TABLE_ADD_COLUMN || + pReq->action == TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION || + pReq->action == TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COLUMN_REF) { if (!tDecodeIsEnd(pDecoder)) { TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pReq->typeMod)); } diff --git a/source/common/src/tcol.c b/source/common/src/tcol.c index db424ffd88f..a629b46f2b3 100644 --- a/source/common/src/tcol.c +++ b/source/common/src/tcol.c @@ -358,10 +358,15 @@ int32_t setColCompressByOption(uint8_t type, uint8_t encode, uint16_t compressTy return TSDB_CODE_SUCCESS; } -bool withExtSchema(uint8_t tableType) { +bool withColCompress(uint8_t tableType) { return TSDB_SUPER_TABLE == tableType || TSDB_NORMAL_TABLE == tableType || TSDB_CHILD_TABLE == tableType; } +bool withExtSchema(uint8_t tableType) { + return TSDB_SUPER_TABLE == tableType || TSDB_NORMAL_TABLE == tableType || TSDB_CHILD_TABLE == tableType || + TSDB_VIRTUAL_NORMAL_TABLE == tableType || TSDB_VIRTUAL_CHILD_TABLE == tableType; +} + bool hasRefCol(uint8_t tableType) { return TSDB_VIRTUAL_NORMAL_TABLE == tableType || TSDB_VIRTUAL_CHILD_TABLE == tableType; } diff --git a/source/dnode/vnode/src/meta/metaEntry.c b/source/dnode/vnode/src/meta/metaEntry.c index d3abca2954f..cabf4297a31 100644 --- a/source/dnode/vnode/src/meta/metaEntry.c +++ b/source/dnode/vnode/src/meta/metaEntry.c @@ -28,22 +28,38 @@ static bool schemasHasTypeMod(const SSchema *pSchema, int32_t nCols) { return false; } -static int32_t metaEncodeExtSchema(SEncoder* pCoder, const SMetaEntry* pME) { - if (pME->pExtSchemas) { - const SSchemaWrapper *pSchWrapper = NULL; - bool hasTypeMods = false; - if (pME->type == TSDB_SUPER_TABLE) { - pSchWrapper = &pME->stbEntry.schemaRow; - } else if (pME->type == TSDB_NORMAL_TABLE) { - pSchWrapper = &pME->ntbEntry.schemaRow; - } else { - return 0; - } - hasTypeMods = schemasHasTypeMod(pSchWrapper->pSchema, pSchWrapper->nCols); +static const SSchemaWrapper *metaGetEntryRowSchema(const SMetaEntry *pME) { + if (pME == NULL) { + return NULL; + } - for (int32_t i = 0; i < pSchWrapper->nCols && hasTypeMods; ++i) { - TAOS_CHECK_RETURN(tEncodeI32v(pCoder, pME->pExtSchemas[i].typeMod)); - } + if (pME->type == TSDB_SUPER_TABLE) { + return &pME->stbEntry.schemaRow; + } + + if (pME->type == TSDB_NORMAL_TABLE || pME->type == TSDB_VIRTUAL_NORMAL_TABLE) { + return &pME->ntbEntry.schemaRow; + } + + return NULL; +} + +static int32_t metaGetEntryRowSchemaNum(const SMetaEntry *pME) { + const SSchemaWrapper *pSchema = metaGetEntryRowSchema(pME); + return pSchema == NULL ? 0 : pSchema->nCols; +} + +static int32_t metaEncodeExtSchema(SEncoder* pCoder, const SMetaEntry* pME) { + const SSchemaWrapper *pSchWrapper = metaGetEntryRowSchema(pME); + bool hasTypeMods = false; + + if (pME->pExtSchemas == NULL || pSchWrapper == NULL) { + return 0; + } + + hasTypeMods = schemasHasTypeMod(pSchWrapper->pSchema, pSchWrapper->nCols); + for (int32_t i = 0; i < pSchWrapper->nCols && hasTypeMods; ++i) { + TAOS_CHECK_RETURN(tEncodeI32v(pCoder, pME->pExtSchemas[i].typeMod)); } return 0; } @@ -82,13 +98,10 @@ int meteEncodeColRefEntry(SEncoder *pCoder, const SMetaEntry *pME) { } static int32_t metaDecodeExtSchemas(SDecoder* pDecoder, SMetaEntry* pME) { - bool hasExtSchema = false; - SSchemaWrapper* pSchWrapper = NULL; - if (pME->type == TSDB_SUPER_TABLE) { - pSchWrapper = &pME->stbEntry.schemaRow; - } else if (pME->type == TSDB_NORMAL_TABLE) { - pSchWrapper = &pME->ntbEntry.schemaRow; - } else { + bool hasExtSchema = false; + const SSchemaWrapper* pSchWrapper = metaGetEntryRowSchema((const SMetaEntry*)pME); + + if (pSchWrapper == NULL) { return 0; } @@ -110,15 +123,13 @@ static int32_t metaDecodeExtSchemas(SDecoder* pDecoder, SMetaEntry* pME) { } SExtSchema* metaGetSExtSchema(const SMetaEntry *pME) { - const SSchemaWrapper *pSchWrapper = NULL; + const SSchemaWrapper *pSchWrapper = metaGetEntryRowSchema(pME); bool hasTypeMods = false; - if (pME->type == TSDB_SUPER_TABLE) { - pSchWrapper = &pME->stbEntry.schemaRow; - } else if (pME->type == TSDB_NORMAL_TABLE) { - pSchWrapper = &pME->ntbEntry.schemaRow; - } else { + + if (pSchWrapper == NULL) { return NULL; } + hasTypeMods = schemasHasTypeMod(pSchWrapper->pSchema, pSchWrapper->nCols); if (hasTypeMods) { @@ -761,14 +772,15 @@ int32_t metaCloneEntry(const SMetaEntry *pEntry, SMetaEntry **ppEntry) { return code; } } - if (pEntry->pExtSchemas && pEntry->colCmpr.nCols > 0) { - (*ppEntry)->pExtSchemas = taosMemoryCalloc(pEntry->colCmpr.nCols, sizeof(SExtSchema)); + int32_t numOfExtSchema = metaGetEntryRowSchemaNum(pEntry); + if (pEntry->pExtSchemas && numOfExtSchema > 0) { + (*ppEntry)->pExtSchemas = taosMemoryCalloc(numOfExtSchema, sizeof(SExtSchema)); if (!(*ppEntry)->pExtSchemas) { code = terrno; metaCloneEntryFree(ppEntry); return code; } - memcpy((*ppEntry)->pExtSchemas, pEntry->pExtSchemas, sizeof(SExtSchema) * pEntry->colCmpr.nCols); + memcpy((*ppEntry)->pExtSchemas, pEntry->pExtSchemas, sizeof(SExtSchema) * numOfExtSchema); } return code; diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index df105a03d79..35c7c0b634b 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -172,10 +172,27 @@ int metaUpdateMetaRsp(tb_uid_t uid, char *tbName, SSchemaWrapper *pSchema, int64 return 0; } -int32_t metaUpdateVtbMetaRsp(SMetaEntry *pEntry, char *tbName, SSchemaWrapper *pSchema, SColRefWrapper *pRef, - int64_t ownerId, STableMetaRsp *pMetaRsp, int8_t tableType) { - int32_t code = TSDB_CODE_SUCCESS; - SHashObj* pColRefHash = NULL; +static int32_t metaFillRspSchemaExt(const SSchemaWrapper *pSchema, const SExtSchema *pExtSchemas, SSchemaExt *pSchemaExt) { + if (pSchema == NULL || pSchemaExt == NULL) { + return TSDB_CODE_SUCCESS; + } + + for (int32_t i = 0; i < pSchema->nCols; ++i) { + pSchemaExt[i].colId = pSchema->pSchema[i].colId; + } + + if (pExtSchemas != NULL) { + for (int32_t i = 0; i < pSchema->nCols; ++i) { + pSchemaExt[i].typeMod = pExtSchemas[i].typeMod; + } + } + + return TSDB_CODE_SUCCESS; +} + +int32_t metaUpdateVtbMetaRsp(SMetaEntry *pEntry, char *tbName, const SSchemaWrapper *pSchema, const SColRefWrapper *pRef, + const SExtSchema *pExtSchemas, int64_t ownerId, STableMetaRsp *pMetaRsp, int8_t tableType) { + int32_t code = TSDB_CODE_SUCCESS; if (!pRef) { return TSDB_CODE_INVALID_PARA; } @@ -186,7 +203,7 @@ int32_t metaUpdateVtbMetaRsp(SMetaEntry *pEntry, char *tbName, SSchemaWrapper *p goto _return; } - pMetaRsp->pSchemaExt = taosMemoryMalloc(pSchema->nCols * sizeof(SSchemaExt)); + pMetaRsp->pSchemaExt = taosMemoryCalloc(pSchema->nCols, sizeof(SSchemaExt)); if (pMetaRsp->pSchemaExt == NULL) { code = terrno; goto _return; @@ -195,30 +212,25 @@ int32_t metaUpdateVtbMetaRsp(SMetaEntry *pEntry, char *tbName, SSchemaWrapper *p pMetaRsp->numOfColumns = pSchema->nCols; pMetaRsp->sversion = pSchema->version; memcpy(pMetaRsp->pSchemas, pSchema->pSchema, pSchema->nCols * sizeof(SSchema)); - } - pMetaRsp->pColRefs = taosMemoryMalloc(pRef->nCols * sizeof(SColRef)); - if (NULL == pMetaRsp->pColRefs) { - code = terrno; - goto _return; - } - pColRefHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); - if (!pColRefHash) { - code = terrno; - goto _return; - } - - for (int32_t i = 0; i < pRef->nCols; i++) { - SColRef* pColRef = &pRef->pColRef[i]; - if (pColRef->hasRef) { - code = taosHashPut(pColRefHash, pColRef->refTableName, strlen(pColRef->refTableName), NULL, 0); - if (code) { - goto _return; - } + code = metaFillRspSchemaExt(pSchema, pExtSchemas, pMetaRsp->pSchemaExt); + if (code != TSDB_CODE_SUCCESS) { + goto _return; } } - memcpy(pMetaRsp->pColRefs, pRef->pColRef, pRef->nCols * sizeof(SColRef)); + if (pRef->nCols > 0) { + pMetaRsp->pColRefs = taosMemoryMalloc(pRef->nCols * sizeof(SColRef)); + if (NULL == pMetaRsp->pColRefs) { + code = terrno; + goto _return; + } + + memcpy(pMetaRsp->pColRefs, pRef->pColRef, pRef->nCols * sizeof(SColRef)); + } else { + pMetaRsp->pColRefs = NULL; + } + tstrncpy(pMetaRsp->tbName, tbName, TSDB_TABLE_NAME_LEN); if (tableType == TSDB_VIRTUAL_NORMAL_TABLE) { pMetaRsp->tuid = pEntry->uid; @@ -247,10 +259,8 @@ int32_t metaUpdateVtbMetaRsp(SMetaEntry *pEntry, char *tbName, SSchemaWrapper *p pMetaRsp->numOfTagRefs = 0; } - taosHashCleanup(pColRefHash); return code; _return: - taosHashCleanup(pColRefHash); taosMemoryFreeClear(pMetaRsp->pSchemaExt); taosMemoryFreeClear(pMetaRsp->pSchemas); taosMemoryFreeClear(pMetaRsp->pColRefs); @@ -1000,7 +1010,7 @@ int32_t metaGetColCmpr(SMeta *pMeta, tb_uid_t uid, SHashObj **ppColCmprObj) { taosHashCleanup(pColCmprObj); return rc; } - if (withExtSchema(e.type)) { + if (withColCompress(e.type)) { SColCmprWrapper *p = &e.colCmpr; for (int32_t i = 0; i < p->nCols; i++) { SColCmpr *pCmpr = &p->pColCmpr[i]; diff --git a/source/dnode/vnode/src/meta/metaTable2.c b/source/dnode/vnode/src/meta/metaTable2.c index 4faa352f170..8084202b657 100644 --- a/source/dnode/vnode/src/meta/metaTable2.c +++ b/source/dnode/vnode/src/meta/metaTable2.c @@ -23,8 +23,10 @@ extern int32_t metaHandleEntry2(SMeta *pMeta, const SMetaEntry *pEntry); extern int32_t metaUpdateMetaRsp(tb_uid_t uid, char *tbName, SSchemaWrapper *pSchema, int64_t ownerId, STableMetaRsp *pMetaRsp); -extern int32_t metaUpdateVtbMetaRsp(SMetaEntry *pEntry, char *tbName, SSchemaWrapper *pSchema, SColRefWrapper *pRef, - int64_t ownerId, STableMetaRsp *pMetaRsp, int8_t tableType); +extern int32_t metaUpdateVtbMetaRsp(SMetaEntry *pEntry, char *tbName, const SSchemaWrapper *pSchema, + const SColRefWrapper *pRef, const SExtSchema *pExtSchemas, int64_t ownerId, + STableMetaRsp *pMetaRsp, + int8_t tableType); extern int32_t metaFetchEntryByUid(SMeta *pMeta, int64_t uid, SMetaEntry **ppEntry); extern int32_t metaFetchEntryByName(SMeta *pMeta, const char *name, SMetaEntry **ppEntry); extern void metaFetchEntryFree(SMetaEntry **ppEntry); @@ -541,8 +543,8 @@ static int32_t metaBuildCreateVirtualNormalTableRsp(SMeta *pMeta, SMetaEntry *pE return terrno; } - code = metaUpdateVtbMetaRsp(pEntry, pEntry->name, &pEntry->ntbEntry.schemaRow, &pEntry->colRef, pEntry->ntbEntry.ownerId, *ppRsp, - TSDB_VIRTUAL_NORMAL_TABLE); + code = metaUpdateVtbMetaRsp(pEntry, pEntry->name, &pEntry->ntbEntry.schemaRow, &pEntry->colRef, pEntry->pExtSchemas, + pEntry->ntbEntry.ownerId, *ppRsp, TSDB_VIRTUAL_NORMAL_TABLE); if (code) { taosMemoryFreeClear(*ppRsp); return code; @@ -573,6 +575,7 @@ static int32_t metaCreateVirtualNormalTable(SMeta *pMeta, int64_t version, SVCre .ntbEntry.schemaRow = pReq->ntb.schemaRow, .ntbEntry.ncid = pReq->ntb.schemaRow.pSchema[pReq->ntb.schemaRow.nCols - 1].colId + 1, .ntbEntry.ownerId = pReq->ntb.userId, + .pExtSchemas = pReq->pExtSchemas, .colRef = pReq->colRef}; code = metaBuildCreateVirtualNormalTableRsp(pMeta, &entry, ppRsp); @@ -598,7 +601,8 @@ static int32_t metaCreateVirtualNormalTable(SMeta *pMeta, int64_t version, SVCre } static int32_t metaBuildCreateVirtualChildTableRsp(SMeta *pMeta, SMetaEntry *pEntry, STableMetaRsp **ppRsp) { - int32_t code = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; + SMetaEntry *pSuper = NULL; if (NULL == ppRsp) { return code; @@ -609,12 +613,22 @@ static int32_t metaBuildCreateVirtualChildTableRsp(SMeta *pMeta, SMetaEntry *pEn return terrno; } - code = metaUpdateVtbMetaRsp(pEntry, pEntry->name, NULL, &pEntry->colRef, 0, *ppRsp, TSDB_VIRTUAL_CHILD_TABLE); - if (code) { + code = metaFetchEntryByUid(pMeta, pEntry->ctbEntry.suid, &pSuper); + if (code != TSDB_CODE_SUCCESS) { taosMemoryFreeClear(*ppRsp); return code; } + + code = metaUpdateVtbMetaRsp(pEntry, pEntry->name, &pSuper->stbEntry.schemaRow, &pEntry->colRef, pSuper->pExtSchemas, + pSuper->stbEntry.ownerId, *ppRsp, TSDB_VIRTUAL_CHILD_TABLE); + if (code) { + metaFetchEntryFree(&pSuper); + taosMemoryFreeClear(*ppRsp); + return code; + } + (*ppRsp)->suid = pEntry->ctbEntry.suid; + metaFetchEntryFree(&pSuper); return code; } @@ -918,8 +932,8 @@ int32_t metaAddTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, ST } if (pEntry->type == TSDB_VIRTUAL_NORMAL_TABLE) { - code = metaUpdateVtbMetaRsp(pEntry, pReq->tbName, pSchema, &pEntry->colRef, pEntry->ntbEntry.ownerId, pRsp, - pEntry->type); + code = metaUpdateVtbMetaRsp(pEntry, pReq->tbName, pSchema, &pEntry->colRef, pEntry->pExtSchemas, + pEntry->ntbEntry.ownerId, pRsp, pEntry->type); if (code) { metaError("vgId:%d, %s failed at %s:%d since %s, uid:%" PRId64 " name:%s version:%" PRId64, TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, tstrerror(code), pEntry->uid, pReq->tbName, version); @@ -1066,8 +1080,8 @@ int32_t metaDropTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, S // build response if (pEntry->type == TSDB_VIRTUAL_NORMAL_TABLE) { - code = metaUpdateVtbMetaRsp(pEntry, pReq->tbName, pSchema, &pEntry->colRef, pEntry->ntbEntry.ownerId, pRsp, - pEntry->type); + code = metaUpdateVtbMetaRsp(pEntry, pReq->tbName, pSchema, &pEntry->colRef, pEntry->pExtSchemas, + pEntry->ntbEntry.ownerId, pRsp, pEntry->type); if (code) { metaError("vgId:%d, %s failed at %s:%d since %s, uid:%" PRId64 " name:%s version:%" PRId64, TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, tstrerror(code), pEntry->uid, pReq->tbName, version); @@ -1171,8 +1185,8 @@ int32_t metaAlterTableColumnName(SMeta *pMeta, int64_t version, SVAlterTbReq *pR // build response if (pEntry->type == TSDB_VIRTUAL_NORMAL_TABLE) { - code = metaUpdateVtbMetaRsp(pEntry, pReq->tbName, pSchema, &pEntry->colRef, pEntry->ntbEntry.ownerId, pRsp, - pEntry->type); + code = metaUpdateVtbMetaRsp(pEntry, pReq->tbName, pSchema, &pEntry->colRef, pEntry->pExtSchemas, + pEntry->ntbEntry.ownerId, pRsp, pEntry->type); if (code) { metaError("vgId:%d, %s failed at %s:%d since %s, uid:%" PRId64 " name:%s version:%" PRId64, TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, tstrerror(code), pEntry->uid, pReq->tbName, version); @@ -1287,8 +1301,8 @@ int32_t metaAlterTableColumnBytes(SMeta *pMeta, int64_t version, SVAlterTbReq *p // build response if (pEntry->type == TSDB_VIRTUAL_NORMAL_TABLE) { - code = metaUpdateVtbMetaRsp(pEntry, pReq->tbName, pSchema, &pEntry->colRef, pEntry->ntbEntry.ownerId, pRsp, - pEntry->type); + code = metaUpdateVtbMetaRsp(pEntry, pReq->tbName, pSchema, &pEntry->colRef, pEntry->pExtSchemas, + pEntry->ntbEntry.ownerId, pRsp, pEntry->type); if (code) { metaError("vgId:%d, %s failed at %s:%d since %s, uid:%" PRId64 " name:%s version:%" PRId64, TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__, tstrerror(code), pEntry->uid, pReq->tbName, version); @@ -2600,6 +2614,7 @@ int32_t metaAlterTableColumnRef(SMeta *pMeta, int64_t version, SVAlterTbReq *pRe // build response code = metaUpdateVtbMetaRsp( pEntry, pReq->tbName, pSchema, &pEntry->colRef, + pEntry->type == TSDB_VIRTUAL_CHILD_TABLE ? pSuper->pExtSchemas : pEntry->pExtSchemas, pEntry->type == TSDB_VIRTUAL_CHILD_TABLE ? pSuper->stbEntry.ownerId : pEntry->ntbEntry.ownerId, pRsp, pEntry->type); if (code) { @@ -2704,6 +2719,7 @@ int32_t metaRemoveTableColumnRef(SMeta *pMeta, int64_t version, SVAlterTbReq *pR // build response code = metaUpdateVtbMetaRsp( pEntry, pReq->tbName, pSchema, &pEntry->colRef, + pEntry->type == TSDB_VIRTUAL_CHILD_TABLE ? pSuper->pExtSchemas : pEntry->pExtSchemas, pEntry->type == TSDB_VIRTUAL_CHILD_TABLE ? pSuper->stbEntry.ownerId : pEntry->ntbEntry.ownerId, pRsp, pEntry->type); if (code) { diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 7b88b9ba74a..ac43f8dfc84 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -35,7 +35,7 @@ void vnodeQueryClose(SVnode *pVnode) { qWorkerDestroy((void **)&pVnode->pQuery); int32_t fillTableColCmpr(SMetaReader *reader, SSchemaExt *pExt, int32_t numOfCol) { int8_t tblType = reader->me.type; - if (withExtSchema(tblType)) { + if (withColCompress(tblType)) { SColCmprWrapper *p = &(reader->me.colCmpr); if (numOfCol != p->nCols) { vError("fillTableColCmpr table type:%d, col num:%d, col cmpr num:%d mismatch", tblType, numOfCol, p->nCols); @@ -236,13 +236,17 @@ int32_t vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { (void)memcpy(metaRsp.pSchemas + schema.nCols, schemaTag.pSchema, sizeof(SSchema) * schemaTag.nCols); } if (metaRsp.pSchemaExt) { - SMetaReader *pReader = mer1.me.type == TSDB_CHILD_TABLE ? &mer2 : &mer1; + SMetaReader *pReader = + (mer1.me.type == TSDB_CHILD_TABLE || mer1.me.type == TSDB_VIRTUAL_CHILD_TABLE) ? &mer2 : &mer1; code = fillTableColCmpr(pReader, metaRsp.pSchemaExt, metaRsp.numOfColumns); if (code < 0) { goto _exit; } - for (int32_t i = 0; i < metaRsp.numOfColumns && pReader->me.pExtSchemas; i++) { - metaRsp.pSchemaExt[i].typeMod = pReader->me.pExtSchemas[i].typeMod; + for (int32_t i = 0; i < metaRsp.numOfColumns; i++) { + metaRsp.pSchemaExt[i].colId = schema.pSchema[i].colId; + if (pReader->me.pExtSchemas) { + metaRsp.pSchemaExt[i].typeMod = pReader->me.pExtSchemas[i].typeMod; + } } } else { code = TSDB_CODE_OUT_OF_MEMORY; @@ -435,7 +439,7 @@ int32_t vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { cfgRsp.numOfColumns = schema.nCols; cfgRsp.virtualStb = false; // vnode don't have super table, so it's always false cfgRsp.pSchemas = (SSchema *)taosMemoryMalloc(sizeof(SSchema) * (cfgRsp.numOfColumns + cfgRsp.numOfTags)); - cfgRsp.pSchemaExt = (SSchemaExt *)taosMemoryMalloc(cfgRsp.numOfColumns * sizeof(SSchemaExt)); + cfgRsp.pSchemaExt = (SSchemaExt *)taosMemoryCalloc(cfgRsp.numOfColumns, sizeof(SSchemaExt)); cfgRsp.pColRefs = (SColRef *)taosMemoryMalloc(sizeof(SColRef) * cfgRsp.numOfColumns); cfgRsp.numOfTagRefs = 0; cfgRsp.pTagRefs = NULL; @@ -449,20 +453,21 @@ int32_t vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { (void)memcpy(cfgRsp.pSchemas + schema.nCols, schemaTag.pSchema, sizeof(SSchema) * schemaTag.nCols); } - SMetaReader *pReader = (mer1.me.type == TSDB_CHILD_TABLE || mer1.me.type == TSDB_VIRTUAL_CHILD_TABLE) ? &mer2 : &mer1; - SColCmprWrapper *pColCmpr = &pReader->me.colCmpr; - SColRefWrapper *pColRef = &mer1.me.colRef; + SMetaReader *pReader = (mer1.me.type == TSDB_CHILD_TABLE || mer1.me.type == TSDB_VIRTUAL_CHILD_TABLE) ? &mer2 : &mer1; + SColRefWrapper *pColRef = &mer1.me.colRef; if (withExtSchema(cfgRsp.tableType)) { + code = fillTableColCmpr(pReader, cfgRsp.pSchemaExt, cfgRsp.numOfColumns); + if (code < 0) { + goto _exit; + } + for (int32_t i = 0; i < cfgRsp.numOfColumns; i++) { - SColCmpr *pCmpr = &pColCmpr->pColCmpr[i]; SSchemaExt *pSchExt = cfgRsp.pSchemaExt + i; - pSchExt->colId = pCmpr->id; - pSchExt->compress = pCmpr->alg; - if (pReader->me.pExtSchemas) + pSchExt->colId = schema.pSchema[i].colId; + if (pReader->me.pExtSchemas) { pSchExt->typeMod = pReader->me.pExtSchemas[i].typeMod; - else - pSchExt->typeMod = 0; + } } } diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 00d6302749c..8931ceb3a8f 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -539,34 +539,8 @@ int32_t ctgCopyTbMeta(SCatalog *pCtg, SCtgTbMetaCtx *ctx, SCtgDBCache **pDb, SCt ctx->tbInfo.tbType = tbMeta->tableType; if (tbMeta->tableType != TSDB_CHILD_TABLE && tbMeta->tableType != TSDB_VIRTUAL_CHILD_TABLE) { - int32_t schemaExtSize = 0; - int32_t colRefSize = 0; - int32_t metaSize = CTG_META_SIZE(tbMeta); - if (withExtSchema(tbMeta->tableType) && tbMeta->schemaExt != NULL) { - schemaExtSize = tbMeta->tableInfo.numOfColumns * sizeof(SSchemaExt); - } - if (hasRefCol(tbMeta->tableType) && tbMeta->colRef != NULL) { - colRefSize += tbMeta->tableInfo.numOfColumns * sizeof(SColRef); - } - *pTableMeta = taosMemoryCalloc(1, metaSize + schemaExtSize + colRefSize); - if (NULL == *pTableMeta) { - CTG_ERR_RET(terrno); - } - - TAOS_MEMCPY(*pTableMeta, tbMeta, metaSize); - if (withExtSchema(tbMeta->tableType) && tbMeta->schemaExt != NULL) { - (*pTableMeta)->schemaExt = (SSchemaExt *)((char *)*pTableMeta + metaSize); - TAOS_MEMCPY((*pTableMeta)->schemaExt, tbMeta->schemaExt, schemaExtSize); - } else { - (*pTableMeta)->schemaExt = NULL; - } - if (hasRefCol(tbMeta->tableType) && tbMeta->colRef) { - (*pTableMeta)->colRef = (SColRef *)((char *)*pTableMeta + metaSize + schemaExtSize); - TAOS_MEMCPY((*pTableMeta)->colRef, tbMeta->colRef, colRefSize); - (*pTableMeta)->numOfColRefs = tbMeta->numOfColRefs; - } else { - (*pTableMeta)->colRef = NULL; - } + int32_t code = cloneTableMeta(tbMeta, pTableMeta); + CTG_ERR_RET(code); ctgDebug("tb:%s, get meta from cache, type:%d, db:%s", ctx->pName->tname, tbMeta->tableType, dbFName); return TSDB_CODE_SUCCESS; @@ -583,6 +557,9 @@ int32_t ctgCopyTbMeta(SCatalog *pCtg, SCtgTbMetaCtx *ctx, SCtgDBCache **pDb, SCt colRefSize += tbMeta->numOfColRefs * sizeof(SColRef); numOfColRefs = tbMeta->numOfColRefs; tmpRef = taosMemoryMalloc(colRefSize); + if (NULL == tmpRef) { + CTG_ERR_RET(terrno); + } TAOS_MEMCPY(tmpRef, tbMeta->colRef, colRefSize); } *pTableMeta = taosMemoryCalloc(1, metaSize + colRefSize); @@ -623,7 +600,7 @@ int32_t ctgCopyTbMeta(SCatalog *pCtg, SCtgTbMetaCtx *ctx, SCtgDBCache **pDb, SCt metaSize = CTG_META_SIZE(stbMeta); int32_t schemaExtSize = 0; - if (stbMeta->schemaExt) { + if (withExtSchema(stbMeta->tableType) && stbMeta->schemaExt) { schemaExtSize = stbMeta->tableInfo.numOfColumns * sizeof(SSchemaExt); } *pTableMeta = taosMemoryRealloc(*pTableMeta, metaSize + schemaExtSize + colRefSize); @@ -633,7 +610,7 @@ int32_t ctgCopyTbMeta(SCatalog *pCtg, SCtgTbMetaCtx *ctx, SCtgDBCache **pDb, SCt } TAOS_MEMCPY(&(*pTableMeta)->numOfColRefs, &stbMeta->numOfColRefs, metaSize - sizeof(SCTableMeta)); - if (stbMeta->schemaExt) { + if (withExtSchema(stbMeta->tableType) && stbMeta->schemaExt) { (*pTableMeta)->schemaExt = (SSchemaExt*)((char*)*pTableMeta + metaSize); TAOS_MEMCPY((*pTableMeta)->schemaExt, stbMeta->schemaExt, schemaExtSize); } else { @@ -3602,35 +3579,10 @@ int32_t ctgGetTbMetasFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgTbMe SMetaRes res = {0}; STableMeta *pTableMeta = NULL; if (tbMeta->tableType != TSDB_CHILD_TABLE && tbMeta->tableType != TSDB_VIRTUAL_CHILD_TABLE) { - int32_t schemaExtSize = 0; - int32_t colRefSize = 0; - int32_t metaSize = CTG_META_SIZE(tbMeta); - if (withExtSchema(tbMeta->tableType) && tbMeta->schemaExt != NULL) { - schemaExtSize = tbMeta->tableInfo.numOfColumns * sizeof(SSchemaExt); - } - if (hasRefCol(tbMeta->tableType) && tbMeta->colRef) { - colRefSize = tbMeta->tableInfo.numOfColumns * sizeof(SColRef); - } - - pTableMeta = taosMemoryCalloc(1, metaSize + schemaExtSize + colRefSize); - if (NULL == pTableMeta) { + int32_t code = cloneTableMeta(tbMeta, &pTableMeta); + if (code != TSDB_CODE_SUCCESS) { ctgReleaseTbMetaToCache(pCtg, dbCache, pCache); - CTG_ERR_RET(terrno); - } - - TAOS_MEMCPY(pTableMeta, tbMeta, metaSize); - if (withExtSchema(tbMeta->tableType) && tbMeta->schemaExt != NULL) { - pTableMeta->schemaExt = (SSchemaExt *)((char *)pTableMeta + metaSize); - TAOS_MEMCPY(pTableMeta->schemaExt, tbMeta->schemaExt, schemaExtSize); - } else { - pTableMeta->schemaExt = NULL; - } - if (hasRefCol(tbMeta->tableType) && tbMeta->colRef) { - pTableMeta->colRef = (SColRef *)((char *)pTableMeta + metaSize + schemaExtSize); - pTableMeta->numOfColRefs = tbMeta->tableInfo.numOfColumns; - TAOS_MEMCPY(pTableMeta->colRef, tbMeta->colRef, colRefSize); - } else { - pTableMeta->colRef = NULL; + CTG_ERR_RET(code); } CTG_UNLOCK(CTG_READ, &pCache->metaLock); @@ -3687,6 +3639,10 @@ int32_t ctgGetTbMetasFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgTbMe colRefNum = tbMeta->numOfColRefs; taosMemoryFreeClear(tmpRef); tmpRef = taosMemoryMalloc(colRefSize); + if (NULL == tmpRef) { + ctgReleaseTbMetaToCache(pCtg, dbCache, pCache); + CTG_ERR_RET(terrno); + } TAOS_MEMCPY(tmpRef, tbMeta->colRef, colRefSize); } @@ -3766,7 +3722,7 @@ int32_t ctgGetTbMetasFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgTbMe } int32_t schemaExtSize = 0; - if (stbMeta->schemaExt != NULL) { + if (withExtSchema(stbMeta->tableType) && stbMeta->schemaExt != NULL) { schemaExtSize = stbMeta->tableInfo.numOfColumns * sizeof(SSchemaExt); } metaSize = CTG_META_SIZE(stbMeta); @@ -3780,6 +3736,7 @@ int32_t ctgGetTbMetasFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgTbMe TAOS_MEMCPY(&pTableMeta->numOfColRefs, &stbMeta->numOfColRefs, metaSize + schemaExtSize - sizeof(SCTableMeta)); if (withExtSchema(stbMeta->tableType) && stbMeta->schemaExt != NULL) { pTableMeta->schemaExt = (SSchemaExt *)((char *)pTableMeta + metaSize); + TAOS_MEMCPY(pTableMeta->schemaExt, stbMeta->schemaExt, schemaExtSize); } else { pTableMeta->schemaExt = NULL; } @@ -4297,4 +4254,3 @@ _return: CTG_RET(code); } - diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index 6451a088554..1af1ee17001 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -1701,7 +1701,7 @@ int32_t ctgCloneMetaOutput(STableMetaOutput* output, STableMetaOutput** pOutput) if (output->vctbMeta) { int32_t metaSize = sizeof(SVCTableMeta); int32_t colRefSize = 0; - if (hasRefCol(output->vctbMeta->tableType) && (*pOutput)->vctbMeta->colRef) { + if (hasRefCol(output->vctbMeta->tableType) && output->vctbMeta->colRef) { colRefSize = output->vctbMeta->numOfColRefs * sizeof(SColRef); } (*pOutput)->vctbMeta = taosMemoryMalloc(metaSize + colRefSize); @@ -1712,7 +1712,7 @@ int32_t ctgCloneMetaOutput(STableMetaOutput* output, STableMetaOutput** pOutput) } TAOS_MEMCPY((*pOutput)->vctbMeta, output->vctbMeta, metaSize); - if (hasRefCol(output->vctbMeta->tableType) && (*pOutput)->vctbMeta->colRef) { + if (hasRefCol(output->vctbMeta->tableType) && output->vctbMeta->colRef) { (*pOutput)->vctbMeta->colRef = (SColRef*)((char*)(*pOutput)->vctbMeta + metaSize); TAOS_MEMCPY((*pOutput)->vctbMeta->colRef, output->vctbMeta->colRef, colRefSize); } else { @@ -1721,37 +1721,11 @@ int32_t ctgCloneMetaOutput(STableMetaOutput* output, STableMetaOutput** pOutput) } if (output->tbMeta) { - int32_t metaSize = CTG_META_SIZE(output->tbMeta); - int32_t schemaExtSize = 0; - int32_t colRefSize = 0; - if (withExtSchema(output->tbMeta->tableType) && (*pOutput)->tbMeta->schemaExt) { - schemaExtSize = output->tbMeta->tableInfo.numOfColumns * sizeof(SSchemaExt); - } - if (hasRefCol(output->tbMeta->tableType) && (*pOutput)->tbMeta->colRef) { - colRefSize = output->tbMeta->tableInfo.numOfColumns * sizeof(SColRef); - } - - (*pOutput)->tbMeta = taosMemoryMalloc(metaSize + schemaExtSize + colRefSize); - qTrace("tbmeta cloned, size:%d, p:%p", metaSize, (*pOutput)->tbMeta); - - if (NULL == (*pOutput)->tbMeta) { - qError("malloc %d failed", (int32_t)sizeof(STableMetaOutput)); + int32_t code = cloneTableMeta(output->tbMeta, &(*pOutput)->tbMeta); + if (TSDB_CODE_SUCCESS != code) { + taosMemoryFreeClear((*pOutput)->vctbMeta); taosMemoryFreeClear(*pOutput); - CTG_ERR_RET(terrno); - } - - TAOS_MEMCPY((*pOutput)->tbMeta, output->tbMeta, metaSize); - if (withExtSchema(output->tbMeta->tableType) && (*pOutput)->tbMeta->schemaExt) { - (*pOutput)->tbMeta->schemaExt = (SSchemaExt*)((char*)(*pOutput)->tbMeta + metaSize); - TAOS_MEMCPY((*pOutput)->tbMeta->schemaExt, output->tbMeta->schemaExt, schemaExtSize); - } else { - (*pOutput)->tbMeta->schemaExt = NULL; - } - if (hasRefCol(output->tbMeta->tableType) && (*pOutput)->tbMeta->colRef) { - (*pOutput)->tbMeta->colRef = (SColRef*)((char*)(*pOutput)->tbMeta + metaSize + schemaExtSize); - TAOS_MEMCPY((*pOutput)->tbMeta->colRef, output->tbMeta->colRef, colRefSize); - } else { - (*pOutput)->tbMeta->colRef = NULL; + CTG_ERR_RET(code); } } @@ -3262,4 +3236,3 @@ int32_t ctgAddTSMAFetch(SArray** pFetchs, int32_t dbIdx, int32_t tbIdx, int32_t* return TSDB_CODE_SUCCESS; } - diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index b9d6fcc2b64..57da16f2263 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -182,14 +182,14 @@ static int32_t setDescResultIntoDataBlock(bool sysInfoUser, SSDataBlock* pBlock, SColumnInfoData* pCol7 = NULL; // colref SColumnInfoData* pCol8 = NULL; - if (withExtSchema(pMeta->tableType)) { + if (withColCompress(pMeta->tableType)) { pCol5 = taosArrayGet(pBlock->pDataBlock, 4); pCol6 = taosArrayGet(pBlock->pDataBlock, 5); pCol7 = taosArrayGet(pBlock->pDataBlock, 6); } if (hasRefCol(pMeta->tableType)) { - pCol5 = taosArrayGet(pBlock->pDataBlock, 4); + pCol8 = taosArrayGet(pBlock->pDataBlock, 4); } int32_t fillTagCol = 0; @@ -226,7 +226,7 @@ static int32_t setDescResultIntoDataBlock(bool sysInfoUser, SSDataBlock* pBlock, STR_TO_VARSTR(buf, "VIEW COL"); } COL_DATA_SET_VAL_AND_CHECK(pCol4, pBlock->info.rows, buf, false); - if (withExtSchema(pMeta->tableType) && pMeta->schemaExt) { + if (withColCompress(pMeta->tableType) && pMeta->schemaExt) { if (i < pMeta->tableInfo.numOfColumns) { STR_TO_VARSTR(buf, columnEncodeStr(COMPRESS_L1_TYPE_U32(pMeta->schemaExt[i].compress))); COL_DATA_SET_VAL_AND_CHECK(pCol5, pBlock->info.rows, buf, false); @@ -259,10 +259,10 @@ static int32_t setDescResultIntoDataBlock(bool sysInfoUser, SSDataBlock* pBlock, } else { STR_TO_VARSTR(buf, ""); } - COL_DATA_SET_VAL_AND_CHECK(pCol5, pBlock->info.rows, buf, false); + COL_DATA_SET_VAL_AND_CHECK(pCol8, pBlock->info.rows, buf, false); } else { STR_TO_VARSTR(buf, ""); - COL_DATA_SET_VAL_AND_CHECK(pCol5, pBlock->info.rows, buf, false); + COL_DATA_SET_VAL_AND_CHECK(pCol8, pBlock->info.rows, buf, false); } } @@ -302,10 +302,10 @@ static int32_t execDescribe(bool sysInfoUser, SNode* pStmt, SRetrieveTableRsp** } if (TSDB_CODE_SUCCESS == code) { if (pDesc->pMeta) { - if (withExtSchema(pDesc->pMeta->tableType) && pDesc->pMeta->schemaExt) { - code = buildRetrieveTableRsp(pBlock, DESCRIBE_RESULT_COLS_COMPRESS, pRsp); - } else if (hasRefCol(pDesc->pMeta->tableType) && pDesc->pMeta->colRef) { + if (hasRefCol(pDesc->pMeta->tableType)) { code = buildRetrieveTableRsp(pBlock, DESCRIBE_RESULT_COLS_REF, pRsp); + } else if (withColCompress(pDesc->pMeta->tableType) && pDesc->pMeta->schemaExt) { + code = buildRetrieveTableRsp(pBlock, DESCRIBE_RESULT_COLS_COMPRESS, pRsp); } else { code = buildRetrieveTableRsp(pBlock, DESCRIBE_RESULT_COLS, pRsp); } @@ -607,7 +607,7 @@ static void appendColumnFields(char* buf, int32_t* len, STableCfg* pCfg) { char expandName[(SHOW_CREATE_TB_RESULT_FIELD1_LEN << 1) + 1] = {0}; for (int32_t i = 0; i < pCfg->numOfColumns; ++i) { SSchema* pSchema = pCfg->pSchemas + i; - SColRef* pRef = pCfg->pColRefs + i; + SColRef* pRef = pCfg->pColRefs ? pCfg->pColRefs + i : NULL; #define LTYPE_LEN \ (32 + 60 + TSDB_COL_FNAME_LEN + TSDB_DB_NAME_LEN + \ 10) // 60 byte for compress info, TSDB_COL_FNAME_LEN + TSDB_DB_NAME_LEN for column ref @@ -621,12 +621,17 @@ static void appendColumnFields(char* buf, int32_t* len, STableCfg* pCfg) { typeLen += snprintf(type + typeLen, LTYPE_LEN - typeLen, "(%d)", (int32_t)((pSchema->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)); } else if (IS_DECIMAL_TYPE(pSchema->type)) { - uint8_t precision, scale; - decimalFromTypeMod(pCfg->pSchemaExt[i].typeMod, &precision, &scale); + uint8_t precision = 0, scale = 0; + if (pCfg->pSchemaExt) { + decimalFromTypeMod(pCfg->pSchemaExt[i].typeMod, &precision, &scale); + } else if ((((uint32_t)pSchema->bytes) >> 24) != 0) { + int32_t bytes = pSchema->bytes; + extractDecimalTypeInfoFromBytes(&bytes, &precision, &scale); + } typeLen += snprintf(type + typeLen, LTYPE_LEN - typeLen, "(%d,%d)", precision, scale); } - if (withExtSchema(pCfg->tableType) && pCfg->pSchemaExt && tsShowFullCreateTableColumn) { + if (tsShowFullCreateTableColumn && pCfg->pSchemaExt && pCfg->pSchemaExt[i].compress != 0) { typeLen += snprintf(type + typeLen, LTYPE_LEN - typeLen, " ENCODE \'%s\'", columnEncodeStr(COMPRESS_L1_TYPE_U32(pCfg->pSchemaExt[i].compress))); typeLen += snprintf(type + typeLen, LTYPE_LEN - typeLen, " COMPRESS \'%s\'", @@ -635,7 +640,7 @@ static void appendColumnFields(char* buf, int32_t* len, STableCfg* pCfg) { columnLevelStr(COMPRESS_L2_TYPE_LEVEL_U32(pCfg->pSchemaExt[i].compress))); } - if (hasRefCol(pCfg->tableType) && pCfg->pColRefs && pRef->hasRef) { + if (hasRefCol(pCfg->tableType) && pRef && pRef->hasRef) { typeLen += snprintf(type + typeLen, LTYPE_LEN - typeLen, " FROM `%s`", pRef->refDbName); typeLen += snprintf(type + typeLen, LTYPE_LEN - typeLen, "."); typeLen += diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 85e2658a490..c1801bcbc3f 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1398,6 +1398,7 @@ static int32_t createVTableScanInfoFromBatchParam(SOperatorInfo* pOperator) { pAPI->metaReaderFn.readerReleaseLock(&orgTable); qDebug("dynamic vtable scan for origin table:%s, %s", pOrgTbInfo->tbName, GET_TASKID(pTaskInfo)); QUERY_CHECK_CODE(code, lino, _return); + SExtSchema *extSchema = NULL; switch (orgTable.me.type) { case TSDB_CHILD_TABLE: pAPI->metaReaderFn.initReader(&superTable, pInfo->base.readHandle.vnode, META_READER_LOCK, &pAPI->metaFn); @@ -1405,9 +1406,11 @@ static int32_t createVTableScanInfoFromBatchParam(SOperatorInfo* pOperator) { pAPI->metaReaderFn.readerReleaseLock(&superTable); QUERY_CHECK_CODE(code, lino, _return); schema = &superTable.me.stbEntry.schemaRow; + extSchema = superTable.me.pExtSchemas; break; case TSDB_NORMAL_TABLE: schema = &orgTable.me.ntbEntry.schemaRow; + extSchema = orgTable.me.pExtSchemas; break; default: qError("invalid table type:%d", orgTable.me.type); @@ -1443,10 +1446,11 @@ static int32_t createVTableScanInfoFromBatchParam(SOperatorInfo* pOperator) { SColIdNameKV* kv = taosArrayGet(pOrgTbInfo->colMap, i); for (int32_t j = 0; j < schema->nCols; j++) { if (strcmp(kv->colName, schema->pSchema[j].name) == 0) { + SDataType refType = {0}; + schemaToRefDataType(&schema->pSchema[j], extSchema ? extSchema[j].typeMod : 0, &refType); SColIdPair pPair = {.vtbColId = kv->colId, .orgColId = (col_id_t)(schema->pSchema[j].colId), - .type.type = schema->pSchema[j].type, - .type.bytes = schema->pSchema[j].bytes}; + .type = refType}; QUERY_CHECK_NULL(taosArrayPush(pColArray, &pPair), code, lino, _return, terrno); qDebug("dynamic vtable scan map col:%s, orgColId:%d, vtbColId:%d, %s", kv->colName, pPair.orgColId, pPair.vtbColId, GET_TASKID(pTaskInfo)); @@ -1473,6 +1477,14 @@ static int32_t createVTableScanInfoFromBatchParam(SOperatorInfo* pOperator) { code = TSDB_CODE_VTABLE_COLUMN_TYPE_MISMATCH; goto _return; } + if (IS_DECIMAL_TYPE(pItem->dataType.type) && + (pItem->dataType.precision != pPair->type.precision || + pItem->dataType.scale != pPair->type.scale)) { + qError("column decimal type does not match for vtable colId:%d, org colId:%d, org table name:%s", + pPair->vtbColId, pPair->orgColId, orgTable.me.name); + code = TSDB_CODE_VTABLE_COLUMN_TYPE_MISMATCH; + goto _return; + } SColIdSlotIdPair colIdSlotIdPair = {.orgColId = pPair->orgColId, .vtbSlotId = pItem->dstSlotId}; QUERY_CHECK_NULL(taosArrayPush(pBlockColArray, &colIdSlotIdPair), code, lino, _return, terrno); qDebug("dynamic vtable scan block col map orgColId:%d, vtbSlotId:%d, %s", colIdSlotIdPair.orgColId, @@ -1699,6 +1711,12 @@ static int32_t createVTableScanInfoFromParam(SOperatorInfo* pOperator) { } pInfo->pBlockColMap = taosArrayInit(schema->nCols, sizeof(SColIdSlotIdPair)); QUERY_CHECK_NULL(pBlockColArray, code, lino, _return, terrno); + SExtSchema *extSchema = NULL; + if (orgTable.me.type == TSDB_CHILD_TABLE) { + extSchema = superTable.me.pExtSchemas; + } else { + extSchema = orgTable.me.pExtSchemas; + } // virtual table's origin table scan do not has ts column. SColIdPair tsPair = {.vtbColId = PRIMARYKEY_TIMESTAMP_COL_ID, @@ -1711,10 +1729,11 @@ static int32_t createVTableScanInfoFromParam(SOperatorInfo* pOperator) { SColIdNameKV* kv = taosArrayGet(pOrgTbInfo->colMap, i); for (int32_t j = 0; j < schema->nCols; j++) { if (strcmp(kv->colName, schema->pSchema[j].name) == 0) { + SDataType refType = {0}; + schemaToRefDataType(&schema->pSchema[j], extSchema ? extSchema[j].typeMod : 0, &refType); SColIdPair pPair = {.vtbColId = kv->colId, .orgColId = (col_id_t)(schema->pSchema[j].colId), - .type.type = schema->pSchema[j].type, - .type.bytes = schema->pSchema[j].bytes}; + .type = refType}; QUERY_CHECK_NULL(taosArrayPush(pColArray, &pPair), code, lino, _return, terrno); break; } @@ -1748,6 +1767,14 @@ static int32_t createVTableScanInfoFromParam(SOperatorInfo* pOperator) { code = TSDB_CODE_VTABLE_COLUMN_TYPE_MISMATCH; goto _return; } + if (IS_DECIMAL_TYPE(pItem->dataType.type) && + (pItem->dataType.precision != pPair->type.precision || + pItem->dataType.scale != pPair->type.scale)) { + qError("column decimal type does not match for vtable colId:%d, org colId:%d, org table name:%s", + pPair->vtbColId, pPair->orgColId, orgTable.me.name); + code = TSDB_CODE_VTABLE_COLUMN_TYPE_MISMATCH; + goto _return; + } SColIdSlotIdPair colIdSlotIdPair = {.orgColId = pPair->orgColId, .vtbSlotId = pItem->dstSlotId}; QUERY_CHECK_NULL(taosArrayPush(pBlockColArray, &colIdSlotIdPair), code, lino, _return, terrno); if (i > 0) { diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 04094bd59a1..6b9eb685704 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -577,6 +577,8 @@ static SSDataBlock* doOptimizeTableNameFilter(SOperatorInfo* pOperator, SSDataBl STR_TO_VARSTR(tableName, pInfo->req.filterTb); SMetaReader smrTable = {0}; + SMetaReader smrSuperTable = {0}; + bool hasSuperTable = false; pAPI->metaReaderFn.initReader(&smrTable, pInfo->readHandle.vnode, META_READER_LOCK, &pAPI->metaFn); int32_t code = pAPI->metaReaderFn.getTableEntryByName(&smrTable, pInfo->req.filterTb); if (code != TSDB_CODE_SUCCESS) { @@ -620,6 +622,17 @@ static SSDataBlock* doOptimizeTableNameFilter(SOperatorInfo* pOperator, SSDataBl colRef = &smrTable.me.colRef; STR_TO_VARSTR(typeName, "VIRTUAL_NORMAL_TABLE"); } else if (smrTable.me.type == TSDB_VIRTUAL_CHILD_TABLE) { + pAPI->metaReaderFn.initReader(&smrSuperTable, pInfo->readHandle.vnode, META_READER_LOCK, &pAPI->metaFn); + hasSuperTable = true; + code = pAPI->metaReaderFn.getTableEntryByUid(&smrSuperTable, smrTable.me.ctbEntry.suid); + if (code != TSDB_CODE_SUCCESS) { + pAPI->metaReaderFn.clearReader(&smrTable); + pAPI->metaReaderFn.clearReader(&smrSuperTable); + pInfo->loadInfo.totalRows = 0; + return NULL; + } + schemaRow = &smrSuperTable.me.stbEntry.schemaRow; + extSchemaRow = smrSuperTable.me.pExtSchemas; colRef = &smrTable.me.colRef; STR_TO_VARSTR(typeName, "VIRTUAL_CHILD_TABLE"); } @@ -627,12 +640,18 @@ static SSDataBlock* doOptimizeTableNameFilter(SOperatorInfo* pOperator, SSDataBl code = sysTableUserColsFillOneTableCols(dbname, &numOfRows, dataBlock, tableName, schemaRow, extSchemaRow, typeName, colRef); if (code != TSDB_CODE_SUCCESS) { + if (hasSuperTable) { + pAPI->metaReaderFn.clearReader(&smrSuperTable); + } pAPI->metaReaderFn.clearReader(&smrTable); pInfo->loadInfo.totalRows = 0; qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); pTaskInfo->code = code; T_LONG_JMP(pTaskInfo->env, code); } + if (hasSuperTable) { + pAPI->metaReaderFn.clearReader(&smrSuperTable); + } pAPI->metaReaderFn.clearReader(&smrTable); if (numOfRows > 0) { @@ -1938,6 +1957,10 @@ static int32_t sysTableUserColsFillOneTableCols(const char* dbname, int32_t* pNu // col type int8_t colType = schemaRow->pSchema[i].type; + SDataType colDataType = {0}; + if (IS_DECIMAL_TYPE(colType)) { + schemaToRefDataType(&schemaRow->pSchema[i], extSchemaRow ? extSchemaRow[i].typeMod : 0, &colDataType); + } pColInfoData = taosArrayGet(dataBlock->pDataBlock, 4); QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); int32_t colStrBufflen = 32; @@ -1955,12 +1978,10 @@ static int32_t sysTableUserColsFillOneTableCols(const char* dbname, int32_t* pNu colTypeLen += snprintf(varDataVal(colTypeStr) + colTypeLen, colStrBufflen, "(%d)", (int32_t)((schemaRow->pSchema[i].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)); } else if (IS_DECIMAL_TYPE(colType)) { - QUERY_CHECK_NULL(extSchemaRow, code, lino, _end, TSDB_CODE_INVALID_PARA); - STypeMod typeMod = extSchemaRow[i].typeMod; - uint8_t prec = 0, scale = 0; - decimalFromTypeMod(typeMod, &prec, &scale); - colTypeLen += snprintf(varDataVal(colTypeStr) + colTypeLen, sizeof(colTypeStr) - colTypeLen - VARSTR_HEADER_SIZE, - "(%d,%d)", prec, scale); + if (colDataType.precision > 0) { + colTypeLen += snprintf(varDataVal(colTypeStr) + colTypeLen, sizeof(colTypeStr) - colTypeLen - VARSTR_HEADER_SIZE, + "(%d,%d)", colDataType.precision, colDataType.scale); + } } varDataSetLen(colTypeStr, colTypeLen); code = colDataSetVal(pColInfoData, numOfRows, (char*)colTypeStr, false); @@ -1972,13 +1993,33 @@ static int32_t sysTableUserColsFillOneTableCols(const char* dbname, int32_t* pNu code = colDataSetVal(pColInfoData, numOfRows, (const char*)&schemaRow->pSchema[i].bytes, false); QUERY_CHECK_CODE(code, lino, _end); - // col precision, col scale, col nullable - for (int32_t j = 6; j <= 8; ++j) { - pColInfoData = taosArrayGet(dataBlock->pDataBlock, j); - QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); + // col precision + pColInfoData = taosArrayGet(dataBlock->pDataBlock, 6); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); + if (IS_DECIMAL_TYPE(colType) && colDataType.precision > 0) { + int32_t precision = colDataType.precision; + code = colDataSetVal(pColInfoData, numOfRows, (const char*)&precision, false); + QUERY_CHECK_CODE(code, lino, _end); + } else { colDataSetNULL(pColInfoData, numOfRows); } + // col scale + pColInfoData = taosArrayGet(dataBlock->pDataBlock, 7); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); + if (IS_DECIMAL_TYPE(colType) && colDataType.precision > 0) { + int32_t scale = colDataType.scale; + code = colDataSetVal(pColInfoData, numOfRows, (const char*)&scale, false); + QUERY_CHECK_CODE(code, lino, _end); + } else { + colDataSetNULL(pColInfoData, numOfRows); + } + + // col nullable + pColInfoData = taosArrayGet(dataBlock->pDataBlock, 8); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); + colDataSetNULL(pColInfoData, numOfRows); + // col data source pColInfoData = taosArrayGet(dataBlock->pDataBlock, 9); QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 3c4ba19839d..d5f9295976b 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -276,12 +276,13 @@ static int32_t tableNodeCopy(const STableNode* pSrc, STableNode* pDst) { } static STableMeta* tableMetaClone(const STableMeta* pSrc) { - int32_t len = TABLE_META_SIZE(pSrc); + int32_t len = TABLE_META_FULL_SIZE(pSrc); STableMeta* pDst = taosMemoryMalloc(len); if (NULL == pDst) { return NULL; } memcpy(pDst, pSrc, len); + tableMetaResetPointers(pDst); return pDst; } diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 54def9b0b1f..7514cf84f5c 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -741,6 +741,9 @@ static const char* jkRefColColId = "ColId"; static const char* jkRefColDbName = "DbName"; static const char* jkRefColTableName = "TableName"; static const char* jkRefColColName = "ColName"; +static const char* jkSchemaExtColId = "ColId"; +static const char* jkSchemaExtCompress = "Compress"; +static const char* jkSchemaExtTypeMod = "TypeMod"; static int32_t refColToJson(const void* pObj, SJson* pJson) { const SColRef* pCol = (const SColRef*)pObj; @@ -782,12 +785,41 @@ static int32_t jsonToRefCol(const SJson* pJson, void* pObj) { return code; } +static int32_t schemaExtToJson(const void* pObj, SJson* pJson) { + const SSchemaExt* pSchemaExt = (const SSchemaExt*)pObj; + + int32_t code = tjsonAddIntegerToObject(pJson, jkSchemaExtColId, pSchemaExt->colId); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkSchemaExtCompress, pSchemaExt->compress); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkSchemaExtTypeMod, pSchemaExt->typeMod); + } + + return code; +} + +static int32_t jsonToSchemaExt(const SJson* pJson, void* pObj) { + SSchemaExt* pSchemaExt = (SSchemaExt*)pObj; + + int32_t code = tjsonGetSmallIntValue(pJson, jkSchemaExtColId, &pSchemaExt->colId); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetUIntValue(pJson, jkSchemaExtCompress, &pSchemaExt->compress); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkSchemaExtTypeMod, &pSchemaExt->typeMod); + } + + return code; +} + static const char* jkTableMetaVgId = "VgId"; static const char* jkTableMetaTableType = "TableType"; static const char* jkTableMetaUid = "Uid"; static const char* jkTableMetaSuid = "Suid"; static const char* jkTableMetaColRefNum = "ColRefNum"; static const char* jkTableMetaRefCols = "RefCols"; +static const char* jkTableMetaSchemaExts = "SchemaExts"; static const char* jkTableMetaSversion = "Sversion"; static const char* jkTableMetaTversion = "Tversion"; static const char* jkTableMetaRversion = "Rversion"; @@ -813,6 +845,10 @@ static int32_t tableMetaToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code && pNode->numOfColRefs > 0 && pNode->colRef) { code = tjsonAddArray(pJson, jkTableMetaRefCols, refColToJson, pNode->colRef, sizeof(SColRef), pNode->numOfColRefs); } + if (TSDB_CODE_SUCCESS == code && pNode->schemaExt) { + code = tjsonAddArray(pJson, jkTableMetaSchemaExts, schemaExtToJson, pNode->schemaExt, sizeof(SSchemaExt), + pNode->tableInfo.numOfColumns); + } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkTableMetaSversion, pNode->sversion); } @@ -835,6 +871,10 @@ static int32_t tableMetaToJson(const void* pObj, SJson* pJson) { static int32_t jsonToTableMeta(const SJson* pJson, void* pObj) { STableMeta* pNode = (STableMeta*)pObj; + SJson* pJsonSchemaExt = tjsonGetObjectItem(pJson, jkTableMetaSchemaExts); + SJson* pJsonRefCols = tjsonGetObjectItem(pJson, jkTableMetaRefCols); + int32_t schemaExtNum = (NULL == pJsonSchemaExt) ? 0 : tjsonGetArraySize(pJsonSchemaExt); + int32_t colRefNum = (NULL == pJsonRefCols) ? 0 : tjsonGetArraySize(pJsonRefCols); int32_t code; tjsonGetNumberValue(pJson, jkTableMetaVgId, pNode->vgId, code); @@ -865,10 +905,22 @@ static int32_t jsonToTableMeta(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonToArray(pJson, jkTableMetaColSchemas, jsonToSchema, pNode->schema, sizeof(SSchema)); } - if (TSDB_CODE_SUCCESS == code && pNode->numOfColRefs > 0) { - pNode->colRef = (SColRef*)((char*)(pNode + 1) + TABLE_TOTAL_COL_NUM(pNode) * sizeof(SSchema)); - code = tjsonToArray(pJson, jkTableMetaRefCols, jsonToRefCol, pNode->colRef, sizeof(SColRef)); + if (TSDB_CODE_SUCCESS == code) { + pNode->schemaExt = schemaExtNum > 0 ? (SSchemaExt *)((char *)pNode + TABLE_META_BASE_SIZE(pNode)) : NULL; + if (NULL != pNode->schemaExt) { + code = tjsonToArray(pJson, jkTableMetaSchemaExts, jsonToSchemaExt, pNode->schemaExt, sizeof(SSchemaExt)); + } } + if (TSDB_CODE_SUCCESS == code) { + pNode->colRef = colRefNum > 0 + ? (SColRef *)((char *)pNode + TABLE_META_BASE_SIZE(pNode) + TABLE_META_SCHEMA_EXT_SIZE(pNode)) + : NULL; + if (NULL != pNode->colRef) { + code = tjsonToArray(pJson, jkTableMetaRefCols, jsonToRefCol, pNode->colRef, sizeof(SColRef)); + } + } + pNode->tagRef = NULL; + pNode->numOfTagRefs = 0; return code; } @@ -6198,7 +6250,7 @@ static int32_t realTableNodeToJson(const void* pObj, SJson* pJson) { int32_t code = tableNodeToJson(pObj, pJson); if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddIntegerToObject(pJson, jkRealTableMetaSize, TABLE_META_SIZE(pNode->pMeta)); + code = tjsonAddIntegerToObject(pJson, jkRealTableMetaSize, TABLE_META_FULL_SIZE(pNode->pMeta)); } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddObject(pJson, jkRealTableMeta, tableMetaToJson, pNode->pMeta); @@ -6328,7 +6380,7 @@ static int32_t virtualTableNodeToJson(const void* pObj, SJson* pJson) { int32_t code = tableNodeToJson(pObj, pJson); if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddIntegerToObject(pJson, jkVirtualTableMetaSize, TABLE_META_SIZE(pNode->pMeta)); + code = tjsonAddIntegerToObject(pJson, jkVirtualTableMetaSize, TABLE_META_FULL_SIZE(pNode->pMeta)); } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddObject(pJson, jkVirtuaTableMeta, tableMetaToJson, pNode->pMeta); @@ -6381,7 +6433,7 @@ static int32_t placeHolderTableNodeToJson(const void* pObj, SJson* pJson) { int32_t code = tableNodeToJson(pObj, pJson); if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddIntegerToObject(pJson, jkPlaceHolderTableMetaSize, TABLE_META_SIZE(pNode->pMeta)); + code = tjsonAddIntegerToObject(pJson, jkPlaceHolderTableMetaSize, TABLE_META_FULL_SIZE(pNode->pMeta)); } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddObject(pJson, jkPlaceHolderTableMeta, tableMetaToJson, pNode->pMeta); diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 25a397c9817..8223a0b3708 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -754,12 +754,7 @@ static int32_t rewriteDropTableWithMetaCache(STranslateContext* pCxt) { } tstrncpy(dbName, pDbStart + 1, pDbEnd - pDbStart); - int32_t metaSize = - sizeof(STableMeta) + sizeof(SSchema) * (pMeta->tableInfo.numOfColumns + pMeta->tableInfo.numOfTags); - int32_t schemaExtSize = - (withExtSchema(pMeta->tableType) && pMeta->schemaExt) ? sizeof(SSchemaExt) * pMeta->tableInfo.numOfColumns : 0; - int32_t colRefSize = (hasRefCol(pMeta->tableType) && pMeta->colRef) ? sizeof(SColRef) * pMeta->numOfColRefs : 0; - const char* pTbName = (const char*)pMeta + metaSize + schemaExtSize + colRefSize; + const char* pTbName = (const char *)pMeta + TABLE_META_FULL_SIZE(pMeta); SName name = {0}; toName(pParCxt->acctId, dbName, pTbName, &name); @@ -1568,6 +1563,12 @@ static void setVtbColumnInfoBySchema(const SVirtualTableNode* pTable, const SSch if (TSDB_DATA_TYPE_TIMESTAMP == pCol->node.resType.type) { pCol->node.resType.precision = pTable->pMeta->tableInfo.precision; } + if (pTable->pMeta->schemaExt && tagFlag < 0) { + int32_t colIdx = (int32_t)(pColSchema - pTable->pMeta->schema); + if (colIdx >= 0 && colIdx < pTable->pMeta->tableInfo.numOfColumns) { + fillTypeFromTypeMod(&pCol->node.resType, pTable->pMeta->schemaExt[colIdx].typeMod); + } + } pCol->tableHasPk = false; pCol->isPk = false; pCol->numOfPKs = 0; @@ -12976,11 +12977,6 @@ static int32_t checkColumnType(SNodeList* pList, int8_t virtualTable) { int32_t blobColNum = 0; FOREACH(pNode, pList) { SColumnDefNode* pCol = (SColumnDefNode*)pNode; - if (virtualTable && IS_DECIMAL_TYPE(pCol->dataType.type)) { - code = TSDB_CODE_VTABLE_NOT_SUPPORT_DATA_TYPE; - break; - } - if (pCol->pOptions && ((SColumnOptions*)pCol->pOptions)->bPrimaryKey && IS_STR_DATA_BLOB(pCol->dataType.type)) { code = TSDB_CODE_BLOB_NOT_SUPPORT_PRIMARY_KEY; break; @@ -13106,11 +13102,6 @@ static int32_t columnDefNodeToField(SNodeList* pList, SArray** pArray, bool calB SNode* pNode; FOREACH(pNode, pList) { SColumnDefNode* pCol = (SColumnDefNode*)pNode; - if (virtualTable && IS_DECIMAL_TYPE(pCol->dataType.type)) { - code = TSDB_CODE_VTABLE_NOT_SUPPORT_DATA_TYPE; - break; - } - if (pCol->pOptions && ((SColumnOptions*)pCol->pOptions)->bPrimaryKey && IS_STR_DATA_BLOB(pCol->dataType.type)) { code = TSDB_CODE_BLOB_NOT_SUPPORT_PRIMARY_KEY; break; @@ -13160,12 +13151,6 @@ static int32_t tagDefNodeToField(SNodeList* pList, SArray** pArray, bool calByte SField field = { .type = pCol->dataType.type, }; - if (virtualTable && IS_DECIMAL_TYPE(pCol->dataType.type)) { - taosArrayDestroy(*pArray); - *pArray = NULL; - return TSDB_CODE_VTABLE_NOT_SUPPORT_DATA_TYPE; - } - if (IS_STR_DATA_BLOB(pCol->dataType.type)) { taosArrayDestroy(*pArray); *pArray = NULL; @@ -14280,18 +14265,6 @@ static const SSchema* getNormalColSchema(const STableMeta* pTableMeta, const cha return NULL; } -static const col_id_t getNormalColSchemaIndex(const STableMeta* pTableMeta, const char* pColName) { - int32_t numOfCols = getNumOfColumns(pTableMeta); - SSchema* pColsSchema = getTableColumnSchema(pTableMeta); - for (int32_t i = 0; i < numOfCols; ++i) { - const SSchema* pSchema = pColsSchema + i; - if (0 == strcmp(pColName, pSchema->name)) { - return (col_id_t)i; - } - } - return -1; -} - static SSchema* getTagSchema(const STableMeta* pTableMeta, const char* pTagName) { int32_t numOfTags = getNumOfTags(pTableMeta); SSchema* pTagsSchema = getTableTagSchema(pTableMeta); @@ -14336,10 +14309,6 @@ static int32_t checkAlterTableByColumnType(STranslateContext* pCxt, SAlterTableS } static int32_t checkAlterSuperTableBySchema(STranslateContext* pCxt, SAlterTableStmt* pStmt, const STableMeta* pTableMeta) { - if (pTableMeta->virtualStb && IS_DECIMAL_TYPE(pStmt->dataType.type)) { - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_VTABLE_NOT_SUPPORT_DATA_TYPE); - } - SSchema* pTagsSchema = getTableTagSchema(pTableMeta); if (getNumOfTags(pTableMeta) == 1 && pTagsSchema->type == TSDB_DATA_TYPE_JSON && (pStmt->alterType == TSDB_ALTER_TABLE_ADD_TAG || pStmt->alterType == TSDB_ALTER_TABLE_DROP_TAG || @@ -14428,6 +14397,10 @@ static int32_t checkAlterSuperTableBySchema(STranslateContext* pCxt, SAlterTable return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TAGS_NUM); } + if (IS_DECIMAL_TYPE(pStmt->dataType.type)) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN, "Decimal type is not allowed for tag"); + } + if (tagsLen + calcTypeBytes(pStmt->dataType) > TSDB_MAX_TAGS_LEN) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TAGS_LENGTH, TSDB_MAX_TAGS_LEN); } @@ -21877,12 +21850,10 @@ static int32_t extractExplainResultSchema(int32_t* numOfCols, SSchema** pSchema) static int32_t extractDescribeResultSchema(STableMeta* pMeta, int32_t* numOfCols, SSchema** pSchema) { *numOfCols = DESCRIBE_RESULT_COLS; if (pMeta) { - if (withExtSchema(pMeta->tableType)) { - *numOfCols = DESCRIBE_RESULT_COLS_COMPRESS; - } else if (hasRefCol(pMeta->tableType)) { + if (hasRefCol(pMeta->tableType)) { *numOfCols = DESCRIBE_RESULT_COLS_REF; - } else { - // DESCRIBE_RESULT_COLS + } else if (withColCompress(pMeta->tableType) && pMeta->schemaExt) { + *numOfCols = DESCRIBE_RESULT_COLS_COMPRESS; } } *pSchema = taosMemoryCalloc((*numOfCols), sizeof(SSchema)); @@ -21907,7 +21878,11 @@ static int32_t extractDescribeResultSchema(STableMeta* pMeta, int32_t* numOfCols tstrncpy((*pSchema)[3].name, "note", TSDB_COL_NAME_LEN); if (pMeta) { - if (withExtSchema(pMeta->tableType)) { + if (hasRefCol(pMeta->tableType)) { + (*pSchema)[4].type = TSDB_DATA_TYPE_BINARY; + (*pSchema)[4].bytes = DESCRIBE_RESULT_COL_REF_LEN; + tstrncpy((*pSchema)[4].name, "ref", TSDB_COL_NAME_LEN); + } else if (withColCompress(pMeta->tableType) && pMeta->schemaExt) { (*pSchema)[4].type = TSDB_DATA_TYPE_BINARY; (*pSchema)[4].bytes = DESCRIBE_RESULT_COPRESS_OPTION_LEN; tstrncpy((*pSchema)[4].name, "encode", TSDB_COL_NAME_LEN); @@ -21919,10 +21894,6 @@ static int32_t extractDescribeResultSchema(STableMeta* pMeta, int32_t* numOfCols (*pSchema)[6].type = TSDB_DATA_TYPE_BINARY; (*pSchema)[6].bytes = DESCRIBE_RESULT_COPRESS_OPTION_LEN; tstrncpy((*pSchema)[6].name, "level", TSDB_COL_NAME_LEN); - } else if (hasRefCol(pMeta->tableType)) { - (*pSchema)[4].type = TSDB_DATA_TYPE_BINARY; - (*pSchema)[4].bytes = DESCRIBE_RESULT_COL_REF_LEN; - tstrncpy((*pSchema)[4].name, "ref", TSDB_COL_NAME_LEN); } } @@ -23190,6 +23161,15 @@ static int32_t buildVirtualTableBatchReq(STranslateContext* pCxt, const SCreateV SColumnDefNode* pColDef = (SColumnDefNode*)pCol; SSchema* pSchema = req.ntb.schemaRow.pSchema + index; toSchema(pColDef, index + 1, pSchema); + if (IS_DECIMAL_TYPE(pColDef->dataType.type)) { + if (NULL == req.pExtSchemas) { + req.pExtSchemas = taosMemoryCalloc(req.ntb.schemaRow.nCols, sizeof(SExtSchema)); + if (NULL == req.pExtSchemas) { + PAR_ERR_JRET(terrno); + } + } + req.pExtSchemas[index].typeMod = calcTypeMod(&pColDef->dataType); + } if (pColDef->pOptions && ((SColumnOptions*)pColDef->pOptions)->hasRef) { PAR_ERR_JRET( setColRef(&req.colRef.pColRef[index], index + 1, NULL, ((SColumnOptions*)pColDef->pOptions)->refColumn, @@ -23252,7 +23232,7 @@ static int32_t buildVirtualSubTableBatchReq(const SCreateVSubTableStmt* pStmt, S if (pStmt->pSpecificColRefs) { FOREACH(pCol, pStmt->pSpecificColRefs) { SColumnRefNode* pColRef = (SColumnRefNode*)pCol; - col_id_t schemaIdx = getNormalColSchemaIndex(pStbMeta, pColRef->colName); + int32_t schemaIdx = getNormalColSchemaIndex(pStbMeta, pColRef->colName); if (schemaIdx == -1) { PAR_ERR_JRET(TSDB_CODE_PAR_INVALID_COLUMN); } @@ -24868,22 +24848,21 @@ static int32_t checkColRef(STranslateContext* pCxt, char* colName, char* pRefDbN "virtual table's column:\"%s\"'s reference can only be normal table or child table", colName)); } - const SSchema* pRefCol = getNormalColSchema(pRefTableMeta, pRefColName); - if (NULL == pRefCol) { + int32_t refColIndex = getNormalColSchemaIndex(pRefTableMeta, pRefColName); + if (-1 == refColIndex) { PAR_ERR_JRET(generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_REF_COLUMN, "virtual table's column:\"%s\"'s reference column:\"%s\" not exist", colName, pRefColName)); } + const SSchema* pRefCol = pRefTableMeta->schema + refColIndex; + const SSchemaExt* pRefExt = + (pRefTableMeta->schemaExt && refColIndex < pRefTableMeta->tableInfo.numOfColumns) + ? pRefTableMeta->schemaExt + refColIndex + : NULL; + SDataType refType = {0}; + schemaToRefDataType(pRefCol, NULL != pRefExt ? pRefExt->typeMod : 0, &refType); - if (pRefCol->type != type.type) { - PAR_ERR_JRET(generateSyntaxErrMsgExt( - &pCxt->msgBuf, TSDB_CODE_PAR_INVALID_REF_COLUMN_TYPE, - "virtual table's column:\"%s\"'s type and reference column:\"%s\"'s type not match", colName, pRefColName)); - } - - // For variable-length types (VARCHAR, NCHAR, etc.), allow different lengths - // Virtual table can have different length than source table - if (!IS_VAR_DATA_TYPE(pRefCol->type) && pRefCol->bytes != type.bytes) { + if (!isSameRefDataType(&type, &refType)) { PAR_ERR_JRET(generateSyntaxErrMsgExt( &pCxt->msgBuf, TSDB_CODE_PAR_INVALID_REF_COLUMN_TYPE, "virtual table's column:\"%s\"'s type and reference column:\"%s\"'s type not match", colName, pRefColName)); @@ -24902,7 +24881,7 @@ static int32_t checkTagRef(STranslateContext* pCxt, char* tagName, char* pRefDbN PAR_ERR_JRET(getTableMeta(pCxt, pRefDbName, pRefTableName, &pRefTableMeta)); // referenced table must be child table (which has tags) - if (pRefTableMeta->tableType != TSDB_CHILD_TABLE) { + if (pRefTableMeta->tableType != TSDB_CHILD_TABLE && pRefTableMeta->tableType != TSDB_VIRTUAL_CHILD_TABLE) { PAR_ERR_JRET(generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_REF_COLUMN, "virtual table's tag:\"%s\"'s reference can only be child table", tagName)); } @@ -24921,13 +24900,10 @@ static int32_t checkTagRef(STranslateContext* pCxt, char* tagName, char* pRefDbN } } - if (pRefTag->type != type.type) { - PAR_ERR_JRET(generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_REF_COLUMN_TYPE, - "virtual table's tag:\"%s\"'s type and reference tag:\"%s\"'s type not match", - tagName, pRefColName)); - } + SDataType refType = {0}; + schemaToRefDataType(pRefTag, 0, &refType); - if (!IS_VAR_DATA_TYPE(pRefTag->type) && pRefTag->bytes != type.bytes) { + if (!isSameRefDataType(&type, &refType)) { PAR_ERR_JRET(generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_REF_COLUMN_TYPE, "virtual table's tag:\"%s\"'s type and reference tag:\"%s\"'s type not match", tagName, pRefColName)); @@ -24949,10 +24925,6 @@ static int32_t buildAddColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, S return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_DUPLICATED_COLUMN); } - if (isVirtualTable(pTableMeta) && IS_DECIMAL_TYPE(pStmt->dataType.type)) { - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_VTABLE_NOT_SUPPORT_DATA_TYPE); - } - if ((TSDB_DATA_TYPE_VARCHAR == pStmt->dataType.type && calcTypeBytes(pStmt->dataType) > TSDB_MAX_BINARY_LEN) || (TSDB_DATA_TYPE_VARBINARY == pStmt->dataType.type && calcTypeBytes(pStmt->dataType) > TSDB_MAX_BINARY_LEN) || (TSDB_DATA_TYPE_NCHAR == pStmt->dataType.type && calcTypeBytes(pStmt->dataType) > TSDB_MAX_NCHAR_LEN)) { @@ -24975,8 +24947,9 @@ static int32_t buildAddColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, S } // check ref column exists and check type - PAR_ERR_RET(checkColRef(pCxt, pStmt->colName, pStmt->refDbName, pStmt->refTableName, pStmt->refColName, - (SDataType){.type = pStmt->dataType.type, .bytes = calcTypeBytes(pStmt->dataType)}, + SDataType colType = pStmt->dataType; + colType.bytes = calcTypeBytes(colType); + PAR_ERR_RET(checkColRef(pCxt, pStmt->colName, pStmt->refDbName, pStmt->refTableName, pStmt->refColName, colType, pTableMeta->tableInfo.precision)); pReq->type = pStmt->dataType.type; @@ -25190,9 +25163,15 @@ static int buildAlterTableColumnRef(STranslateContext* pCxt, SAlterTableStmt* pS if (NULL == pSchema) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN, pStmt->colName); } + int32_t schemaIdx = getNormalColSchemaIndex(pTableMeta, pStmt->colName); + const SSchemaExt* pSchemaExt = + (schemaIdx >= 0 && pTableMeta->schemaExt && schemaIdx < pTableMeta->tableInfo.numOfColumns) + ? pTableMeta->schemaExt + schemaIdx + : NULL; + SDataType colType = {0}; + schemaToRefDataType(pSchema, NULL != pSchemaExt ? pSchemaExt->typeMod : 0, &colType); - PAR_ERR_JRET(checkColRef(pCxt, pStmt->colName, pStmt->refDbName, pStmt->refTableName, pStmt->refColName, - (SDataType){.type = pSchema->type, .bytes = pSchema->bytes}, + PAR_ERR_JRET(checkColRef(pCxt, pStmt->colName, pStmt->refDbName, pStmt->refTableName, pStmt->refColName, colType, pTableMeta->tableInfo.precision)); pReq->colName = taosStrdup(pStmt->colName); @@ -26124,7 +26103,6 @@ static int32_t rewriteCreateVirtualTable(STranslateContext* pCxt, SQuery* pQuery SNode* pNode = NULL; int32_t index = 0; SDbCfgInfo dbCfg = {0}; - int8_t precision = 0; PAR_ERR_JRET(checkCreateVirtualTable(pCxt, pStmt)); @@ -26134,6 +26112,7 @@ static int32_t rewriteCreateVirtualTable(STranslateContext* pCxt, SQuery* pQuery } toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name); + PAR_ERR_JRET(getDBCfg(pCxt, pStmt->dbName, &dbCfg)); FOREACH(pNode, pStmt->pCols) { SColumnDefNode* pColNode = (SColumnDefNode*)pNode; @@ -26142,12 +26121,11 @@ static int32_t rewriteCreateVirtualTable(STranslateContext* pCxt, SQuery* pQuery if (index == 0) { PAR_ERR_JRET(generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_VTABLE_PRIMTS_HAS_REF)); } - if (IS_DECIMAL_TYPE(pColNode->dataType.type)) { - PAR_ERR_JRET(generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_VTABLE_NOT_SUPPORT_DATA_TYPE)); - } + SDataType colType = pColNode->dataType; + colType.bytes = calcTypeBytes(colType); PAR_ERR_JRET(checkColRef( pCxt, pColNode->colName, pColOptions->refDb, pColOptions->refTable, pColOptions->refColumn, - (SDataType){.type = pColNode->dataType.type, .bytes = calcTypeBytes(pColNode->dataType)}, dbCfg.precision)); + colType, dbCfg.precision)); } index++; } @@ -26222,8 +26200,10 @@ static int32_t checkAndReplaceTagRefs(STranslateContext* pCxt, SNodeList* pSpeci } // Validate the tag reference + SDataType tagType = {0}; + schemaToRefDataType(pSchema, 0, &tagType); PAR_ERR_JRET(checkTagRef(pCxt, (char*)pSchema->name, pColRef->refDbName, pColRef->refTableName, - pColRef->refColName, (SDataType){.type = pSchema->type, .bytes = pSchema->bytes})); + pColRef->refColName, tagType)); // Store the tag reference info (with tag name filled in) if (NULL == pTagRefNodes) { @@ -26302,18 +26282,28 @@ static int32_t rewriteCreateVirtualSubTable(STranslateContext* pCxt, SQuery* pQu if (pSchema->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { PAR_ERR_JRET(TSDB_CODE_VTABLE_PRIMTS_HAS_REF); } + int32_t schemaIdx = getNormalColSchemaIndex(pSuperTableMeta, pColRef->colName); + const SSchemaExt* pSchemaExt = + (schemaIdx >= 0 && pSuperTableMeta->schemaExt && schemaIdx < pSuperTableMeta->tableInfo.numOfColumns) + ? pSuperTableMeta->schemaExt + schemaIdx + : NULL; + SDataType colType = {0}; + schemaToRefDataType(pSchema, NULL != pSchemaExt ? pSchemaExt->typeMod : 0, &colType); PAR_ERR_JRET(checkColRef(pCxt, pColRef->colName, pColRef->refDbName, pColRef->refTableName, pColRef->refColName, - (SDataType){.type = pSchema->type, .bytes = pSchema->bytes}, - pSuperTableMeta->tableInfo.precision)); + colType, pSuperTableMeta->tableInfo.precision)); } } else if (pStmt->pColRefs) { int32_t index = 1; FOREACH(pCol, pStmt->pColRefs) { SColumnRefNode* pColRef = (SColumnRefNode*)pCol; - PAR_ERR_JRET(checkColRef( - pCxt, pColRef->colName, pColRef->refDbName, pColRef->refTableName, pColRef->refColName, - (SDataType){.type = pSuperTableMeta->schema[index].type, .bytes = pSuperTableMeta->schema[index].bytes}, - pSuperTableMeta->tableInfo.precision)); + const SSchemaExt* pSchemaExt = + (pSuperTableMeta->schemaExt && index < pSuperTableMeta->tableInfo.numOfColumns) + ? pSuperTableMeta->schemaExt + index + : NULL; + SDataType colType = {0}; + schemaToRefDataType(&pSuperTableMeta->schema[index], NULL != pSchemaExt ? pSchemaExt->typeMod : 0, &colType); + PAR_ERR_JRET(checkColRef(pCxt, pColRef->colName, pColRef->refDbName, pColRef->refTableName, pColRef->refColName, + colType, pSuperTableMeta->tableInfo.precision)); index++; } } else { diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 762c98aa632..2db22a8288d 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -413,26 +413,12 @@ STableMeta* tableMetaDup(const STableMeta* pTableMeta) { return NULL; } - bool hasSchemaExt = pTableMeta->schemaExt == NULL ? false : true; - size_t schemaExtSize = hasSchemaExt ? pTableMeta->tableInfo.numOfColumns * sizeof(SSchemaExt) : 0; - bool hasColRef = pTableMeta->colRef == NULL ? false : true; - size_t colRefSize = hasColRef ? pTableMeta->numOfColRefs * sizeof(SColRef) : 0; - - size_t size = sizeof(STableMeta) + numOfFields * sizeof(SSchema); - STableMeta* p = taosMemoryMalloc(size + schemaExtSize + colRefSize); + size_t size = TABLE_META_FULL_SIZE(pTableMeta); + STableMeta* p = taosMemoryMalloc(size); if (NULL == p) return NULL; - memcpy(p, pTableMeta, colRefSize + schemaExtSize + size); - if (hasSchemaExt) { - p->schemaExt = (SSchemaExt*)(((char*)p) + size); - } else { - p->schemaExt = NULL; - } - if (hasColRef) { - p->colRef = (SColRef*)(((char*)p) + size + schemaExtSize); - } else { - p->colRef = NULL; - } + memcpy(p, pTableMeta, size); + tableMetaResetPointers(p); return p; } @@ -1256,12 +1242,7 @@ int32_t getTableNameFromCache(SParseMetaCache* pMetaCache, const SName* pName, c code = getMetaDataFromHash(fullName, strlen(fullName), pMetaCache->pTableName, (void**)&pMeta); if (TSDB_CODE_SUCCESS == code) { if (!pMeta) code = TSDB_CODE_PAR_INTERNAL_ERROR; - int32_t metaSize = - sizeof(STableMeta) + sizeof(SSchema) * (pMeta->tableInfo.numOfColumns + pMeta->tableInfo.numOfTags); - int32_t schemaExtSize = - (withExtSchema(pMeta->tableType) && pMeta->schemaExt) ? sizeof(SSchemaExt) * pMeta->tableInfo.numOfColumns : 0; - int32_t colRefSize = (hasRefCol(pMeta->tableType) && pMeta->colRef) ? sizeof(SColRef) * pMeta->numOfColRefs : 0; - const char* pTableName = (const char*)pMeta + metaSize + schemaExtSize + colRefSize; + const char* pTableName = (const char *)pMeta + TABLE_META_FULL_SIZE(pMeta); tstrncpy(pTbName, pTableName, TSDB_TABLE_NAME_LEN); } @@ -1867,4 +1848,3 @@ int32_t updateExprSubQueryType(SNode* pNode, ESubQueryType* type) { return code; } - diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 9a6b52f2d47..2944c01716c 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -982,17 +982,31 @@ _return: return code; } -static int32_t checkColRefType(const SSchema* vtbSchema, const SSchema* refSchema) { - if (vtbSchema->type != refSchema->type) { +static int32_t checkColRefType(const SSchema* vtbSchema, const SSchemaExt* vtbSchemaExt, const SSchema* refSchema, + const SSchemaExt* refSchemaExt) { + SDataType vtbType = {0}; + SDataType refType = {0}; + schemaToRefDataType(vtbSchema, NULL != vtbSchemaExt ? vtbSchemaExt->typeMod : 0, &vtbType); + schemaToRefDataType(refSchema, NULL != refSchemaExt ? refSchemaExt->typeMod : 0, &refType); + + if (vtbType.type != refType.type) { qError("virtual table column:%s type mismatch, virtual table column type:%d, bytes:%d, " "ref table column:%s, type:%d, bytes:%d", - vtbSchema->name, vtbSchema->type, vtbSchema->bytes, refSchema->name, refSchema->type, refSchema->bytes); + vtbSchema->name, vtbType.type, vtbType.bytes, refSchema->name, refType.type, refType.bytes); return TSDB_CODE_PAR_INVALID_REF_COLUMN_TYPE; } - if (!IS_VAR_DATA_TYPE(vtbSchema->type) && vtbSchema->bytes != refSchema->bytes) { + if (!IS_VAR_DATA_TYPE(vtbType.type) && vtbType.bytes != refType.bytes) { qError("virtual table column:%s bytes mismatch, virtual table column type:%d, bytes:%d, " "ref table column:%s, type:%d, bytes:%d", - vtbSchema->name, vtbSchema->type, vtbSchema->bytes, refSchema->name, refSchema->type, refSchema->bytes); + vtbSchema->name, vtbType.type, vtbType.bytes, refSchema->name, refType.type, refType.bytes); + return TSDB_CODE_PAR_INVALID_REF_COLUMN_TYPE; + } + if (IS_DECIMAL_TYPE(vtbType.type) && + (vtbType.precision != refType.precision || vtbType.scale != refType.scale)) { + qError("virtual table column:%s decimal type mismatch, virtual table column type:%d, precision:%u, scale:%u, " + "ref table column:%s, type:%d, precision:%u, scale:%u", + vtbSchema->name, vtbType.type, vtbType.precision, vtbType.scale, refSchema->name, refType.type, + refType.precision, refType.scale); return TSDB_CODE_PAR_INVALID_REF_COLUMN_TYPE; } return TSDB_CODE_SUCCESS; @@ -1021,15 +1035,25 @@ static int32_t addSubScanNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SVi SLogicNode **ppRefScan = (SLogicNode **)taosHashGet(refTablesMap, &tableNameKey, strlen(tableNameKey)); const SSchema* pRefColSchema = &((SRealTableNode*)pRefTable)->pMeta->schema[colIdx]; + const SSchemaExt* pRefSchemaExt = + (((SRealTableNode*)pRefTable)->pMeta->schemaExt && colIdx < ((SRealTableNode*)pRefTable)->pMeta->tableInfo.numOfColumns) + ? ((SRealTableNode*)pRefTable)->pMeta->schemaExt + colIdx + : NULL; + const SSchemaExt* pVtbSchemaExt = + (pVirtualTable->pMeta->schemaExt && schemaIndex < pVirtualTable->pMeta->tableInfo.numOfColumns) + ? pVirtualTable->pMeta->schemaExt + schemaIndex + : NULL; if (NULL == ppRefScan) { PLAN_ERR_JRET(createRefScanLogicNode(pCxt, pSelect, (SRealTableNode*)pRefTable, &pRefScan)); - PLAN_ERR_JRET(checkColRefType(&pVirtualTable->pMeta->schema[schemaIndex], pRefColSchema)); + PLAN_ERR_JRET(checkColRefType(&pVirtualTable->pMeta->schema[schemaIndex], pVtbSchemaExt, pRefColSchema, + pRefSchemaExt)); PLAN_ERR_JRET(scanAddCol(pRefScan, pColRef, &pVirtualTable->table, &pVirtualTable->pMeta->schema[schemaIndex], colId, pRefColSchema)); PLAN_ERR_JRET(taosHashPut(refTablesMap, &tableNameKey, strlen(tableNameKey), &pRefScan, POINTER_BYTES)); put = true; } else { pRefScan = *ppRefScan; - PLAN_ERR_JRET(checkColRefType(&pVirtualTable->pMeta->schema[schemaIndex], pRefColSchema)); + PLAN_ERR_JRET(checkColRefType(&pVirtualTable->pMeta->schema[schemaIndex], pVtbSchemaExt, pRefColSchema, + pRefSchemaExt)); PLAN_ERR_JRET(scanAddCol(pRefScan, pColRef, &pVirtualTable->table, &pVirtualTable->pMeta->schema[schemaIndex], colId, pRefColSchema)); } diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index b475d1f9ec0..29fb5c4a091 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -58,6 +58,60 @@ const SSchema* tGetTbnameColumnSchema() { return &_s; } +bool hasDecimalBytesTypeInfo(int32_t bytes) { return (((uint32_t)bytes >> 24) != 0); } + +/* + * Build a comparable ref-column type from schema metadata. + * + * typeMod carries decimal precision/scale when schemaExt is present. When it is + * absent, fall back to the legacy decimal metadata encoded in schema bytes. + */ +void schemaToRefDataType(const SSchema* pSchema, STypeMod typeMod, SDataType* pType) { + if (NULL == pSchema || NULL == pType) { + return; + } + + memset(pType, 0, sizeof(*pType)); + pType->type = pSchema->type; + pType->bytes = pSchema->bytes; + if (IS_DECIMAL_TYPE(pSchema->type)) { + if (typeMod != 0) { + fillTypeFromTypeMod(pType, typeMod); + } else if ((((uint32_t)pSchema->bytes) >> 24) != 0) { + extractDecimalTypeInfoFromBytes(&pType->bytes, &pType->precision, &pType->scale); + } + } +} + +bool isSameRefDataType(const SDataType* pLeft, const SDataType* pRight) { + if (pLeft->type != pRight->type) { + return false; + } + if (!IS_VAR_DATA_TYPE(pLeft->type) && pLeft->bytes != pRight->bytes) { + return false; + } + if (IS_DECIMAL_TYPE(pLeft->type) && + (pLeft->precision != pRight->precision || pLeft->scale != pRight->scale)) { + return false; + } + + return true; +} + +int32_t getNormalColSchemaIndex(const STableMeta* pTableMeta, const char* pColName) { + if (NULL == pTableMeta || NULL == pColName) { + return -1; + } + + for (int32_t i = 0; i < pTableMeta->tableInfo.numOfColumns; ++i) { + if (0 == strcmp(pColName, pTableMeta->schema[i].name)) { + return i; + } + } + + return -1; +} + static bool doValidateSchema(SSchema* pSchema, int32_t numOfCols, int32_t maxLen) { if (!pSchema) { return false; @@ -626,44 +680,13 @@ int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst) { return TSDB_CODE_INVALID_PARA; } - int32_t metaSize = sizeof(STableMeta) + numOfField * sizeof(SSchema); - int32_t schemaExtSize = 0; - int32_t colRefSize = 0; - int32_t tagRefSize = 0; - if (withExtSchema(pSrc->tableType) && pSrc->schemaExt) { - schemaExtSize = pSrc->tableInfo.numOfColumns * sizeof(SSchemaExt); - } - if (hasRefCol(pSrc->tableType) && pSrc->colRef) { - colRefSize = pSrc->numOfColRefs * sizeof(SColRef); - } - if (hasRefCol(pSrc->tableType) && pSrc->tagRef) { - tagRefSize = pSrc->numOfTagRefs * sizeof(SColRef); - } - *pDst = taosMemoryMalloc(metaSize + schemaExtSize + colRefSize + tagRefSize); + int32_t metaSize = TABLE_META_FULL_SIZE(pSrc); + *pDst = taosMemoryMalloc(metaSize); if (NULL == *pDst) { return terrno; } memcpy(*pDst, pSrc, metaSize); - if (withExtSchema(pSrc->tableType) && pSrc->schemaExt) { - (*pDst)->schemaExt = (SSchemaExt*)((char*)*pDst + metaSize); - memcpy((*pDst)->schemaExt, pSrc->schemaExt, schemaExtSize); - } else { - (*pDst)->schemaExt = NULL; - } - if (hasRefCol(pSrc->tableType) && pSrc->colRef) { - (*pDst)->colRef = (SColRef*)((char*)*pDst + metaSize + schemaExtSize); - memcpy((*pDst)->colRef, pSrc->colRef, colRefSize); - } else { - (*pDst)->colRef = NULL; - } - if (hasRefCol(pSrc->tableType) && pSrc->tagRef) { - (*pDst)->tagRef = (SColRef*)((char*)*pDst + metaSize + schemaExtSize + colRefSize); - memcpy((*pDst)->tagRef, pSrc->tagRef, tagRefSize); - (*pDst)->numOfTagRefs = pSrc->numOfTagRefs; - } else { - (*pDst)->tagRef = NULL; - (*pDst)->numOfTagRefs = 0; - } + tableMetaResetPointers(*pDst); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index ccd09900dc0..47b398b7f6f 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -807,6 +807,7 @@ int32_t queryCreateTableMetaExFromMsg(STableMetaRsp *msg, bool isStb, STableMeta int32_t pColRefSize = (hasRefCol(msg->tableType) && msg->pColRefs) ? sizeof(SColRef) * msg->numOfColRefs : 0; int32_t tbNameSize = strlen(msg->tbName) + 1; + STableMeta *pTableMeta = taosMemoryCalloc(1, metaSize + schemaExtSize + pColRefSize + tbNameSize); if (NULL == pTableMeta) { qError("calloc size[%d] failed", metaSize); @@ -839,8 +840,8 @@ int32_t queryCreateTableMetaExFromMsg(STableMetaRsp *msg, bool isStb, STableMeta pTableMeta->schemaExt = NULL; } - if (hasRefCol(msg->tableType) && msg->pColRefs) { - pTableMeta->colRef = (SColRef *)((char *)pTableMeta + metaSize + schemaExtSize); + if (hasRefCol(msg->tableType) && msg->pColRefs && !isStb) { + pTableMeta->colRef = pColRef; memcpy(pTableMeta->colRef, msg->pColRefs, pColRefSize); } else { pTableMeta->colRef = NULL; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index b9b6ba88893..d6eb5bd73fa 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -1097,7 +1097,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_VTABLE_SCAN_INTERNAL_ERROR, "Virtual table scan TAOS_DEFINE_ERROR(TSDB_CODE_VTABLE_SCAN_INVALID_DOWNSTREAM, "Virtual table scan invalid downstream operator type") TAOS_DEFINE_ERROR(TSDB_CODE_VTABLE_PRIMTS_HAS_REF, "Virtual table prim timestamp column should not has ref column") TAOS_DEFINE_ERROR(TSDB_CODE_VTABLE_NOT_VIRTUAL_SUPER_TABLE, "Create virtual child table must use virtual super table") -TAOS_DEFINE_ERROR(TSDB_CODE_VTABLE_NOT_SUPPORT_DATA_TYPE, "Virtual table not support decimal type") +TAOS_DEFINE_ERROR(TSDB_CODE_VTABLE_NOT_SUPPORT_DATA_TYPE, "Virtual table does not support this data type") TAOS_DEFINE_ERROR(TSDB_CODE_VTABLE_NOT_SUPPORT_STMT, "Virtual table not support in STMT query and STMT insert") TAOS_DEFINE_ERROR(TSDB_CODE_VTABLE_NOT_SUPPORT_TOPIC, "Virtual table not support in topic") TAOS_DEFINE_ERROR(TSDB_CODE_VTABLE_NOT_SUPPORT_CROSS_DB, "Virtual super table query not support origin table from different databases") diff --git a/test/cases/05-VirtualTables/test_vtable_alter.py b/test/cases/05-VirtualTables/test_vtable_alter.py index 7b7933b0ec2..64897c713a1 100644 --- a/test/cases/05-VirtualTables/test_vtable_alter.py +++ b/test/cases/05-VirtualTables/test_vtable_alter.py @@ -484,9 +484,6 @@ class TestVtableAlter(): # 1.6. change column length when column reference exists tdSql.error("alter vtable vtb_virtual_ntb0 modify column nchar_16_col nchar(32);") - # 1.7. add column with decimal type - tdSql.error("alter vtable vtb_virtual_ntb0 add column extra_decimal decimal(38,38)") - # 2. child table # 2.1. change column reference with wrong type tdSql.error("alter vtable vtb_virtual_ctb0 alter column int_col set vtb_org_child_19.tinyint_col") @@ -502,14 +499,7 @@ class TestVtableAlter(): tdSql.execute("alter stable vtb_virtual_stb modify column nchar_16_col nchar(32);") #tdSql.error("select nchar_16_col from vtb_virtual_ctb0;") - # 3.3. add column with decimal type - tdSql.error("alter stable vtb_virtual_stb add column extra_decimal decimal(38,38)") - - # 3.4. add tag with decimal type - tdSql.error("alter stable vtb_virtual_stb add tag extra_decimal_tag decimal(38,38)") - - # 3.5 drop virtual child table's origin table and create a new origin table with same name but different column type + # 3.3 drop virtual child table's origin table and create a new origin table with same name but different column type tdSql.execute("drop table vtb_org_child_10") tdSql.execute("create table vtb_org_child_10(ts timestamp, bool_col float)") tdSql.error("select * from vtb_virtual_stb") - diff --git a/test/cases/05-VirtualTables/test_vtable_create.py b/test/cases/05-VirtualTables/test_vtable_create.py index 238e1827d07..a709a78a195 100644 --- a/test/cases/05-VirtualTables/test_vtable_create.py +++ b/test/cases/05-VirtualTables/test_vtable_create.py @@ -780,75 +780,3 @@ class TestVtableCreate: "geo_32_col FROM vtb_org_child_18.geo_32_col)" "USING vtb_virtual_stb TAGS (13, false, 13, 13, 'vchild13', 'vchild13')") - # 11. create virtual table using decimal - # 11.1 super table - # 11.1.1 decimal column - tdSql.error(f"CREATE STABLE `vtb_virtual_stb_error` (" - "ts timestamp, " - "u_tinyint_col tinyint unsigned, " - "u_smallint_col smallint unsigned, " - "u_int_col int unsigned, " - "u_bigint_col bigint unsigned, " - "tinyint_col tinyint, " - "smallint_col smallint, " - "int_col int, " - "bigint_col bigint, " - "float_col float, " - "double_col double, " - "bool_col bool, " - "binary_16_col binary(16)," - "binary_32_col binary(32)," - "nchar_16_col nchar(16)," - "nchar_32_col nchar(32)," - "varbinary_16_col varbinary(16)," - "varbinary_32_col varbinary(32)," - "geo_16_col geometry(16)," - "geo_32_col geometry(32)," - "decimal_col decimal(38,38)" - ") TAGS (" - "int_tag int," - "bool_tag bool," - "float_tag float," - "double_tag double," - "nchar_32_tag nchar(32)," - "binary_32_tag binary(32))" - "VIRTUAL 1") - # 11.1.2 decimal tag - tdSql.error(f"CREATE STABLE `vtb_virtual_stb_error` (" - "ts timestamp, " - "u_tinyint_col tinyint unsigned, " - "u_smallint_col smallint unsigned, " - "u_int_col int unsigned, " - "u_bigint_col bigint unsigned, " - "tinyint_col tinyint, " - "smallint_col smallint, " - "int_col int, " - "bigint_col bigint, " - "float_col float, " - "double_col double, " - "bool_col bool, " - "binary_16_col binary(16)," - "binary_32_col binary(32)," - "nchar_16_col nchar(16)," - "nchar_32_col nchar(32)," - "varbinary_16_col varbinary(16)," - "varbinary_32_col varbinary(32)," - "geo_16_col geometry(16)," - "geo_32_col geometry(32)" - ") TAGS (" - "int_tag int," - "bool_tag bool," - "float_tag float," - "double_tag double," - "nchar_32_tag nchar(32)," - "binary_32_tag binary(32))," - "decimal_tag decimal(38,38)" - "VIRTUAL 1") - - # 11.2 virtual normal table - tdSql.error("CREATE VTABLE `error_vtb_virtual_ntb8` (" - "ts timestamp FROM vtb_org_normal_0.ts, " - "u_tinyint_col tinyint unsigned from vtb_org_normal_0.u_tinyint_col, " - "u_smallint_col smallint unsigned from vtb_org_normal_1.u_smallint_col, " - "decimal_col decimal(38,38))") - diff --git a/test/cases/05-VirtualTables/test_vtable_decimal.py b/test/cases/05-VirtualTables/test_vtable_decimal.py new file mode 100644 index 00000000000..84930e88a71 --- /dev/null +++ b/test/cases/05-VirtualTables/test_vtable_decimal.py @@ -0,0 +1,894 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- +from decimal import Decimal + +from new_test_framework.utils import tdLog, tdSql, tdDnodes + + +DB_NAME = "test_vtable_decimal" + + +class TestVTableDecimal: + + @staticmethod + def find_row(field_name): + for row in range(tdSql.getRows()): + if tdSql.getData(row, 0) == field_name: + return row + raise AssertionError(f"field {field_name} not found in result set") + + @staticmethod + def check_decimal(row, col, expect): + value = tdSql.getData(row, col) + assert Decimal(str(value)) == Decimal(expect), f"expect {expect}, got {value}" + + @staticmethod + def check_decimal_close(row, col, expect, tolerance="0.000001"): + value = Decimal(str(tdSql.getData(row, col))) + delta = abs(value - Decimal(expect)) + assert delta <= Decimal(tolerance), f"expect {expect} +/- {tolerance}, got {value}" + + @staticmethod + def check_column_meta(table_name, col_name, precision, scale, source): + tdSql.query( + "select col_precision, col_scale, col_source " + "from information_schema.ins_columns " + f"where db_name='{DB_NAME}' and table_name='{table_name}' and col_name='{col_name}'" + ) + tdSql.checkRows(1) + tdSql.checkData(0, 0, precision) + tdSql.checkData(0, 1, scale) + tdSql.checkData(0, 2, source) + + @staticmethod + def check_non_decimal_meta(table_name, col_name, source): + tdSql.query( + "select col_precision, col_scale, col_source " + "from information_schema.ins_columns " + f"where db_name='{DB_NAME}' and table_name='{table_name}' and col_name='{col_name}'" + ) + tdSql.checkRows(1) + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, source) + + @staticmethod + def check_no_column_meta(table_name, col_name): + tdSql.query( + "select col_name from information_schema.ins_columns " + f"where db_name='{DB_NAME}' and table_name='{table_name}' and col_name='{col_name}'" + ) + tdSql.checkRows(0) + + @staticmethod + def check_projected_rows(expect_rows): + tdSql.checkRows(len(expect_rows)) + for row_idx, (ts_val, dec64_val, dec128_val, metric) in enumerate(expect_rows): + tdSql.checkData(row_idx, 0, ts_val) + TestVTableDecimal.check_decimal(row_idx, 1, dec64_val) + TestVTableDecimal.check_decimal(row_idx, 2, dec128_val) + tdSql.checkData(row_idx, 3, metric) + + @classmethod + def setup_class(cls): + tdLog.info("prepare decimal origin tables.") + tdSql.execute(f"drop database if exists {DB_NAME}") + tdSql.execute(f"create database {DB_NAME}") + tdSql.execute(f"use {DB_NAME}") + + tdSql.execute( + "create table org_ntb_0 (" + "ts timestamp, " + "dec64_col decimal(18,2), " + "dec128_col decimal(20,4), " + "dec64_max decimal(18,18), " + "dec128_max decimal(38,10), " + "metric int)" + ) + tdSql.execute( + "create table org_ntb_1 (" + "ts timestamp, " + "dec64_col decimal(10,3), " + "dec128_col decimal(30,8), " + "metric float)" + ) + tdSql.execute( + "create stable org_stb (" + "ts timestamp, " + "dec64_col decimal(18,2), " + "dec128_col decimal(20,4), " + "metric int" + ") tags (group_id int)" + ) + tdSql.execute("create table org_ctb_0 using org_stb tags (0)") + tdSql.execute("create table org_ctb_1 using org_stb tags (1)") + + org_ntb_0_rows = [ + (1700000000000, "1.25", "100.1234", "0.123456789012345678", "12345678901234567890.1234567890", 10), + (1700000001000, "2.50", "200.5678", "0.000000000000000001", "99999999999999999999.9999999999", 20), + (1700000002000, "3.75", "300.0001", "0.999999999999999999", "0.0000000001", 30), + ] + for ts_val, dec64_val, dec128_val, dec64_max, dec128_max, metric in org_ntb_0_rows: + tdSql.execute( + "insert into org_ntb_0 values " + f"({ts_val}, {dec64_val}, {dec128_val}, {dec64_max}, {dec128_max}, {metric})" + ) + + org_ntb_1_rows = [ + (1700000000000, "1.234", "12345678901234567890.12345678", "1.5"), + (1700000001000, "9.876", "99999999999999999999.99999999", "2.5"), + ] + for ts_val, dec64_val, dec128_val, metric in org_ntb_1_rows: + tdSql.execute( + f"insert into org_ntb_1 values ({ts_val}, {dec64_val}, {dec128_val}, {metric})" + ) + + org_ctb_0_rows = [ + (1700000000000, "1.25", "100.1234", 10), + (1700000001000, "2.50", "200.5678", 20), + (1700000002000, "3.75", "300.0001", 30), + ] + for ts_val, dec64_val, dec128_val, metric in org_ctb_0_rows: + tdSql.execute( + f"insert into org_ctb_0 values ({ts_val}, {dec64_val}, {dec128_val}, {metric})" + ) + + org_ctb_1_rows = [ + (1700000000000, "10.00", "1000.0000", 100), + (1700000001000, "20.00", "2000.0000", 200), + ] + for ts_val, dec64_val, dec128_val, metric in org_ctb_1_rows: + tdSql.execute( + f"insert into org_ctb_1 values ({ts_val}, {dec64_val}, {dec128_val}, {metric})" + ) + + tdLog.info("prepare decimal virtual tables.") + tdSql.execute( + "create stable vstb_decimal (" + "ts timestamp, " + "dec64_col decimal(18,2), " + "dec128_col decimal(20,4), " + "metric int" + ") tags (tag_plain int) virtual 1" + ) + tdSql.execute( + "create vtable vntb_ref_ntb (" + "ts timestamp, " + "dec64_col decimal(18,2) from org_ntb_0.dec64_col, " + "dec128_col decimal(20,4) from org_ntb_0.dec128_col, " + "metric int from org_ntb_0.metric)" + ) + tdSql.execute( + "create vtable vntb_ref_ctb (" + "ts timestamp, " + "dec64_col decimal(18,2) from org_ctb_0.dec64_col, " + "dec128_col decimal(20,4) from org_ctb_0.dec128_col, " + "metric int from org_ctb_0.metric)" + ) + tdSql.execute( + "create vtable vntb_multi_src (" + "ts timestamp, " + "dec64_from_ctb decimal(18,2) from org_ctb_0.dec64_col, " + "dec128_from_ntb decimal(20,4) from org_ntb_0.dec128_col, " + "metric_from_ctb int from org_ctb_0.metric)" + ) + tdSql.execute( + "create vtable vntb_local_dec (" + "ts timestamp, " + "dec64_col decimal(18,2), " + "dec128_col decimal(38,10))" + ) + tdSql.execute( + "create vtable vntb_mixed_dec (" + "ts timestamp, " + "dec64_ref decimal(18,2) from org_ctb_0.dec64_col, " + "dec128_local decimal(30,8), " + "metric int from org_ctb_0.metric)" + ) + tdSql.execute( + "create vtable vntb_precision_decimal (" + "ts timestamp, " + "dec64_max decimal(18,18) from org_ntb_0.dec64_max, " + "dec128_max decimal(38,10) from org_ntb_0.dec128_max)" + ) + tdSql.execute( + "create vtable vctb_full (" + "dec64_col from org_ctb_0.dec64_col, " + "dec128_col from org_ntb_0.dec128_col, " + "metric from org_ctb_0.metric)" + " using vstb_decimal tags (1)" + ) + tdSql.execute( + "create vtable vctb_partial (" + "dec64_col from org_ctb_0.dec64_col)" + " using vstb_decimal tags (2)" + ) + tdSql.execute("create vtable vctb_empty using vstb_decimal tags (3)") + tdSql.execute( + "create vtable vctb_src0 (" + "dec64_col from org_ctb_0.dec64_col, " + "dec128_col from org_ctb_0.dec128_col, " + "metric from org_ctb_0.metric)" + " using vstb_decimal tags (10)" + ) + tdSql.execute( + "create vtable vctb_src1 (" + "dec64_col from org_ctb_1.dec64_col, " + "dec128_col from org_ctb_1.dec128_col, " + "metric from org_ctb_1.metric)" + " using vstb_decimal tags (20)" + ) + + def test_decimal_create_and_ref_validation(self): + """Create: decimal create boundaries and reference validation + + 1. validate virtual stable decimal precision and scale boundaries + 2. validate virtual normal table decimal reference matching + 3. validate virtual child table decimal reference matching + 4. verify decimal precision and scale metadata are persisted on creation + + Catalog: + - VirtualTable + + Since: v3.4.1.0 + + Labels: virtual, decimal, create + + Jira: None + + History: + - 2026-4-1 Joey Sima Expanded by test plan + + """ + tdSql.execute(f"use {DB_NAME}") + + tdSql.execute("create stable vstb_p1s0 (ts timestamp, c1 decimal(1,0)) tags (t1 int) virtual 1") + tdSql.execute("create stable vstb_p18s18 (ts timestamp, c1 decimal(18,18)) tags (t1 int) virtual 1") + tdSql.execute("create stable vstb_p18s0 (ts timestamp, c1 decimal(18,0)) tags (t1 int) virtual 1") + tdSql.execute("create stable vstb_p19s0 (ts timestamp, c1 decimal(19,0)) tags (t1 int) virtual 1") + tdSql.execute("create stable vstb_p38s0 (ts timestamp, c1 decimal(38,0)) tags (t1 int) virtual 1") + tdSql.execute("create stable vstb_p38s38 (ts timestamp, c1 decimal(38,38)) tags (t1 int) virtual 1") + + tdSql.error("create stable vstb_bad_p0 (ts timestamp, c1 decimal(0,0)) tags (t1 int) virtual 1") + tdSql.error("create stable vstb_bad_p39 (ts timestamp, c1 decimal(39,0)) tags (t1 int) virtual 1") + tdSql.error("create stable vstb_bad_s19 (ts timestamp, c1 decimal(18,19)) tags (t1 int) virtual 1") + tdSql.error("create stable vstb_bad_s39 (ts timestamp, c1 decimal(38,39)) tags (t1 int) virtual 1") + + tdSql.query(f"show create stable {DB_NAME}.vstb_p18s18") + tdSql.checkRows(1) + assert "DECIMAL(18,18)" in tdSql.getData(0, 1) + + tdSql.query(f"show create stable {DB_NAME}.vstb_p38s38") + tdSql.checkRows(1) + assert "DECIMAL(38,38)" in tdSql.getData(0, 1) + + tdSql.query(f"describe {DB_NAME}.vstb_p19s0") + tdSql.checkCols(7) + tdSql.checkRows(3) + dec19_row = self.find_row("c1") + tdSql.checkData(dec19_row, 1, "DECIMAL(19, 0)") + + tdSql.query(f"describe {DB_NAME}.vstb_p38s38") + tdSql.checkCols(7) + tdSql.checkRows(3) + dec38_row = self.find_row("c1") + tdSql.checkData(dec38_row, 1, "DECIMAL(38, 38)") + + tdSql.error( + "create stable vstb_dec_tag_bad (" + "ts timestamp, metric int" + ") tags (group_id int, tag_dec64 decimal(18,2), tag_dec128 decimal(30,6)) virtual 1" + ) + tdSql.error( + "create stable vstb_dec_both_bad (" + "ts timestamp, dec64_col decimal(18,2), dec128_col decimal(38,10)" + ") tags (tag_dec decimal(10,2)) virtual 1" + ) + tdSql.error( + "create stable vstb_dec_tag_ref_bad (" + "ts timestamp, metric int" + ") tags (group_id int, tag_dec decimal(10,2)) virtual 1" + ) + tdSql.error( + "create stable stb_dec_tag_bad (" + "ts timestamp, metric int" + ") tags (tag_dec decimal(10,2))" + ) + tdSql.error( + "create stable vstb_dec_tag_bad2 (" + "ts timestamp, metric int" + ") tags (tag_dec decimal(10,2)) virtual 1" + ) + + tdSql.error( + "create vtable vntb_bad_type (" + "ts timestamp, " + "dec64_col decimal(18,2) from org_ntb_0.dec128_col)" + ) + tdSql.error( + "create vtable vntb_bad_prec (" + "ts timestamp, " + "dec64_col decimal(10,2) from org_ntb_0.dec64_col)" + ) + tdSql.error( + "create vtable vntb_bad_scale (" + "ts timestamp, " + "dec64_col decimal(18,3) from org_ntb_0.dec64_col)" + ) + tdSql.error( + "create vtable vntb_bad_dec_to_int (" + "ts timestamp, " + "dec64_col decimal(18,2) from org_ntb_0.metric)" + ) + tdSql.error( + "create vtable vntb_bad_int_to_dec (" + "ts timestamp, " + "metric int from org_ntb_0.dec64_col)" + ) + tdSql.error( + "create vtable vctb_bad_ref (" + "dec64_col from org_ntb_1.dec64_col, " + "dec128_col from org_ntb_0.dec128_col, " + "metric from org_ntb_0.metric)" + " using vstb_decimal tags (99)" + ) + + self.check_column_meta("vntb_ref_ntb", "dec64_col", 18, 2, f"{DB_NAME}.org_ntb_0.dec64_col") + self.check_column_meta("vntb_ref_ntb", "dec128_col", 20, 4, f"{DB_NAME}.org_ntb_0.dec128_col") + self.check_non_decimal_meta("vntb_ref_ntb", "metric", f"{DB_NAME}.org_ntb_0.metric") + self.check_column_meta("vntb_precision_decimal", "dec64_max", 18, 18, f"{DB_NAME}.org_ntb_0.dec64_max") + self.check_column_meta("vntb_precision_decimal", "dec128_max", 38, 10, f"{DB_NAME}.org_ntb_0.dec128_max") + self.check_column_meta("vctb_full", "dec64_col", 18, 2, f"{DB_NAME}.org_ctb_0.dec64_col") + self.check_column_meta("vctb_full", "dec128_col", 20, 4, f"{DB_NAME}.org_ntb_0.dec128_col") + self.check_column_meta("vctb_partial", "dec64_col", 18, 2, f"{DB_NAME}.org_ctb_0.dec64_col") + + def test_decimal_alter_paths(self): + """Alter: decimal add, drop, and set reference + + 1. validate add column on virtual normal tables with and without references + 2. validate alter set keeps decimal precision and scale rules + 3. validate drop column removes decimal metadata + 4. validate virtual stable add and drop decimal columns propagate to child tables + + Catalog: + - VirtualTable + + Since: v3.4.1.0 + + Labels: virtual, decimal, alter + + Jira: None + + History: + - 2026-4-1 Joey Sima Expanded by test plan + + """ + tdSql.execute(f"use {DB_NAME}") + + tdSql.execute( + "create vtable vntb_decimal_alter (" + "ts timestamp, " + "dec64_col decimal(18,2) from org_ctb_0.dec64_col, " + "dec128_local decimal(30,8), " + "metric int from org_ctb_0.metric)" + ) + tdSql.execute("alter vtable vntb_decimal_alter add column extra_dec decimal(38,18)") + tdSql.execute( + "alter vtable vntb_decimal_alter add column extra_dec128 decimal(20,4) from org_ntb_0.dec128_col" + ) + tdSql.error( + "alter vtable vntb_decimal_alter add column bad_dec_scale decimal(10,3) from org_ntb_0.dec64_col" + ) + tdSql.error( + "alter vtable vntb_decimal_alter add column bad_dec_type decimal(18,2) from org_ntb_0.dec128_col" + ) + tdSql.execute( + "alter vtable vntb_decimal_alter alter column dec128_local set org_ntb_1.dec128_col" + ) + tdSql.error( + "alter vtable vntb_decimal_alter alter column dec64_col set org_ntb_1.dec64_col" + ) + + self.check_column_meta("vntb_decimal_alter", "extra_dec", 38, 18, None) + self.check_column_meta( + "vntb_decimal_alter", "extra_dec128", 20, 4, f"{DB_NAME}.org_ntb_0.dec128_col" + ) + self.check_column_meta( + "vntb_decimal_alter", "dec128_local", 30, 8, f"{DB_NAME}.org_ntb_1.dec128_col" + ) + + tdSql.query(f"describe {DB_NAME}.vntb_decimal_alter") + tdSql.checkCols(5) + extra_dec_row = self.find_row("extra_dec") + tdSql.checkData(extra_dec_row, 1, "DECIMAL(38, 18)") + + tdSql.execute("alter vtable vntb_decimal_alter drop column extra_dec") + self.check_no_column_meta("vntb_decimal_alter", "extra_dec") + + tdSql.execute( + "create stable vstb_decimal_alter (" + "ts timestamp, " + "dec64_col decimal(18,2), " + "dec128_col decimal(20,4), " + "metric int" + ") tags (tag_plain int) virtual 1" + ) + tdSql.execute( + "create vtable vctb_decimal_alter (" + "dec64_col from org_ctb_0.dec64_col, " + "dec128_col from org_ctb_0.dec128_col, " + "metric from org_ctb_0.metric)" + " using vstb_decimal_alter tags (100)" + ) + tdSql.execute("alter stable vstb_decimal_alter add column new_dec decimal(38,10)") + + self.check_column_meta("vctb_decimal_alter", "new_dec", 38, 10, None) + + tdSql.query(f"describe {DB_NAME}.vstb_decimal_alter") + tdSql.checkCols(7) + stable_new_dec_row = self.find_row("new_dec") + tdSql.checkData(stable_new_dec_row, 1, "DECIMAL(38, 10)") + + tdSql.query(f"describe {DB_NAME}.vctb_decimal_alter") + tdSql.checkCols(5) + new_dec_row = self.find_row("new_dec") + tdSql.checkData(new_dec_row, 1, "DECIMAL(38, 10)") + + tdSql.error("alter stable vstb_decimal add tag new_dec_tag decimal(18,6)") + + tdSql.query(f"select dec64_col, new_dec from {DB_NAME}.vctb_decimal_alter order by ts") + tdSql.checkRows(3) + for row_idx in range(3): + self.check_decimal(row_idx, 0, ["1.25", "2.50", "3.75"][row_idx]) + tdSql.checkData(row_idx, 1, None) + + tdSql.execute("alter stable vstb_decimal_alter drop column new_dec") + self.check_no_column_meta("vstb_decimal_alter", "new_dec") + self.check_no_column_meta("vctb_decimal_alter", "new_dec") + + def test_decimal_describe_show_create_and_information_schema(self): + """Meta: describe, show create, and ins_columns for decimal virtual tables + + 1. verify describe keeps the current decimal virtual-table result shape + 2. verify show create renders DECIMAL with references + 3. verify information_schema reports precision and scale for decimal columns + 4. verify non-decimal columns keep null precision and scale + + Catalog: + - VirtualTable + + Since: v3.4.1.0 + + Labels: virtual, decimal, metadata + + Jira: None + + History: + - 2026-4-1 Joey Sima Expanded by test plan + + """ + tdSql.execute(f"use {DB_NAME}") + + tdSql.query(f"describe {DB_NAME}.vntb_ref_ntb") + tdSql.checkCols(5) + tdSql.checkRows(4) + dec64_row = self.find_row("dec64_col") + tdSql.checkData(dec64_row, 1, "DECIMAL(18, 2)") + dec128_row = self.find_row("dec128_col") + tdSql.checkData(dec128_row, 1, "DECIMAL(20, 4)") + + tdSql.query(f"describe {DB_NAME}.vctb_full") + tdSql.checkCols(5) + tdSql.checkRows(5) + child_dec64_row = self.find_row("dec64_col") + tdSql.checkData(child_dec64_row, 1, "DECIMAL(18, 2)") + child_dec128_row = self.find_row("dec128_col") + tdSql.checkData(child_dec128_row, 1, "DECIMAL(20, 4)") + tag_row = self.find_row("tag_plain") + tdSql.checkData(tag_row, 3, "TAG") + + tdSql.query(f"describe {DB_NAME}.vstb_decimal") + tdSql.checkCols(7) + tdSql.checkRows(5) + stb_dec128_row = self.find_row("dec128_col") + tdSql.checkData(stb_dec128_row, 1, "DECIMAL(20, 4)") + + tdSql.query(f"show create vtable {DB_NAME}.vntb_ref_ntb") + tdSql.checkRows(1) + create_sql = tdSql.getData(0, 1) + assert "DECIMAL(18,2)" in create_sql + assert "DECIMAL(20,4)" in create_sql + assert "FROM `test_vtable_decimal`.`org_ntb_0`.`dec64_col`" in create_sql + assert "FROM `test_vtable_decimal`.`org_ntb_0`.`dec128_col`" in create_sql + + tdSql.query(f"show create stable {DB_NAME}.vstb_decimal") + tdSql.checkRows(1) + create_sql = tdSql.getData(0, 1) + assert "`dec64_col` DECIMAL(18,2)" in create_sql + assert "`dec128_col` DECIMAL(20,4)" in create_sql + + tdSql.query(f"show create vtable {DB_NAME}.vctb_full") + tdSql.checkRows(1) + create_sql = tdSql.getData(0, 1) + assert "FROM `test_vtable_decimal`.`org_ctb_0`.`dec64_col`" in create_sql + assert "FROM `test_vtable_decimal`.`org_ntb_0`.`dec128_col`" in create_sql + + self.check_column_meta("vntb_ref_ntb", "dec64_col", 18, 2, f"{DB_NAME}.org_ntb_0.dec64_col") + self.check_column_meta("vntb_ref_ntb", "dec128_col", 20, 4, f"{DB_NAME}.org_ntb_0.dec128_col") + self.check_column_meta("vctb_full", "dec64_col", 18, 2, f"{DB_NAME}.org_ctb_0.dec64_col") + self.check_column_meta("vctb_full", "dec128_col", 20, 4, f"{DB_NAME}.org_ntb_0.dec128_col") + self.check_non_decimal_meta("vntb_ref_ntb", "metric", f"{DB_NAME}.org_ntb_0.metric") + + tdSql.query( + "select stable_name, columns, `tags` " + "from information_schema.ins_stables " + f"where db_name='{DB_NAME}' and stable_name='vstb_decimal'" + ) + tdSql.checkRows(1) + tdSql.checkData(0, 0, "vstb_decimal") + tdSql.checkData(0, 1, 4) + tdSql.checkData(0, 2, 1) + + def test_decimal_query_paths(self): + """Query: decimal projection, merge, and null-path behavior + + 1. validate select * on virtual normal and child tables with decimal columns + 2. validate multi-source decimal projection on virtual normal tables + 3. validate virtual stable merge across multiple child mappings + 4. validate local decimal columns remain null or empty without data sources + + Catalog: + - VirtualTable + + Since: v3.4.1.0 + + Labels: virtual, decimal, query + + Jira: None + + History: + - 2026-4-1 Joey Sima Expanded by test plan + + """ + tdSql.execute(f"use {DB_NAME}") + + expect_rows = [ + (1700000000000, "1.25", "100.1234", 10), + (1700000001000, "2.50", "200.5678", 20), + (1700000002000, "3.75", "300.0001", 30), + ] + + tdSql.query(f"select * from {DB_NAME}.vntb_ref_ntb order by ts") + tdSql.checkCols(4) + self.check_projected_rows(expect_rows) + + tdSql.query(f"select * from {DB_NAME}.vctb_full order by ts") + tdSql.checkCols(4) + self.check_projected_rows(expect_rows) + + tdSql.query(f"select ts, dec64_col, dec128_col, metric from {DB_NAME}.vctb_partial order by ts") + tdSql.checkRows(3) + for row_idx, (ts_val, dec64_val, _, _) in enumerate(expect_rows): + tdSql.checkData(row_idx, 0, ts_val) + self.check_decimal(row_idx, 1, dec64_val) + tdSql.checkData(row_idx, 2, None) + tdSql.checkData(row_idx, 3, None) + + tdSql.query(f"describe {DB_NAME}.vctb_empty") + tdSql.checkCols(5) + tdSql.checkRows(5) + empty_dec64_row = self.find_row("dec64_col") + empty_dec128_row = self.find_row("dec128_col") + empty_metric_row = self.find_row("metric") + tdSql.checkData(empty_dec64_row, 1, "DECIMAL(18, 2)") + tdSql.checkData(empty_dec128_row, 1, "DECIMAL(20, 4)") + tdSql.checkData(empty_metric_row, 1, "INT") + + self.check_column_meta("vctb_empty", "dec64_col", 18, 2, None) + self.check_column_meta("vctb_empty", "dec128_col", 20, 4, None) + self.check_non_decimal_meta("vctb_empty", "metric", None) + + tdSql.query(f"select ts, dec64_col, dec128_col, metric from {DB_NAME}.vctb_empty") + if tdSql.getRows() == 0: + tdSql.checkRows(0) + else: + for row_idx in range(tdSql.getRows()): + tdSql.checkData(row_idx, 0, None) + tdSql.checkData(row_idx, 1, None) + tdSql.checkData(row_idx, 2, None) + tdSql.checkData(row_idx, 3, None) + + tdSql.query( + "select ts, dec64_from_ctb, dec128_from_ntb, metric_from_ctb " + f"from {DB_NAME}.vntb_multi_src order by ts" + ) + tdSql.checkCols(4) + self.check_projected_rows(expect_rows) + + tdSql.query( + "select ts, dec64_ref, dec128_local, metric " + f"from {DB_NAME}.vntb_mixed_dec order by ts" + ) + tdSql.checkRows(3) + for row_idx, (ts_val, dec64_val, _, metric) in enumerate(expect_rows): + tdSql.checkData(row_idx, 0, ts_val) + self.check_decimal(row_idx, 1, dec64_val) + tdSql.checkData(row_idx, 2, None) + tdSql.checkData(row_idx, 3, metric) + + tdSql.query(f"select * from {DB_NAME}.vntb_local_dec") + tdSql.checkRows(0) + + tdSql.query( + "select tbname, tag_plain, ts, dec64_col, dec128_col, metric " + f"from {DB_NAME}.vstb_decimal where tag_plain in (10, 20) order by tag_plain, ts" + ) + tdSql.checkRows(5) + stable_expect_rows = [ + ("vctb_src0", 10, 1700000000000, "1.25", "100.1234", 10), + ("vctb_src0", 10, 1700000001000, "2.50", "200.5678", 20), + ("vctb_src0", 10, 1700000002000, "3.75", "300.0001", 30), + ("vctb_src1", 20, 1700000000000, "10.00", "1000.0000", 100), + ("vctb_src1", 20, 1700000001000, "20.00", "2000.0000", 200), + ] + for row_idx, (tbname, tag_plain, ts_val, dec64_val, dec128_val, metric) in enumerate(stable_expect_rows): + tdSql.checkData(row_idx, 0, tbname) + tdSql.checkData(row_idx, 1, tag_plain) + tdSql.checkData(row_idx, 2, ts_val) + self.check_decimal(row_idx, 3, dec64_val) + self.check_decimal(row_idx, 4, dec128_val) + tdSql.checkData(row_idx, 5, metric) + + def test_decimal_filter_aggregate_and_advanced_query(self): + """Query: decimal filters, aggregates, and advanced operators + + 1. validate decimal equality, range, and mixed-type filters + 2. validate aggregate, first, and last on decimal columns + 3. validate stable-level group aggregates across child mappings + 4. validate limit, order by, distinct, subquery, union all, and arithmetic expressions + + Catalog: + - VirtualTable + + Since: v3.4.1.0 + + Labels: virtual, decimal, aggregate + + Jira: None + + History: + - 2026-4-1 Joey Sima Expanded by test plan + + """ + tdSql.execute(f"use {DB_NAME}") + + tdSql.query(f"select count(*) from {DB_NAME}.vntb_multi_src where dec64_from_ctb = 1.25") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1) + + tdSql.query(f"select count(*) from {DB_NAME}.vntb_multi_src where dec64_from_ctb >= 2.50") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 2) + + tdSql.query( + f"select count(*) from {DB_NAME}.vntb_multi_src " + "where dec128_from_ntb between 100.0000 and 250.0000" + ) + tdSql.checkRows(1) + tdSql.checkData(0, 0, 2) + + tdSql.query(f"select count(*) from {DB_NAME}.vntb_multi_src where dec64_from_ctb > 2") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 2) + + tdSql.query( + f"select count(*) from {DB_NAME}.vntb_multi_src " + "where dec64_from_ctb >= 2.50 and dec128_from_ntb < 300.0000" + ) + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1) + + tdSql.query( + "select count(*), sum(dec64_from_ctb), avg(dec64_from_ctb), " + "min(dec64_from_ctb), max(dec64_from_ctb), " + f"min(dec128_from_ntb), max(dec128_from_ntb) from {DB_NAME}.vntb_multi_src" + ) + tdSql.checkRows(1) + tdSql.checkData(0, 0, 3) + self.check_decimal(0, 1, "7.50") + self.check_decimal(0, 2, "2.50") + self.check_decimal(0, 3, "1.25") + self.check_decimal(0, 4, "3.75") + self.check_decimal(0, 5, "100.1234") + self.check_decimal(0, 6, "300.0001") + + tdSql.query( + "select first(dec64_from_ctb), last(dec64_from_ctb), " + f"first(dec128_from_ntb), last(dec128_from_ntb) from {DB_NAME}.vntb_multi_src" + ) + tdSql.checkRows(1) + self.check_decimal(0, 0, "1.25") + self.check_decimal(0, 1, "3.75") + self.check_decimal(0, 2, "100.1234") + self.check_decimal(0, 3, "300.0001") + + tdSql.error( + f"select stddev(dec64_col), variance(dec64_col) from {DB_NAME}.vntb_ref_ntb" + ) + + tdSql.query( + f"select sum(dec64_col) from {DB_NAME}.vntb_ref_ntb where dec64_col > 1.25" + ) + tdSql.checkRows(1) + self.check_decimal(0, 0, "6.25") + + tdSql.query( + f"select avg(dec128_col) from {DB_NAME}.vstb_decimal " + "where tag_plain = 20 and metric > 15" + ) + tdSql.checkRows(1) + self.check_decimal_close(0, 0, "1500.0000", "0.000001") + + tdSql.query( + "select tag_plain, count(*), sum(dec64_col) " + f"from {DB_NAME}.vstb_decimal where tag_plain in (10, 20) " + "group by tag_plain order by tag_plain" + ) + tdSql.checkRows(2) + tdSql.checkData(0, 0, 10) + tdSql.checkData(0, 1, 3) + self.check_decimal(0, 2, "7.50") + tdSql.checkData(1, 0, 20) + tdSql.checkData(1, 1, 2) + self.check_decimal(1, 2, "30.00") + + tdSql.query( + f"select dec64_from_ctb, dec128_from_ntb from {DB_NAME}.vntb_multi_src " + "order by ts limit 2 offset 1" + ) + tdSql.checkRows(2) + self.check_decimal(0, 0, "2.50") + self.check_decimal(0, 1, "200.5678") + self.check_decimal(1, 0, "3.75") + self.check_decimal(1, 1, "300.0001") + + tdSql.query(f"select dec64_from_ctb from {DB_NAME}.vntb_multi_src order by dec64_from_ctb desc") + tdSql.checkRows(3) + self.check_decimal(0, 0, "3.75") + self.check_decimal(2, 0, "1.25") + + tdSql.query( + f"select distinct dec64_from_ctb from {DB_NAME}.vntb_multi_src order by dec64_from_ctb" + ) + tdSql.checkRows(3) + self.check_decimal(0, 0, "1.25") + self.check_decimal(1, 0, "2.50") + self.check_decimal(2, 0, "3.75") + + tdSql.query( + "select * from (" + f"select dec64_from_ctb, dec128_from_ntb from {DB_NAME}.vntb_multi_src" + ") order by dec64_from_ctb" + ) + tdSql.checkRows(3) + self.check_decimal(0, 0, "1.25") + self.check_decimal(0, 1, "100.1234") + + tdSql.query( + f"select dec64_from_ctb from {DB_NAME}.vntb_multi_src " + f"union all select dec64_col from {DB_NAME}.vctb_full" + ) + tdSql.checkRows(6) + + tdSql.query( + "select dec64_from_ctb + 1.00, dec64_from_ctb * 2, dec128_from_ntb - 50.0000 " + f"from {DB_NAME}.vntb_multi_src order by ts" + ) + tdSql.checkRows(3) + self.check_decimal_close(0, 0, "2.25") + self.check_decimal_close(0, 1, "2.50") + self.check_decimal_close(0, 2, "50.1234") + self.check_decimal_close(2, 0, "4.75") + self.check_decimal_close(2, 1, "7.50") + self.check_decimal_close(2, 2, "250.0001") + + def test_decimal_precision_and_restart(self): + """Meta: decimal precision values and restart persistence + + 1. validate extreme precision and scale decimal values query correctly + 2. validate decimal metadata survives taosd restart + 3. validate decimal queries still return exact values after restart + + Catalog: + - VirtualTable + + Since: v3.4.1.0 + + Labels: virtual, decimal, restart + + Jira: None + + History: + - 2026-4-1 Joey Sima Expanded by test plan + + """ + tdSql.execute(f"use {DB_NAME}") + + tdSql.query(f"select dec64_max, dec128_max from {DB_NAME}.vntb_precision_decimal order by ts") + tdSql.checkRows(3) + self.check_decimal(0, 0, "0.123456789012345678") + self.check_decimal(0, 1, "12345678901234567890.1234567890") + self.check_decimal(1, 0, "0.000000000000000001") + self.check_decimal(1, 1, "99999999999999999999.9999999999") + self.check_decimal(2, 0, "0.999999999999999999") + self.check_decimal(2, 1, "0.0000000001") + + tdSql.query(f"select count(*) from {DB_NAME}.vntb_precision_decimal where dec64_max > 0.5") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1) + + tdSql.query(f"select max(dec128_max) from {DB_NAME}.vntb_precision_decimal") + tdSql.checkRows(1) + self.check_decimal(0, 0, "99999999999999999999.9999999999") + + tdSql.query(f"select sum(dec64_max) from {DB_NAME}.vntb_precision_decimal") + tdSql.checkRows(1) + self.check_decimal(0, 0, "1.123456789012345678") + + tdSql.query(f"select min(dec128_max), max(dec128_max) from {DB_NAME}.vntb_precision_decimal") + tdSql.checkRows(1) + self.check_decimal(0, 0, "0.0000000001") + self.check_decimal(0, 1, "99999999999999999999.9999999999") + + tdDnodes.stoptaosd(1) + tdDnodes.starttaosd(1) + + tdSql.execute(f"use {DB_NAME}") + + tdSql.query(f"describe {DB_NAME}.vctb_full") + tdSql.checkCols(5) + dec64_row = self.find_row("dec64_col") + tdSql.checkData(dec64_row, 1, "DECIMAL(18, 2)") + + tdSql.query(f"show create vtable {DB_NAME}.vctb_full") + tdSql.checkRows(1) + create_sql = tdSql.getData(0, 1) + assert "FROM `test_vtable_decimal`.`org_ctb_0`.`dec64_col`" in create_sql + assert "FROM `test_vtable_decimal`.`org_ntb_0`.`dec128_col`" in create_sql + + self.check_column_meta("vntb_precision_decimal", "dec64_max", 18, 18, f"{DB_NAME}.org_ntb_0.dec64_max") + self.check_column_meta("vntb_precision_decimal", "dec128_max", 38, 10, f"{DB_NAME}.org_ntb_0.dec128_max") + + tdSql.query(f"select sum(dec64_col), max(dec128_col) from {DB_NAME}.vntb_ref_ntb") + tdSql.checkRows(1) + self.check_decimal(0, 0, "7.50") + self.check_decimal(0, 1, "300.0001") + + tdSql.query( + "select tag_plain, count(*), sum(dec64_col) " + f"from {DB_NAME}.vstb_decimal where tag_plain in (10, 20) " + "group by tag_plain order by tag_plain" + ) + tdSql.checkRows(2) + tdSql.checkData(0, 0, 10) + self.check_decimal(0, 2, "7.50") + tdSql.checkData(1, 0, 20) + self.check_decimal(1, 2, "30.00") + + tdSql.execute("alter vtable vntb_ref_ntb add column post_restart_dec decimal(15,5)") + self.check_column_meta("vntb_ref_ntb", "post_restart_dec", 15, 5, None) + + tdSql.query(f"describe {DB_NAME}.vntb_ref_ntb") + tdSql.checkCols(5) + post_restart_row = self.find_row("post_restart_dec") + tdSql.checkData(post_restart_row, 1, "DECIMAL(15, 5)") diff --git a/test/ci/cases.task b/test/ci/cases.task index d8d8e99adff..71588a4347d 100644 --- a/test/ci/cases.task +++ b/test/ci/cases.task @@ -227,6 +227,7 @@ ,,y,.,./ci/pytest.sh pytest cases/05-VirtualTables/test_vtable_schema_is_old.py ,,y,.,./ci/pytest.sh pytest cases/05-VirtualTables/test_vtable_show_tag.py ,,y,.,./ci/pytest.sh pytest cases/05-VirtualTables/test_vtable_show_create.py +,,y,.,./ci/pytest.sh pytest cases/05-VirtualTables/test_vtable_decimal.py #,,y,.,./ci/pytest.sh pytest cases/05-VirtualTables/test_vtable_tag_ref.py ,,y,.,./ci/pytest.sh pytest cases/05-VirtualTables/test_vtable_nchar_length.py ,,y,.,./ci/pytest.sh pytest cases/05-VirtualTables/test_vtable_validate_referencing.py