TDengine/source/dnode/mnode/impl/src/mndTopic.c

1052 lines
34 KiB
C
Raw Normal View History

2021-12-23 12:28:08 +00:00
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
2023-09-01 05:24:47 +00:00
*f
2021-12-23 12:28:08 +00:00
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
2022-01-06 02:28:34 +00:00
#include "mndTopic.h"
#include "audit.h"
2022-05-30 14:42:18 +00:00
#include "mndConsumer.h"
2021-12-23 12:28:08 +00:00
#include "mndDb.h"
#include "mndDnode.h"
#include "mndMnode.h"
2022-06-25 04:03:15 +00:00
#include "mndPrivilege.h"
2021-12-23 12:28:08 +00:00
#include "mndShow.h"
2021-12-25 05:27:37 +00:00
#include "mndStb.h"
2022-05-16 20:09:25 +00:00
#include "mndSubscribe.h"
2021-12-23 12:28:08 +00:00
#include "mndTrans.h"
#include "mndUser.h"
#include "mndVgroup.h"
#include "osMemPool.h"
2022-03-26 12:12:45 +00:00
#include "parser.h"
#include "tlockfree.h"
2021-12-23 12:28:08 +00:00
#include "tname.h"
#define MND_TOPIC_VER_NUMBER 4
2021-12-25 05:27:37 +00:00
#define MND_TOPIC_RESERVE_SIZE 64
2021-12-23 12:28:08 +00:00
SHashObj *topicsToReload = NULL;
2023-02-23 13:59:55 +00:00
SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic);
SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw);
2022-01-06 02:28:34 +00:00
static int32_t mndTopicActionInsert(SSdb *pSdb, SMqTopicObj *pTopic);
static int32_t mndTopicActionDelete(SSdb *pSdb, SMqTopicObj *pTopic);
2023-02-21 07:32:17 +00:00
static int32_t mndTopicActionUpdate(SSdb *pSdb, SMqTopicObj *pOldTopic, SMqTopicObj *pNewTopic);
2022-05-16 06:55:31 +00:00
static int32_t mndProcessCreateTopicReq(SRpcMsg *pReq);
static int32_t mndProcessDropTopicReq(SRpcMsg *pReq);
2022-05-09 10:09:55 +00:00
2022-05-16 06:55:31 +00:00
static int32_t mndRetrieveTopic(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
2022-01-06 02:28:34 +00:00
static void mndCancelGetNextTopic(SMnode *pMnode, void *pIter);
static int32_t processAst(SMqTopicObj *topicObj, const char *ast);
2021-12-23 12:28:08 +00:00
int32_t mndInitTopic(SMnode *pMnode) {
SSdbTable table = {
.sdbType = SDB_TOPIC,
.keyType = SDB_KEY_BINARY,
.encodeFp = (SdbEncodeFp)mndTopicActionEncode,
.decodeFp = (SdbDecodeFp)mndTopicActionDecode,
.insertFp = (SdbInsertFp)mndTopicActionInsert,
.updateFp = (SdbUpdateFp)mndTopicActionUpdate,
.deleteFp = (SdbDeleteFp)mndTopicActionDelete,
};
2021-12-23 12:28:08 +00:00
if (pMnode == NULL) {
return TSDB_CODE_INVALID_PARA;
}
2022-10-31 10:37:28 +00:00
mndSetMsgHandle(pMnode, TDMT_MND_TMQ_CREATE_TOPIC, mndProcessCreateTopicReq);
mndSetMsgHandle(pMnode, TDMT_MND_TMQ_DROP_TOPIC, mndProcessDropTopicReq);
2021-12-23 12:28:08 +00:00
2022-04-22 06:26:36 +00:00
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_TOPICS, mndRetrieveTopic);
2022-04-14 12:54:43 +00:00
mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_TOPICS, mndCancelGetNextTopic);
2022-02-16 12:18:18 +00:00
2021-12-23 12:28:08 +00:00
return sdbSetTable(pMnode->pSdb, table);
}
void mndCleanupTopic(SMnode *pMnode) {}
void mndTopicGetShowName(const char *fullTopic, char *topic) {
if (fullTopic == NULL) {
return;
}
char *tmp = strchr(fullTopic, '.');
if (tmp == NULL) {
tstrncpy(topic, fullTopic, TSDB_TOPIC_FNAME_LEN);
} else {
tstrncpy(topic, tmp + 1, TSDB_TOPIC_FNAME_LEN);
}
2022-05-09 08:35:59 +00:00
}
2022-01-06 02:56:25 +00:00
SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) {
if (pTopic == NULL) {
return NULL;
}
2024-07-22 08:33:42 +00:00
int32_t code = 0;
int32_t lino = 0;
2022-01-06 10:17:08 +00:00
terrno = TSDB_CODE_OUT_OF_MEMORY;
void * swBuf = NULL;
2022-05-17 09:00:21 +00:00
int32_t physicalPlanLen = 0;
if (pTopic->physicalPlan) {
physicalPlanLen = strlen(pTopic->physicalPlan) + 1;
}
2023-02-23 13:59:55 +00:00
2022-05-17 09:00:21 +00:00
int32_t schemaLen = 0;
if (pTopic->schema.nCols) {
2022-05-17 09:36:01 +00:00
schemaLen = taosEncodeSSchemaWrapper(NULL, &pTopic->schema);
2022-05-17 09:00:21 +00:00
}
2022-07-22 08:05:28 +00:00
int32_t size = sizeof(SMqTopicObj) + physicalPlanLen + pTopic->sqlLen + schemaLen + MND_TOPIC_RESERVE_SIZE;
2021-12-25 05:27:37 +00:00
SSdbRaw *pRaw = sdbAllocRaw(SDB_TOPIC, MND_TOPIC_VER_NUMBER, size);
2023-02-23 13:59:55 +00:00
if (pRaw == NULL) {
goto TOPIC_ENCODE_OVER;
}
2021-12-23 12:28:08 +00:00
int32_t dataPos = 0;
2022-02-28 12:46:23 +00:00
SDB_SET_BINARY(pRaw, dataPos, pTopic->name, TSDB_TOPIC_FNAME_LEN, TOPIC_ENCODE_OVER);
2022-01-06 10:17:08 +00:00
SDB_SET_BINARY(pRaw, dataPos, pTopic->db, TSDB_DB_FNAME_LEN, TOPIC_ENCODE_OVER);
2022-12-06 04:02:32 +00:00
SDB_SET_BINARY(pRaw, dataPos, pTopic->createUser, TSDB_USER_LEN, TOPIC_ENCODE_OVER);
2022-01-06 10:17:08 +00:00
SDB_SET_INT64(pRaw, dataPos, pTopic->createTime, TOPIC_ENCODE_OVER);
SDB_SET_INT64(pRaw, dataPos, pTopic->updateTime, TOPIC_ENCODE_OVER);
SDB_SET_INT64(pRaw, dataPos, pTopic->uid, TOPIC_ENCODE_OVER);
SDB_SET_INT64(pRaw, dataPos, pTopic->dbUid, TOPIC_ENCODE_OVER);
SDB_SET_INT32(pRaw, dataPos, pTopic->version, TOPIC_ENCODE_OVER);
2022-04-21 08:37:55 +00:00
SDB_SET_INT8(pRaw, dataPos, pTopic->subType, TOPIC_ENCODE_OVER);
2022-06-21 05:50:33 +00:00
SDB_SET_INT8(pRaw, dataPos, pTopic->withMeta, TOPIC_ENCODE_OVER);
2022-05-11 06:42:20 +00:00
2022-06-02 03:01:05 +00:00
SDB_SET_INT64(pRaw, dataPos, pTopic->stbUid, TOPIC_ENCODE_OVER);
SDB_SET_BINARY(pRaw, dataPos, pTopic->stbName, TSDB_TABLE_FNAME_LEN, TOPIC_ENCODE_OVER);
2022-01-06 10:17:08 +00:00
SDB_SET_INT32(pRaw, dataPos, pTopic->sqlLen, TOPIC_ENCODE_OVER);
SDB_SET_BINARY(pRaw, dataPos, pTopic->sql, pTopic->sqlLen, TOPIC_ENCODE_OVER);
2022-01-24 10:21:25 +00:00
SDB_SET_INT32(pRaw, dataPos, physicalPlanLen, TOPIC_ENCODE_OVER);
2022-05-17 09:00:21 +00:00
if (physicalPlanLen) {
SDB_SET_BINARY(pRaw, dataPos, pTopic->physicalPlan, physicalPlanLen, TOPIC_ENCODE_OVER);
2022-03-26 12:12:45 +00:00
}
2022-04-14 12:34:42 +00:00
SDB_SET_INT32(pRaw, dataPos, schemaLen, TOPIC_ENCODE_OVER);
2022-05-17 09:00:21 +00:00
if (schemaLen) {
swBuf = taosMemoryMalloc(schemaLen);
if (swBuf == NULL) {
goto TOPIC_ENCODE_OVER;
}
void *aswBuf = swBuf;
if (taosEncodeSSchemaWrapper(&aswBuf, &pTopic->schema) < 0) {
goto TOPIC_ENCODE_OVER;
}
2022-05-17 09:00:21 +00:00
SDB_SET_BINARY(pRaw, dataPos, swBuf, schemaLen, TOPIC_ENCODE_OVER);
}
2023-02-23 13:59:55 +00:00
2022-01-06 10:17:08 +00:00
SDB_SET_RESERVE(pRaw, dataPos, MND_TOPIC_RESERVE_SIZE, TOPIC_ENCODE_OVER);
SDB_SET_DATALEN(pRaw, dataPos, TOPIC_ENCODE_OVER);
terrno = TSDB_CODE_SUCCESS;
TOPIC_ENCODE_OVER:
2022-05-17 09:00:21 +00:00
if (swBuf) taosMemoryFree(swBuf);
2022-01-06 10:17:08 +00:00
if (terrno != TSDB_CODE_SUCCESS) {
mError("topic:%s, failed to encode to raw:%p since %s", pTopic->name, pRaw, terrstr());
sdbFreeRaw(pRaw);
return NULL;
}
mDebug("topic:%s, encode to raw:%p, row:%p", pTopic->name, pRaw, pTopic);
2021-12-23 12:28:08 +00:00
return pRaw;
}
2022-01-06 02:28:34 +00:00
SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) {
if (pRaw == NULL) return NULL;
2024-07-22 08:33:42 +00:00
int32_t code = 0;
int32_t lino = 0;
2021-12-31 06:22:50 +00:00
terrno = TSDB_CODE_OUT_OF_MEMORY;
SSdbRow * pRow = NULL;
2022-12-01 08:04:39 +00:00
SMqTopicObj *pTopic = NULL;
void * buf = NULL;
char* ast = NULL;
2021-12-23 12:28:08 +00:00
int8_t sver = 0;
2021-12-31 06:22:50 +00:00
if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto TOPIC_DECODE_OVER;
2021-12-23 12:28:08 +00:00
2023-06-01 07:10:08 +00:00
if (sver < 1 || sver > MND_TOPIC_VER_NUMBER) {
2021-12-23 12:28:08 +00:00
terrno = TSDB_CODE_SDB_INVALID_DATA_VER;
2021-12-31 06:22:50 +00:00
goto TOPIC_DECODE_OVER;
2021-12-23 12:28:08 +00:00
}
2022-12-01 08:04:39 +00:00
pRow = sdbAllocRow(sizeof(SMqTopicObj));
2021-12-31 06:22:50 +00:00
if (pRow == NULL) goto TOPIC_DECODE_OVER;
2022-12-01 08:04:39 +00:00
pTopic = sdbGetRowObj(pRow);
2021-12-31 06:22:50 +00:00
if (pTopic == NULL) goto TOPIC_DECODE_OVER;
2021-12-23 12:28:08 +00:00
int32_t len = 0;
2021-12-23 12:28:08 +00:00
int32_t dataPos = 0;
2022-02-28 12:46:23 +00:00
SDB_GET_BINARY(pRaw, dataPos, pTopic->name, TSDB_TOPIC_FNAME_LEN, TOPIC_DECODE_OVER);
2022-01-06 02:44:34 +00:00
SDB_GET_BINARY(pRaw, dataPos, pTopic->db, TSDB_DB_FNAME_LEN, TOPIC_DECODE_OVER);
2022-12-06 04:02:32 +00:00
if (sver >= 2) {
SDB_GET_BINARY(pRaw, dataPos, pTopic->createUser, TSDB_USER_LEN, TOPIC_DECODE_OVER);
}
2022-01-06 02:44:34 +00:00
SDB_GET_INT64(pRaw, dataPos, &pTopic->createTime, TOPIC_DECODE_OVER);
SDB_GET_INT64(pRaw, dataPos, &pTopic->updateTime, TOPIC_DECODE_OVER);
SDB_GET_INT64(pRaw, dataPos, &pTopic->uid, TOPIC_DECODE_OVER);
SDB_GET_INT64(pRaw, dataPos, &pTopic->dbUid, TOPIC_DECODE_OVER);
SDB_GET_INT32(pRaw, dataPos, &pTopic->version, TOPIC_DECODE_OVER);
2022-04-21 08:37:55 +00:00
SDB_GET_INT8(pRaw, dataPos, &pTopic->subType, TOPIC_DECODE_OVER);
2022-06-21 05:50:33 +00:00
SDB_GET_INT8(pRaw, dataPos, &pTopic->withMeta, TOPIC_DECODE_OVER);
2022-05-11 06:42:20 +00:00
2022-06-02 03:01:05 +00:00
SDB_GET_INT64(pRaw, dataPos, &pTopic->stbUid, TOPIC_DECODE_OVER);
2023-06-01 07:10:08 +00:00
if (sver >= 3) {
SDB_GET_BINARY(pRaw, dataPos, pTopic->stbName, TSDB_TABLE_FNAME_LEN, TOPIC_DECODE_OVER);
}
2022-04-14 12:34:42 +00:00
SDB_GET_INT32(pRaw, dataPos, &pTopic->sqlLen, TOPIC_DECODE_OVER);
pTopic->sql = taosMemoryCalloc(pTopic->sqlLen, sizeof(char));
if (pTopic->sql == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto TOPIC_DECODE_OVER;
}
2022-02-15 02:11:34 +00:00
SDB_GET_BINARY(pRaw, dataPos, pTopic->sql, pTopic->sqlLen, TOPIC_DECODE_OVER);
2022-01-18 01:53:11 +00:00
if (sver < 4) {
int32_t astLen = 0;
SDB_GET_INT32(pRaw, dataPos, &astLen, TOPIC_DECODE_OVER);
if (astLen) {
ast = taosMemoryCalloc(astLen, sizeof(char));
if (ast == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto TOPIC_DECODE_OVER;
}
SDB_GET_BINARY(pRaw, dataPos, ast, astLen, TOPIC_DECODE_OVER);
terrno = processAst(pTopic, ast);
if (terrno != TSDB_CODE_SUCCESS) {
goto TOPIC_DECODE_OVER;
}
2022-05-17 09:00:21 +00:00
}
} else {
SDB_GET_INT32(pRaw, dataPos, &len, TOPIC_DECODE_OVER);
if (len) {
pTopic->physicalPlan = taosMemoryCalloc(len, sizeof(char));
if (pTopic->physicalPlan == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto TOPIC_DECODE_OVER;
}
SDB_GET_BINARY(pRaw, dataPos, pTopic->physicalPlan, len, TOPIC_DECODE_OVER);
} else {
pTopic->physicalPlan = NULL;
2022-05-17 09:00:21 +00:00
}
2021-12-31 06:22:50 +00:00
SDB_GET_INT32(pRaw, dataPos, &len, TOPIC_DECODE_OVER);
if (len) {
buf = taosMemoryMalloc(len);
if (buf == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto TOPIC_DECODE_OVER;
}
SDB_GET_BINARY(pRaw, dataPos, buf, len, TOPIC_DECODE_OVER);
if (taosDecodeSSchemaWrapper(buf, &pTopic->schema) == NULL) {
goto TOPIC_DECODE_OVER;
}
} else {
pTopic->schema.nCols = 0;
pTopic->schema.version = 0;
pTopic->schema.pSchema = NULL;
}
2022-07-22 08:05:28 +00:00
}
2022-03-26 12:12:45 +00:00
2022-01-19 08:28:13 +00:00
SDB_GET_RESERVE(pRaw, dataPos, MND_TOPIC_RESERVE_SIZE, TOPIC_DECODE_OVER);
2022-01-06 10:17:08 +00:00
terrno = TSDB_CODE_SUCCESS;
2021-12-31 06:22:50 +00:00
TOPIC_DECODE_OVER:
taosMemoryFreeClear(buf);
taosMemoryFreeClear(ast);
2022-01-06 10:17:08 +00:00
if (terrno != TSDB_CODE_SUCCESS) {
2022-12-01 08:04:39 +00:00
mError("topic:%s, failed to decode from raw:%p since %s", pTopic == NULL ? "null" : pTopic->name, pRaw, terrstr());
2022-03-25 16:29:53 +00:00
taosMemoryFreeClear(pRow);
2021-12-31 06:22:50 +00:00
return NULL;
}
2021-12-23 12:28:08 +00:00
mDebug("topic:%s, decode from raw:%p, row:%p", pTopic->name, pRaw, pTopic);
2021-12-23 12:28:08 +00:00
return pRow;
}
2022-01-06 02:28:34 +00:00
static int32_t mndTopicActionInsert(SSdb *pSdb, SMqTopicObj *pTopic) {
mDebug("topic:%s perform insert action", pTopic != NULL ? pTopic->name : "null");
2021-12-23 12:28:08 +00:00
return 0;
}
2022-01-06 02:28:34 +00:00
static int32_t mndTopicActionDelete(SSdb *pSdb, SMqTopicObj *pTopic) {
if (pTopic == NULL) return 0;
mDebug("%p topic:%s perform delete action", pTopic, pTopic->name);
2022-09-16 06:36:01 +00:00
taosMemoryFreeClear(pTopic->sql);
taosMemoryFreeClear(pTopic->physicalPlan);
if (pTopic->schema.nCols) taosMemoryFreeClear(pTopic->schema.pSchema);
2021-12-23 12:28:08 +00:00
return 0;
}
2022-01-06 02:28:34 +00:00
static int32_t mndTopicActionUpdate(SSdb *pSdb, SMqTopicObj *pOldTopic, SMqTopicObj *pNewTopic) {
if (pOldTopic == NULL || pNewTopic == NULL) return 0;
mDebug("topic:%s perform update action", pOldTopic->name);
taosWLockLatch(&pOldTopic->lock);
SMqTopicObj tmpTopic = *pOldTopic;
(void)memcpy(pOldTopic, pNewTopic, offsetof(SMqTopicObj, lock));
*pNewTopic = tmpTopic;
taosWUnLockLatch(&pOldTopic->lock);
2021-12-23 12:28:08 +00:00
return 0;
}
int32_t mndAcquireTopic(SMnode *pMnode, const char *topicName, SMqTopicObj **pTopic) {
if (pMnode == NULL || topicName == NULL || pTopic == NULL) {
return TSDB_CODE_INVALID_PARA;
}
SSdb *pSdb = pMnode->pSdb;
*pTopic = sdbAcquire(pSdb, SDB_TOPIC, topicName);
if (*pTopic == NULL) {
return TSDB_CODE_MND_TOPIC_NOT_EXIST;
2021-12-23 12:28:08 +00:00
}
feat: new stream (#31678) * fix: windows compile issue * test: add vtable cases (#31829) * fix: windows compile issues * test:add test cases * fix: windows compile issue * case: em-4 stream case submit * test: stream4_sub1 found bug2 * test: submit test_scene_meters_bug2.py * add stream parameters example * feat: [TS-6100] Do not translate const value as column. * Feat/ts 6100 3.0 zlv (#31747) * modify asan exampel * modify asan exampel * add example * add example * modify case example --------- Co-authored-by: zelv01 <1101510017@qq.com> * feat(stream): fix memory leak * modify sliding example * test: update test case. * feat(stream): fix conflicts * fix: add offset case 10a 10s 10m 10h 10d * feat(stream): fix conflicts * chore(stream): rename case name #TS-6100 * add case * modify example * fix: windows compile issues * fix: data null check * feat: [TS-6100] Forbid where when using %%trows (#31827) * feat: [TS-6100] Forbid where when using %%trows * test: update cases * feat: [TS-6100] Fix leaks. --------- Co-authored-by: Simon Guan <guanshengliang@qq.com> * test: reproduce bugs * test: update test case. * test: update test case. * feat: [TS-6100] Fix leaks. * test: add cases * Feat/ts 6100 3.0.pw10 (#31841) * enh: add operator reset func * fix: merge join reset issue * fix: memory issues * fix: add debug assert * fix: memory issues * fix: memory leak * fix: memory issues * fix taos log miss * fix: case issue * fix: case issue * fix: case issues * fix: drop dnode issue * fix: memory issues * fix: memory issues * fix: memory leak issues * fix: recalculate time range issue * fix: add debug log * fix: memory issues * fix: enable case asan * Update streamlist_for_ci.task * fix: case asan issue * fix: stream name issue * fix: external window compile issues * fix: deploy memory issue * fix: ahandle issue * fix: ahandle issue * fix: ahandle issue * fix: virtual table reader list issue * fix: log info * fix: msg error * fix: virtual table addr list issue * fix: memory issues * fix: memory leak issue * fix: memory issues * fix: memory free issues * fix: memory issues * fix: snode deploy issue * fix: mnode reader issue * fix: memory issues * fix: add debug test * enh: add ignore nodata trigger * fix: memory leaks * fix: configuration issue * fix: memory issue * fix: external window issue * fix: external window issues * fix: external window placeholder issue * fix: placeholder function init issues * fix: memory leak issue * fix: add debug log * fix: compile issues * fix: double free issue * fix: runner addr update issue * fix: msg rsp issue * fix: external window reset issue * fix: configuration issue * fix: deploy msg issue * fix: compile issue --------- Co-authored-by: huohong <sallyhuo@taosdata.com> * test: reproduce bugs * fix: add sliding interval combine case * test: add cases * test: add recalc test. * test: reproduce bugs * case : add vt ts is null check * modify case * bug: submit test_idmp_meters_bug3.py * test: add test for recalc. * test: add cases * fix: error code check * test: add cases * fix(stream): scan wal with schema in that version * add case * test: add cases * test: update test case. * fix: windows compile issues * add case * test: add cases (#31845) * modify case * fix: reset interpPrev * test: add test_idmp_meters bug4 and bug3 * add case * fix(stream): opti wal interface * fix: remove test_idmp_meters_bug5.py * test: add cases * fix(stream): fix ts data fetch for virtual tables * cancel asan case * test: update test case. * test: update test case. * add case * test: add cases * test: add cases * test: add case test_idmp_meters_bug5.py * test: update test case. * fix(stream): tmq error * test: add cases * feat: [TS-6100] Restore deleted code in mndSma.c since they are still in use. * fix(stream): optimize val scan logic * test: add test_recalc_expired_time.py to ci. * test: update test case. * test: update test case. * feat: [TS-6100] Fix fill range check * fix(stream): optimize val scan logic * add case * test: modify for partition by %%1 * test: add fun case stream4_sub7 * fix(stream): optimize val scan logic * add case * feat: [TS-6100] Rename OPTIONS to STREAM_OPTIONS. * test: add test for recalc. * test: use stream_options. * fix: some cases error. * test: remove recalc from ci. * fix: ci case issues (#31880) * enh: add operator reset func * fix: merge join reset issue * fix: memory issues * fix: add debug assert * fix: memory issues * fix: memory leak * fix: memory issues * fix taos log miss * fix: case issue * fix: case issue * fix: case issues * fix: drop dnode issue * fix: memory issues * fix: memory issues * fix: memory leak issues * fix: recalculate time range issue * fix: add debug log * fix: memory issues * fix: enable case asan * Update streamlist_for_ci.task * fix: case asan issue * fix: stream name issue * fix: external window compile issues * fix: deploy memory issue * fix: ahandle issue * fix: ahandle issue * fix: ahandle issue * fix: virtual table reader list issue * fix: log info * fix: msg error * fix: virtual table addr list issue * fix: memory issues * fix: memory leak issue * fix: memory issues * fix: memory free issues * fix: memory issues * fix: snode deploy issue * fix: mnode reader issue * fix: memory issues * fix: add debug test * enh: add ignore nodata trigger * fix: memory leaks * fix: configuration issue * fix: memory issue * fix: external window issue * fix: external window issues * fix: external window placeholder issue * fix: placeholder function init issues * fix: memory leak issue * fix: add debug log * fix: compile issues * fix: double free issue * fix: runner addr update issue * fix: msg rsp issue * fix: external window reset issue * fix: configuration issue * fix: deploy msg issue * fix: compile issue * fix: external window idx issue * fix: ci issues --------- Co-authored-by: huohong <sallyhuo@taosdata.com> * fix(stream): fix compilation error * fix(stream): optimize val scan logic * test:add test cases * test: modify case * fix: external agg error * test(stream): tobacco scene testing #TD-36514 * test: add stream cases (#31885) * fix: windows compile issue * fix: calc timerange * fix: windows compile issue * modify case * fix(stream): compile error * test: remove one debug test case file * test: modify * test: add test cases * test: reproduce bugs * test: reproduce bugs * feat: [TS-6100] Placeholder function should only appera in SELECT and… (#31868) * feat: [TS-6100] Placeholder function should only appera in SELECT and WHERE and FROM. * test: update case --------- Co-authored-by: Simon Guan <guanshengliang@qq.com> * add example * add example * modify case example * modify case * test:alter sql * test: add stream5 case * fix(stream): get schema error with version * test: add delete recalc test py. * test: remove bug cases * test: stream5 case test passed * test: add state cases (#31893) * fix(stream): compile error * test: modify case * test: add cases * test: add test. * test: update test case. * chore(test): fix case err * test: update test case. * fix: align data get * fix(stream): fix row index of datablock written into data cache * fix: put align data * test: update test case. * test: add test cases for virtual table * chore(test): fix case err #TD-36514 * add case * test: add test for water mark. * test: add meters bug6 for stream5 * test: add cases (#31903) * test: add test for recalc. * feat: [TS-6100] %%trows can only be used when event type is window close. * test: add precision of database for ms/us/ns * modify case * add case * add case * test: add test to ci. * modify case * fix: ci case issues (#31904) * enh: add operator reset func * fix: merge join reset issue * fix: memory issues * fix: add debug assert * fix: memory issues * fix: memory leak * fix: memory issues * fix taos log miss * fix: case issue * fix: case issue * fix: case issues * fix: drop dnode issue * fix: memory issues * fix: memory issues * fix: memory leak issues * fix: recalculate time range issue * fix: add debug log * fix: memory issues * fix: enable case asan * Update streamlist_for_ci.task * fix: case asan issue * fix: stream name issue * fix: external window compile issues * fix: deploy memory issue * fix: ahandle issue * fix: ahandle issue * fix: ahandle issue * fix: virtual table reader list issue * fix: log info * fix: msg error * fix: virtual table addr list issue * fix: memory issues * fix: memory leak issue * fix: memory issues * fix: memory free issues * fix: memory issues * fix: snode deploy issue * fix: mnode reader issue * fix: memory issues * fix: add debug test * enh: add ignore nodata trigger * fix: memory leaks * fix: configuration issue * fix: memory issue * fix: external window issue * fix: external window issues * fix: external window placeholder issue * fix: placeholder function init issues * fix: memory leak issue * fix: add debug log * fix: compile issues * fix: double free issue * fix: runner addr update issue * fix: msg rsp issue * fix: external window reset issue * fix: configuration issue * fix: deploy msg issue * fix: compile issue * fix: external window idx issue * fix: ci issues * fix: ci case issues * fix: drop dnode issue --------- Co-authored-by: huohong <sallyhuo@taosdata.com> * fix(stream): ci error * test: update test case. * feat: [TS-6100] Disable some failed UT. * feat: [TS-6100] Fix virtual table * test: add bug 5. * test: add test delete recalc to ci. * test: add bug 6. * test(stream): tobacco scene #TD-36514 * fix: reqCids,reqCols memory leak in SSTriggerRealtimeContext Co-authored-by: Tony Zhang <tonyzhang@taosdata.com> * test: add case stream6 * fix(stream): implement some pending features in trigger task * modify case * modify case * fix: case issues * modify case * test: add recalc for warter mark. * fix(stream): fix count window trigger of virtual tables * fix(stream): memory leak * test: fix run err. * test: add stream6 bug7 * fix: adjust format * test(stream): tobacco scene testing #TD-36514 * test: change bug7 with update window1 and 2 * test: add test bug 7. * case: restore write 3 window * fix: windows compile issue * fix: notify * test: add cases * modify case * test: update test case. * test(stream): toobacco scene testing #TD-36514 --------- Co-authored-by: Simon Guan <slguan@taosdata.com> Co-authored-by: plum-lihui <huili@taosdata.com> Co-authored-by: Alex Duan <417921451@qq.com> Co-authored-by: zelv01 <1101510017@qq.com> Co-authored-by: Jing Sima <simondominic9997@outlook.com> Co-authored-by: xiangyang guo <66111494+happyguoxy@users.noreply.github.com> Co-authored-by: wangmm0220 <wangmm0220@gmail.com> Co-authored-by: Haojun Liao <hjliao@taosdata.com> Co-authored-by: zyyang90 <zyyang@taosdata.com> Co-authored-by: Alex Duan <51781608+DuanKuanJun@users.noreply.github.com> Co-authored-by: facetosea <285808407@qq.com> Co-authored-by: Simon Guan <guanshengliang@qq.com> Co-authored-by: huohong <sallyhuo@taosdata.com> Co-authored-by: Li Hui <52318143+plum-lihui@users.noreply.github.com> Co-authored-by: Jinqing Kuang <kuangjinqingcn@gmail.com> Co-authored-by: xiao-77 <berylbao@taosdata.com> Co-authored-by: Zhixiao Bao <62235797+xiao-77@users.noreply.github.com> Co-authored-by: happyguoxy <happy_guoxy@163.com> Co-authored-by: Tony Zhang <34825804+Tony2h@users.noreply.github.com> Co-authored-by: Tony Zhang <tonyzhang@taosdata.com>
2025-07-16 06:42:16 +00:00
return TSDB_CODE_SUCCESS;
2021-12-23 12:28:08 +00:00
}
2022-01-06 02:28:34 +00:00
void mndReleaseTopic(SMnode *pMnode, SMqTopicObj *pTopic) {
if (pMnode == NULL) return;
2021-12-23 12:28:08 +00:00
SSdb *pSdb = pMnode->pSdb;
sdbRelease(pSdb, pTopic);
}
2022-03-10 09:15:45 +00:00
static int32_t mndCheckCreateTopicReq(SCMCreateTopicReq *pCreate) {
if (pCreate == NULL) return TSDB_CODE_INVALID_PARA;
if (pCreate->sql == NULL) return TSDB_CODE_MND_INVALID_TOPIC;
2022-05-31 08:20:40 +00:00
if (pCreate->subType == TOPIC_SUB_TYPE__COLUMN) {
if (pCreate->ast == NULL || pCreate->ast[0] == 0) return TSDB_CODE_MND_INVALID_TOPIC;
2022-05-31 08:20:40 +00:00
} else if (pCreate->subType == TOPIC_SUB_TYPE__TABLE) {
if (pCreate->subStbName[0] == 0) return TSDB_CODE_MND_INVALID_TOPIC;
2022-05-31 08:20:40 +00:00
} else if (pCreate->subType == TOPIC_SUB_TYPE__DB) {
if (pCreate->subDbName[0] == 0) return TSDB_CODE_MND_INVALID_TOPIC;
2022-02-16 12:18:18 +00:00
}
2022-04-21 08:37:55 +00:00
return 0;
}
static int32_t processAst(SMqTopicObj *topicObj, const char *ast) {
SNode * pAst = NULL;
SQueryPlan *pPlan = NULL;
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
2024-03-18 06:31:40 +00:00
PRINT_LOG_START
if (ast == NULL) {
topicObj->physicalPlan = taosStrdup("");
goto END;
}
qDebugL("%s topic:%s ast %s", __func__, topicObj->name, ast);
MND_TMQ_RETURN_CHECK(nodesStringToNode(ast, &pAst));
MND_TMQ_RETURN_CHECK(qExtractResultSchema(pAst, &topicObj->schema.nCols, &topicObj->schema.pSchema));
SPlanContext cxt = {.pAstRoot = pAst, .topicQuery = true};
MND_TMQ_RETURN_CHECK(qCreateQueryPlan(&cxt, &pPlan, NULL));
if (pPlan == NULL) {
code = TSDB_CODE_MND_INVALID_TOPIC_QUERY;
goto END;
}
int32_t levelNum = LIST_LENGTH(pPlan->pSubplans);
if (levelNum != 1) {
code = TSDB_CODE_MND_INVALID_TOPIC_QUERY;
goto END;
}
2024-03-18 06:31:40 +00:00
SNodeListNode *pNodeListNode = (SNodeListNode *)nodesListGetNode(pPlan->pSubplans, 0);
MND_TMQ_NULL_CHECK(pNodeListNode);
int32_t opNum = LIST_LENGTH(pNodeListNode->pNodeList);
if (opNum != 1) {
code = TSDB_CODE_MND_INVALID_TOPIC_QUERY;
goto END;
2024-03-18 06:31:40 +00:00
}
code = nodesNodeToString(nodesListGetNode(pNodeListNode->pNodeList, 0), false, &topicObj->physicalPlan, NULL);
2024-03-18 06:31:40 +00:00
END:
nodesDestroyNode(pAst);
qDestroyQueryPlan(pPlan);
PRINT_LOG_END
2024-03-18 06:31:40 +00:00
return code;
}
2022-12-06 04:02:32 +00:00
static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq *pCreate, SDbObj *pDb,
const char *userName) {
if (pMnode == NULL || pReq == NULL || pCreate == NULL || pDb == NULL || userName == NULL)
return TSDB_CODE_INVALID_PARA;
STrans * pTrans = NULL;
int32_t code = 0;
int32_t lino = 0;
2022-01-06 02:28:34 +00:00
SMqTopicObj topicObj = {0};
PRINT_LOG_START
mInfo("start to create topic:%s", pCreate->name);
pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq, "create-topic");
MND_TMQ_NULL_CHECK(pTrans);
mndTransSetDbName(pTrans, pDb->name, NULL);
MND_TMQ_RETURN_CHECK(mndTransCheckConflict(pMnode, pTrans));
2022-01-14 10:35:57 +00:00
tstrncpy(topicObj.name, pCreate->name, TSDB_TOPIC_FNAME_LEN);
2021-12-28 08:06:01 +00:00
tstrncpy(topicObj.db, pDb->name, TSDB_DB_FNAME_LEN);
2022-12-06 04:02:32 +00:00
tstrncpy(topicObj.createUser, userName, TSDB_USER_LEN);
2022-12-06 05:23:59 +00:00
MND_TMQ_RETURN_CHECK(mndCheckTopicPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_TOPIC, &topicObj));
2022-12-06 05:23:59 +00:00
2021-12-23 12:28:08 +00:00
topicObj.createTime = taosGetTimestampMs();
topicObj.updateTime = topicObj.createTime;
2022-01-25 10:19:51 +00:00
topicObj.uid = mndGenerateUid(pCreate->name, strlen(pCreate->name));
2021-12-23 12:28:08 +00:00
topicObj.dbUid = pDb->uid;
topicObj.version = 1;
topicObj.sql = taosStrdup(pCreate->sql);
MND_TMQ_NULL_CHECK(topicObj.sql);
2022-04-14 12:34:42 +00:00
topicObj.sqlLen = strlen(pCreate->sql) + 1;
2022-05-31 08:20:40 +00:00
topicObj.subType = pCreate->subType;
2022-06-21 05:50:33 +00:00
topicObj.withMeta = pCreate->withMeta;
taosInitRWLatch(&topicObj.lock);
2023-02-23 06:09:56 +00:00
MND_TMQ_RETURN_CHECK(processAst(&topicObj, pCreate->ast));
2022-07-22 08:05:28 +00:00
if (pCreate->subStbName[0] != 0) {
tstrncpy(topicObj.stbName, pCreate->subStbName, TSDB_TABLE_FNAME_LEN);
SStbObj *pStb = mndAcquireStb(pMnode, topicObj.stbName);
MND_TMQ_NULL_CHECK(pStb);
2022-06-02 03:01:05 +00:00
topicObj.stbUid = pStb->uid;
2022-11-27 14:23:40 +00:00
mndReleaseStb(pMnode, pStb);
2022-03-26 12:12:45 +00:00
}
2022-05-23 13:08:00 +00:00
SSdbRaw *pCommitRaw = mndTopicActionEncode(&topicObj);
MND_TMQ_NULL_CHECK(pCommitRaw);
code = mndTransAppendCommitlog(pTrans, pCommitRaw);
if (code != 0) {
sdbFreeRaw(pCommitRaw);
goto END;
2022-02-16 12:18:18 +00:00
}
2023-02-23 06:09:56 +00:00
MND_TMQ_RETURN_CHECK(sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY));
MND_TMQ_RETURN_CHECK(mndTransPrepare(pMnode, pTrans));
END:
PRINT_LOG_END
2022-09-16 06:36:01 +00:00
taosMemoryFreeClear(topicObj.sql);
taosMemoryFreeClear(topicObj.physicalPlan);
2023-02-23 06:09:56 +00:00
if (topicObj.schema.nCols) {
taosMemoryFreeClear(topicObj.schema.pSchema);
}
2022-02-16 12:18:18 +00:00
mndTransDrop(pTrans);
return code;
2021-12-23 12:28:08 +00:00
}
static int32_t mndReloadTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq *pCreate, SDbObj *pDb,
const char *userName, SMqTopicObj *topicObjOri) {
if (pMnode == NULL || pReq == NULL || pCreate == NULL || pDb == NULL || userName == NULL)
return TSDB_CODE_INVALID_PARA;
STrans * pTrans = NULL;
int32_t code = 0;
int32_t lino = 0;
SMqTopicObj topicObj = {0};
2025-04-24 03:45:37 +00:00
PRINT_LOG_START
mInfo("start to reload topic:%s", pCreate->name);
pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq, "reload-topic");
MND_TMQ_NULL_CHECK(pTrans);
mndTransSetDbName(pTrans, pDb->name, NULL);
MND_TMQ_RETURN_CHECK(mndTransCheckConflict(pMnode, pTrans));
tstrncpy(topicObj.name, pCreate->name, TSDB_TOPIC_FNAME_LEN);
tstrncpy(topicObj.db, pDb->name, TSDB_DB_FNAME_LEN);
tstrncpy(topicObj.createUser, userName, TSDB_USER_LEN);
MND_TMQ_RETURN_CHECK(mndCheckTopicPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_TOPIC, &topicObj));
taosRLockLatch(&topicObjOri->lock);
topicObj.createTime = topicObjOri->createTime;
topicObj.updateTime = taosGetTimestampMs();
topicObj.uid = topicObjOri->uid;
topicObj.dbUid = pDb->uid;
topicObj.version = topicObjOri->version + 1;
topicObj.sql = taosStrdup(pCreate->sql);
topicObj.sqlLen = strlen(pCreate->sql) + 1;
topicObj.subType = pCreate->subType;
topicObj.withMeta = pCreate->withMeta;
taosInitRWLatch(&topicObj.lock);
taosRUnLockLatch(&topicObjOri->lock);
MND_TMQ_RETURN_CHECK(processAst(&topicObj, pCreate->ast));
if (pCreate->subStbName[0] != 0) {
tstrncpy(topicObj.stbName, pCreate->subStbName, TSDB_TABLE_FNAME_LEN);
SStbObj *pStb = mndAcquireStb(pMnode, topicObj.stbName);
MND_TMQ_NULL_CHECK(pStb);
topicObj.stbUid = pStb->uid;
mndReleaseStb(pMnode, pStb);
}
SSdbRaw *pCommitRaw = mndTopicActionEncode(&topicObj);
MND_TMQ_NULL_CHECK(pCommitRaw);
code = mndTransAppendCommitlog(pTrans, pCommitRaw);
if (code != 0) {
sdbFreeRaw(pCommitRaw);
goto END;
2022-02-16 12:18:18 +00:00
}
2022-01-14 10:35:57 +00:00
MND_TMQ_RETURN_CHECK(sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY));
MND_TMQ_RETURN_CHECK(mndTransPrepare(pMnode, pTrans));
2021-12-23 12:28:08 +00:00
END:
PRINT_LOG_END
taosMemoryFreeClear(topicObj.sql);
taosMemoryFreeClear(topicObj.physicalPlan);
if (topicObj.schema.nCols) {
taosMemoryFreeClear(topicObj.schema.pSchema);
}
mndTransDrop(pTrans);
return code;
}
2021-12-23 12:28:08 +00:00
static int32_t creatTopic(SRpcMsg *pReq, SCMCreateTopicReq *createTopicReq) {
SMqTopicObj *pTopic = NULL;
SDbObj * pDb = NULL;
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SMnode * pMnode = pReq->info.node;
2025-12-16 08:27:11 +00:00
int64_t tss = taosGetTimestampMs();
PRINT_LOG_START
mInfo("topic:%s start to create, sql:%s", createTopicReq->name, createTopicReq->sql);
code = mndAcquireTopic(pMnode, createTopicReq->name, &pTopic);
feat: new stream (#31678) * fix: windows compile issue * test: add vtable cases (#31829) * fix: windows compile issues * test:add test cases * fix: windows compile issue * case: em-4 stream case submit * test: stream4_sub1 found bug2 * test: submit test_scene_meters_bug2.py * add stream parameters example * feat: [TS-6100] Do not translate const value as column. * Feat/ts 6100 3.0 zlv (#31747) * modify asan exampel * modify asan exampel * add example * add example * modify case example --------- Co-authored-by: zelv01 <1101510017@qq.com> * feat(stream): fix memory leak * modify sliding example * test: update test case. * feat(stream): fix conflicts * fix: add offset case 10a 10s 10m 10h 10d * feat(stream): fix conflicts * chore(stream): rename case name #TS-6100 * add case * modify example * fix: windows compile issues * fix: data null check * feat: [TS-6100] Forbid where when using %%trows (#31827) * feat: [TS-6100] Forbid where when using %%trows * test: update cases * feat: [TS-6100] Fix leaks. --------- Co-authored-by: Simon Guan <guanshengliang@qq.com> * test: reproduce bugs * test: update test case. * test: update test case. * feat: [TS-6100] Fix leaks. * test: add cases * Feat/ts 6100 3.0.pw10 (#31841) * enh: add operator reset func * fix: merge join reset issue * fix: memory issues * fix: add debug assert * fix: memory issues * fix: memory leak * fix: memory issues * fix taos log miss * fix: case issue * fix: case issue * fix: case issues * fix: drop dnode issue * fix: memory issues * fix: memory issues * fix: memory leak issues * fix: recalculate time range issue * fix: add debug log * fix: memory issues * fix: enable case asan * Update streamlist_for_ci.task * fix: case asan issue * fix: stream name issue * fix: external window compile issues * fix: deploy memory issue * fix: ahandle issue * fix: ahandle issue * fix: ahandle issue * fix: virtual table reader list issue * fix: log info * fix: msg error * fix: virtual table addr list issue * fix: memory issues * fix: memory leak issue * fix: memory issues * fix: memory free issues * fix: memory issues * fix: snode deploy issue * fix: mnode reader issue * fix: memory issues * fix: add debug test * enh: add ignore nodata trigger * fix: memory leaks * fix: configuration issue * fix: memory issue * fix: external window issue * fix: external window issues * fix: external window placeholder issue * fix: placeholder function init issues * fix: memory leak issue * fix: add debug log * fix: compile issues * fix: double free issue * fix: runner addr update issue * fix: msg rsp issue * fix: external window reset issue * fix: configuration issue * fix: deploy msg issue * fix: compile issue --------- Co-authored-by: huohong <sallyhuo@taosdata.com> * test: reproduce bugs * fix: add sliding interval combine case * test: add cases * test: add recalc test. * test: reproduce bugs * case : add vt ts is null check * modify case * bug: submit test_idmp_meters_bug3.py * test: add test for recalc. * test: add cases * fix: error code check * test: add cases * fix(stream): scan wal with schema in that version * add case * test: add cases * test: update test case. * fix: windows compile issues * add case * test: add cases (#31845) * modify case * fix: reset interpPrev * test: add test_idmp_meters bug4 and bug3 * add case * fix(stream): opti wal interface * fix: remove test_idmp_meters_bug5.py * test: add cases * fix(stream): fix ts data fetch for virtual tables * cancel asan case * test: update test case. * test: update test case. * add case * test: add cases * test: add cases * test: add case test_idmp_meters_bug5.py * test: update test case. * fix(stream): tmq error * test: add cases * feat: [TS-6100] Restore deleted code in mndSma.c since they are still in use. * fix(stream): optimize val scan logic * test: add test_recalc_expired_time.py to ci. * test: update test case. * test: update test case. * feat: [TS-6100] Fix fill range check * fix(stream): optimize val scan logic * add case * test: modify for partition by %%1 * test: add fun case stream4_sub7 * fix(stream): optimize val scan logic * add case * feat: [TS-6100] Rename OPTIONS to STREAM_OPTIONS. * test: add test for recalc. * test: use stream_options. * fix: some cases error. * test: remove recalc from ci. * fix: ci case issues (#31880) * enh: add operator reset func * fix: merge join reset issue * fix: memory issues * fix: add debug assert * fix: memory issues * fix: memory leak * fix: memory issues * fix taos log miss * fix: case issue * fix: case issue * fix: case issues * fix: drop dnode issue * fix: memory issues * fix: memory issues * fix: memory leak issues * fix: recalculate time range issue * fix: add debug log * fix: memory issues * fix: enable case asan * Update streamlist_for_ci.task * fix: case asan issue * fix: stream name issue * fix: external window compile issues * fix: deploy memory issue * fix: ahandle issue * fix: ahandle issue * fix: ahandle issue * fix: virtual table reader list issue * fix: log info * fix: msg error * fix: virtual table addr list issue * fix: memory issues * fix: memory leak issue * fix: memory issues * fix: memory free issues * fix: memory issues * fix: snode deploy issue * fix: mnode reader issue * fix: memory issues * fix: add debug test * enh: add ignore nodata trigger * fix: memory leaks * fix: configuration issue * fix: memory issue * fix: external window issue * fix: external window issues * fix: external window placeholder issue * fix: placeholder function init issues * fix: memory leak issue * fix: add debug log * fix: compile issues * fix: double free issue * fix: runner addr update issue * fix: msg rsp issue * fix: external window reset issue * fix: configuration issue * fix: deploy msg issue * fix: compile issue * fix: external window idx issue * fix: ci issues --------- Co-authored-by: huohong <sallyhuo@taosdata.com> * fix(stream): fix compilation error * fix(stream): optimize val scan logic * test:add test cases * test: modify case * fix: external agg error * test(stream): tobacco scene testing #TD-36514 * test: add stream cases (#31885) * fix: windows compile issue * fix: calc timerange * fix: windows compile issue * modify case * fix(stream): compile error * test: remove one debug test case file * test: modify * test: add test cases * test: reproduce bugs * test: reproduce bugs * feat: [TS-6100] Placeholder function should only appera in SELECT and… (#31868) * feat: [TS-6100] Placeholder function should only appera in SELECT and WHERE and FROM. * test: update case --------- Co-authored-by: Simon Guan <guanshengliang@qq.com> * add example * add example * modify case example * modify case * test:alter sql * test: add stream5 case * fix(stream): get schema error with version * test: add delete recalc test py. * test: remove bug cases * test: stream5 case test passed * test: add state cases (#31893) * fix(stream): compile error * test: modify case * test: add cases * test: add test. * test: update test case. * chore(test): fix case err * test: update test case. * fix: align data get * fix(stream): fix row index of datablock written into data cache * fix: put align data * test: update test case. * test: add test cases for virtual table * chore(test): fix case err #TD-36514 * add case * test: add test for water mark. * test: add meters bug6 for stream5 * test: add cases (#31903) * test: add test for recalc. * feat: [TS-6100] %%trows can only be used when event type is window close. * test: add precision of database for ms/us/ns * modify case * add case * add case * test: add test to ci. * modify case * fix: ci case issues (#31904) * enh: add operator reset func * fix: merge join reset issue * fix: memory issues * fix: add debug assert * fix: memory issues * fix: memory leak * fix: memory issues * fix taos log miss * fix: case issue * fix: case issue * fix: case issues * fix: drop dnode issue * fix: memory issues * fix: memory issues * fix: memory leak issues * fix: recalculate time range issue * fix: add debug log * fix: memory issues * fix: enable case asan * Update streamlist_for_ci.task * fix: case asan issue * fix: stream name issue * fix: external window compile issues * fix: deploy memory issue * fix: ahandle issue * fix: ahandle issue * fix: ahandle issue * fix: virtual table reader list issue * fix: log info * fix: msg error * fix: virtual table addr list issue * fix: memory issues * fix: memory leak issue * fix: memory issues * fix: memory free issues * fix: memory issues * fix: snode deploy issue * fix: mnode reader issue * fix: memory issues * fix: add debug test * enh: add ignore nodata trigger * fix: memory leaks * fix: configuration issue * fix: memory issue * fix: external window issue * fix: external window issues * fix: external window placeholder issue * fix: placeholder function init issues * fix: memory leak issue * fix: add debug log * fix: compile issues * fix: double free issue * fix: runner addr update issue * fix: msg rsp issue * fix: external window reset issue * fix: configuration issue * fix: deploy msg issue * fix: compile issue * fix: external window idx issue * fix: ci issues * fix: ci case issues * fix: drop dnode issue --------- Co-authored-by: huohong <sallyhuo@taosdata.com> * fix(stream): ci error * test: update test case. * feat: [TS-6100] Disable some failed UT. * feat: [TS-6100] Fix virtual table * test: add bug 5. * test: add test delete recalc to ci. * test: add bug 6. * test(stream): tobacco scene #TD-36514 * fix: reqCids,reqCols memory leak in SSTriggerRealtimeContext Co-authored-by: Tony Zhang <tonyzhang@taosdata.com> * test: add case stream6 * fix(stream): implement some pending features in trigger task * modify case * modify case * fix: case issues * modify case * test: add recalc for warter mark. * fix(stream): fix count window trigger of virtual tables * fix(stream): memory leak * test: fix run err. * test: add stream6 bug7 * fix: adjust format * test(stream): tobacco scene testing #TD-36514 * test: change bug7 with update window1 and 2 * test: add test bug 7. * case: restore write 3 window * fix: windows compile issue * fix: notify * test: add cases * modify case * test: update test case. * test(stream): toobacco scene testing #TD-36514 --------- Co-authored-by: Simon Guan <slguan@taosdata.com> Co-authored-by: plum-lihui <huili@taosdata.com> Co-authored-by: Alex Duan <417921451@qq.com> Co-authored-by: zelv01 <1101510017@qq.com> Co-authored-by: Jing Sima <simondominic9997@outlook.com> Co-authored-by: xiangyang guo <66111494+happyguoxy@users.noreply.github.com> Co-authored-by: wangmm0220 <wangmm0220@gmail.com> Co-authored-by: Haojun Liao <hjliao@taosdata.com> Co-authored-by: zyyang90 <zyyang@taosdata.com> Co-authored-by: Alex Duan <51781608+DuanKuanJun@users.noreply.github.com> Co-authored-by: facetosea <285808407@qq.com> Co-authored-by: Simon Guan <guanshengliang@qq.com> Co-authored-by: huohong <sallyhuo@taosdata.com> Co-authored-by: Li Hui <52318143+plum-lihui@users.noreply.github.com> Co-authored-by: Jinqing Kuang <kuangjinqingcn@gmail.com> Co-authored-by: xiao-77 <berylbao@taosdata.com> Co-authored-by: Zhixiao Bao <62235797+xiao-77@users.noreply.github.com> Co-authored-by: happyguoxy <happy_guoxy@163.com> Co-authored-by: Tony Zhang <34825804+Tony2h@users.noreply.github.com> Co-authored-by: Tony Zhang <tonyzhang@taosdata.com>
2025-07-16 06:42:16 +00:00
if (code == TSDB_CODE_SUCCESS) {
mndReleaseTopic(pMnode, pTopic);
if (createTopicReq->igExists) {
mInfo("topic:%s already exist, ignore exist is set", createTopicReq->name);
2021-12-23 12:28:08 +00:00
} else {
code = TSDB_CODE_MND_TOPIC_ALREADY_EXIST;
2021-12-23 12:28:08 +00:00
}
goto END;
} else if (code != TSDB_CODE_MND_TOPIC_NOT_EXIST) {
goto END;
2021-12-23 12:28:08 +00:00
}
pDb = mndAcquireDb(pMnode, createTopicReq->subDbName);
MND_TMQ_NULL_CHECK(pDb);
2022-02-16 12:18:18 +00:00
if (pDb->cfg.walRetentionPeriod == 0) {
code = TSDB_CODE_MND_DB_RETENTION_PERIOD_ZERO;
goto END;
}
if (sdbGetSize(pMnode->pSdb, SDB_TOPIC) >= tmqMaxTopicNum) {
code = TSDB_CODE_TMQ_TOPIC_OUT_OF_RANGE;
goto END;
}
MND_TMQ_RETURN_CHECK(grantCheck(TSDB_GRANT_SUBSCRIPTION));
MND_TMQ_RETURN_CHECK(mndCreateTopic(pMnode, pReq, createTopicReq, pDb, pReq->info.conn.user));
2025-12-16 08:27:11 +00:00
if (tsAuditLevel >= AUDIT_LEVEL_DATABASE) {
int64_t tse = taosGetTimestampMs();
double duration = (double)(tse - tss);
duration = duration / 1000;
auditRecord(pReq, pMnode->clusterId, "createTopic", createTopicReq->subDbName, createTopicReq->name,
createTopicReq->sql, strlen(createTopicReq->sql), duration, 0);
}
code = TSDB_CODE_ACTION_IN_PROGRESS;
END:
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
mError("%s failed, topic:%s since %s", __func__, createTopicReq->name, tstrerror(code));
} else {
mInfo("topic:%s create successfully", createTopicReq->name);
2023-02-23 06:09:56 +00:00
}
mndReleaseDb(pMnode, pDb);
return code;
}
2022-02-16 12:18:18 +00:00
static int32_t reloadTopic(SRpcMsg *pReq, SCMCreateTopicReq *createTopicReq) {
SMnode * pMnode = pReq->info.node;
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SDbObj * pDb = NULL;
SMqTopicObj *pTopic = NULL;
2025-12-16 08:27:11 +00:00
int64_t tss = taosGetTimestampMs();
PRINT_LOG_START
code = mndAcquireTopic(pMnode, createTopicReq->name, &pTopic);
if (code != 0) {
if (createTopicReq->igExists) {
mInfo("topic:%s, not exist, ignore not exist is set", createTopicReq->name);
code = 0;
goto END;
} else {
mError("topic:%s, failed to reload since %s", createTopicReq->name, tstrerror(code));
goto END;
2024-09-06 02:16:15 +00:00
}
}
2023-09-01 05:24:47 +00:00
pDb = mndAcquireDb(pMnode, createTopicReq->subDbName);
MND_TMQ_NULL_CHECK(pDb);
MND_TMQ_RETURN_CHECK(grantCheck(TSDB_GRANT_SUBSCRIPTION));
MND_TMQ_RETURN_CHECK(mndReloadTopic(pMnode, pReq, createTopicReq, pDb, pReq->info.conn.user, pTopic));
2025-12-16 08:27:11 +00:00
if (tsAuditLevel >= AUDIT_LEVEL_DATABASE) {
int64_t tse = taosGetTimestampMs();
double duration = (double)(tse - tss);
duration = duration / 1000;
auditRecord(pReq, pMnode->clusterId, "reloadTopic", createTopicReq->subDbName, createTopicReq->name,
createTopicReq->sql, strlen(createTopicReq->sql), duration, 0);
}
code = TSDB_CODE_ACTION_IN_PROGRESS;
if (topicsToReload == NULL) {
topicsToReload = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
MND_TMQ_NULL_CHECK(topicsToReload);
}
MND_TMQ_RETURN_CHECK(
taosHashPut(topicsToReload, createTopicReq->name, strlen(createTopicReq->name), createTopicReq->name, 1));
mInfo("topic:%s, marked to reload", createTopicReq->name);
END:
2022-05-21 08:35:24 +00:00
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
mError("%s failed, topic:%s since %s", __func__, createTopicReq->name, tstrerror(code));
} else {
mInfo("topic:%s create successfully", createTopicReq->name);
2021-12-23 12:28:08 +00:00
}
2022-02-16 12:18:18 +00:00
mndReleaseTopic(pMnode, pTopic);
2021-12-23 12:28:08 +00:00
mndReleaseDb(pMnode, pDb);
return code;
}
static int32_t mndProcessCreateTopicReq(SRpcMsg *pReq) {
if (pReq == NULL || pReq->contLen <= 0) {
return TSDB_CODE_INVALID_MSG;
}
SMnode *pMnode = pReq->info.node;
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SCMCreateTopicReq createTopicReq = {0};
PRINT_LOG_START
MND_TMQ_RETURN_CHECK(tDeserializeSCMCreateTopicReq(pReq->pCont, pReq->contLen, &createTopicReq));
mInfo("topic:%s start to create, sql:%s", createTopicReq.name, createTopicReq.sql);
MND_TMQ_RETURN_CHECK(mndCheckCreateTopicReq(&createTopicReq));
if (createTopicReq.reload) {
MND_TMQ_RETURN_CHECK(reloadTopic(pReq, &createTopicReq));
} else {
MND_TMQ_RETURN_CHECK(creatTopic(pReq, &createTopicReq));
}
END:
2022-03-10 09:15:45 +00:00
tFreeSCMCreateTopicReq(&createTopicReq);
2022-02-16 12:18:18 +00:00
return code;
}
static int32_t mndDropTopic(SMnode *pMnode, STrans *pTrans, SRpcMsg *pReq, SMqTopicObj *pTopic) {
if (pMnode == NULL || pTrans == NULL || pReq == NULL || pTopic == NULL) {
return TSDB_CODE_INVALID_MSG;
}
int32_t code = 0;
int32_t lino = 0;
SSdbRaw *pCommitRaw = NULL;
PRINT_LOG_START
MND_TMQ_RETURN_CHECK(mndUserRemoveTopic(pMnode, pTrans, pTopic->name));
pCommitRaw = mndTopicActionEncode(pTopic);
MND_TMQ_NULL_CHECK(pCommitRaw);
code = mndTransAppendCommitlog(pTrans, pCommitRaw);
if (code != 0) {
sdbFreeRaw(pCommitRaw);
goto END;
}
MND_TMQ_RETURN_CHECK(sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED));
MND_TMQ_RETURN_CHECK(mndTransPrepare(pMnode, pTrans));
2021-12-23 12:28:08 +00:00
END:
PRINT_LOG_END
return code;
2021-12-23 12:28:08 +00:00
}
bool checkTopic(SArray *topics, char *topicName) {
if (topics == NULL || topicName == NULL) {
return false;
}
2024-03-12 11:42:26 +00:00
int32_t sz = taosArrayGetSize(topics);
for (int32_t i = 0; i < sz; i++) {
char *name = taosArrayGetP(topics, i);
if (name && strcmp(name, topicName) == 0) {
2024-03-12 11:42:26 +00:00
return true;
}
}
return false;
}
static int32_t checkConsumer(STrans *pTrans, SMqConsumerObj *pConsumer, bool deleteConsumer, char *topicName) {
int32_t code = 0;
int32_t lino = 0;
SMqConsumerObj *pConsumerNew = NULL;
taosRLockLatch(&pConsumer->lock);
bool found1 = checkTopic(pConsumer->assignedTopics, topicName);
bool found2 = checkTopic(pConsumer->rebRemovedTopics, topicName);
bool found3 = checkTopic(pConsumer->rebNewTopics, topicName);
if (found1 || found2 || found3) {
if (deleteConsumer) {
MND_TMQ_RETURN_CHECK(tNewSMqConsumerObj(pConsumer->consumerId, pConsumer->cgroup, CONSUMER_CLEAR, NULL, NULL, &pConsumerNew));
MND_TMQ_RETURN_CHECK(mndSetConsumerDropLogs(pTrans, pConsumerNew));
tDeleteSMqConsumerObj(pConsumerNew);
pConsumerNew = NULL;
} else {
mError("topic:%s, failed to drop since subscribed by consumer:0x%" PRIx64 ", in consumer group %s", topicName,
pConsumer->consumerId, pConsumer->cgroup);
code = TSDB_CODE_MND_TOPIC_SUBSCRIBED;
goto END;
}
}
END:
taosRUnLockLatch(&pConsumer->lock);
tDeleteSMqConsumerObj(pConsumerNew);
return code;
}
2024-03-12 11:42:26 +00:00
static int32_t mndCheckConsumerByTopic(SMnode *pMnode, STrans *pTrans, char *topicName, bool deleteConsumer) {
if (pMnode == NULL || pTrans == NULL || topicName == NULL) {
return TSDB_CODE_INVALID_MSG;
}
int32_t code = 0;
int32_t lino = 0;
SSdb * pSdb = pMnode->pSdb;
void * pIter = NULL;
SMqConsumerObj *pConsumer = NULL;
PRINT_LOG_START
2024-03-12 11:42:26 +00:00
while (1) {
pIter = sdbFetch(pSdb, SDB_CONSUMER, pIter, (void **)&pConsumer);
if (pIter == NULL) {
break;
2024-03-12 11:42:26 +00:00
}
MND_TMQ_RETURN_CHECK(checkConsumer(pTrans, pConsumer, deleteConsumer, topicName));
sdbRelease(pSdb, pConsumer);
2024-03-12 11:42:26 +00:00
}
END:
PRINT_LOG_END
sdbRelease(pSdb, pConsumer);
2024-03-12 11:42:26 +00:00
sdbCancelFetch(pSdb, pIter);
return code;
}
2022-05-16 06:55:31 +00:00
static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) {
if (pReq == NULL) {
return TSDB_CODE_INVALID_MSG;
}
SMnode * pMnode = pReq->info.node;
2022-02-16 02:25:14 +00:00
SMDropTopicReq dropReq = {0};
int32_t code = 0;
int32_t lino = 0;
SMqTopicObj * pTopic = NULL;
STrans * pTrans = NULL;
2025-12-16 08:27:11 +00:00
int64_t tss = taosGetTimestampMs();
2021-12-23 12:28:08 +00:00
2025-04-24 03:45:37 +00:00
PRINT_LOG_START
MND_TMQ_RETURN_CHECK(tDeserializeSMDropTopicReq(pReq->pCont, pReq->contLen, &dropReq));
2021-12-23 12:28:08 +00:00
code = mndAcquireTopic(pMnode, dropReq.name, &pTopic);
if (code != 0) {
2022-05-19 03:20:47 +00:00
if (dropReq.igNotExists) {
2022-09-23 07:42:36 +00:00
mInfo("topic:%s, not exist, ignore not exist is set", dropReq.name);
code = 0;
2022-05-19 03:20:47 +00:00
}
goto END;
2022-05-19 03:20:47 +00:00
}
taosRLockLatch(&pTopic->lock);
2022-05-17 13:20:43 +00:00
pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq, "drop-topic");
MND_TMQ_NULL_CHECK(pTrans);
mndTransSetDbName(pTrans, pTopic->db, NULL);
MND_TMQ_RETURN_CHECK(mndTransCheckConflict(pMnode, pTrans));
mInfo("trans:%d, used to drop topic:%s, force:%d", pTrans->id, pTopic->name, dropReq.force);
MND_TMQ_RETURN_CHECK(mndCheckTopicPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_TOPIC, pTopic));
MND_TMQ_RETURN_CHECK(mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_READ_DB, pTopic->db));
MND_TMQ_RETURN_CHECK(mndCheckConsumerByTopic(pMnode, pTrans, dropReq.name, dropReq.force));
MND_TMQ_RETURN_CHECK(mndDropSubByTopic(pMnode, pTrans, dropReq.name, dropReq.force));
MND_TMQ_RETURN_CHECK(mndDropTopic(pMnode, pTrans, pReq, pTopic));
2025-12-16 08:27:11 +00:00
if (tsAuditLevel >= AUDIT_LEVEL_DATABASE) {
int64_t tse = taosGetTimestampMs();
double duration = (double)(tse - tss);
duration = duration / 1000;
auditRecord(pReq, pMnode->clusterId, "dropTopic", pTopic->db, dropReq.name, dropReq.sql, dropReq.sqlLen, duration,
0);
}
code = TSDB_CODE_ACTION_IN_PROGRESS;
END:
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
mError("%s failed, topic:%s since %s", __func__, dropReq.name, tstrerror(code));
} else {
mInfo("topic:%s dropped successfully", dropReq.name);
2021-12-23 12:28:08 +00:00
}
if (pTopic != NULL) {
taosRUnLockLatch(&pTopic->lock);
2024-09-06 02:16:15 +00:00
}
mndReleaseTopic(pMnode, pTopic);
mndTransDrop(pTrans);
2023-09-06 03:00:24 +00:00
tFreeSMDropTopicReq(&dropReq);
return code;
2021-12-23 12:28:08 +00:00
}
int32_t mndGetNumOfTopics(SMnode *pMnode, char *dbName, int32_t *pNumOfTopics) {
if (pMnode == NULL || dbName == NULL || pNumOfTopics == NULL) {
return TSDB_CODE_INVALID_MSG;
}
2023-02-23 06:09:56 +00:00
*pNumOfTopics = 0;
SSdb * pSdb = pMnode->pSdb;
2022-03-01 07:56:11 +00:00
SDbObj *pDb = mndAcquireDb(pMnode, dbName);
if (pDb == NULL) {
return TSDB_CODE_MND_DB_NOT_SELECTED;
2022-03-01 07:56:11 +00:00
}
int32_t numOfTopics = 0;
void * pIter = NULL;
2022-03-01 07:56:11 +00:00
while (1) {
SMqTopicObj *pTopic = NULL;
pIter = sdbFetch(pSdb, SDB_TOPIC, pIter, (void **)&pTopic);
2023-02-23 06:09:56 +00:00
if (pIter == NULL) {
break;
}
taosRLockLatch(&pTopic->lock);
2022-03-01 07:56:11 +00:00
if (pTopic->dbUid == pDb->uid) {
numOfTopics++;
}
taosRUnLockLatch(&pTopic->lock);
2022-03-01 07:56:11 +00:00
sdbRelease(pSdb, pTopic);
}
*pNumOfTopics = numOfTopics;
mndReleaseDb(pMnode, pDb);
return 0;
}
static void schemaToJson(SSchema *schema, int32_t nCols, char *schemaJson) {
if (schema == NULL || schemaJson == NULL) {
return;
}
char * string = NULL;
int32_t code = 0;
int32_t lino = 0;
cJSON *cbytes = NULL;
cJSON *ctype = NULL;
cJSON *cname = NULL;
cJSON *column = NULL;
cJSON *columns = cJSON_CreateArray();
MND_TMQ_NULL_CHECK(columns);
for (int i = 0; i < nCols; i++) {
column = cJSON_CreateObject();
MND_TMQ_NULL_CHECK(column);
SSchema *s = schema + i;
cname = cJSON_CreateString(s->name);
MND_TMQ_NULL_CHECK(cname);
MND_TMQ_CONDITION_CHECK(cJSON_AddItemToObject(column, "name", cname), 0);
cname = NULL; // ownership transferred to column object
ctype = cJSON_CreateString(tDataTypes[s->type].name);
MND_TMQ_NULL_CHECK(ctype);
MND_TMQ_CONDITION_CHECK(cJSON_AddItemToObject(column, "type", ctype), 0);
ctype = NULL; // ownership transferred to column object
int32_t length = 0;
if (s->type == TSDB_DATA_TYPE_BINARY || s->type == TSDB_DATA_TYPE_VARBINARY || s->type == TSDB_DATA_TYPE_GEOMETRY) {
length = s->bytes - VARSTR_HEADER_SIZE;
} else if (s->type == TSDB_DATA_TYPE_NCHAR || s->type == TSDB_DATA_TYPE_JSON) {
length = (s->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE;
} else {
length = s->bytes;
}
cbytes = cJSON_CreateNumber(length);
MND_TMQ_NULL_CHECK(cbytes);
MND_TMQ_CONDITION_CHECK(cJSON_AddItemToObject(column, "length", cbytes), 0);
cbytes = NULL; // ownership transferred to column object
MND_TMQ_CONDITION_CHECK(cJSON_AddItemToArray(columns, column), 0);
column = NULL; // ownership transferred to columns array
}
END:
string = cJSON_PrintUnformatted(columns);
cJSON_Delete(columns);
cJSON_Delete(column);
cJSON_Delete(cname);
cJSON_Delete(ctype);
cJSON_Delete(cbytes);
size_t len = strlen(string);
if (string && len <= TSDB_SHOW_SCHEMA_JSON_LEN) {
STR_TO_VARSTR(schemaJson, string);
} else {
mError("mndRetrieveTopic build schema error json:%p, json len:%zu", string, len);
STR_TO_VARSTR(schemaJson, "NULL");
}
taosMemoryFree(string);
}
static int32_t buildResult(SMqTopicObj *pTopic, int32_t *numOfRows, SMnode *pMnode, SSDataBlock *pBlock) {
SColumnInfoData *pColInfo = NULL;
SName n = {0};
int32_t cols = 0;
char * schemaJson = NULL;
char * sql = NULL;
int32_t code = 0;
int32_t lino = 0;
taosRLockLatch(&pTopic->lock);
char topicName[TSDB_TOPIC_NAME_LEN + VARSTR_HEADER_SIZE + 5] = {0};
const char *pName = mndGetDbStr(pTopic->name);
STR_TO_VARSTR(topicName, pName);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
MND_TMQ_NULL_CHECK(pColInfo);
MND_TMQ_RETURN_CHECK(colDataSetVal(pColInfo, *numOfRows, (const char *)topicName, false));
char dbName[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
MND_TMQ_RETURN_CHECK(tNameFromString(&n, pTopic->db, T_NAME_ACCT | T_NAME_DB));
MND_TMQ_RETURN_CHECK(tNameGetDbName(&n, varDataVal(dbName)));
varDataSetLen(dbName, strlen(varDataVal(dbName)));
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
MND_TMQ_NULL_CHECK(pColInfo);
MND_TMQ_RETURN_CHECK(colDataSetVal(pColInfo, *numOfRows, (const char *)dbName, false));
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
MND_TMQ_NULL_CHECK(pColInfo);
MND_TMQ_RETURN_CHECK(colDataSetVal(pColInfo, *numOfRows, (const char *)&pTopic->createTime, false));
sql = taosMemoryMalloc(strlen(pTopic->sql) + VARSTR_HEADER_SIZE);
MND_TMQ_NULL_CHECK(sql);
STR_TO_VARSTR(sql, pTopic->sql);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
MND_TMQ_NULL_CHECK(pColInfo);
MND_TMQ_RETURN_CHECK(colDataSetVal(pColInfo, *numOfRows, (const char *)sql, false));
taosMemoryFreeClear(sql);
schemaJson = taosMemoryMalloc(TSDB_SHOW_SCHEMA_JSON_LEN + VARSTR_HEADER_SIZE);
MND_TMQ_NULL_CHECK(schemaJson);
if (pTopic->subType == TOPIC_SUB_TYPE__COLUMN) {
schemaToJson(pTopic->schema.pSchema, pTopic->schema.nCols, schemaJson);
} else if (pTopic->subType == TOPIC_SUB_TYPE__TABLE) {
SStbObj *pStb = mndAcquireStb(pMnode, pTopic->stbName);
if (pStb == NULL) {
STR_TO_VARSTR(schemaJson, "NULL");
mError("mndRetrieveTopic mndAcquireStb null stbName:%s", pTopic->stbName);
} else {
schemaToJson(pStb->pColumns, pStb->numOfColumns, schemaJson);
mndReleaseStb(pMnode, pStb);
}
} else {
STR_TO_VARSTR(schemaJson, "NULL");
}
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
MND_TMQ_NULL_CHECK(pColInfo);
MND_TMQ_RETURN_CHECK(colDataSetVal(pColInfo, *numOfRows, (const char *)schemaJson, false));
taosMemoryFreeClear(schemaJson);
(*numOfRows)++;
END:
taosRUnLockLatch(&pTopic->lock);
taosMemoryFreeClear(sql);
taosMemoryFreeClear(schemaJson);
return code;
}
2022-05-16 06:55:31 +00:00
static int32_t mndRetrieveTopic(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rowsCapacity) {
if (pReq == NULL || pShow == NULL || pBlock == NULL) {
return TSDB_CODE_INVALID_MSG;
}
SMnode * pMnode = pReq->info.node;
SSdb * pSdb = pMnode->pSdb;
2022-01-06 02:28:34 +00:00
int32_t numOfRows = 0;
SMqTopicObj *pTopic = NULL;
int32_t code = 0;
int32_t lino = 0;
PRINT_LOG_START
2022-02-16 02:34:29 +00:00
2022-04-22 06:26:36 +00:00
while (numOfRows < rowsCapacity) {
2021-12-23 12:28:08 +00:00
pShow->pIter = sdbFetch(pSdb, SDB_TOPIC, pShow->pIter, (void **)&pTopic);
if (pShow->pIter == NULL) break;
MND_TMQ_RETURN_CHECK(buildResult(pTopic, &numOfRows, pMnode, pBlock));
2021-12-23 12:28:08 +00:00
sdbRelease(pSdb, pTopic);
pTopic = NULL;
2021-12-23 12:28:08 +00:00
}
pShow->numOfRows += numOfRows;
2022-05-16 20:09:25 +00:00
END:
sdbCancelFetch(pSdb, pShow->pIter);
sdbRelease(pSdb, pTopic);
if (code != TSDB_CODE_SUCCESS) {
mError("%s failed since %s", __func__, tstrerror(code));
return code;
} else {
mDebug("%s retrieved %d rows successfully", __func__, numOfRows);
return numOfRows;
}
2022-04-29 12:11:58 +00:00
}
2021-12-23 12:28:08 +00:00
static void mndCancelGetNextTopic(SMnode *pMnode, void *pIter) {
if (pMnode == NULL) return;
2021-12-23 12:28:08 +00:00
SSdb *pSdb = pMnode->pSdb;
sdbCancelFetchByType(pSdb, pIter, SDB_TOPIC);
2021-12-23 12:28:08 +00:00
}
2022-04-29 12:11:58 +00:00
2023-02-23 13:59:55 +00:00
bool mndTopicExistsForDb(SMnode *pMnode, SDbObj *pDb) {
if (pMnode == NULL || pDb == NULL) {
return false;
}
SSdb * pSdb = pMnode->pSdb;
void * pIter = NULL;
SMqTopicObj *pTopic = NULL;
2023-02-23 13:59:55 +00:00
while (1) {
pIter = sdbFetch(pSdb, SDB_TOPIC, pIter, (void **)&pTopic);
2023-02-23 13:59:55 +00:00
if (pIter == NULL) {
break;
}
taosRLockLatch(&pTopic->lock);
bool found = pTopic->dbUid == pDb->uid;
taosRUnLockLatch(&pTopic->lock);
if (found) {
sdbRelease(pSdb, pTopic);
sdbCancelFetch(pSdb, pIter);
2023-02-23 13:59:55 +00:00
return true;
}
sdbRelease(pSdb, pTopic);
}
2023-02-23 13:59:55 +00:00
return false;
}