TDengine/source/dnode/vnode/inc/vnode.h

438 lines
20 KiB
C
Raw Normal View History

2021-09-22 05:21:07 +00:00
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
2021-09-22 12:29:24 +00:00
*/
#ifndef _TD_VNODE_H_
#define _TD_VNODE_H_
2021-11-09 05:24:22 +00:00
#include "os.h"
2022-03-21 11:08:25 +00:00
#include "tmsgcb.h"
2022-03-29 10:58:15 +00:00
#include "tqueue.h"
#include "trpc.h"
2021-11-09 05:24:22 +00:00
2022-04-19 09:07:42 +00:00
#include "sync.h"
2021-11-11 03:41:16 +00:00
#include "tarray.h"
2022-01-19 06:04:01 +00:00
#include "tfs.h"
2021-11-24 08:29:02 +00:00
#include "wal.h"
2021-11-01 11:49:44 +00:00
2023-05-23 03:46:02 +00:00
#include "filter.h"
2022-04-14 02:01:04 +00:00
#include "tcommon.h"
#include "tfs.h"
2022-07-23 07:32:18 +00:00
#include "tgrant.h"
2022-04-08 06:09:15 +00:00
#include "tmsg.h"
#include "trow.h"
#include "storageapi.h"
2023-08-18 08:38:26 +00:00
#include "tdb.h"
2022-04-24 06:19:12 +00:00
2025-06-30 02:33:34 +00:00
#include "libs/metrics/metrics.h"
2021-09-22 12:29:24 +00:00
#ifdef __cplusplus
extern "C" {
#endif
2022-05-01 16:30:47 +00:00
2022-04-14 02:01:04 +00:00
// vnode
2025-06-30 02:33:34 +00:00
typedef struct SVnode SVnode;
typedef struct STsdbCfg STsdbCfg; // todo: remove
typedef struct SVnodeCfg SVnodeCfg;
typedef struct SVSnapReader SVSnapReader;
typedef struct SVSnapWriter SVSnapWriter;
typedef struct SVnodeWriteMetrics SVnodeWriteMetrics;
2022-04-14 02:10:56 +00:00
2022-04-19 01:57:46 +00:00
extern const SVnodeCfg vnodeCfgDefault;
2024-08-15 03:55:20 +00:00
typedef void (*StopDnodeFp)();
2024-12-19 06:51:21 +00:00
int32_t vnodeInit(StopDnodeFp stopDnodeFp);
2022-04-14 02:01:04 +00:00
void vnodeCleanup();
int32_t vnodeCreate(const char *path, SVnodeCfg *pCfg, int32_t diskPrimary, STfs *pTfs);
bool vnodeShouldRemoveWal(SVnode *pVnode);
int32_t vnodeAlterReplica(const char *path, SAlterVnodeReplicaReq *pReq, int32_t diskPrimary, STfs *pTfs);
int32_t vnodeAlterHashRange(const char *srcPath, const char *dstPath, SAlterVnodeHashRangeReq *pReq,
int32_t diskPrimary, STfs *pTfs);
int32_t vnodeRestoreVgroupId(const char *srcPath, const char *dstPath, int32_t srcVgId, int32_t dstVgId,
int32_t diskPrimary, STfs *pTfs);
void vnodeDestroy(int32_t vgId, const char *path, STfs *pTfs, int32_t nodeId);
2025-07-12 07:39:29 +00:00
SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, STfs *pMountTfs, SMsgCb msgCb, bool force);
void vnodePreClose(SVnode *pVnode);
void vnodePostClose(SVnode *pVnode);
void vnodeSyncCheckTimeout(SVnode *pVnode);
2022-04-14 02:01:04 +00:00
void vnodeClose(SVnode *pVnode);
2023-01-31 02:35:27 +00:00
int32_t vnodeSyncCommit(SVnode *pVnode);
2023-02-13 03:40:40 +00:00
int32_t vnodeBegin(SVnode *pVnode);
2022-07-06 07:15:55 +00:00
2023-08-18 08:38:26 +00:00
int32_t vnodeStart(SVnode *pVnode);
void vnodeStop(SVnode *pVnode);
int64_t vnodeGetSyncHandle(SVnode *pVnode);
int32_t vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot);
int32_t vnodeSetWalKeepVersion(SVnode *pVnode, int64_t keepVersion);
2023-08-18 08:38:26 +00:00
void vnodeGetInfo(void *pVnode, const char **dbname, int32_t *vgId, int64_t *numOfTables, int64_t *numOfNormalTables);
int32_t vnodeGetTableList(void *pVnode, int8_t type, SArray *pList);
int32_t vnodeGetAllTableList(SVnode *pVnode, uint64_t uid, SArray *list);
int32_t vnodeIsCatchUp(SVnode *pVnode);
2023-04-24 02:23:43 +00:00
ESyncRole vnodeGetRole(SVnode *pVnode);
int32_t vnodeGetArbToken(SVnode *pVnode, char *outToken);
2024-11-06 07:25:50 +00:00
int32_t vnodeGetDBSize(void *pVnode, SDbSizeStatisInfo *pInfo);
int32_t vnodeUpdateArbTerm(SVnode *pVnode, int64_t arbTerm);
2022-09-28 14:07:16 +00:00
int32_t vnodeGetCtbIdList(void *pVnode, int64_t suid, SArray *list);
2022-09-28 14:07:16 +00:00
int32_t vnodeGetCtbIdListByFilter(SVnode *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg), void *arg);
2022-08-25 08:22:55 +00:00
int32_t vnodeGetStbIdList(SVnode *pVnode, int64_t suid, SArray *list);
2023-08-18 08:38:26 +00:00
int32_t vnodeGetStbIdListByFilter(SVnode *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg, void *arg1),
void *arg);
void *vnodeGetIdx(void *pVnode);
void *vnodeGetIvtIdx(void *pVnode);
2022-04-19 09:07:42 +00:00
int32_t vnodeGetCtbNum(SVnode *pVnode, int64_t suid, int64_t *num);
int32_t vnodeGetStbColumnNum(SVnode *pVnode, tb_uid_t suid, int *num);
int32_t vnodeGetStbInfo(SVnode *pVnode, tb_uid_t suid, int64_t *keep, int8_t *flags);
int32_t vnodeGetTimeSeriesNum(SVnode *pVnode, int64_t *num);
int32_t vnodeGetAllCtbNum(SVnode *pVnode, int64_t *num);
2025-06-21 07:11:09 +00:00
int32_t vnodeGetTableSchema(void *pVnode, int64_t uid, STSchema **pSchema, int64_t *suid, SSchemaWrapper **pTagSchema);
2022-10-11 10:22:29 +00:00
void vnodeResetLoad(SVnode *pVnode, SVnodeLoad *pLoad);
int32_t vnodeSetSyncTimeout(SVnode *pVnode, int32_t ms);
2022-07-06 07:15:55 +00:00
int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad);
int32_t vnodeGetLoadLite(SVnode *pVnode, SVnodeLoadLite *pLoad);
2022-07-06 07:15:55 +00:00
int32_t vnodeValidateTableHash(SVnode *pVnode, char *tableFName);
int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg);
int32_t vnodePreprocessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg);
int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg *pRsp);
int32_t vnodeProcessSyncMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp);
2024-11-05 11:57:13 +00:00
int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo);
2022-07-06 07:15:55 +00:00
int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo);
feat: new stream (#31678) * fix: windows compile issue * test: add vtable cases (#31829) * fix: windows compile issues * test:add test cases * fix: windows compile issue * case: em-4 stream case submit * test: stream4_sub1 found bug2 * test: submit test_scene_meters_bug2.py * add stream parameters example * feat: [TS-6100] Do not translate const value as column. * Feat/ts 6100 3.0 zlv (#31747) * modify asan exampel * modify asan exampel * add example * add example * modify case example --------- Co-authored-by: zelv01 <1101510017@qq.com> * feat(stream): fix memory leak * modify sliding example * test: update test case. * feat(stream): fix conflicts * fix: add offset case 10a 10s 10m 10h 10d * feat(stream): fix conflicts * chore(stream): rename case name #TS-6100 * add case * modify example * fix: windows compile issues * fix: data null check * feat: [TS-6100] Forbid where when using %%trows (#31827) * feat: [TS-6100] Forbid where when using %%trows * test: update cases * feat: [TS-6100] Fix leaks. --------- Co-authored-by: Simon Guan <guanshengliang@qq.com> * test: reproduce bugs * test: update test case. * test: update test case. * feat: [TS-6100] Fix leaks. * test: add cases * Feat/ts 6100 3.0.pw10 (#31841) * enh: add operator reset func * fix: merge join reset issue * fix: memory issues * fix: add debug assert * fix: memory issues * fix: memory leak * fix: memory issues * fix taos log miss * fix: case issue * fix: case issue * fix: case issues * fix: drop dnode issue * fix: memory issues * fix: memory issues * fix: memory leak issues * fix: recalculate time range issue * fix: add debug log * fix: memory issues * fix: enable case asan * Update streamlist_for_ci.task * fix: case asan issue * fix: stream name issue * fix: external window compile issues * fix: deploy memory issue * fix: ahandle issue * fix: ahandle issue * fix: ahandle issue * fix: virtual table reader list issue * fix: log info * fix: msg error * fix: virtual table addr list issue * fix: memory issues * fix: memory leak issue * fix: memory issues * fix: memory free issues * fix: memory issues * fix: snode deploy issue * fix: mnode reader issue * fix: memory issues * fix: add debug test * enh: add ignore nodata trigger * fix: memory leaks * fix: configuration issue * fix: memory issue * fix: external window issue * fix: external window issues * fix: external window placeholder issue * fix: placeholder function init issues * fix: memory leak issue * fix: add debug log * fix: compile issues * fix: double free issue * fix: runner addr update issue * fix: msg rsp issue * fix: external window reset issue * fix: configuration issue * fix: deploy msg issue * fix: compile issue --------- Co-authored-by: huohong <sallyhuo@taosdata.com> * test: reproduce bugs * fix: add sliding interval combine case * test: add cases * test: add recalc test. * test: reproduce bugs * case : add vt ts is null check * modify case * bug: submit test_idmp_meters_bug3.py * test: add test for recalc. * test: add cases * fix: error code check * test: add cases * fix(stream): scan wal with schema in that version * add case * test: add cases * test: update test case. * fix: windows compile issues * add case * test: add cases (#31845) * modify case * fix: reset interpPrev * test: add test_idmp_meters bug4 and bug3 * add case * fix(stream): opti wal interface * fix: remove test_idmp_meters_bug5.py * test: add cases * fix(stream): fix ts data fetch for virtual tables * cancel asan case * test: update test case. * test: update test case. * add case * test: add cases * test: add cases * test: add case test_idmp_meters_bug5.py * test: update test case. * fix(stream): tmq error * test: add cases * feat: [TS-6100] Restore deleted code in mndSma.c since they are still in use. * fix(stream): optimize val scan logic * test: add test_recalc_expired_time.py to ci. * test: update test case. * test: update test case. * feat: [TS-6100] Fix fill range check * fix(stream): optimize val scan logic * add case * test: modify for partition by %%1 * test: add fun case stream4_sub7 * fix(stream): optimize val scan logic * add case * feat: [TS-6100] Rename OPTIONS to STREAM_OPTIONS. * test: add test for recalc. * test: use stream_options. * fix: some cases error. * test: remove recalc from ci. * fix: ci case issues (#31880) * enh: add operator reset func * fix: merge join reset issue * fix: memory issues * fix: add debug assert * fix: memory issues * fix: memory leak * fix: memory issues * fix taos log miss * fix: case issue * fix: case issue * fix: case issues * fix: drop dnode issue * fix: memory issues * fix: memory issues * fix: memory leak issues * fix: recalculate time range issue * fix: add debug log * fix: memory issues * fix: enable case asan * Update streamlist_for_ci.task * fix: case asan issue * fix: stream name issue * fix: external window compile issues * fix: deploy memory issue * fix: ahandle issue * fix: ahandle issue * fix: ahandle issue * fix: virtual table reader list issue * fix: log info * fix: msg error * fix: virtual table addr list issue * fix: memory issues * fix: memory leak issue * fix: memory issues * fix: memory free issues * fix: memory issues * fix: snode deploy issue * fix: mnode reader issue * fix: memory issues * fix: add debug test * enh: add ignore nodata trigger * fix: memory leaks * fix: configuration issue * fix: memory issue * fix: external window issue * fix: external window issues * fix: external window placeholder issue * fix: placeholder function init issues * fix: memory leak issue * fix: add debug log * fix: compile issues * fix: double free issue * fix: runner addr update issue * fix: msg rsp issue * fix: external window reset issue * fix: configuration issue * fix: deploy msg issue * fix: compile issue * fix: external window idx issue * fix: ci issues --------- Co-authored-by: huohong <sallyhuo@taosdata.com> * fix(stream): fix compilation error * fix(stream): optimize val scan logic * test:add test cases * test: modify case * fix: external agg error * test(stream): tobacco scene testing #TD-36514 * test: add stream cases (#31885) * fix: windows compile issue * fix: calc timerange * fix: windows compile issue * modify case * fix(stream): compile error * test: remove one debug test case file * test: modify * test: add test cases * test: reproduce bugs * test: reproduce bugs * feat: [TS-6100] Placeholder function should only appera in SELECT and… (#31868) * feat: [TS-6100] Placeholder function should only appera in SELECT and WHERE and FROM. * test: update case --------- Co-authored-by: Simon Guan <guanshengliang@qq.com> * add example * add example * modify case example * modify case * test:alter sql * test: add stream5 case * fix(stream): get schema error with version * test: add delete recalc test py. * test: remove bug cases * test: stream5 case test passed * test: add state cases (#31893) * fix(stream): compile error * test: modify case * test: add cases * test: add test. * test: update test case. * chore(test): fix case err * test: update test case. * fix: align data get * fix(stream): fix row index of datablock written into data cache * fix: put align data * test: update test case. * test: add test cases for virtual table * chore(test): fix case err #TD-36514 * add case * test: add test for water mark. * test: add meters bug6 for stream5 * test: add cases (#31903) * test: add test for recalc. * feat: [TS-6100] %%trows can only be used when event type is window close. * test: add precision of database for ms/us/ns * modify case * add case * add case * test: add test to ci. * modify case * fix: ci case issues (#31904) * enh: add operator reset func * fix: merge join reset issue * fix: memory issues * fix: add debug assert * fix: memory issues * fix: memory leak * fix: memory issues * fix taos log miss * fix: case issue * fix: case issue * fix: case issues * fix: drop dnode issue * fix: memory issues * fix: memory issues * fix: memory leak issues * fix: recalculate time range issue * fix: add debug log * fix: memory issues * fix: enable case asan * Update streamlist_for_ci.task * fix: case asan issue * fix: stream name issue * fix: external window compile issues * fix: deploy memory issue * fix: ahandle issue * fix: ahandle issue * fix: ahandle issue * fix: virtual table reader list issue * fix: log info * fix: msg error * fix: virtual table addr list issue * fix: memory issues * fix: memory leak issue * fix: memory issues * fix: memory free issues * fix: memory issues * fix: snode deploy issue * fix: mnode reader issue * fix: memory issues * fix: add debug test * enh: add ignore nodata trigger * fix: memory leaks * fix: configuration issue * fix: memory issue * fix: external window issue * fix: external window issues * fix: external window placeholder issue * fix: placeholder function init issues * fix: memory leak issue * fix: add debug log * fix: compile issues * fix: double free issue * fix: runner addr update issue * fix: msg rsp issue * fix: external window reset issue * fix: configuration issue * fix: deploy msg issue * fix: compile issue * fix: external window idx issue * fix: ci issues * fix: ci case issues * fix: drop dnode issue --------- Co-authored-by: huohong <sallyhuo@taosdata.com> * fix(stream): ci error * test: update test case. * feat: [TS-6100] Disable some failed UT. * feat: [TS-6100] Fix virtual table * test: add bug 5. * test: add test delete recalc to ci. * test: add bug 6. * test(stream): tobacco scene #TD-36514 * fix: reqCids,reqCols memory leak in SSTriggerRealtimeContext Co-authored-by: Tony Zhang <tonyzhang@taosdata.com> * test: add case stream6 * fix(stream): implement some pending features in trigger task * modify case * modify case * fix: case issues * modify case * test: add recalc for warter mark. * fix(stream): fix count window trigger of virtual tables * fix(stream): memory leak * test: fix run err. * test: add stream6 bug7 * fix: adjust format * test(stream): tobacco scene testing #TD-36514 * test: change bug7 with update window1 and 2 * test: add test bug 7. * case: restore write 3 window * fix: windows compile issue * fix: notify * test: add cases * modify case * test: update test case. * test(stream): toobacco scene testing #TD-36514 --------- Co-authored-by: Simon Guan <slguan@taosdata.com> Co-authored-by: plum-lihui <huili@taosdata.com> Co-authored-by: Alex Duan <417921451@qq.com> Co-authored-by: zelv01 <1101510017@qq.com> Co-authored-by: Jing Sima <simondominic9997@outlook.com> Co-authored-by: xiangyang guo <66111494+happyguoxy@users.noreply.github.com> Co-authored-by: wangmm0220 <wangmm0220@gmail.com> Co-authored-by: Haojun Liao <hjliao@taosdata.com> Co-authored-by: zyyang90 <zyyang@taosdata.com> Co-authored-by: Alex Duan <51781608+DuanKuanJun@users.noreply.github.com> Co-authored-by: facetosea <285808407@qq.com> Co-authored-by: Simon Guan <guanshengliang@qq.com> Co-authored-by: huohong <sallyhuo@taosdata.com> Co-authored-by: Li Hui <52318143+plum-lihui@users.noreply.github.com> Co-authored-by: Jinqing Kuang <kuangjinqingcn@gmail.com> Co-authored-by: xiao-77 <berylbao@taosdata.com> Co-authored-by: Zhixiao Bao <62235797+xiao-77@users.noreply.github.com> Co-authored-by: happyguoxy <happy_guoxy@163.com> Co-authored-by: Tony Zhang <34825804+Tony2h@users.noreply.github.com> Co-authored-by: Tony Zhang <tonyzhang@taosdata.com>
2025-07-16 06:42:16 +00:00
int32_t vnodeProcessStreamReaderMsg(SVnode *pVnode, SRpcMsg *pMsg);
2022-07-06 07:15:55 +00:00
void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs);
void vnodeApplyWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs);
void vnodeProposeCommitOnNeed(SVnode *pVnode, bool atExit);
2022-06-08 06:44:42 +00:00
2022-04-14 02:01:04 +00:00
// meta
2023-08-18 08:38:26 +00:00
void _metaReaderInit(SMetaReader *pReader, void *pVnode, int32_t flags, SStoreMeta *pAPI);
void metaReaderReleaseLock(SMetaReader *pReader);
2022-05-12 11:21:44 +00:00
void metaReaderClear(SMetaReader *pReader);
int32_t metaReaderGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid);
int32_t metaReaderGetTableEntryByVersionUid(SMetaReader *pReader, int64_t version, tb_uid_t uid);
int32_t metaReaderGetTableEntryByUidCache(SMetaReader *pReader, tb_uid_t uid);
2023-05-24 13:53:40 +00:00
int32_t metaGetTableTags(void *pVnode, uint64_t suid, SArray *uidList);
2023-08-18 08:38:26 +00:00
int32_t metaGetTableTagsByUids(void *pVnode, int64_t suid, SArray *uidList);
2022-05-20 02:30:48 +00:00
int32_t metaReadNext(SMetaReader *pReader);
const void *metaGetTableTagVal(const void *tag, int16_t type, STagVal *tagVal);
2024-09-16 04:59:21 +00:00
int32_t metaGetTableNameByUid(void *pVnode, uint64_t uid, char *tbName);
int metaGetTableSzNameByUid(void *meta, uint64_t uid, char *tbName);
2023-05-24 15:49:26 +00:00
int metaGetTableUidByName(void *pVnode, char *tbName, uint64_t *uid);
2025-06-30 02:33:34 +00:00
int metaGetTableTypeSuidByName(void *meta, char *tbName, ETableType *tbType, uint64_t *suid);
int metaGetTableTtlByUid(void *meta, uint64_t uid, int64_t *ttlDays);
2023-08-18 08:38:26 +00:00
bool metaIsTableExist(void *pVnode, tb_uid_t uid);
2023-05-29 02:12:27 +00:00
int32_t metaGetCachedTableUidList(void *pVnode, tb_uid_t suid, const uint8_t *key, int32_t keyLen, SArray *pList,
bool *acquired);
2023-05-29 02:12:27 +00:00
int32_t metaUidFilterCachePut(void *pVnode, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload,
int32_t payloadLen, double selectivityRatio);
2025-11-10 02:20:29 +00:00
int32_t metaStableTagFilterCacheGet(void* pVnode, tb_uid_t suid,
const uint8_t* pTagCondKey, int32_t tagCondKeyLen,
const uint8_t* pKey, int32_t keyLen, SArray* pList, bool* acquired);
int32_t metaStableTagFilterCachePut(void* pVnode, uint64_t suid,
const void* pTagCondKey, int32_t tagCondKeyLen,
2025-11-10 08:53:15 +00:00
const void* pKey, int32_t keyLen, SArray* pUidList, SArray** pTagColIds);
2025-11-10 02:20:29 +00:00
int32_t metaStableTagFilterCacheDropSTable(SMeta* pMeta, tb_uid_t suid);
typedef enum {
STABLE_TAG_FILTER_CACHE_DROP_TABLE = 1,
STABLE_TAG_FILTER_CACHE_ADD_TABLE = 2,
} ETagFilterCacheAction;
int32_t metaStableTagFilterCacheUpdateUid(
SMeta* pMeta, const SMetaEntry* pDroppedTable,
const SMetaEntry* pSuperTable, ETagFilterCacheAction action);
2025-11-10 02:20:29 +00:00
int32_t metaStableTagFilterCacheDropTag(
SMeta* pMeta, tb_uid_t suid, col_id_t tagColId);
tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name);
2023-05-29 02:50:50 +00:00
int32_t metaGetCachedTbGroup(void *pVnode, tb_uid_t suid, const uint8_t *pKey, int32_t keyLen, SArray **pList);
2023-08-18 08:38:26 +00:00
int32_t metaPutTbGroupToCache(void *pVnode, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload,
2023-05-23 03:46:02 +00:00
int32_t payloadLen);
bool metaTbInFilterCache(SMeta *pMeta, const void *key, int8_t type);
int32_t metaPutTbToFilterCache(SMeta *pMeta, const void *key, int8_t type);
2023-10-07 01:30:32 +00:00
int32_t metaSizeOfTbFilterCache(SMeta *pMeta, int8_t type);
int32_t metaInitTbFilterCache(SMeta *pMeta);
2023-05-09 09:04:20 +00:00
int32_t metaGetStbStats(void *pVnode, int64_t uid, int64_t *numOfTables, int32_t *numOfCols, int8_t *flags);
2022-04-08 06:09:15 +00:00
2025-06-30 02:33:34 +00:00
int32_t metaGetCachedRefDbs(void *pVnode, tb_uid_t suid, SArray *pList);
int32_t metaPutRefDbsToCache(void *pVnode, tb_uid_t suid, SArray *pList);
2022-04-14 02:01:04 +00:00
// tsdb
2022-06-16 09:18:30 +00:00
typedef struct STsdbReader STsdbReader;
2022-04-14 02:10:56 +00:00
2022-09-08 10:36:08 +00:00
#define TSDB_DEFAULT_STT_FILE 8
#define TSDB_DEFAULT_PAGE_SIZE 4096
2022-09-08 08:56:29 +00:00
2022-07-25 03:18:30 +00:00
#define TIMEWINDOW_RANGE_CONTAINED 1
#define TIMEWINDOW_RANGE_EXTERNAL 2
2022-04-14 02:53:10 +00:00
#define CACHESCAN_RETRIEVE_TYPE_ALL 0x1
#define CACHESCAN_RETRIEVE_TYPE_SINGLE 0x2
#define CACHESCAN_RETRIEVE_LAST_ROW 0x4
#define CACHESCAN_RETRIEVE_LAST 0x8
2022-06-28 07:22:32 +00:00
2024-12-04 09:39:28 +00:00
int32_t tsdbReaderOpen2(void *pVnode, SQueryTableDataCond *pCond, void *pTableList, int32_t numOfTables,
SSDataBlock *pResBlock, void **ppReader, const char *idstr, SHashObj **pIgnoreTables);
feat: new stream (#31678) * fix: windows compile issue * test: add vtable cases (#31829) * fix: windows compile issues * test:add test cases * fix: windows compile issue * case: em-4 stream case submit * test: stream4_sub1 found bug2 * test: submit test_scene_meters_bug2.py * add stream parameters example * feat: [TS-6100] Do not translate const value as column. * Feat/ts 6100 3.0 zlv (#31747) * modify asan exampel * modify asan exampel * add example * add example * modify case example --------- Co-authored-by: zelv01 <1101510017@qq.com> * feat(stream): fix memory leak * modify sliding example * test: update test case. * feat(stream): fix conflicts * fix: add offset case 10a 10s 10m 10h 10d * feat(stream): fix conflicts * chore(stream): rename case name #TS-6100 * add case * modify example * fix: windows compile issues * fix: data null check * feat: [TS-6100] Forbid where when using %%trows (#31827) * feat: [TS-6100] Forbid where when using %%trows * test: update cases * feat: [TS-6100] Fix leaks. --------- Co-authored-by: Simon Guan <guanshengliang@qq.com> * test: reproduce bugs * test: update test case. * test: update test case. * feat: [TS-6100] Fix leaks. * test: add cases * Feat/ts 6100 3.0.pw10 (#31841) * enh: add operator reset func * fix: merge join reset issue * fix: memory issues * fix: add debug assert * fix: memory issues * fix: memory leak * fix: memory issues * fix taos log miss * fix: case issue * fix: case issue * fix: case issues * fix: drop dnode issue * fix: memory issues * fix: memory issues * fix: memory leak issues * fix: recalculate time range issue * fix: add debug log * fix: memory issues * fix: enable case asan * Update streamlist_for_ci.task * fix: case asan issue * fix: stream name issue * fix: external window compile issues * fix: deploy memory issue * fix: ahandle issue * fix: ahandle issue * fix: ahandle issue * fix: virtual table reader list issue * fix: log info * fix: msg error * fix: virtual table addr list issue * fix: memory issues * fix: memory leak issue * fix: memory issues * fix: memory free issues * fix: memory issues * fix: snode deploy issue * fix: mnode reader issue * fix: memory issues * fix: add debug test * enh: add ignore nodata trigger * fix: memory leaks * fix: configuration issue * fix: memory issue * fix: external window issue * fix: external window issues * fix: external window placeholder issue * fix: placeholder function init issues * fix: memory leak issue * fix: add debug log * fix: compile issues * fix: double free issue * fix: runner addr update issue * fix: msg rsp issue * fix: external window reset issue * fix: configuration issue * fix: deploy msg issue * fix: compile issue --------- Co-authored-by: huohong <sallyhuo@taosdata.com> * test: reproduce bugs * fix: add sliding interval combine case * test: add cases * test: add recalc test. * test: reproduce bugs * case : add vt ts is null check * modify case * bug: submit test_idmp_meters_bug3.py * test: add test for recalc. * test: add cases * fix: error code check * test: add cases * fix(stream): scan wal with schema in that version * add case * test: add cases * test: update test case. * fix: windows compile issues * add case * test: add cases (#31845) * modify case * fix: reset interpPrev * test: add test_idmp_meters bug4 and bug3 * add case * fix(stream): opti wal interface * fix: remove test_idmp_meters_bug5.py * test: add cases * fix(stream): fix ts data fetch for virtual tables * cancel asan case * test: update test case. * test: update test case. * add case * test: add cases * test: add cases * test: add case test_idmp_meters_bug5.py * test: update test case. * fix(stream): tmq error * test: add cases * feat: [TS-6100] Restore deleted code in mndSma.c since they are still in use. * fix(stream): optimize val scan logic * test: add test_recalc_expired_time.py to ci. * test: update test case. * test: update test case. * feat: [TS-6100] Fix fill range check * fix(stream): optimize val scan logic * add case * test: modify for partition by %%1 * test: add fun case stream4_sub7 * fix(stream): optimize val scan logic * add case * feat: [TS-6100] Rename OPTIONS to STREAM_OPTIONS. * test: add test for recalc. * test: use stream_options. * fix: some cases error. * test: remove recalc from ci. * fix: ci case issues (#31880) * enh: add operator reset func * fix: merge join reset issue * fix: memory issues * fix: add debug assert * fix: memory issues * fix: memory leak * fix: memory issues * fix taos log miss * fix: case issue * fix: case issue * fix: case issues * fix: drop dnode issue * fix: memory issues * fix: memory issues * fix: memory leak issues * fix: recalculate time range issue * fix: add debug log * fix: memory issues * fix: enable case asan * Update streamlist_for_ci.task * fix: case asan issue * fix: stream name issue * fix: external window compile issues * fix: deploy memory issue * fix: ahandle issue * fix: ahandle issue * fix: ahandle issue * fix: virtual table reader list issue * fix: log info * fix: msg error * fix: virtual table addr list issue * fix: memory issues * fix: memory leak issue * fix: memory issues * fix: memory free issues * fix: memory issues * fix: snode deploy issue * fix: mnode reader issue * fix: memory issues * fix: add debug test * enh: add ignore nodata trigger * fix: memory leaks * fix: configuration issue * fix: memory issue * fix: external window issue * fix: external window issues * fix: external window placeholder issue * fix: placeholder function init issues * fix: memory leak issue * fix: add debug log * fix: compile issues * fix: double free issue * fix: runner addr update issue * fix: msg rsp issue * fix: external window reset issue * fix: configuration issue * fix: deploy msg issue * fix: compile issue * fix: external window idx issue * fix: ci issues --------- Co-authored-by: huohong <sallyhuo@taosdata.com> * fix(stream): fix compilation error * fix(stream): optimize val scan logic * test:add test cases * test: modify case * fix: external agg error * test(stream): tobacco scene testing #TD-36514 * test: add stream cases (#31885) * fix: windows compile issue * fix: calc timerange * fix: windows compile issue * modify case * fix(stream): compile error * test: remove one debug test case file * test: modify * test: add test cases * test: reproduce bugs * test: reproduce bugs * feat: [TS-6100] Placeholder function should only appera in SELECT and… (#31868) * feat: [TS-6100] Placeholder function should only appera in SELECT and WHERE and FROM. * test: update case --------- Co-authored-by: Simon Guan <guanshengliang@qq.com> * add example * add example * modify case example * modify case * test:alter sql * test: add stream5 case * fix(stream): get schema error with version * test: add delete recalc test py. * test: remove bug cases * test: stream5 case test passed * test: add state cases (#31893) * fix(stream): compile error * test: modify case * test: add cases * test: add test. * test: update test case. * chore(test): fix case err * test: update test case. * fix: align data get * fix(stream): fix row index of datablock written into data cache * fix: put align data * test: update test case. * test: add test cases for virtual table * chore(test): fix case err #TD-36514 * add case * test: add test for water mark. * test: add meters bug6 for stream5 * test: add cases (#31903) * test: add test for recalc. * feat: [TS-6100] %%trows can only be used when event type is window close. * test: add precision of database for ms/us/ns * modify case * add case * add case * test: add test to ci. * modify case * fix: ci case issues (#31904) * enh: add operator reset func * fix: merge join reset issue * fix: memory issues * fix: add debug assert * fix: memory issues * fix: memory leak * fix: memory issues * fix taos log miss * fix: case issue * fix: case issue * fix: case issues * fix: drop dnode issue * fix: memory issues * fix: memory issues * fix: memory leak issues * fix: recalculate time range issue * fix: add debug log * fix: memory issues * fix: enable case asan * Update streamlist_for_ci.task * fix: case asan issue * fix: stream name issue * fix: external window compile issues * fix: deploy memory issue * fix: ahandle issue * fix: ahandle issue * fix: ahandle issue * fix: virtual table reader list issue * fix: log info * fix: msg error * fix: virtual table addr list issue * fix: memory issues * fix: memory leak issue * fix: memory issues * fix: memory free issues * fix: memory issues * fix: snode deploy issue * fix: mnode reader issue * fix: memory issues * fix: add debug test * enh: add ignore nodata trigger * fix: memory leaks * fix: configuration issue * fix: memory issue * fix: external window issue * fix: external window issues * fix: external window placeholder issue * fix: placeholder function init issues * fix: memory leak issue * fix: add debug log * fix: compile issues * fix: double free issue * fix: runner addr update issue * fix: msg rsp issue * fix: external window reset issue * fix: configuration issue * fix: deploy msg issue * fix: compile issue * fix: external window idx issue * fix: ci issues * fix: ci case issues * fix: drop dnode issue --------- Co-authored-by: huohong <sallyhuo@taosdata.com> * fix(stream): ci error * test: update test case. * feat: [TS-6100] Disable some failed UT. * feat: [TS-6100] Fix virtual table * test: add bug 5. * test: add test delete recalc to ci. * test: add bug 6. * test(stream): tobacco scene #TD-36514 * fix: reqCids,reqCols memory leak in SSTriggerRealtimeContext Co-authored-by: Tony Zhang <tonyzhang@taosdata.com> * test: add case stream6 * fix(stream): implement some pending features in trigger task * modify case * modify case * fix: case issues * modify case * test: add recalc for warter mark. * fix(stream): fix count window trigger of virtual tables * fix(stream): memory leak * test: fix run err. * test: add stream6 bug7 * fix: adjust format * test(stream): tobacco scene testing #TD-36514 * test: change bug7 with update window1 and 2 * test: add test bug 7. * case: restore write 3 window * fix: windows compile issue * fix: notify * test: add cases * modify case * test: update test case. * test(stream): toobacco scene testing #TD-36514 --------- Co-authored-by: Simon Guan <slguan@taosdata.com> Co-authored-by: plum-lihui <huili@taosdata.com> Co-authored-by: Alex Duan <417921451@qq.com> Co-authored-by: zelv01 <1101510017@qq.com> Co-authored-by: Jing Sima <simondominic9997@outlook.com> Co-authored-by: xiangyang guo <66111494+happyguoxy@users.noreply.github.com> Co-authored-by: wangmm0220 <wangmm0220@gmail.com> Co-authored-by: Haojun Liao <hjliao@taosdata.com> Co-authored-by: zyyang90 <zyyang@taosdata.com> Co-authored-by: Alex Duan <51781608+DuanKuanJun@users.noreply.github.com> Co-authored-by: facetosea <285808407@qq.com> Co-authored-by: Simon Guan <guanshengliang@qq.com> Co-authored-by: huohong <sallyhuo@taosdata.com> Co-authored-by: Li Hui <52318143+plum-lihui@users.noreply.github.com> Co-authored-by: Jinqing Kuang <kuangjinqingcn@gmail.com> Co-authored-by: xiao-77 <berylbao@taosdata.com> Co-authored-by: Zhixiao Bao <62235797+xiao-77@users.noreply.github.com> Co-authored-by: happyguoxy <happy_guoxy@163.com> Co-authored-by: Tony Zhang <34825804+Tony2h@users.noreply.github.com> Co-authored-by: Tony Zhang <tonyzhang@taosdata.com>
2025-07-16 06:42:16 +00:00
int32_t tsdbSetTableList2(void *pReader, const void *pTableList, int32_t num);
2024-12-04 09:39:28 +00:00
int32_t tsdbReaderSetId(void *pReader, const char *idstr);
feat: new stream (#31678) * fix: windows compile issue * test: add vtable cases (#31829) * fix: windows compile issues * test:add test cases * fix: windows compile issue * case: em-4 stream case submit * test: stream4_sub1 found bug2 * test: submit test_scene_meters_bug2.py * add stream parameters example * feat: [TS-6100] Do not translate const value as column. * Feat/ts 6100 3.0 zlv (#31747) * modify asan exampel * modify asan exampel * add example * add example * modify case example --------- Co-authored-by: zelv01 <1101510017@qq.com> * feat(stream): fix memory leak * modify sliding example * test: update test case. * feat(stream): fix conflicts * fix: add offset case 10a 10s 10m 10h 10d * feat(stream): fix conflicts * chore(stream): rename case name #TS-6100 * add case * modify example * fix: windows compile issues * fix: data null check * feat: [TS-6100] Forbid where when using %%trows (#31827) * feat: [TS-6100] Forbid where when using %%trows * test: update cases * feat: [TS-6100] Fix leaks. --------- Co-authored-by: Simon Guan <guanshengliang@qq.com> * test: reproduce bugs * test: update test case. * test: update test case. * feat: [TS-6100] Fix leaks. * test: add cases * Feat/ts 6100 3.0.pw10 (#31841) * enh: add operator reset func * fix: merge join reset issue * fix: memory issues * fix: add debug assert * fix: memory issues * fix: memory leak * fix: memory issues * fix taos log miss * fix: case issue * fix: case issue * fix: case issues * fix: drop dnode issue * fix: memory issues * fix: memory issues * fix: memory leak issues * fix: recalculate time range issue * fix: add debug log * fix: memory issues * fix: enable case asan * Update streamlist_for_ci.task * fix: case asan issue * fix: stream name issue * fix: external window compile issues * fix: deploy memory issue * fix: ahandle issue * fix: ahandle issue * fix: ahandle issue * fix: virtual table reader list issue * fix: log info * fix: msg error * fix: virtual table addr list issue * fix: memory issues * fix: memory leak issue * fix: memory issues * fix: memory free issues * fix: memory issues * fix: snode deploy issue * fix: mnode reader issue * fix: memory issues * fix: add debug test * enh: add ignore nodata trigger * fix: memory leaks * fix: configuration issue * fix: memory issue * fix: external window issue * fix: external window issues * fix: external window placeholder issue * fix: placeholder function init issues * fix: memory leak issue * fix: add debug log * fix: compile issues * fix: double free issue * fix: runner addr update issue * fix: msg rsp issue * fix: external window reset issue * fix: configuration issue * fix: deploy msg issue * fix: compile issue --------- Co-authored-by: huohong <sallyhuo@taosdata.com> * test: reproduce bugs * fix: add sliding interval combine case * test: add cases * test: add recalc test. * test: reproduce bugs * case : add vt ts is null check * modify case * bug: submit test_idmp_meters_bug3.py * test: add test for recalc. * test: add cases * fix: error code check * test: add cases * fix(stream): scan wal with schema in that version * add case * test: add cases * test: update test case. * fix: windows compile issues * add case * test: add cases (#31845) * modify case * fix: reset interpPrev * test: add test_idmp_meters bug4 and bug3 * add case * fix(stream): opti wal interface * fix: remove test_idmp_meters_bug5.py * test: add cases * fix(stream): fix ts data fetch for virtual tables * cancel asan case * test: update test case. * test: update test case. * add case * test: add cases * test: add cases * test: add case test_idmp_meters_bug5.py * test: update test case. * fix(stream): tmq error * test: add cases * feat: [TS-6100] Restore deleted code in mndSma.c since they are still in use. * fix(stream): optimize val scan logic * test: add test_recalc_expired_time.py to ci. * test: update test case. * test: update test case. * feat: [TS-6100] Fix fill range check * fix(stream): optimize val scan logic * add case * test: modify for partition by %%1 * test: add fun case stream4_sub7 * fix(stream): optimize val scan logic * add case * feat: [TS-6100] Rename OPTIONS to STREAM_OPTIONS. * test: add test for recalc. * test: use stream_options. * fix: some cases error. * test: remove recalc from ci. * fix: ci case issues (#31880) * enh: add operator reset func * fix: merge join reset issue * fix: memory issues * fix: add debug assert * fix: memory issues * fix: memory leak * fix: memory issues * fix taos log miss * fix: case issue * fix: case issue * fix: case issues * fix: drop dnode issue * fix: memory issues * fix: memory issues * fix: memory leak issues * fix: recalculate time range issue * fix: add debug log * fix: memory issues * fix: enable case asan * Update streamlist_for_ci.task * fix: case asan issue * fix: stream name issue * fix: external window compile issues * fix: deploy memory issue * fix: ahandle issue * fix: ahandle issue * fix: ahandle issue * fix: virtual table reader list issue * fix: log info * fix: msg error * fix: virtual table addr list issue * fix: memory issues * fix: memory leak issue * fix: memory issues * fix: memory free issues * fix: memory issues * fix: snode deploy issue * fix: mnode reader issue * fix: memory issues * fix: add debug test * enh: add ignore nodata trigger * fix: memory leaks * fix: configuration issue * fix: memory issue * fix: external window issue * fix: external window issues * fix: external window placeholder issue * fix: placeholder function init issues * fix: memory leak issue * fix: add debug log * fix: compile issues * fix: double free issue * fix: runner addr update issue * fix: msg rsp issue * fix: external window reset issue * fix: configuration issue * fix: deploy msg issue * fix: compile issue * fix: external window idx issue * fix: ci issues --------- Co-authored-by: huohong <sallyhuo@taosdata.com> * fix(stream): fix compilation error * fix(stream): optimize val scan logic * test:add test cases * test: modify case * fix: external agg error * test(stream): tobacco scene testing #TD-36514 * test: add stream cases (#31885) * fix: windows compile issue * fix: calc timerange * fix: windows compile issue * modify case * fix(stream): compile error * test: remove one debug test case file * test: modify * test: add test cases * test: reproduce bugs * test: reproduce bugs * feat: [TS-6100] Placeholder function should only appera in SELECT and… (#31868) * feat: [TS-6100] Placeholder function should only appera in SELECT and WHERE and FROM. * test: update case --------- Co-authored-by: Simon Guan <guanshengliang@qq.com> * add example * add example * modify case example * modify case * test:alter sql * test: add stream5 case * fix(stream): get schema error with version * test: add delete recalc test py. * test: remove bug cases * test: stream5 case test passed * test: add state cases (#31893) * fix(stream): compile error * test: modify case * test: add cases * test: add test. * test: update test case. * chore(test): fix case err * test: update test case. * fix: align data get * fix(stream): fix row index of datablock written into data cache * fix: put align data * test: update test case. * test: add test cases for virtual table * chore(test): fix case err #TD-36514 * add case * test: add test for water mark. * test: add meters bug6 for stream5 * test: add cases (#31903) * test: add test for recalc. * feat: [TS-6100] %%trows can only be used when event type is window close. * test: add precision of database for ms/us/ns * modify case * add case * add case * test: add test to ci. * modify case * fix: ci case issues (#31904) * enh: add operator reset func * fix: merge join reset issue * fix: memory issues * fix: add debug assert * fix: memory issues * fix: memory leak * fix: memory issues * fix taos log miss * fix: case issue * fix: case issue * fix: case issues * fix: drop dnode issue * fix: memory issues * fix: memory issues * fix: memory leak issues * fix: recalculate time range issue * fix: add debug log * fix: memory issues * fix: enable case asan * Update streamlist_for_ci.task * fix: case asan issue * fix: stream name issue * fix: external window compile issues * fix: deploy memory issue * fix: ahandle issue * fix: ahandle issue * fix: ahandle issue * fix: virtual table reader list issue * fix: log info * fix: msg error * fix: virtual table addr list issue * fix: memory issues * fix: memory leak issue * fix: memory issues * fix: memory free issues * fix: memory issues * fix: snode deploy issue * fix: mnode reader issue * fix: memory issues * fix: add debug test * enh: add ignore nodata trigger * fix: memory leaks * fix: configuration issue * fix: memory issue * fix: external window issue * fix: external window issues * fix: external window placeholder issue * fix: placeholder function init issues * fix: memory leak issue * fix: add debug log * fix: compile issues * fix: double free issue * fix: runner addr update issue * fix: msg rsp issue * fix: external window reset issue * fix: configuration issue * fix: deploy msg issue * fix: compile issue * fix: external window idx issue * fix: ci issues * fix: ci case issues * fix: drop dnode issue --------- Co-authored-by: huohong <sallyhuo@taosdata.com> * fix(stream): ci error * test: update test case. * feat: [TS-6100] Disable some failed UT. * feat: [TS-6100] Fix virtual table * test: add bug 5. * test: add test delete recalc to ci. * test: add bug 6. * test(stream): tobacco scene #TD-36514 * fix: reqCids,reqCols memory leak in SSTriggerRealtimeContext Co-authored-by: Tony Zhang <tonyzhang@taosdata.com> * test: add case stream6 * fix(stream): implement some pending features in trigger task * modify case * modify case * fix: case issues * modify case * test: add recalc for warter mark. * fix(stream): fix count window trigger of virtual tables * fix(stream): memory leak * test: fix run err. * test: add stream6 bug7 * fix: adjust format * test(stream): tobacco scene testing #TD-36514 * test: change bug7 with update window1 and 2 * test: add test bug 7. * case: restore write 3 window * fix: windows compile issue * fix: notify * test: add cases * modify case * test: update test case. * test(stream): toobacco scene testing #TD-36514 --------- Co-authored-by: Simon Guan <slguan@taosdata.com> Co-authored-by: plum-lihui <huili@taosdata.com> Co-authored-by: Alex Duan <417921451@qq.com> Co-authored-by: zelv01 <1101510017@qq.com> Co-authored-by: Jing Sima <simondominic9997@outlook.com> Co-authored-by: xiangyang guo <66111494+happyguoxy@users.noreply.github.com> Co-authored-by: wangmm0220 <wangmm0220@gmail.com> Co-authored-by: Haojun Liao <hjliao@taosdata.com> Co-authored-by: zyyang90 <zyyang@taosdata.com> Co-authored-by: Alex Duan <51781608+DuanKuanJun@users.noreply.github.com> Co-authored-by: facetosea <285808407@qq.com> Co-authored-by: Simon Guan <guanshengliang@qq.com> Co-authored-by: huohong <sallyhuo@taosdata.com> Co-authored-by: Li Hui <52318143+plum-lihui@users.noreply.github.com> Co-authored-by: Jinqing Kuang <kuangjinqingcn@gmail.com> Co-authored-by: xiao-77 <berylbao@taosdata.com> Co-authored-by: Zhixiao Bao <62235797+xiao-77@users.noreply.github.com> Co-authored-by: happyguoxy <happy_guoxy@163.com> Co-authored-by: Tony Zhang <34825804+Tony2h@users.noreply.github.com> Co-authored-by: Tony Zhang <tonyzhang@taosdata.com>
2025-07-16 06:42:16 +00:00
void tsdbReaderClose2(void *pReader);
int32_t tsdbNextDataBlock2(void *pReader, bool *hasNext);
2024-12-04 09:39:28 +00:00
int32_t tsdbRetrieveDatablockSMA2(STsdbReader *pReader, SSDataBlock *pDataBlock, bool *allHave, bool *hasNullSMA);
feat: new stream (#31678) * fix: windows compile issue * test: add vtable cases (#31829) * fix: windows compile issues * test:add test cases * fix: windows compile issue * case: em-4 stream case submit * test: stream4_sub1 found bug2 * test: submit test_scene_meters_bug2.py * add stream parameters example * feat: [TS-6100] Do not translate const value as column. * Feat/ts 6100 3.0 zlv (#31747) * modify asan exampel * modify asan exampel * add example * add example * modify case example --------- Co-authored-by: zelv01 <1101510017@qq.com> * feat(stream): fix memory leak * modify sliding example * test: update test case. * feat(stream): fix conflicts * fix: add offset case 10a 10s 10m 10h 10d * feat(stream): fix conflicts * chore(stream): rename case name #TS-6100 * add case * modify example * fix: windows compile issues * fix: data null check * feat: [TS-6100] Forbid where when using %%trows (#31827) * feat: [TS-6100] Forbid where when using %%trows * test: update cases * feat: [TS-6100] Fix leaks. --------- Co-authored-by: Simon Guan <guanshengliang@qq.com> * test: reproduce bugs * test: update test case. * test: update test case. * feat: [TS-6100] Fix leaks. * test: add cases * Feat/ts 6100 3.0.pw10 (#31841) * enh: add operator reset func * fix: merge join reset issue * fix: memory issues * fix: add debug assert * fix: memory issues * fix: memory leak * fix: memory issues * fix taos log miss * fix: case issue * fix: case issue * fix: case issues * fix: drop dnode issue * fix: memory issues * fix: memory issues * fix: memory leak issues * fix: recalculate time range issue * fix: add debug log * fix: memory issues * fix: enable case asan * Update streamlist_for_ci.task * fix: case asan issue * fix: stream name issue * fix: external window compile issues * fix: deploy memory issue * fix: ahandle issue * fix: ahandle issue * fix: ahandle issue * fix: virtual table reader list issue * fix: log info * fix: msg error * fix: virtual table addr list issue * fix: memory issues * fix: memory leak issue * fix: memory issues * fix: memory free issues * fix: memory issues * fix: snode deploy issue * fix: mnode reader issue * fix: memory issues * fix: add debug test * enh: add ignore nodata trigger * fix: memory leaks * fix: configuration issue * fix: memory issue * fix: external window issue * fix: external window issues * fix: external window placeholder issue * fix: placeholder function init issues * fix: memory leak issue * fix: add debug log * fix: compile issues * fix: double free issue * fix: runner addr update issue * fix: msg rsp issue * fix: external window reset issue * fix: configuration issue * fix: deploy msg issue * fix: compile issue --------- Co-authored-by: huohong <sallyhuo@taosdata.com> * test: reproduce bugs * fix: add sliding interval combine case * test: add cases * test: add recalc test. * test: reproduce bugs * case : add vt ts is null check * modify case * bug: submit test_idmp_meters_bug3.py * test: add test for recalc. * test: add cases * fix: error code check * test: add cases * fix(stream): scan wal with schema in that version * add case * test: add cases * test: update test case. * fix: windows compile issues * add case * test: add cases (#31845) * modify case * fix: reset interpPrev * test: add test_idmp_meters bug4 and bug3 * add case * fix(stream): opti wal interface * fix: remove test_idmp_meters_bug5.py * test: add cases * fix(stream): fix ts data fetch for virtual tables * cancel asan case * test: update test case. * test: update test case. * add case * test: add cases * test: add cases * test: add case test_idmp_meters_bug5.py * test: update test case. * fix(stream): tmq error * test: add cases * feat: [TS-6100] Restore deleted code in mndSma.c since they are still in use. * fix(stream): optimize val scan logic * test: add test_recalc_expired_time.py to ci. * test: update test case. * test: update test case. * feat: [TS-6100] Fix fill range check * fix(stream): optimize val scan logic * add case * test: modify for partition by %%1 * test: add fun case stream4_sub7 * fix(stream): optimize val scan logic * add case * feat: [TS-6100] Rename OPTIONS to STREAM_OPTIONS. * test: add test for recalc. * test: use stream_options. * fix: some cases error. * test: remove recalc from ci. * fix: ci case issues (#31880) * enh: add operator reset func * fix: merge join reset issue * fix: memory issues * fix: add debug assert * fix: memory issues * fix: memory leak * fix: memory issues * fix taos log miss * fix: case issue * fix: case issue * fix: case issues * fix: drop dnode issue * fix: memory issues * fix: memory issues * fix: memory leak issues * fix: recalculate time range issue * fix: add debug log * fix: memory issues * fix: enable case asan * Update streamlist_for_ci.task * fix: case asan issue * fix: stream name issue * fix: external window compile issues * fix: deploy memory issue * fix: ahandle issue * fix: ahandle issue * fix: ahandle issue * fix: virtual table reader list issue * fix: log info * fix: msg error * fix: virtual table addr list issue * fix: memory issues * fix: memory leak issue * fix: memory issues * fix: memory free issues * fix: memory issues * fix: snode deploy issue * fix: mnode reader issue * fix: memory issues * fix: add debug test * enh: add ignore nodata trigger * fix: memory leaks * fix: configuration issue * fix: memory issue * fix: external window issue * fix: external window issues * fix: external window placeholder issue * fix: placeholder function init issues * fix: memory leak issue * fix: add debug log * fix: compile issues * fix: double free issue * fix: runner addr update issue * fix: msg rsp issue * fix: external window reset issue * fix: configuration issue * fix: deploy msg issue * fix: compile issue * fix: external window idx issue * fix: ci issues --------- Co-authored-by: huohong <sallyhuo@taosdata.com> * fix(stream): fix compilation error * fix(stream): optimize val scan logic * test:add test cases * test: modify case * fix: external agg error * test(stream): tobacco scene testing #TD-36514 * test: add stream cases (#31885) * fix: windows compile issue * fix: calc timerange * fix: windows compile issue * modify case * fix(stream): compile error * test: remove one debug test case file * test: modify * test: add test cases * test: reproduce bugs * test: reproduce bugs * feat: [TS-6100] Placeholder function should only appera in SELECT and… (#31868) * feat: [TS-6100] Placeholder function should only appera in SELECT and WHERE and FROM. * test: update case --------- Co-authored-by: Simon Guan <guanshengliang@qq.com> * add example * add example * modify case example * modify case * test:alter sql * test: add stream5 case * fix(stream): get schema error with version * test: add delete recalc test py. * test: remove bug cases * test: stream5 case test passed * test: add state cases (#31893) * fix(stream): compile error * test: modify case * test: add cases * test: add test. * test: update test case. * chore(test): fix case err * test: update test case. * fix: align data get * fix(stream): fix row index of datablock written into data cache * fix: put align data * test: update test case. * test: add test cases for virtual table * chore(test): fix case err #TD-36514 * add case * test: add test for water mark. * test: add meters bug6 for stream5 * test: add cases (#31903) * test: add test for recalc. * feat: [TS-6100] %%trows can only be used when event type is window close. * test: add precision of database for ms/us/ns * modify case * add case * add case * test: add test to ci. * modify case * fix: ci case issues (#31904) * enh: add operator reset func * fix: merge join reset issue * fix: memory issues * fix: add debug assert * fix: memory issues * fix: memory leak * fix: memory issues * fix taos log miss * fix: case issue * fix: case issue * fix: case issues * fix: drop dnode issue * fix: memory issues * fix: memory issues * fix: memory leak issues * fix: recalculate time range issue * fix: add debug log * fix: memory issues * fix: enable case asan * Update streamlist_for_ci.task * fix: case asan issue * fix: stream name issue * fix: external window compile issues * fix: deploy memory issue * fix: ahandle issue * fix: ahandle issue * fix: ahandle issue * fix: virtual table reader list issue * fix: log info * fix: msg error * fix: virtual table addr list issue * fix: memory issues * fix: memory leak issue * fix: memory issues * fix: memory free issues * fix: memory issues * fix: snode deploy issue * fix: mnode reader issue * fix: memory issues * fix: add debug test * enh: add ignore nodata trigger * fix: memory leaks * fix: configuration issue * fix: memory issue * fix: external window issue * fix: external window issues * fix: external window placeholder issue * fix: placeholder function init issues * fix: memory leak issue * fix: add debug log * fix: compile issues * fix: double free issue * fix: runner addr update issue * fix: msg rsp issue * fix: external window reset issue * fix: configuration issue * fix: deploy msg issue * fix: compile issue * fix: external window idx issue * fix: ci issues * fix: ci case issues * fix: drop dnode issue --------- Co-authored-by: huohong <sallyhuo@taosdata.com> * fix(stream): ci error * test: update test case. * feat: [TS-6100] Disable some failed UT. * feat: [TS-6100] Fix virtual table * test: add bug 5. * test: add test delete recalc to ci. * test: add bug 6. * test(stream): tobacco scene #TD-36514 * fix: reqCids,reqCols memory leak in SSTriggerRealtimeContext Co-authored-by: Tony Zhang <tonyzhang@taosdata.com> * test: add case stream6 * fix(stream): implement some pending features in trigger task * modify case * modify case * fix: case issues * modify case * test: add recalc for warter mark. * fix(stream): fix count window trigger of virtual tables * fix(stream): memory leak * test: fix run err. * test: add stream6 bug7 * fix: adjust format * test(stream): tobacco scene testing #TD-36514 * test: change bug7 with update window1 and 2 * test: add test bug 7. * case: restore write 3 window * fix: windows compile issue * fix: notify * test: add cases * modify case * test: update test case. * test(stream): toobacco scene testing #TD-36514 --------- Co-authored-by: Simon Guan <slguan@taosdata.com> Co-authored-by: plum-lihui <huili@taosdata.com> Co-authored-by: Alex Duan <417921451@qq.com> Co-authored-by: zelv01 <1101510017@qq.com> Co-authored-by: Jing Sima <simondominic9997@outlook.com> Co-authored-by: xiangyang guo <66111494+happyguoxy@users.noreply.github.com> Co-authored-by: wangmm0220 <wangmm0220@gmail.com> Co-authored-by: Haojun Liao <hjliao@taosdata.com> Co-authored-by: zyyang90 <zyyang@taosdata.com> Co-authored-by: Alex Duan <51781608+DuanKuanJun@users.noreply.github.com> Co-authored-by: facetosea <285808407@qq.com> Co-authored-by: Simon Guan <guanshengliang@qq.com> Co-authored-by: huohong <sallyhuo@taosdata.com> Co-authored-by: Li Hui <52318143+plum-lihui@users.noreply.github.com> Co-authored-by: Jinqing Kuang <kuangjinqingcn@gmail.com> Co-authored-by: xiao-77 <berylbao@taosdata.com> Co-authored-by: Zhixiao Bao <62235797+xiao-77@users.noreply.github.com> Co-authored-by: happyguoxy <happy_guoxy@163.com> Co-authored-by: Tony Zhang <34825804+Tony2h@users.noreply.github.com> Co-authored-by: Tony Zhang <tonyzhang@taosdata.com>
2025-07-16 06:42:16 +00:00
void tsdbReleaseDataBlock2(void *pReader);
2025-11-07 07:45:12 +00:00
int32_t tsdbRetrieveDataBlock2(void *pReader, SSDataBlock **pBlock);
feat: new stream (#31678) * fix: windows compile issue * test: add vtable cases (#31829) * fix: windows compile issues * test:add test cases * fix: windows compile issue * case: em-4 stream case submit * test: stream4_sub1 found bug2 * test: submit test_scene_meters_bug2.py * add stream parameters example * feat: [TS-6100] Do not translate const value as column. * Feat/ts 6100 3.0 zlv (#31747) * modify asan exampel * modify asan exampel * add example * add example * modify case example --------- Co-authored-by: zelv01 <1101510017@qq.com> * feat(stream): fix memory leak * modify sliding example * test: update test case. * feat(stream): fix conflicts * fix: add offset case 10a 10s 10m 10h 10d * feat(stream): fix conflicts * chore(stream): rename case name #TS-6100 * add case * modify example * fix: windows compile issues * fix: data null check * feat: [TS-6100] Forbid where when using %%trows (#31827) * feat: [TS-6100] Forbid where when using %%trows * test: update cases * feat: [TS-6100] Fix leaks. --------- Co-authored-by: Simon Guan <guanshengliang@qq.com> * test: reproduce bugs * test: update test case. * test: update test case. * feat: [TS-6100] Fix leaks. * test: add cases * Feat/ts 6100 3.0.pw10 (#31841) * enh: add operator reset func * fix: merge join reset issue * fix: memory issues * fix: add debug assert * fix: memory issues * fix: memory leak * fix: memory issues * fix taos log miss * fix: case issue * fix: case issue * fix: case issues * fix: drop dnode issue * fix: memory issues * fix: memory issues * fix: memory leak issues * fix: recalculate time range issue * fix: add debug log * fix: memory issues * fix: enable case asan * Update streamlist_for_ci.task * fix: case asan issue * fix: stream name issue * fix: external window compile issues * fix: deploy memory issue * fix: ahandle issue * fix: ahandle issue * fix: ahandle issue * fix: virtual table reader list issue * fix: log info * fix: msg error * fix: virtual table addr list issue * fix: memory issues * fix: memory leak issue * fix: memory issues * fix: memory free issues * fix: memory issues * fix: snode deploy issue * fix: mnode reader issue * fix: memory issues * fix: add debug test * enh: add ignore nodata trigger * fix: memory leaks * fix: configuration issue * fix: memory issue * fix: external window issue * fix: external window issues * fix: external window placeholder issue * fix: placeholder function init issues * fix: memory leak issue * fix: add debug log * fix: compile issues * fix: double free issue * fix: runner addr update issue * fix: msg rsp issue * fix: external window reset issue * fix: configuration issue * fix: deploy msg issue * fix: compile issue --------- Co-authored-by: huohong <sallyhuo@taosdata.com> * test: reproduce bugs * fix: add sliding interval combine case * test: add cases * test: add recalc test. * test: reproduce bugs * case : add vt ts is null check * modify case * bug: submit test_idmp_meters_bug3.py * test: add test for recalc. * test: add cases * fix: error code check * test: add cases * fix(stream): scan wal with schema in that version * add case * test: add cases * test: update test case. * fix: windows compile issues * add case * test: add cases (#31845) * modify case * fix: reset interpPrev * test: add test_idmp_meters bug4 and bug3 * add case * fix(stream): opti wal interface * fix: remove test_idmp_meters_bug5.py * test: add cases * fix(stream): fix ts data fetch for virtual tables * cancel asan case * test: update test case. * test: update test case. * add case * test: add cases * test: add cases * test: add case test_idmp_meters_bug5.py * test: update test case. * fix(stream): tmq error * test: add cases * feat: [TS-6100] Restore deleted code in mndSma.c since they are still in use. * fix(stream): optimize val scan logic * test: add test_recalc_expired_time.py to ci. * test: update test case. * test: update test case. * feat: [TS-6100] Fix fill range check * fix(stream): optimize val scan logic * add case * test: modify for partition by %%1 * test: add fun case stream4_sub7 * fix(stream): optimize val scan logic * add case * feat: [TS-6100] Rename OPTIONS to STREAM_OPTIONS. * test: add test for recalc. * test: use stream_options. * fix: some cases error. * test: remove recalc from ci. * fix: ci case issues (#31880) * enh: add operator reset func * fix: merge join reset issue * fix: memory issues * fix: add debug assert * fix: memory issues * fix: memory leak * fix: memory issues * fix taos log miss * fix: case issue * fix: case issue * fix: case issues * fix: drop dnode issue * fix: memory issues * fix: memory issues * fix: memory leak issues * fix: recalculate time range issue * fix: add debug log * fix: memory issues * fix: enable case asan * Update streamlist_for_ci.task * fix: case asan issue * fix: stream name issue * fix: external window compile issues * fix: deploy memory issue * fix: ahandle issue * fix: ahandle issue * fix: ahandle issue * fix: virtual table reader list issue * fix: log info * fix: msg error * fix: virtual table addr list issue * fix: memory issues * fix: memory leak issue * fix: memory issues * fix: memory free issues * fix: memory issues * fix: snode deploy issue * fix: mnode reader issue * fix: memory issues * fix: add debug test * enh: add ignore nodata trigger * fix: memory leaks * fix: configuration issue * fix: memory issue * fix: external window issue * fix: external window issues * fix: external window placeholder issue * fix: placeholder function init issues * fix: memory leak issue * fix: add debug log * fix: compile issues * fix: double free issue * fix: runner addr update issue * fix: msg rsp issue * fix: external window reset issue * fix: configuration issue * fix: deploy msg issue * fix: compile issue * fix: external window idx issue * fix: ci issues --------- Co-authored-by: huohong <sallyhuo@taosdata.com> * fix(stream): fix compilation error * fix(stream): optimize val scan logic * test:add test cases * test: modify case * fix: external agg error * test(stream): tobacco scene testing #TD-36514 * test: add stream cases (#31885) * fix: windows compile issue * fix: calc timerange * fix: windows compile issue * modify case * fix(stream): compile error * test: remove one debug test case file * test: modify * test: add test cases * test: reproduce bugs * test: reproduce bugs * feat: [TS-6100] Placeholder function should only appera in SELECT and… (#31868) * feat: [TS-6100] Placeholder function should only appera in SELECT and WHERE and FROM. * test: update case --------- Co-authored-by: Simon Guan <guanshengliang@qq.com> * add example * add example * modify case example * modify case * test:alter sql * test: add stream5 case * fix(stream): get schema error with version * test: add delete recalc test py. * test: remove bug cases * test: stream5 case test passed * test: add state cases (#31893) * fix(stream): compile error * test: modify case * test: add cases * test: add test. * test: update test case. * chore(test): fix case err * test: update test case. * fix: align data get * fix(stream): fix row index of datablock written into data cache * fix: put align data * test: update test case. * test: add test cases for virtual table * chore(test): fix case err #TD-36514 * add case * test: add test for water mark. * test: add meters bug6 for stream5 * test: add cases (#31903) * test: add test for recalc. * feat: [TS-6100] %%trows can only be used when event type is window close. * test: add precision of database for ms/us/ns * modify case * add case * add case * test: add test to ci. * modify case * fix: ci case issues (#31904) * enh: add operator reset func * fix: merge join reset issue * fix: memory issues * fix: add debug assert * fix: memory issues * fix: memory leak * fix: memory issues * fix taos log miss * fix: case issue * fix: case issue * fix: case issues * fix: drop dnode issue * fix: memory issues * fix: memory issues * fix: memory leak issues * fix: recalculate time range issue * fix: add debug log * fix: memory issues * fix: enable case asan * Update streamlist_for_ci.task * fix: case asan issue * fix: stream name issue * fix: external window compile issues * fix: deploy memory issue * fix: ahandle issue * fix: ahandle issue * fix: ahandle issue * fix: virtual table reader list issue * fix: log info * fix: msg error * fix: virtual table addr list issue * fix: memory issues * fix: memory leak issue * fix: memory issues * fix: memory free issues * fix: memory issues * fix: snode deploy issue * fix: mnode reader issue * fix: memory issues * fix: add debug test * enh: add ignore nodata trigger * fix: memory leaks * fix: configuration issue * fix: memory issue * fix: external window issue * fix: external window issues * fix: external window placeholder issue * fix: placeholder function init issues * fix: memory leak issue * fix: add debug log * fix: compile issues * fix: double free issue * fix: runner addr update issue * fix: msg rsp issue * fix: external window reset issue * fix: configuration issue * fix: deploy msg issue * fix: compile issue * fix: external window idx issue * fix: ci issues * fix: ci case issues * fix: drop dnode issue --------- Co-authored-by: huohong <sallyhuo@taosdata.com> * fix(stream): ci error * test: update test case. * feat: [TS-6100] Disable some failed UT. * feat: [TS-6100] Fix virtual table * test: add bug 5. * test: add test delete recalc to ci. * test: add bug 6. * test(stream): tobacco scene #TD-36514 * fix: reqCids,reqCols memory leak in SSTriggerRealtimeContext Co-authored-by: Tony Zhang <tonyzhang@taosdata.com> * test: add case stream6 * fix(stream): implement some pending features in trigger task * modify case * modify case * fix: case issues * modify case * test: add recalc for warter mark. * fix(stream): fix count window trigger of virtual tables * fix(stream): memory leak * test: fix run err. * test: add stream6 bug7 * fix: adjust format * test(stream): tobacco scene testing #TD-36514 * test: change bug7 with update window1 and 2 * test: add test bug 7. * case: restore write 3 window * fix: windows compile issue * fix: notify * test: add cases * modify case * test: update test case. * test(stream): toobacco scene testing #TD-36514 --------- Co-authored-by: Simon Guan <slguan@taosdata.com> Co-authored-by: plum-lihui <huili@taosdata.com> Co-authored-by: Alex Duan <417921451@qq.com> Co-authored-by: zelv01 <1101510017@qq.com> Co-authored-by: Jing Sima <simondominic9997@outlook.com> Co-authored-by: xiangyang guo <66111494+happyguoxy@users.noreply.github.com> Co-authored-by: wangmm0220 <wangmm0220@gmail.com> Co-authored-by: Haojun Liao <hjliao@taosdata.com> Co-authored-by: zyyang90 <zyyang@taosdata.com> Co-authored-by: Alex Duan <51781608+DuanKuanJun@users.noreply.github.com> Co-authored-by: facetosea <285808407@qq.com> Co-authored-by: Simon Guan <guanshengliang@qq.com> Co-authored-by: huohong <sallyhuo@taosdata.com> Co-authored-by: Li Hui <52318143+plum-lihui@users.noreply.github.com> Co-authored-by: Jinqing Kuang <kuangjinqingcn@gmail.com> Co-authored-by: xiao-77 <berylbao@taosdata.com> Co-authored-by: Zhixiao Bao <62235797+xiao-77@users.noreply.github.com> Co-authored-by: happyguoxy <happy_guoxy@163.com> Co-authored-by: Tony Zhang <34825804+Tony2h@users.noreply.github.com> Co-authored-by: Tony Zhang <tonyzhang@taosdata.com>
2025-07-16 06:42:16 +00:00
int32_t tsdbReaderReset2(void *pReader, SQueryTableDataCond *pCond);
2024-12-04 09:39:28 +00:00
int32_t tsdbGetFileBlocksDistInfo2(STsdbReader *pReader, STableBlockDistInfo *pTableBlockInfo);
2025-10-17 05:18:37 +00:00
void tsdbGetDataBlock(STsdbReader* pReader, SSDataBlock** pBlock);
void tsdbSetDataBlock(STsdbReader* pReader, SSDataBlock* pBlock);
2024-12-04 09:39:28 +00:00
int64_t tsdbGetNumOfRowsInMemTable2(STsdbReader *pHandle, uint32_t *rows);
void *tsdbGetIdx2(SMeta *pMeta);
void *tsdbGetIvtIdx2(SMeta *pMeta);
uint64_t tsdbGetReaderMaxVersion2(STsdbReader *pReader);
void tsdbReaderSetCloseFlag(STsdbReader *pReader);
int64_t tsdbGetLastTimestamp2(SVnode *pVnode, void *pTableList, int32_t numOfTables, const char *pIdStr);
void tsdbSetFilesetDelimited(STsdbReader *pReader);
void tsdbReaderSetNotifyCb(STsdbReader *pReader, TsdReaderNotifyCbFn notifyFn, void *param);
2025-11-12 03:02:17 +00:00
int32_t tsdbCreateFirstLastTsIter(void *pVnode, STimeWindow *pWindow, SVersionRange *pVerRange, uint64_t suid, void *pTableList,
int32_t numOfTables, int32_t order, void **pIter, const char *idstr);
int32_t tsdbNextFirstLastTsBlock(void *pIter, SSDataBlock *pRes, bool* hasNext);
2025-11-12 03:02:17 +00:00
void tsdbDestroyFirstLastTsIter(void *pIter);
2023-06-15 09:31:59 +00:00
2023-05-23 03:46:02 +00:00
int32_t tsdbReuseCacherowsReader(void *pReader, void *pTableIdList, int32_t numOfTables);
2022-10-29 10:27:22 +00:00
int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols,
SArray *pCidList, int32_t *pSlotIds, uint64_t suid, void **pReader, const char *idstr,
SArray *pFuncTypeList, SColumnInfo *pkCol, int32_t numOfPks);
int32_t tsdbRetrieveCacheRows(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, const int32_t *dstSlotIds,
SArray *pTableUids, bool *pGotAllRows);
void tsdbCacherowsReaderClose(void *pReader);
void tsdbCacheSetCapacity(SVnode *pVnode, size_t capacity);
size_t tsdbCacheGetCapacity(SVnode *pVnode);
size_t tsdbCacheGetUsage(SVnode *pVnode);
int32_t tsdbCacheGetElems(SVnode *pVnode);
2022-04-14 02:01:04 +00:00
//// tq
typedef struct SIdInfo {
int64_t version;
int32_t index;
} SIdInfo;
2022-08-09 11:06:24 +00:00
typedef struct SVTColInfo {
int32_t vColId; // column id of virtual table
int32_t pColId; // column id of physical table
int64_t pTbUid; // uid of physical table
} SVTColInfo;
typedef struct SVTSourceScanInfo {
SHashObj *pVirtualTables; // source column info of each vtable column. key: vtUid, value: SArray<SVTColInfo>*
SHashObj *pPhysicalTables; // set of vtables for each ptable. Key: ptUid, value: SArray<vtUid>*
SLRUCache *pPhyTblSchemaCache; // cache for physical table schema
int32_t nextVirtualTableIdx; // index in the value of pVirtualTables
uint64_t metaFetch;
uint64_t cacheHit;
} SVTSourceScanInfo;
2022-07-07 09:16:12 +00:00
typedef struct STqReader {
2025-06-30 02:33:34 +00:00
SPackedData msg;
SSubmitReq2 submit;
int32_t nextBlk;
int64_t lastBlkUid;
SWalReader *pWalReader;
SMeta *pVnodeMeta;
SHashObj *tbIdHash;
SArray *pColIdList; // SArray<int16_t>
int32_t cachedSchemaVer;
int64_t cachedSchemaSuid;
int64_t cachedSchemaUid;
SSchemaWrapper *pSchemaWrapper;
SSDataBlock *pResBlock;
int64_t lastTs;
bool hasPrimaryKey;
SExtSchema *extSchema;
SVTSourceScanInfo vtSourceScanInfo;
2022-07-07 09:16:12 +00:00
} STqReader;
2023-04-27 08:06:06 +00:00
STqReader *tqReaderOpen(SVnode *pVnode);
void tqReaderClose(STqReader *);
2022-07-07 09:16:12 +00:00
bool tqGetTablePrimaryKey(STqReader *pReader);
void tqSetTablePrimaryKey(STqReader *pReader, int64_t uid);
int32_t tqReaderSetColIdList(STqReader *pReader, SArray *pColIdList, const char *id);
int32_t tqReaderSetTbUidList(STqReader *pReader, const SArray *tbUidList, const char *id);
2025-06-30 02:33:34 +00:00
void tqReaderAddTbUidList(STqReader *pReader, const SArray *pTableUidList);
void tqReaderRemoveTbUidList(STqReader *pReader, const SArray *tbUidList);
2022-07-07 09:16:12 +00:00
2023-08-18 08:38:26 +00:00
bool tqReaderIsQueriedTable(STqReader *pReader, uint64_t uid);
bool tqCurrentBlockConsumed(const STqReader *pReader);
2023-08-18 08:38:26 +00:00
int32_t tqReaderSeek(STqReader *pReader, int64_t ver, const char *id);
bool tqNextBlockInWal(STqReader *pReader, const char *idstr, int sourceExcluded);
2023-08-18 08:38:26 +00:00
bool tqNextBlockImpl(STqReader *pReader, const char *idstr);
SWalReader *tqGetWalReader(STqReader *pReader);
SSDataBlock *tqGetResultBlock(STqReader *pReader);
2023-10-08 08:06:15 +00:00
int64_t tqGetResultBlockTime(STqReader *pReader);
2022-07-08 09:48:34 +00:00
int32_t tqReaderSetSubmitMsg(STqReader *pReader, void *msgStr, int32_t msgLen, int64_t ver, SArray* rawList, SDecoder* decoder);
void tqReaderClearSubmitMsg(STqReader *pReader);
2023-04-27 01:28:20 +00:00
bool tqNextDataBlockFilterOut(STqReader *pReader, SHashObj *filterOutUids);
2023-08-18 08:38:26 +00:00
int32_t tqRetrieveDataBlock(STqReader *pReader, SSDataBlock **pRes, const char *idstr);
feat: new stream (#31678) * fix: windows compile issue * test: add vtable cases (#31829) * fix: windows compile issues * test:add test cases * fix: windows compile issue * case: em-4 stream case submit * test: stream4_sub1 found bug2 * test: submit test_scene_meters_bug2.py * add stream parameters example * feat: [TS-6100] Do not translate const value as column. * Feat/ts 6100 3.0 zlv (#31747) * modify asan exampel * modify asan exampel * add example * add example * modify case example --------- Co-authored-by: zelv01 <1101510017@qq.com> * feat(stream): fix memory leak * modify sliding example * test: update test case. * feat(stream): fix conflicts * fix: add offset case 10a 10s 10m 10h 10d * feat(stream): fix conflicts * chore(stream): rename case name #TS-6100 * add case * modify example * fix: windows compile issues * fix: data null check * feat: [TS-6100] Forbid where when using %%trows (#31827) * feat: [TS-6100] Forbid where when using %%trows * test: update cases * feat: [TS-6100] Fix leaks. --------- Co-authored-by: Simon Guan <guanshengliang@qq.com> * test: reproduce bugs * test: update test case. * test: update test case. * feat: [TS-6100] Fix leaks. * test: add cases * Feat/ts 6100 3.0.pw10 (#31841) * enh: add operator reset func * fix: merge join reset issue * fix: memory issues * fix: add debug assert * fix: memory issues * fix: memory leak * fix: memory issues * fix taos log miss * fix: case issue * fix: case issue * fix: case issues * fix: drop dnode issue * fix: memory issues * fix: memory issues * fix: memory leak issues * fix: recalculate time range issue * fix: add debug log * fix: memory issues * fix: enable case asan * Update streamlist_for_ci.task * fix: case asan issue * fix: stream name issue * fix: external window compile issues * fix: deploy memory issue * fix: ahandle issue * fix: ahandle issue * fix: ahandle issue * fix: virtual table reader list issue * fix: log info * fix: msg error * fix: virtual table addr list issue * fix: memory issues * fix: memory leak issue * fix: memory issues * fix: memory free issues * fix: memory issues * fix: snode deploy issue * fix: mnode reader issue * fix: memory issues * fix: add debug test * enh: add ignore nodata trigger * fix: memory leaks * fix: configuration issue * fix: memory issue * fix: external window issue * fix: external window issues * fix: external window placeholder issue * fix: placeholder function init issues * fix: memory leak issue * fix: add debug log * fix: compile issues * fix: double free issue * fix: runner addr update issue * fix: msg rsp issue * fix: external window reset issue * fix: configuration issue * fix: deploy msg issue * fix: compile issue --------- Co-authored-by: huohong <sallyhuo@taosdata.com> * test: reproduce bugs * fix: add sliding interval combine case * test: add cases * test: add recalc test. * test: reproduce bugs * case : add vt ts is null check * modify case * bug: submit test_idmp_meters_bug3.py * test: add test for recalc. * test: add cases * fix: error code check * test: add cases * fix(stream): scan wal with schema in that version * add case * test: add cases * test: update test case. * fix: windows compile issues * add case * test: add cases (#31845) * modify case * fix: reset interpPrev * test: add test_idmp_meters bug4 and bug3 * add case * fix(stream): opti wal interface * fix: remove test_idmp_meters_bug5.py * test: add cases * fix(stream): fix ts data fetch for virtual tables * cancel asan case * test: update test case. * test: update test case. * add case * test: add cases * test: add cases * test: add case test_idmp_meters_bug5.py * test: update test case. * fix(stream): tmq error * test: add cases * feat: [TS-6100] Restore deleted code in mndSma.c since they are still in use. * fix(stream): optimize val scan logic * test: add test_recalc_expired_time.py to ci. * test: update test case. * test: update test case. * feat: [TS-6100] Fix fill range check * fix(stream): optimize val scan logic * add case * test: modify for partition by %%1 * test: add fun case stream4_sub7 * fix(stream): optimize val scan logic * add case * feat: [TS-6100] Rename OPTIONS to STREAM_OPTIONS. * test: add test for recalc. * test: use stream_options. * fix: some cases error. * test: remove recalc from ci. * fix: ci case issues (#31880) * enh: add operator reset func * fix: merge join reset issue * fix: memory issues * fix: add debug assert * fix: memory issues * fix: memory leak * fix: memory issues * fix taos log miss * fix: case issue * fix: case issue * fix: case issues * fix: drop dnode issue * fix: memory issues * fix: memory issues * fix: memory leak issues * fix: recalculate time range issue * fix: add debug log * fix: memory issues * fix: enable case asan * Update streamlist_for_ci.task * fix: case asan issue * fix: stream name issue * fix: external window compile issues * fix: deploy memory issue * fix: ahandle issue * fix: ahandle issue * fix: ahandle issue * fix: virtual table reader list issue * fix: log info * fix: msg error * fix: virtual table addr list issue * fix: memory issues * fix: memory leak issue * fix: memory issues * fix: memory free issues * fix: memory issues * fix: snode deploy issue * fix: mnode reader issue * fix: memory issues * fix: add debug test * enh: add ignore nodata trigger * fix: memory leaks * fix: configuration issue * fix: memory issue * fix: external window issue * fix: external window issues * fix: external window placeholder issue * fix: placeholder function init issues * fix: memory leak issue * fix: add debug log * fix: compile issues * fix: double free issue * fix: runner addr update issue * fix: msg rsp issue * fix: external window reset issue * fix: configuration issue * fix: deploy msg issue * fix: compile issue * fix: external window idx issue * fix: ci issues --------- Co-authored-by: huohong <sallyhuo@taosdata.com> * fix(stream): fix compilation error * fix(stream): optimize val scan logic * test:add test cases * test: modify case * fix: external agg error * test(stream): tobacco scene testing #TD-36514 * test: add stream cases (#31885) * fix: windows compile issue * fix: calc timerange * fix: windows compile issue * modify case * fix(stream): compile error * test: remove one debug test case file * test: modify * test: add test cases * test: reproduce bugs * test: reproduce bugs * feat: [TS-6100] Placeholder function should only appera in SELECT and… (#31868) * feat: [TS-6100] Placeholder function should only appera in SELECT and WHERE and FROM. * test: update case --------- Co-authored-by: Simon Guan <guanshengliang@qq.com> * add example * add example * modify case example * modify case * test:alter sql * test: add stream5 case * fix(stream): get schema error with version * test: add delete recalc test py. * test: remove bug cases * test: stream5 case test passed * test: add state cases (#31893) * fix(stream): compile error * test: modify case * test: add cases * test: add test. * test: update test case. * chore(test): fix case err * test: update test case. * fix: align data get * fix(stream): fix row index of datablock written into data cache * fix: put align data * test: update test case. * test: add test cases for virtual table * chore(test): fix case err #TD-36514 * add case * test: add test for water mark. * test: add meters bug6 for stream5 * test: add cases (#31903) * test: add test for recalc. * feat: [TS-6100] %%trows can only be used when event type is window close. * test: add precision of database for ms/us/ns * modify case * add case * add case * test: add test to ci. * modify case * fix: ci case issues (#31904) * enh: add operator reset func * fix: merge join reset issue * fix: memory issues * fix: add debug assert * fix: memory issues * fix: memory leak * fix: memory issues * fix taos log miss * fix: case issue * fix: case issue * fix: case issues * fix: drop dnode issue * fix: memory issues * fix: memory issues * fix: memory leak issues * fix: recalculate time range issue * fix: add debug log * fix: memory issues * fix: enable case asan * Update streamlist_for_ci.task * fix: case asan issue * fix: stream name issue * fix: external window compile issues * fix: deploy memory issue * fix: ahandle issue * fix: ahandle issue * fix: ahandle issue * fix: virtual table reader list issue * fix: log info * fix: msg error * fix: virtual table addr list issue * fix: memory issues * fix: memory leak issue * fix: memory issues * fix: memory free issues * fix: memory issues * fix: snode deploy issue * fix: mnode reader issue * fix: memory issues * fix: add debug test * enh: add ignore nodata trigger * fix: memory leaks * fix: configuration issue * fix: memory issue * fix: external window issue * fix: external window issues * fix: external window placeholder issue * fix: placeholder function init issues * fix: memory leak issue * fix: add debug log * fix: compile issues * fix: double free issue * fix: runner addr update issue * fix: msg rsp issue * fix: external window reset issue * fix: configuration issue * fix: deploy msg issue * fix: compile issue * fix: external window idx issue * fix: ci issues * fix: ci case issues * fix: drop dnode issue --------- Co-authored-by: huohong <sallyhuo@taosdata.com> * fix(stream): ci error * test: update test case. * feat: [TS-6100] Disable some failed UT. * feat: [TS-6100] Fix virtual table * test: add bug 5. * test: add test delete recalc to ci. * test: add bug 6. * test(stream): tobacco scene #TD-36514 * fix: reqCids,reqCols memory leak in SSTriggerRealtimeContext Co-authored-by: Tony Zhang <tonyzhang@taosdata.com> * test: add case stream6 * fix(stream): implement some pending features in trigger task * modify case * modify case * fix: case issues * modify case * test: add recalc for warter mark. * fix(stream): fix count window trigger of virtual tables * fix(stream): memory leak * test: fix run err. * test: add stream6 bug7 * fix: adjust format * test(stream): tobacco scene testing #TD-36514 * test: change bug7 with update window1 and 2 * test: add test bug 7. * case: restore write 3 window * fix: windows compile issue * fix: notify * test: add cases * modify case * test: update test case. * test(stream): toobacco scene testing #TD-36514 --------- Co-authored-by: Simon Guan <slguan@taosdata.com> Co-authored-by: plum-lihui <huili@taosdata.com> Co-authored-by: Alex Duan <417921451@qq.com> Co-authored-by: zelv01 <1101510017@qq.com> Co-authored-by: Jing Sima <simondominic9997@outlook.com> Co-authored-by: xiangyang guo <66111494+happyguoxy@users.noreply.github.com> Co-authored-by: wangmm0220 <wangmm0220@gmail.com> Co-authored-by: Haojun Liao <hjliao@taosdata.com> Co-authored-by: zyyang90 <zyyang@taosdata.com> Co-authored-by: Alex Duan <51781608+DuanKuanJun@users.noreply.github.com> Co-authored-by: facetosea <285808407@qq.com> Co-authored-by: Simon Guan <guanshengliang@qq.com> Co-authored-by: huohong <sallyhuo@taosdata.com> Co-authored-by: Li Hui <52318143+plum-lihui@users.noreply.github.com> Co-authored-by: Jinqing Kuang <kuangjinqingcn@gmail.com> Co-authored-by: xiao-77 <berylbao@taosdata.com> Co-authored-by: Zhixiao Bao <62235797+xiao-77@users.noreply.github.com> Co-authored-by: happyguoxy <happy_guoxy@163.com> Co-authored-by: Tony Zhang <34825804+Tony2h@users.noreply.github.com> Co-authored-by: Tony Zhang <tonyzhang@taosdata.com>
2025-07-16 06:42:16 +00:00
int32_t tqRetrieveTaosxBlock(STqReader *pReader, SMqDataRsp* pRsp, SArray *blocks, SArray *schemas, SSubmitTbData **pSubmitTbDataRet, SArray* rawList, int8_t fetchMeta);
2022-04-14 02:01:04 +00:00
int32_t tqReaderSetVtableInfo(STqReader *pReader, void *vnode, void *pAPI, SSHashObj *pVtableInfos,
SSDataBlock **ppResBlock, const char *idstr);
int32_t tqRetrieveVTableDataBlock(STqReader *pReader, SSDataBlock **pRes, const char *idstr);
bool tqNextVTableSourceBlockImpl(STqReader *pReader, const char *idstr);
bool tqReaderIsQueriedSourceTable(STqReader *pReader, uint64_t uid);
int32_t tqCommitOffset(void* p);
2022-06-01 11:06:58 +00:00
// sma
int32_t smaGetTSmaDays(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days);
2022-07-04 06:47:41 +00:00
// SVSnapReader
int32_t vnodeSnapReaderOpen(SVnode *pVnode, SSnapshotParam *pParam, SVSnapReader **ppReader);
void vnodeSnapReaderClose(SVSnapReader *pReader);
2022-07-04 12:04:40 +00:00
int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData);
// SVSnapWriter
int32_t vnodeSnapWriterOpen(SVnode *pVnode, SSnapshotParam *pParam, SVSnapWriter **ppWriter);
int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback, SSnapshot *pSnapshot);
2022-07-05 11:19:37 +00:00
int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData);
2022-04-08 06:09:15 +00:00
2024-11-05 11:57:13 +00:00
bool taosXGetTablePrimaryKey(SSnapContext *ctx);
void taosXSetTablePrimaryKey(SSnapContext *ctx, int64_t uid);
int32_t buildSnapContext(SVnode *pVnode, int64_t snapVersion, int64_t suid, int8_t subType, int8_t withMeta,
SSnapContext **ctxRet);
int32_t getTableInfoFromSnapshot(SSnapContext *ctx, void **pBuf, int32_t *contLen, int16_t *type, int64_t *uid);
int32_t getMetaTableInfoFromSnapshot(SSnapContext *ctx, SMetaTableInfo *info);
int32_t setForSnapShot(SSnapContext *ctx, int64_t uid);
void destroySnapContext(SSnapContext *ctx);
2022-04-14 02:01:04 +00:00
// structs
struct STsdbCfg {
2022-05-08 09:09:00 +00:00
int8_t precision;
int8_t update;
int8_t compression;
int8_t slLevel;
int32_t minRows;
int32_t maxRows;
int32_t days; // just for save config, don't use in tsdbRead/tsdbCommit/..., and use STsdbKeepCfg in STsdb instead
int32_t keep0; // just for save config, don't use in tsdbRead/tsdbCommit/..., and use STsdbKeepCfg in STsdb instead
int32_t keep1; // just for save config, don't use in tsdbRead/tsdbCommit/..., and use STsdbKeepCfg in STsdb instead
int32_t keep2; // just for save config, don't use in tsdbRead/tsdbCommit/..., and use STsdbKeepCfg in STsdb instead
int32_t keepTimeOffset; // just for save config, use STsdbKeepCfg in STsdb instead
2022-05-07 14:33:19 +00:00
SRetention retentions[TSDB_RETENTION_MAX];
2025-12-05 05:26:28 +00:00
int32_t encryptAlgr;
SEncryptData encryptData;
2022-05-07 14:33:19 +00:00
};
typedef struct {
int64_t numOfSTables;
int64_t numOfCTables;
int64_t numOfNTables;
int64_t numOfVTables;
int64_t numOfVCTables;
2025-09-26 09:32:32 +00:00
int64_t numOfRSMAs;
int64_t numOfReportedTimeSeries;
2022-08-25 08:22:55 +00:00
int64_t numOfNTimeSeries;
int64_t numOfTimeSeries;
// int64_t itvTimeSeries;
int64_t pointsWritten;
int64_t totalStorage;
int64_t compStorage;
int64_t storageLastUpd;
} SVnodeStats;
2022-04-14 02:01:04 +00:00
struct SVnodeCfg {
int32_t vgId;
2025-07-12 07:39:29 +00:00
int32_t mountVgId;
char dbname[TSDB_DB_FNAME_LEN];
uint64_t dbId;
int32_t cacheLastSize;
int32_t szPage;
int32_t szCache;
uint64_t szBuf;
bool isHeap;
bool isWeak;
int8_t cacheLast;
int8_t isTsma;
int8_t isRsma;
int8_t hashMethod;
int8_t standby;
STsdbCfg tsdbCfg;
SWalCfg walCfg;
SSyncCfg syncCfg;
SVnodeStats vndStats;
uint32_t hashBegin;
uint32_t hashEnd;
2023-05-04 05:51:48 +00:00
bool hashChange;
2022-09-08 08:56:29 +00:00
int16_t sttTrigger;
int16_t hashPrefix;
int16_t hashSuffix;
2022-09-08 10:36:08 +00:00
int32_t tsdbPageSize;
2025-12-05 05:26:28 +00:00
int32_t tdbEncryptAlgr;
SEncryptData tdbEncryptData;
feat[ts-6107]: shared storage (#31552) * add API to use s3 as shared storage * support using local file system as shared storage * upload file to shared storage * support read, compact and drop * finish basic mnode & vnode msg processing * follower sync migration state * implement mnode transaction, and improve log * send migration progress msg to dnode to avoid deadlock * implement following migration * remove mcount * avoid redo migration on startup * avoid follower deadlock when leader is down * trigger migrate by timer, avoid compact after migration * comment out the usage of 'tcs' functions in stream * change config item prefix from s3 to ss * change db option prefix from s3 to ss * rename s3 data struct, function, file to ss * rename s3 macro to ss * update s3 sql to ss * rename remaining s3 items to ss * check ss configruation, improve s3 retry * grant object storage -> shared storage, check ssEnabled * fix memory leaks * update build options * omit sensitive information when dump config * fix backward compatibility issue * fix issues found in ci-checks * fix some failed test cases * avoid follower timeout and improve log * fix: follower timeout because migration status not updated * refuse migration if there's an in progress one * fix ss test case * remove garbage files and other minor improvement * fix failed test cases * update unit test * fix failed test case * fix failed test case * update document * update document and fix failed test cases * fix minor issues in code, test and document * check new commit after migration task is scheduled * fix several issus 1. migrate information cannot be dropped sometimes because progress response was put into read queue. 2. memory leak in rare cases 3. data corruption in rare cases 4. failed test case * add shared storage upgrade tool * fix compile error
2025-07-14 08:33:53 +00:00
int32_t ssChunkSize;
int32_t ssKeepLocal;
int8_t ssCompact;
2022-04-14 02:01:04 +00:00
};
2021-11-07 07:58:32 +00:00
2025-09-26 09:32:32 +00:00
#define TABLE_ROLLUP_ON ((int8_t)0x1)
#define TABLE_IS_ROLLUP(FLG) (((FLG) & (TABLE_ROLLUP_ON)) != 0)
#define TABLE_SET_ROLLUP(FLG) ((FLG) |= TABLE_ROLLUP_ON)
#define TABLE_RESET_ROLLUP(FLG) ((FLG) &= ~TABLE_ROLLUP_ON)
2022-04-24 06:19:12 +00:00
2024-03-06 12:06:26 +00:00
#define TABLE_COL_COMPRESSED ((int8_t)0x2)
#define TABLE_IS_COL_COMPRESSED(FLG) (((FLG) & (TABLE_COL_COMPRESSED)) != 0)
#define TABLE_SET_COL_COMPRESSED(FLG) ((FLG) |= TABLE_COL_COMPRESSED)
#define TABLE_VIRTUAL ((int8_t)0x4)
#define TABLE_IS_VIRTUAL(FLG) (((FLG) & (TABLE_VIRTUAL)) != 0)
#define TABLE_SET_VIRTUAL(FLG) ((FLG) |= TABLE_VIRTUAL)
2024-11-27 09:53:53 +00:00
struct SFileSetReader;
int32_t tsdbFileSetReaderOpen(void *pVnode, struct SFileSetReader **ppReader);
int32_t tsdbFileSetReaderNext(struct SFileSetReader *pReader);
int32_t tsdbFileSetGetEntryField(struct SFileSetReader *pReader, const char *field, void *value);
void tsdbFileSetReaderClose(struct SFileSetReader **ppReader);
int32_t metaFetchEntryByUid(SMeta *pMeta, int64_t uid, SMetaEntry **ppEntry);
void metaFetchEntryFree(SMetaEntry **ppEntry);
2025-06-30 02:33:34 +00:00
/**
* @brief Get raw write metrics for a vnode
*
* @param pVnode Pointer to the vnode object
* @param pRawMetrics Pointer to the SRawWriteMetrics struct to fill with raw metrics
* @return 0 on success, non-zero on error
*/
int32_t vnodeGetRawWriteMetrics(void *pVnode, SRawWriteMetrics *pRawMetrics);
/**
* @brief Reset raw write metrics for a vnode by subtracting old values
*
* @param pVnode Pointer to the vnode object
* @param pOldMetrics Pointer to the SRawWriteMetrics struct containing values to subtract
* @return 0 on success, non-zero on error
*/
int32_t vnodeResetRawWriteMetrics(void *pVnode, const SRawWriteMetrics *pOldMetrics);
2021-09-22 12:29:24 +00:00
#ifdef __cplusplus
}
#endif
#endif /*_TD_VNODE_H_*/