mirror of
https://github.com/taosdata/TDengine
synced 2026-05-24 10:09:01 +00:00
2de6736160
* opt stream build twa result * opt force window close memory * feat(stream):optimize new interval and scan operator * adj log * opt code * opt code * fill history * fix issue for fill history * add ci * feat(stream): add new stream nonblock interval operator * adjust code * use new scan operator * use new scan operator * add log * fix issue * recover stream scan next function * fix issue * fix issue * fix issue * ignore disorder data * fix issue * fix issue for interval sliding * fix issue * fix ci issue * fix ci issue * fix ci issue * add semi && final nonblock interval operator * fix issue * fix issue * fix issue * fix issue * fix issue * fix issue * fix issue * fix issue * refactor(stream): track the msgId for each upstream tasks. * fix(stream): fix race condition. * fix(stream): update the task last msgId when putting into input queue succ. * fix issue * fix issue * put recalculate data to rocksdb * fix issue * fix issue * enh(query)[TD-33071]: add support for saving and restoring tsdbReader scan progress - Implement functionality to save scan progress during tsdbReader operations - Enable resuming scans from the last saved position * fix issue * fix issue * fix issue * fix issue * fix issue * add rec interval check * enh(stream):add recalculate tasks. * enh(stream): support the re-calculate the tasks. * fix issue && do refactor * do refactor * fix issue * fix issue * update backend opt * add new interface * add new session operator * support blob * add new session operator * fix issue * add rec state for rec task * fix invalid read * add new session window * enh(stream): update the stream tasks backend. * new session operator * add pull data * fix(stream): fix error in expand stream backend. * fix issue * fix issue * fix issue * merge code * fix issue * fix(stream): check for null ptr. * fix(stream): add more check. * fix issue * fix issue * fix issue * add debug code * fix issue * fix issue * fix issue * set rec end flag * fix(stream): opt re-calculate stream tasks. * fix issue * fix issue * add new operator * enh(stream): dispatch recalculate block to agg tasks. * fix issue * fix issue * fix(stream): adjust filter. * fix issue * refactor * refactor(stream): adjust the recalculate end block. * fix issue * fix(stream): set correct create trigger block. * fix issue * fix(stream): fix error in build recalculate end block. * fix(stream): check null ptr. * add stream client && fix issue * fix mem leak * fix(stream): free msg. * add stream client * fix(stream): fix error. * add stream client && fix issue * add stream client * refactor(stream): set the recalculate task complete. * add wend and group_id for session window dest table * feat(stream): refactor and set the recalcul agg task complete. * add cfg for adapter * fix issue * add state && event operator * feat(stream): support fill-history task. * add new fill operator * fix(stream): set correct backend when updating fill-history task to recalculate task. * add new fill operator * fix(stream): s2025-03-06 11:10:31.272 et ops always open in scan tsdb * fix(stream):set the correct taskType for sink task. * new fill operator * adj stream fill operator * fix issue * fix issue * fix issue * fix issue * fix issue * fix issue * fix issue * fix issue * adj test * fix issue * fix(stream): fix issue * fix(steam): fix issue * fix(steam): fix issue * fix(steam): fix issue * fix(steam): fix issue * fix(stream): fix issue * fix(stream): fix issue * fix(stream): fix issue * fix: ut com error * fix(stream): fix mem leak and adjust operator type check rule * fix(stream): fix mem leak and adjust test case * refactor code * fix(stream): free items. * fix(stream): free fix memory leak. * fix(stream): fix syntax error. * fix: ignore unexpect block * fix: adjust op type --------- Co-authored-by: Haojun Liao <hjliao@taosdata.com> Co-authored-by: Jinqing Kuang <kuangjinqingcn@gmail.com> Co-authored-by: yihaoDeng <luomoxyz@126.com>
195 lines
12 KiB
C
195 lines
12 KiB
C
/*
|
|
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
|
|
*
|
|
* This program is free software: you can use, redistribute, and/or modify
|
|
* it under the terms of the GNU Affero General Public License, version 3
|
|
* or later ("AGPL"), as published by the Free Software Foundation.
|
|
*
|
|
* This program is distributed in the hope that it will be useful, but WITHOUT
|
|
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
|
* FITNESS FOR A PARTICULAR PURPOSE.
|
|
*
|
|
* You should have received a copy of the GNU Affero General Public License
|
|
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
*/
|
|
|
|
#ifndef _STREAM_FILE_STATE_H_
|
|
#define _STREAM_FILE_STATE_H_
|
|
|
|
#include "storageapi.h"
|
|
#include "tarray.h"
|
|
#include "tdef.h"
|
|
#include "tlist.h"
|
|
|
|
#ifdef __cplusplus
|
|
extern "C" {
|
|
#endif
|
|
|
|
typedef struct SStreamFileState SStreamFileState;
|
|
typedef SList SStreamSnapshot;
|
|
|
|
typedef void* (*_state_buff_get_fn)(void* pRowBuff, const void* pKey, size_t keyLen);
|
|
typedef int32_t (*_state_buff_remove_fn)(void* pRowBuff, const void* pKey, size_t keyLen);
|
|
typedef void (*_state_buff_remove_by_pos_fn)(SStreamFileState* pState, SRowBuffPos* pPos);
|
|
typedef void (*_state_buff_cleanup_fn)(void* pRowBuff);
|
|
typedef void* (*_state_buff_create_statekey_fn)(SRowBuffPos* pPos, int64_t num);
|
|
|
|
typedef int32_t (*_state_file_remove_fn)(SStreamFileState* pFileState, const void* pKey);
|
|
typedef int32_t (*_state_file_get_fn)(SStreamFileState* pFileState, void* pKey, void** data, int32_t* pDataLen);
|
|
typedef int32_t (*_state_file_clear_fn)(SStreamState* pState);
|
|
|
|
typedef int32_t (*_state_fun_get_fn)(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal,
|
|
int32_t* pVLen, int32_t* pWinCode);
|
|
|
|
typedef int32_t (*range_cmpr_fn)(const SSessionKey* pWin1, const SSessionKey* pWin2);
|
|
|
|
typedef int (*__session_compare_fn_t)(const void* pWin, const void* pDatas, int pos);
|
|
|
|
int32_t streamFileStateInit(int64_t memSize, uint32_t keySize, uint32_t rowSize, uint32_t selectRowSize, GetTsFun fp,
|
|
void* pFile, TSKEY delMark, const char* taskId, int64_t checkpointId, int8_t type,
|
|
struct SStreamFileState** ppFileState);
|
|
void streamFileStateDestroy(SStreamFileState* pFileState);
|
|
void streamFileStateClear(SStreamFileState* pFileState);
|
|
bool needClearDiskBuff(SStreamFileState* pFileState);
|
|
void streamFileStateReleaseBuff(SStreamFileState* pFileState, SRowBuffPos* pPos, bool used);
|
|
void streamFileStateClearBuff(SStreamFileState* pFileState, SRowBuffPos* pPos);
|
|
|
|
int32_t addRowBuffIfNotExist(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen,
|
|
int32_t* pWinCode);
|
|
int32_t createRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen);
|
|
|
|
int32_t getRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen,
|
|
int32_t* pWinCode);
|
|
void deleteRowBuff(SStreamFileState* pFileState, const void* pKey, int32_t keyLen);
|
|
void deleteRowBuffByGroupId(SStreamFileState* pFileState, uint64_t groupId);
|
|
int32_t getRowBuffByPos(SStreamFileState* pFileState, SRowBuffPos* pPos, void** pVal);
|
|
bool hasRowBuff(SStreamFileState* pFileState, const SWinKey* pKey, bool hasLimit, bool* pIsLast);
|
|
int32_t putFreeBuff(SStreamFileState* pFileState, SRowBuffPos* pPos);
|
|
|
|
SStreamSnapshot* getSnapshot(SStreamFileState* pFileState);
|
|
void flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot, bool flushState);
|
|
int32_t recoverSnapshot(SStreamFileState* pFileState, int64_t ckId);
|
|
|
|
int32_t deleteExpiredCheckPoint(SStreamFileState* pFileState, TSKEY mark);
|
|
int32_t streamFileStateGetSelectRowSize(SStreamFileState* pFileState);
|
|
void streamFileStateReloadInfo(SStreamFileState* pFileState, TSKEY ts);
|
|
|
|
void* getRowStateBuff(SStreamFileState* pFileState);
|
|
void* getSearchBuff(SStreamFileState* pFileState);
|
|
void* getStateFileStore(SStreamFileState* pFileState);
|
|
bool isDeteled(SStreamFileState* pFileState, TSKEY ts);
|
|
bool isFlushedState(SStreamFileState* pFileState, TSKEY ts, TSKEY gap);
|
|
TSKEY getFlushMark(SStreamFileState* pFileState);
|
|
SRowBuffPos* getNewRowPosForWrite(SStreamFileState* pFileState);
|
|
int32_t getRowStateRowSize(SStreamFileState* pFileState);
|
|
void freeArrayPtr(void* ptr);
|
|
|
|
// session window
|
|
int32_t getSessionWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, TSKEY gap, void** pVal, int32_t* pVLen,
|
|
int32_t* pWinCode);
|
|
int32_t putSessionWinResultBuff(SStreamFileState* pFileState, SRowBuffPos* pPos);
|
|
int32_t getSessionFlushedBuff(SStreamFileState* pFileState, SSessionKey* pKey, void** pVal, int32_t* pVLen,
|
|
int32_t* pWinCode);
|
|
int32_t deleteSessionWinStateBuffFn(void* pBuff, const void* key, size_t keyLen);
|
|
void deleteSessionWinStateBuffByPosFn(SStreamFileState* pFileState, SRowBuffPos* pPos);
|
|
int32_t allocSessioncWinBuffByNextPosition(SStreamFileState* pFileState, SStreamStateCur* pCur,
|
|
const SSessionKey* pWinKey, void** ppVal, int32_t* pVLen);
|
|
|
|
SRowBuffPos* createSessionWinBuff(SStreamFileState* pFileState, SSessionKey* pKey, void* p, int32_t* pVLen);
|
|
int32_t recoverSession(SStreamFileState* pFileState, int64_t ckId);
|
|
|
|
void sessionWinStateClear(SStreamFileState* pFileState);
|
|
void sessionWinStateCleanup(void* pBuff);
|
|
|
|
SStreamStateCur* createStateCursor(SStreamFileState* pFileState);
|
|
SStreamStateCur* sessionWinStateSeekKeyCurrentPrev(SStreamFileState* pFileState, const SSessionKey* pWinKey);
|
|
SStreamStateCur* sessionWinStateSeekKeyCurrentNext(SStreamFileState* pFileState, const SSessionKey* pWinKey);
|
|
SStreamStateCur* sessionWinStateSeekKeyPrev(SStreamFileState* pFileState, const SSessionKey* pWinKey);
|
|
SStreamStateCur* sessionWinStateSeekKeyNext(SStreamFileState* pFileState, const SSessionKey* pWinKey);
|
|
SStreamStateCur* countWinStateSeekKeyPrev(SStreamFileState* pFileState, const SSessionKey* pWinKey, COUNT_TYPE count);
|
|
int32_t sessionWinStateGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, void** pVal, int32_t* pVLen);
|
|
void sessionWinStateMoveToNext(SStreamStateCur* pCur);
|
|
void sessionWinStateMoveToPrev(SStreamStateCur* pCur);
|
|
int32_t sessionWinStateGetKeyByRange(SStreamFileState* pFileState, const SSessionKey* key, SSessionKey* curKey,
|
|
range_cmpr_fn cmpFn);
|
|
|
|
int32_t binarySearch(void* keyList, int num, const void* key, __session_compare_fn_t cmpFn);
|
|
|
|
// state window
|
|
int32_t getStateWinResultBuff(SStreamFileState* pFileState, SSessionKey* key, char* pKeyData, int32_t keyDataLen,
|
|
state_key_cmpr_fn fn, void** pVal, int32_t* pVLen, int32_t* pWinCode);
|
|
|
|
// count window
|
|
int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, COUNT_TYPE winCount, void** pVal,
|
|
int32_t* pVLen, int32_t* pWinCode);
|
|
int32_t createCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, COUNT_TYPE winCount, void** pVal, int32_t* pVLen);
|
|
|
|
// function
|
|
int32_t getSessionRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen,
|
|
int32_t* pWinCode);
|
|
int32_t getFunctionRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen);
|
|
|
|
// time slice
|
|
int32_t getHashSortRowBuff(SStreamFileState* pFileState, const SWinKey* pKey, void** pVal, int32_t* pVLen,
|
|
int32_t* pWinCode);
|
|
int32_t hashSortFileGetFn(SStreamFileState* pFileState, void* pKey, void** data, int32_t* pDataLen);
|
|
int32_t hashSortFileRemoveFn(SStreamFileState* pFileState, const void* pKey);
|
|
void clearSearchBuff(SStreamFileState* pFileState);
|
|
int32_t getHashSortNextRow(SStreamFileState* pFileState, const SWinKey* pKey, SWinKey* pResKey, void** pVal,
|
|
int32_t* pVLen, int32_t* pWinCode);
|
|
int32_t getHashSortPrevRow(SStreamFileState* pFileState, const SWinKey* pKey, SWinKey* pResKey, void** ppVal,
|
|
int32_t* pVLen, int32_t* pWinCode);
|
|
int32_t recoverFillSnapshot(SStreamFileState* pFileState, int64_t ckId);
|
|
void deleteHashSortRowBuff(SStreamFileState* pFileState, const SWinKey* pKey);
|
|
|
|
//group
|
|
int32_t streamFileStateGroupPut(SStreamFileState* pFileState, int64_t groupId, void* value, int32_t vLen);
|
|
void streamFileStateGroupCurNext(SStreamStateCur* pCur);
|
|
int32_t streamFileStateGroupGetKVByCur(SStreamStateCur* pCur, int64_t* pKey, void** pVal, int32_t* pVLen);
|
|
SSHashObj* getGroupIdCache(SStreamFileState* pFileState);
|
|
int fillStateKeyCompare(const void* pWin1, const void* pDatas, int pos);
|
|
int32_t getRowStatePrevRow(SStreamFileState* pFileState, const SWinKey* pKey, SWinKey* pResKey, void** ppVal,
|
|
int32_t* pVLen, int32_t* pWinCode);
|
|
int32_t addSearchItem(SStreamFileState* pFileState, SArray* pWinStates, const SWinKey* pKey, bool* pIsEnd);
|
|
int32_t getRowStateAllPrevRow(SStreamFileState* pFileState, const SWinKey* pKey, SArray* pResArray, int32_t maxNum);
|
|
|
|
//twa
|
|
void setFillInfo(SStreamFileState* pFileState);
|
|
void clearExpiredState(SStreamFileState* pFileState, int32_t numOfKeep, TSKEY minTs);
|
|
int32_t addArrayBuffIfNotExist(SSHashObj* pSearchBuff, uint64_t groupId, SArray** ppResStates);
|
|
int32_t recoverHashSortBuff(SStreamFileState* pFileState, SArray* pWinStates, uint64_t groupId);
|
|
|
|
int32_t getAndSetTsData(STableTsDataState* pTsDataState, uint64_t tableUid, TSKEY* pCurTs, void** ppCurPkVal,
|
|
TSKEY lastTs, void* pLastPkVal, int32_t lastPkLen, int32_t* pWinCode);
|
|
int32_t doTsDataCommit(STableTsDataState* pTsDataState);
|
|
int32_t doRangeDataCommit(STableTsDataState* pTsDataState);
|
|
int32_t initTsDataState(STableTsDataState** ppTsDataState, int8_t pkType, int32_t pkLen, void* pState,
|
|
void* pOtherState);
|
|
void destroyTsDataState(STableTsDataState* pTsDataState);
|
|
int32_t recoverTsData(STableTsDataState* pTsDataState);
|
|
int32_t mergeAndSaveScanRange(STableTsDataState* pTsDataState, STimeWindow* pWin, uint64_t gpId, SRecDataInfo* pRecData,
|
|
int32_t len);
|
|
int32_t mergeAllScanRange(STableTsDataState* pTsDataState);
|
|
int32_t popScanRange(STableTsDataState* pTsDataState, SScanRange* pRange);
|
|
|
|
// continuous
|
|
typedef void* (*getStateBuffFn)(SStreamFileState* pFileState);
|
|
SStreamStateCur* getLastStateCur(SStreamFileState* pFileState, getStateBuffFn fn);
|
|
void moveLastStateCurNext(SStreamStateCur* pCur, getStateBuffFn fn);
|
|
void moveOneStateCurNext(SStreamStateCur* pCur);
|
|
int32_t getNLastStateKVByCur(SStreamStateCur* pCur, int32_t num, SArray* pRes);
|
|
int32_t getNLastSessionStateKVByCur(SStreamStateCur* pCur, int32_t num, SArray* pRes);
|
|
int32_t reloadTsDataState(STableTsDataState* pTsDataState);
|
|
int32_t setStateRecFlag(SStreamFileState* pState, const void* pKey, int32_t keyLen, int32_t mode);
|
|
int32_t getStateRecFlag(SStreamFileState* pFileState, const void* pKey, int32_t keyLen, int32_t* pMode);
|
|
void clearExpiredSessionState(struct SStreamFileState* pFileState, int32_t numOfKeep, TSKEY minTs,
|
|
SSHashObj* pFlushGroup);
|
|
bool hasSessionState(SStreamFileState* pFileState, SSessionKey* pKey, TSKEY gap, bool* pIsLast);
|
|
int32_t saveRecInfoToDisk(STableTsDataState* pTsDataState, SSessionKey* pKey, SRecDataInfo* pVal, int32_t vLen);
|
|
int32_t flushRemainRecInfoToDisk(STableTsDataState* pTsDataState);
|
|
|
|
#ifdef __cplusplus
|
|
}
|
|
#endif
|
|
|
|
#endif // _STREAM_FILE_STATE_H_
|