TDengine/tests/script/tsim/stream/nonblockIntervalBasic.sim
liuyao 2de6736160
feat(stream): add new trigger continuous_window_close (#30125)
* opt stream build twa result

* opt force window close memory

* feat(stream):optimize new interval and scan operator

* adj log

* opt code

* opt code

* fill history

* fix issue for fill history

* add ci

* feat(stream): add new stream nonblock interval operator

* adjust code

* use new scan operator

* use new scan operator

* add log

* fix issue

* recover stream scan next function

* fix issue

* fix issue

* fix issue

* ignore disorder data

* fix issue

* fix issue for interval sliding

* fix issue

* fix ci issue

* fix ci issue

* fix ci issue

* add semi && final nonblock interval operator

* fix issue

* fix issue

* fix issue

* fix issue

* fix issue

* fix issue

* fix issue

* fix issue

* refactor(stream): track the msgId for each upstream tasks.

* fix(stream): fix race condition.

* fix(stream): update the task last msgId when putting into input queue succ.

* fix issue

* fix issue

* put recalculate data to rocksdb

* fix issue

* fix issue

* enh(query)[TD-33071]: add support for saving and restoring tsdbReader scan progress

- Implement functionality to save scan progress during tsdbReader operations
- Enable resuming scans from the last saved position

* fix issue

* fix issue

* fix issue

* fix issue

* fix issue

* add rec interval check

* enh(stream):add recalculate tasks.

* enh(stream): support the re-calculate the tasks.

* fix issue && do refactor

* do refactor

* fix issue

* fix issue

* update backend opt

* add new interface

* add new session operator

* support blob

* add new session operator

* fix issue

* add rec state for rec task

* fix invalid read

* add new session window

* enh(stream): update the stream tasks backend.

* new session operator

* add pull data

* fix(stream): fix error in expand stream backend.

* fix issue

* fix issue

* fix issue

* merge code

* fix issue

* fix(stream): check for null ptr.

* fix(stream): add more check.

* fix issue

* fix issue

* fix issue

* add debug code

* fix issue

* fix issue

* fix issue

* set rec end flag

* fix(stream): opt re-calculate stream tasks.

* fix issue

* fix issue

* add new operator

* enh(stream): dispatch recalculate block to agg tasks.

* fix issue

* fix issue

* fix(stream): adjust filter.

* fix issue

* refactor

* refactor(stream): adjust the recalculate end block.

* fix issue

* fix(stream): set correct create trigger block.

* fix issue

* fix(stream): fix error in build recalculate end block.

* fix(stream): check null ptr.

* add stream client && fix issue

* fix mem leak

* fix(stream): free msg.

* add stream client

* fix(stream): fix error.

* add stream client && fix issue

* add stream client

* refactor(stream): set the recalculate task complete.

* add wend and group_id for session window dest table

* feat(stream): refactor and set the recalcul agg task complete.

* add cfg for adapter

* fix issue

* add state && event operator

* feat(stream): support fill-history task.

* add new fill operator

* fix(stream): set correct backend when updating fill-history task to recalculate task.

* add new fill operator

* fix(stream): s2025-03-06 11:10:31.272
et ops always open in scan tsdb

* fix(stream):set the correct taskType for sink task.

* new fill operator

* adj stream fill operator

* fix issue

* fix issue

* fix issue

* fix issue

* fix issue

* fix issue

* fix issue

* fix issue

* adj test

* fix issue

* fix(stream): fix issue

* fix(steam): fix issue

* fix(steam): fix issue

* fix(steam): fix issue

* fix(steam): fix issue

* fix(stream): fix issue

* fix(stream): fix issue

* fix(stream): fix issue

* fix: ut com error

* fix(stream): fix mem leak and adjust operator type check rule

* fix(stream): fix mem leak and adjust test case

* refactor code

* fix(stream): free items.

* fix(stream): free fix memory leak.

* fix(stream): fix syntax error.

* fix: ignore unexpect block

* fix: adjust op type

---------

Co-authored-by: Haojun Liao <hjliao@taosdata.com>
Co-authored-by: Jinqing Kuang <kuangjinqingcn@gmail.com>
Co-authored-by: yihaoDeng <luomoxyz@126.com>
2025-03-14 20:14:01 +08:00

641 lines
17 KiB
Text

system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/exec.sh -n dnode1 -s start
sleep 1000
sql connect
print ========== interval window
sql drop database if exists test;
sql create database test vgroups 1;
sql use test;
sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create table t1 using st tags(1,1,1);
sql create table t2 using st tags(2,2,2);
sql create stream streams1 trigger continuous_window_close ignore update 0 ignore expired 0 into streamt1 as select _wstart, count(*) c1, sum(b) c2 from st partition by tbname interval(10s) ;
run tsim/stream/checkTaskStatus.sim
sql insert into t1 values(1648791211000,1,2,3);
sleep 500
sql insert into t1 values(1648791221000,1,2,3);
sql select _wstart, count(*) c1, sum(b) c2 from st partition by tbname interval(10s) ;
print $data00 $data01 $data02 $data03 $data04
print $data10 $data11 $data12 $data13 $data14
print $data20 $data21 $data22 $data23 $data24
print $data30 $data31 $data32 $data33 $data34
print $data40 $data41 $data42 $data43 $data44
print $data50 $data51 $data52 $data53 $data54
$loop_count = 0
loop0:
sleep 500
sql select * from streamt1;
print $data00 $data01 $data02 $data03 $data04
print $data10 $data11 $data12 $data13 $data14
print $data20 $data21 $data22 $data23 $data24
print $data30 $data31 $data32 $data33 $data34
print $data40 $data41 $data42 $data43 $data44
print $data50 $data51 $data52 $data53 $data54
$loop_count = $loop_count + 1
if $loop_count == 20 then
return -1
endi
if $rows != 1 then
print =====rows=$rows
goto loop0
endi
if $data01 != 1 then
print =====data01=$data01
goto loop0
endi
if $data02 != 2 then
print =====data02=$data02
goto loop0
endi
print ============================end
print ========== interval window step2
sql drop database if exists test2;
sql create database test2 vgroups 1;
sql use test2;
sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create table t1 using st tags(1,1,1);
sql create table t2 using st tags(2,2,2);
sql create stream streams2 trigger continuous_window_close ignore update 0 ignore expired 0 into streamt2 as select _wstart, count(*) c1, max(a) c2 from st partition by tbname interval(10s) sliding(5s) ;
run tsim/stream/checkTaskStatus.sim
sql insert into t1 values(1648791211000,1,2,3);
sql insert into t1 values(1648791214000,2,2,3);
sql insert into t1 values(1648791215000,3,2,3);
sql insert into t1 values(1648791219000,4,2,3);
sql insert into t1 values(1648791220000,5,2,3);
sleep 2000
sql insert into t1 values(1648791420000,6,2,3);
sql select _wstart, count(*) c1, max(a) c2 from st partition by tbname interval(10s) sliding(5s) ;
print $data00 $data01 $data02 $data03 $data04
print $data10 $data11 $data12 $data13 $data14
print $data20 $data21 $data22 $data23 $data24
print $data30 $data31 $data32 $data33 $data34
print $data40 $data41 $data42 $data43 $data44
print $data50 $data51 $data52 $data53 $data54
$loop_count = 0
loop1:
sleep 500
sql select * from streamt2 order by 1,2;
print $data00 $data01 $data02 $data03 $data04
print $data10 $data11 $data12 $data13 $data14
print $data20 $data21 $data22 $data23 $data24
print $data30 $data31 $data32 $data33 $data34
print $data40 $data41 $data42 $data43 $data44
print $data50 $data51 $data52 $data53 $data54
$loop_count = $loop_count + 1
if $loop_count == 20 then
return -1
endi
if $rows != 4 then
print =====rows=$rows
goto loop1
endi
if $data01 != 2 then
print =====data01=$data01
goto loop1
endi
if $data11 != 4 then
print =====data11=$data11
goto loop1
endi
if $data21 != 3 then
print =====data21=$data21
goto loop1
endi
if $data31 != 1 then
print =====data31=$data31
goto loop1
endi
print ========== interval window step3
sql drop database if exists test3;
sql create database test3 vgroups 2;
sql use test3;
sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create table t1 using st tags(1,1,1);
sql create table t2 using st tags(2,2,2);
sql create stream streams3 trigger continuous_window_close ignore update 0 ignore expired 0 into streamt3 as select _wstart, count(*) c1, sum(b) c2 from st interval(10s);
run tsim/stream/checkTaskStatus.sim
sql insert into t1 values(1648791211000,1,2,3);
sleep 500
sql insert into t1 values(1648791221000,1,2,3);
sql select _wstart, count(*) c1, sum(b) c2 from st interval(10s) ;
print $data00 $data01 $data02 $data03 $data04
print $data10 $data11 $data12 $data13 $data14
print $data20 $data21 $data22 $data23 $data24
print $data30 $data31 $data32 $data33 $data34
print $data40 $data41 $data42 $data43 $data44
print $data50 $data51 $data52 $data53 $data54
$loop_count = 0
loop2:
sleep 500
sql select * from streamt3;
print $data00 $data01 $data02 $data03 $data04
print $data10 $data11 $data12 $data13 $data14
print $data20 $data21 $data22 $data23 $data24
print $data30 $data31 $data32 $data33 $data34
print $data40 $data41 $data42 $data43 $data44
print $data50 $data51 $data52 $data53 $data54
$loop_count = $loop_count + 1
if $loop_count == 20 then
return -1
endi
if $rows != 1 then
print =====rows=$rows
goto loop2
endi
if $data01 != 1 then
print =====data01=$data01
goto loop2
endi
if $data02 != 2 then
print =====data02=$data02
goto loop2
endi
sql insert into t2 values(1648791211000,1,2,3);
sql select _wstart, count(*) c1, sum(b) c2 from st interval(10s) ;
print $data00 $data01 $data02 $data03 $data04
print $data10 $data11 $data12 $data13 $data14
print $data20 $data21 $data22 $data23 $data24
print $data30 $data31 $data32 $data33 $data34
print $data40 $data41 $data42 $data43 $data44
print $data50 $data51 $data52 $data53 $data54
$loop_count = 0
loop3:
sleep 500
sql select * from streamt3;
print $data00 $data01 $data02 $data03 $data04
print $data10 $data11 $data12 $data13 $data14
print $data20 $data21 $data22 $data23 $data24
print $data30 $data31 $data32 $data33 $data34
print $data40 $data41 $data42 $data43 $data44
print $data50 $data51 $data52 $data53 $data54
$loop_count = $loop_count + 1
if $loop_count == 20 then
return -1
endi
if $rows != 1 then
print =====rows=$rows
goto loop3
endi
if $data01 != 2 then
print =====data01=$data01
goto loop3
endi
if $data02 != 4 then
print =====data02=$data02
goto loop3
endi
print ========== interval window step4
sql drop database if exists test4;
sql create database test4 vgroups 2;
sql use test4;
sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create table t1 using st tags(1,1,1);
sql create table t2 using st tags(2,2,2);
sql create stream streams4 trigger continuous_window_close ignore update 0 ignore expired 0 into streamt4 as select _wstart, count(*) c1, max(a) c2 from st interval(10s) sliding(5s) ;
run tsim/stream/checkTaskStatus.sim
sql insert into t1 values(1648791211000,1,2,3);
sql insert into t1 values(1648791214000,2,2,3);
sql insert into t1 values(1648791215000,3,2,3);
sql insert into t1 values(1648791219000,4,2,3);
sql insert into t1 values(1648791220000,5,2,3);
sleep 2000
sql insert into t1 values(1648791420000,6,2,3);
sql select _wstart, count(*) c1, max(a) c2 from st partition by tbname interval(10s) sliding(5s) ;
print $data00 $data01 $data02 $data03 $data04
print $data10 $data11 $data12 $data13 $data14
print $data20 $data21 $data22 $data23 $data24
print $data30 $data31 $data32 $data33 $data34
print $data40 $data41 $data42 $data43 $data44
print $data50 $data51 $data52 $data53 $data54
$loop_count = 0
loop4:
sleep 500
sql select * from streamt4 order by 1,2;
print $data00 $data01 $data02 $data03 $data04
print $data10 $data11 $data12 $data13 $data14
print $data20 $data21 $data22 $data23 $data24
print $data30 $data31 $data32 $data33 $data34
print $data40 $data41 $data42 $data43 $data44
print $data50 $data51 $data52 $data53 $data54
$loop_count = $loop_count + 1
if $loop_count == 20 then
return -1
endi
if $rows != 4 then
print =====rows=$rows
goto loop4
endi
if $data01 != 2 then
print =====data01=$data01
goto loop4
endi
if $data11 != 4 then
print =====data11=$data11
goto loop4
endi
if $data21 != 3 then
print =====data21=$data21
goto loop4
endi
if $data31 != 1 then
print =====data31=$data31
goto loop4
endi
print ========== interval window step5
sql drop database if exists test5;
sql create database test5 vgroups 2;
sql use test5;
sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create table t1 using st tags(1,1,1);
sql create table t2 using st tags(2,2,2);
sql create stream streams5 trigger continuous_window_close ignore update 0 ignore expired 0 into streamt5 as select _wstart, count(*) c1, max(a) c2, b from st partition by b interval(10s);
run tsim/stream/checkTaskStatus.sim
sql insert into t1 values(1648791211000,1,1,3);
sql insert into t1 values(1648791214000,2,2,3);
sql insert into t1 values(1648791215000,3,1,3);
sql insert into t1 values(1648791219000,4,2,3);
sql insert into t2 values(1648791211000,1,1,3);
sql insert into t2 values(1648791214000,2,2,3);
sql insert into t2 values(1648791215000,3,1,3);
sql insert into t2 values(1648791219000,4,2,3);
sql insert into t2 values(1648791220000,5,1,3);
sql insert into t2 values(1648791220001,6,2,3);
sleep 2000
sql insert into t1 values(1648791420000,6,2,3);
print loop5 select _wstart, count(*) c1, max(a) c2, b from st partition by b interval(10s) order by 1,4;
sql select _wstart, count(*) c1, max(a) c2, b from st partition by b interval(10s) order by 1,4;
print $data00 $data01 $data02 $data03 $data04
print $data10 $data11 $data12 $data13 $data14
print $data20 $data21 $data22 $data23 $data24
print $data30 $data31 $data32 $data33 $data34
print $data40 $data41 $data42 $data43 $data44
print $data50 $data51 $data52 $data53 $data54
$loop_count = 0
loop5:
sleep 500
print sql select * from streamt5 order by 1,4;
sql select * from streamt5 order by 1,4;
print $data00 $data01 $data02 $data03 $data04
print $data10 $data11 $data12 $data13 $data14
print $data20 $data21 $data22 $data23 $data24
print $data30 $data31 $data32 $data33 $data34
print $data40 $data41 $data42 $data43 $data44
print $data50 $data51 $data52 $data53 $data54
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
if $rows != 4 then
print =====rows=$rows
goto loop5
endi
if $data01 != 4 then
print =====data01=$data01
goto loop5
endi
if $data11 != 4 then
print =====data11=$data11
goto loop5
endi
if $data21 != 1 then
print =====data21=$data21
goto loop5
endi
if $data31 != 1 then
print =====data31=$data31
goto loop5
endi
print ========== interval window step6
sql drop database if exists test6;
sql create database test6 vgroups 2;
sql use test6;
sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create table t1 using st tags(1,1,1);
sql create table t2 using st tags(2,2,2);
sql create stream streams6 trigger continuous_window_close ignore update 0 ignore expired 0 into streamt6 TAGS(dd varchar(100)) SUBTABLE(concat("streams6-tbn-", cast(dd as varchar(10)) )) as select _wstart, count(*) c1, max(b) c2 from st partition by tbname, ta as dd interval(10s);
sql create stream streams7 trigger continuous_window_close ignore update 0 ignore expired 0 into streamt7 TAGS(dd varchar(100)) SUBTABLE(concat("streams7-tbn-", cast(dd as varchar(10)) )) as select _wstart, count(*) c1, max(b) c2 from st partition by a as dd interval(10s);
run tsim/stream/checkTaskStatus.sim
sql insert into t1 values(1648791211000,1,1,3);
sql insert into t2 values(1648791211000,2,2,3);
sql insert into t1 values(1648791221000,1,3,3);
sql insert into t2 values(1648791221000,2,4,3);
sql show tables;
$loop_count = 0
loop6:
sleep 500
print sql show tables;
sql show tables;
print $data00 $data01 $data02 $data03 $data04
print $data10 $data11 $data12 $data13 $data14
print $data20 $data21 $data22 $data23 $data24
print $data30 $data31 $data32 $data33 $data34
print $data40 $data41 $data42 $data43 $data44
print $data50 $data51 $data52 $data53 $data54
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
if $rows != 6 then
print =====rows=$rows
goto loop6
endi
loop7:
sleep 500
print sql select * from information_schema.ins_tables where table_name like "streams6-tbn-%";
sql select * from information_schema.ins_tables where table_name like "streams6-tbn-%";
print $data00 $data01 $data02 $data03 $data04
print $data10 $data11 $data12 $data13 $data14
print $data20 $data21 $data22 $data23 $data24
print $data30 $data31 $data32 $data33 $data34
print $data40 $data41 $data42 $data43 $data44
print $data50 $data51 $data52 $data53 $data54
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
if $rows != 2 then
print =====rows=$rows
goto loop7
endi
loop8:
sleep 500
print sql select * from information_schema.ins_tables where table_name like "streams7-tbn-%";
sql select * from information_schema.ins_tables where table_name like "streams7-tbn-%";
print $data00 $data01 $data02 $data03 $data04
print $data10 $data11 $data12 $data13 $data14
print $data20 $data21 $data22 $data23 $data24
print $data30 $data31 $data32 $data33 $data34
print $data40 $data41 $data42 $data43 $data44
print $data50 $data51 $data52 $data53 $data54
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
if $rows != 2 then
print =====rows=$rows
goto loop8
endi
loop9:
sleep 500
print sql select * from streamt6;
sql select * from streamt6;
print $data00 $data01 $data02 $data03 $data04
print $data10 $data11 $data12 $data13 $data14
print $data20 $data21 $data22 $data23 $data24
print $data30 $data31 $data32 $data33 $data34
print $data40 $data41 $data42 $data43 $data44
print $data50 $data51 $data52 $data53 $data54
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
if $rows != 2 then
print =====rows=$rows
goto loop9
endi
loop10:
sleep 500
print sql select * from streamt7;
sql select * from streamt7;
print $data00 $data01 $data02 $data03 $data04
print $data10 $data11 $data12 $data13 $data14
print $data20 $data21 $data22 $data23 $data24
print $data30 $data31 $data32 $data33 $data34
print $data40 $data41 $data42 $data43 $data44
print $data50 $data51 $data52 $data53 $data54
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
if $rows != 2 then
print =====rows=$rows
goto loop10
endi
print ========== interval window step6
sql drop database if exists test8;
sql create database test8 vgroups 2;
sql use test8;
sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
sql create table t1 using st tags(1,1,1);
sql create table t2 using st tags(2,2,2);
sql create table streamt8(ts timestamp, a int primary key, b bigint ) tags(ta varchar(100));
sql create stream streams8 trigger continuous_window_close ignore update 0 ignore expired 0 into streamt8 tags(ta) as select _wstart, count(*) c1, max(b) c2 from st partition by tbname, a as ta interval(10s);
sql create stream streams9 trigger continuous_window_close ignore update 0 ignore expired 0 into streamt9(c1, c2 primary key, c3) as select _wstart, count(*) c1, max(b) c2 from st interval(10s);
run tsim/stream/checkTaskStatus.sim
sql insert into t1 values(1648791211000,1,1,3);
sql insert into t2 values(1648791211000,2,2,3);
sql insert into t1 values(1648791221000,1,3,3);
loop11:
sleep 500
print sql select * from streamt9;
sql select * from streamt9;
print $data00 $data01 $data02 $data03 $data04
print $data10 $data11 $data12 $data13 $data14
print $data20 $data21 $data22 $data23 $data24
print $data30 $data31 $data32 $data33 $data34
print $data40 $data41 $data42 $data43 $data44
print $data50 $data51 $data52 $data53 $data54
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
if $rows != 1 then
print =====rows=$rows
goto loop11
endi
if $data01 != 2 then
print =====rows=$rows
goto loop11
endi
sql insert into t2 values(1648791211001,2,4,3);
loop12:
sleep 500
print sql select * from streamt8;
sql select * from streamt8;
print $data00 $data01 $data02 $data03 $data04
print $data10 $data11 $data12 $data13 $data14
print $data20 $data21 $data22 $data23 $data24
print $data30 $data31 $data32 $data33 $data34
print $data40 $data41 $data42 $data43 $data44
print $data50 $data51 $data52 $data53 $data54
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
if $rows != 1 then
print =====rows=$rows
goto loop12
endi
loop13:
sleep 500
print sql select * from streamt9;
sql select * from streamt9;
print $data00 $data01 $data02 $data03 $data04
print $data10 $data11 $data12 $data13 $data14
print $data20 $data21 $data22 $data23 $data24
print $data30 $data31 $data32 $data33 $data34
print $data40 $data41 $data42 $data43 $data44
print $data50 $data51 $data52 $data53 $data54
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
if $rows != 1 then
print =====rows=$rows
goto loop13
endi
if $data01 != 3 then
print =====rows=$rows
goto loop13
endi
system sh/stop_dnodes.sh