mirror of
https://github.com/taosdata/TDengine
synced 2026-05-24 10:09:01 +00:00
259 lines
9.1 KiB
Python
259 lines
9.1 KiB
Python
###################################################################
|
|
# Copyright (c) 2016 by TAOS Technologies, Inc.
|
|
# All rights reserved.
|
|
#
|
|
# This file is proprietary and confidential to TAOS Technologies.
|
|
# No part of this file may be reproduced, stored, transmitted,
|
|
# disclosed or used in any form or by any means other than as
|
|
# expressly provided by the written permission from Jianhui Tao
|
|
#
|
|
###################################################################
|
|
|
|
# -*- coding: utf-8 -*-
|
|
|
|
|
|
from new_test_framework.utils import tdLog, tdSql, tdStream, streamUtil,StreamTableType, StreamTable, cluster
|
|
|
|
|
|
import random
|
|
import time
|
|
import traceback
|
|
import os
|
|
from os import path
|
|
import psutil
|
|
|
|
|
|
class TestStreamBasicCase:
|
|
caseName = "TestStreamBasicCase"
|
|
currentDir = os.path.dirname(os.path.abspath(__file__))
|
|
runAll = False
|
|
dbname = "test"
|
|
trigTbname = ""
|
|
calcTbname = ""
|
|
outTbname = ""
|
|
stName = ""
|
|
resultIdx = ""
|
|
sliding = 1
|
|
subTblNum = 3
|
|
tblRowNum = 10
|
|
tableList = []
|
|
|
|
def setup_class(cls):
|
|
tdLog.info(f"start to excute {__file__}")
|
|
|
|
|
|
|
|
# run
|
|
def test_stream_basic(self):
|
|
"""OldPy: basic test 1
|
|
|
|
1. test stream basic
|
|
|
|
Catalog:
|
|
- Streams:OldPyCases
|
|
|
|
Since: v3.0.0.0
|
|
|
|
Labels: common, ci
|
|
|
|
Jira: None
|
|
|
|
History:
|
|
- 2025-7-21 lvze Migrated from community/tests/system-test/8-stream/stream_basic.py
|
|
- new stream outputtable name can not contain '.'
|
|
- When there is too much history data, the stream calculation speed is too slow, especially when debugflag is 135.
|
|
|
|
"""
|
|
tdSql.execute("create snode on dnode 1")
|
|
self.case1()
|
|
# gen data
|
|
random.seed(int(time.time()))
|
|
os.system(" taosBenchmark -d db -t 2 -v 2 -n 1000 -y")
|
|
# create stream
|
|
tdSql.execute("use db")
|
|
# tdSql.execute("create stream stream3 fill_history 1 into sta as select count(*) as cnt from meters interval(10a);",show=True)
|
|
tdSql.execute("""
|
|
create stream db.stream3 interval(10a) sliding(10a) from db.meters
|
|
stream_options(fill_history)
|
|
into db.sta
|
|
as select
|
|
_twstart ts,
|
|
count(*) cnt
|
|
from
|
|
%%trows;
|
|
""",queryTimes=2,show=True)
|
|
self.checkStreamRunning()
|
|
sql = "select count(*) from sta"
|
|
# loop wait max 60s to check count is ok
|
|
tdLog.info("loop wait result ...")
|
|
while True:
|
|
tdSql.query(f"select count(*) from information_schema.ins_tables where table_name='sta';")
|
|
if tdSql.getData(0,0) == 1:
|
|
tdLog.info("stream result table is ok ")
|
|
break
|
|
tdLog.info(f"wait stream out table ...")
|
|
time.sleep(1)
|
|
|
|
tdSql.checkDataLoop(0, 0, 100, sql, loopCount=50, waitTime=0.5)
|
|
|
|
# time.sleep(5)
|
|
|
|
# check all data is correct
|
|
sql = "select * from sta where cnt != 20;"
|
|
tdSql.query(sql)
|
|
tdSql.checkRows(0)
|
|
|
|
# check ts interval is correct
|
|
sql = "select * from ( select diff(ts) as tsdif from sta ) where tsdif != 10;"
|
|
tdSql.query(sql)
|
|
tdSql.checkRows(0)
|
|
|
|
# self.caseDropStream()
|
|
|
|
|
|
|
|
def case1(self):
|
|
tdSql.execute(f'create database if not exists d1 vgroups 1')
|
|
tdSql.execute(f'use d1')
|
|
tdSql.execute(f'create table st(ts timestamp, i int) tags(t int)')
|
|
tdSql.execute(f'insert into t1 using st tags(1) values("2025-08-01 00:00:00", 1) ("2025-08-01 00:00:01", 2)')
|
|
tdSql.execute(f'insert into t2 using st tags(2) values("2025-08-01 00:00:00", 1) ("2025-08-01 00:00:01", 2)')
|
|
tdSql.execute(f'insert into t3 using st tags(3) values("2025-08-01 00:00:00", 1) ("2025-08-01 00:00:01", 2)')
|
|
|
|
tdSql.execute("""create stream d1.stream1 interval(1m) sliding(1m) from d1.st partition by tbname
|
|
stream_options(fill_history)
|
|
into d1.sta output_subtable(concat('neew-', tbname))
|
|
tags(tname varchar(100) as tbname)
|
|
as select
|
|
_twstart,
|
|
count(*),
|
|
avg(i)
|
|
from
|
|
%%tbname;""", queryTimes=2,show=True)
|
|
|
|
tdSql.execute("""
|
|
create stream d1.stream2 interval(1m) sliding(1m) from d1.st partition by tbname
|
|
stream_options(fill_history)
|
|
into d1.stb output_subtable(concat('new-', tbname))
|
|
tags(tname varchar(100) as tbname)
|
|
as select
|
|
_twstart,
|
|
count(*),
|
|
avg(i)
|
|
from
|
|
%%tbname;
|
|
""", queryTimes=2,show=True)
|
|
self.checkStreamRunning()
|
|
|
|
sql= "select * from sta"
|
|
tdSql.checkRowsLoop(3, sql, loopCount=100, waitTime=0.5)
|
|
tdSql.query("select tbname from sta order by tbname")
|
|
if not tdSql.getData(0, 0).startswith('neew-t1'):
|
|
tdLog.exit("error1")
|
|
|
|
if not tdSql.getData(1, 0).startswith('neew-t2'):
|
|
tdLog.exit("error2")
|
|
|
|
if not tdSql.getData(2, 0).startswith('neew-t3'):
|
|
tdLog.exit("error3")
|
|
|
|
sql= "select * from stb"
|
|
tdSql.checkRowsLoop(3, sql, loopCount=100, waitTime=0.5)
|
|
tdSql.query("select tbname from stb order by tbname")
|
|
if not tdSql.getData(0, 0).startswith('new-t1'):
|
|
tdLog.exit("error4")
|
|
|
|
if not tdSql.getData(1, 0).startswith('new-t2'):
|
|
tdLog.exit("error5")
|
|
|
|
if not tdSql.getData(2, 0).startswith('new-t3'):
|
|
tdLog.exit("error6")
|
|
|
|
def caseDropStream(self):
|
|
tdLog.info(f"start caseDropStream")
|
|
sql = "drop database if exists d1;"
|
|
tdSql.query(sql)
|
|
sql = "drop database if exists db;"
|
|
tdSql.query(sql)
|
|
|
|
sql ="show streams;"
|
|
tdSql.query(sql)
|
|
tdSql.check_rows_loop(0, sql, loopCount=100, waitTime=0.5)
|
|
|
|
sql ="select * from information_schema.ins_stream_tasks;"
|
|
tdSql.query(sql)
|
|
tdSql.check_rows_loop(0, sql, loopCount=100, waitTime=0.5)
|
|
|
|
self.taosBenchmark(" -d db -t 2 -v 4 -n 1000000 -y")
|
|
# create stream
|
|
tdSql.execute("use db;")
|
|
tdSql.execute("create stream stream4 fill_history 1 into sta4 as select _wstart, sum(current),avg(current),last(current),min(voltage),first(voltage),last(phase),max(phase),count(phase), _wend, _wduration from meters partition by tbname, ts interval(10a);", show=True)
|
|
|
|
time.sleep(10)
|
|
|
|
sql ="select * from information_schema.ins_stream_tasks where status == 'ready';"
|
|
tdSql.query(sql, show=True)
|
|
tdSql.check_rows_loop(4, sql, loopCount=100, waitTime=0.5)
|
|
|
|
pl = psutil.pids()
|
|
for pid in pl:
|
|
try:
|
|
if psutil.Process(pid).name() == 'taosd':
|
|
taosdPid = pid
|
|
break
|
|
except psutil.NoSuchProcess:
|
|
pass
|
|
tdLog.info("taosd pid:{}".format(taosdPid))
|
|
p = psutil.Process(taosdPid)
|
|
|
|
cpuInfo = p.cpu_percent(interval=5)
|
|
tdLog.info("taosd cpu:{}".format(cpuInfo))
|
|
|
|
tdSql.execute("drop stream stream4;", show=True)
|
|
|
|
sql ="show streams;"
|
|
tdSql.query(sql, show=True)
|
|
tdSql.check_rows_loop(0, sql, loopCount=100, waitTime=0.5)
|
|
|
|
sql ="select * from information_schema.ins_stream_tasks;"
|
|
tdSql.query(sql, show=True)
|
|
tdSql.check_rows_loop(0, sql, loopCount=100, waitTime=0.5)
|
|
|
|
for i in range(10):
|
|
cpuInfo = p.cpu_percent(interval=5)
|
|
tdLog.info("taosd cpu:{}".format(cpuInfo))
|
|
if cpuInfo < 10:
|
|
return
|
|
else:
|
|
time.sleep(1)
|
|
continue
|
|
cpuInfo = p.cpu_percent(interval=5)
|
|
tdLog.info("taosd cpu:{}".format(cpuInfo))
|
|
if cpuInfo > 10:
|
|
tdLog.exit("drop stream failed, stream tasks are still running")
|
|
|
|
def checkStreamRunning(self):
|
|
tdLog.info(f"check stream running status:")
|
|
|
|
timeout = 60
|
|
start_time = time.time()
|
|
|
|
while True:
|
|
if time.time() - start_time > timeout:
|
|
tdLog.error("Timeout waiting for all streams to be running.")
|
|
tdLog.error(f"Final stream running status: {streamRunning}")
|
|
raise TimeoutError(f"Stream status did not reach 'Running' within {timeout}s timeout.")
|
|
|
|
tdSql.query(f"select status from information_schema.ins_streams order by stream_name;")
|
|
streamRunning=tdSql.getColData(0)
|
|
|
|
if all(status == "Running" for status in streamRunning):
|
|
tdLog.info("All Stream running!")
|
|
tdLog.info(f"stream running status: {streamRunning}")
|
|
return
|
|
else:
|
|
tdLog.info("Stream not running! Wait stream running ...")
|
|
tdLog.info(f"stream running status: {streamRunning}")
|
|
time.sleep(1)
|
|
|
|
|