TDengine/test/cases/18-StreamProcessing/30-OldPyCases/test_oldcase_stream_basic.py

###################################################################
#           Copyright (c) 2016 by TAOS Technologies, Inc.
#                     All rights reserved.
#
#  This file is proprietary and confidential to TAOS Technologies.
#  No part of this file may be reproduced, stored, transmitted,
#  disclosed or used in any form or by any means other than as
#  expressly provided by the written permission from Jianhui Tao
#
###################################################################

# -*- coding: utf-8 -*-


from new_test_framework.utils import tdLog, tdSql, tdStream, streamUtil,StreamTableType, StreamTable, cluster


import random
import time
import traceback
import os
from   os import path
import psutil


class TestStreamBasicCase:
    caseName = "TestStreamBasicCase"
    currentDir = os.path.dirname(os.path.abspath(__file__))
    runAll = False
    dbname = "test"
    trigTbname = ""
    calcTbname = ""
    outTbname = ""
    stName = ""
    resultIdx = ""
    sliding = 1
    subTblNum = 3
    tblRowNum = 10
    tableList = []

    def setup_class(cls):
        tdLog.info(f"start to excute {__file__}")
        
        
        
    # run
    def test_stream_basic(self):
        """OldPy: basic test 1

        1. test stream basic

        Catalog:
            - Streams:OldPyCases

        Since: v3.0.0.0

        Labels: common, ci

        Jira: None

        History:
            - 2025-7-21 lvze Migrated from community/tests/system-test/8-stream/stream_basic.py
            - new stream outputtable name can not contain '.'
            - When there is too much history data, the stream calculation speed is too slow, especially when debugflag is 135.

        """
        tdSql.execute("create snode on dnode 1")
        self.case1()
        # gen data
        random.seed(int(time.time()))
        os.system(" taosBenchmark -d db -t 2 -v 2 -n 1000 -y")
        # create stream
        tdSql.execute("use db")
        # tdSql.execute("create stream stream3 fill_history 1 into sta as select count(*) as cnt from meters interval(10a);",show=True)
        tdSql.execute("""
                    create stream db.stream3 interval(10a) sliding(10a) from db.meters 
                    stream_options(fill_history)
                    into db.sta 
                    as select
                        _twstart ts, 
                        count(*) cnt
                    from
                        %%trows;
                    """,queryTimes=2,show=True)
        self.checkStreamRunning()
        sql = "select count(*) from sta"
        # loop wait max 60s to check count is ok
        tdLog.info("loop wait result ...")
        while True:
            tdSql.query(f"select count(*) from information_schema.ins_tables where table_name='sta';")
            if tdSql.getData(0,0) == 1:
                tdLog.info("stream result table is ok ")
                break
            tdLog.info(f"wait stream out table ...")
            time.sleep(1)

        tdSql.checkDataLoop(0, 0, 100, sql, loopCount=50, waitTime=0.5)

        # time.sleep(5)

        # check all data is correct
        sql = "select * from sta where cnt != 20;"
        tdSql.query(sql)
        tdSql.checkRows(0)

        # check ts interval is correct
        sql = "select * from ( select diff(ts) as tsdif from sta ) where tsdif != 10;"
        tdSql.query(sql)
        tdSql.checkRows(0)

        # self.caseDropStream()
                        


    def case1(self):
        tdSql.execute(f'create database if not exists d1 vgroups 1')
        tdSql.execute(f'use d1')
        tdSql.execute(f'create table st(ts timestamp, i int) tags(t int)')
        tdSql.execute(f'insert into t1 using st tags(1) values("2025-08-01 00:00:00", 1) ("2025-08-01 00:00:01", 2)')
        tdSql.execute(f'insert into t2 using st tags(2) values("2025-08-01 00:00:00", 1) ("2025-08-01 00:00:01", 2)')
        tdSql.execute(f'insert into t3 using st tags(3) values("2025-08-01 00:00:00", 1) ("2025-08-01 00:00:01", 2)')

        tdSql.execute("""create stream d1.stream1 interval(1m) sliding(1m) from d1.st partition by tbname
                    stream_options(fill_history)
                    into d1.sta output_subtable(concat('neew-', tbname)) 
                    tags(tname varchar(100) as tbname)
                    as select
                        _twstart,
                        count(*),
                        avg(i)
                    from
                        %%tbname;""", queryTimes=2,show=True)

        tdSql.execute("""
                    create stream d1.stream2 interval(1m) sliding(1m) from d1.st partition by tbname
                    stream_options(fill_history)
                    into d1.stb output_subtable(concat('new-', tbname)) 
                    tags(tname varchar(100) as tbname)
                    as select
                        _twstart,
                        count(*),
                        avg(i)
                    from
                        %%tbname;
                    """, queryTimes=2,show=True)
        self.checkStreamRunning()

        sql= "select * from sta"
        tdSql.checkRowsLoop(3, sql, loopCount=100, waitTime=0.5)
        tdSql.query("select tbname from sta order by tbname")
        if not tdSql.getData(0, 0).startswith('neew-t1'):
            tdLog.exit("error1")

        if not tdSql.getData(1, 0).startswith('neew-t2'):
            tdLog.exit("error2")

        if not tdSql.getData(2, 0).startswith('neew-t3'):
            tdLog.exit("error3")

        sql= "select * from stb"
        tdSql.checkRowsLoop(3, sql, loopCount=100, waitTime=0.5)
        tdSql.query("select tbname from stb order by tbname")
        if not tdSql.getData(0, 0).startswith('new-t1'):
            tdLog.exit("error4")

        if not tdSql.getData(1, 0).startswith('new-t2'):
            tdLog.exit("error5")

        if not tdSql.getData(2, 0).startswith('new-t3'):
            tdLog.exit("error6")

    def caseDropStream(self):
        tdLog.info(f"start caseDropStream")
        sql = "drop database if exists d1;"
        tdSql.query(sql)
        sql = "drop database if exists db;"
        tdSql.query(sql)

        sql ="show streams;"
        tdSql.query(sql)
        tdSql.check_rows_loop(0, sql, loopCount=100, waitTime=0.5)

        sql ="select * from information_schema.ins_stream_tasks;"
        tdSql.query(sql)
        tdSql.check_rows_loop(0, sql, loopCount=100, waitTime=0.5)

        self.taosBenchmark(" -d db -t 2 -v 4 -n 1000000 -y")
        # create stream
        tdSql.execute("use db;")
        tdSql.execute("create stream stream4 fill_history 1 into sta4 as select _wstart, sum(current),avg(current),last(current),min(voltage),first(voltage),last(phase),max(phase),count(phase), _wend, _wduration from meters partition by tbname, ts interval(10a);", show=True)
        
        time.sleep(10)

        sql ="select * from information_schema.ins_stream_tasks where status == 'ready';"
        tdSql.query(sql, show=True)
        tdSql.check_rows_loop(4, sql, loopCount=100, waitTime=0.5)

        pl = psutil.pids()
        for pid in pl:
            try:
                if psutil.Process(pid).name() == 'taosd':
                    taosdPid = pid
                    break
            except psutil.NoSuchProcess:
                pass
        tdLog.info("taosd pid:{}".format(taosdPid))
        p = psutil.Process(taosdPid)

        cpuInfo = p.cpu_percent(interval=5)
        tdLog.info("taosd cpu:{}".format(cpuInfo))

        tdSql.execute("drop stream stream4;", show=True)

        sql ="show streams;"
        tdSql.query(sql, show=True)
        tdSql.check_rows_loop(0, sql, loopCount=100, waitTime=0.5)

        sql ="select * from information_schema.ins_stream_tasks;"
        tdSql.query(sql, show=True)
        tdSql.check_rows_loop(0, sql, loopCount=100, waitTime=0.5)

        for i in range(10):
            cpuInfo = p.cpu_percent(interval=5)
            tdLog.info("taosd cpu:{}".format(cpuInfo))
            if cpuInfo < 10:
                return
            else:
                time.sleep(1)
                continue
        cpuInfo = p.cpu_percent(interval=5)
        tdLog.info("taosd cpu:{}".format(cpuInfo))
        if cpuInfo > 10:
            tdLog.exit("drop stream failed, stream tasks are still running")

    def checkStreamRunning(self):
        tdLog.info(f"check stream running status:")

        timeout = 60 
        start_time = time.time()

        while True:
            if time.time() - start_time > timeout:
                tdLog.error("Timeout waiting for all streams to be running.")
                tdLog.error(f"Final stream running status: {streamRunning}")
                raise TimeoutError(f"Stream status did not reach 'Running' within {timeout}s timeout.")
            
            tdSql.query(f"select status from information_schema.ins_streams order by stream_name;")
            streamRunning=tdSql.getColData(0)

            if all(status == "Running" for status in streamRunning):
                tdLog.info("All Stream running!")
                tdLog.info(f"stream running status: {streamRunning}")
                return
            else:
                tdLog.info("Stream not running! Wait stream running ...")
                tdLog.info(f"stream running status: {streamRunning}")
                time.sleep(1)