mirror of
https://github.com/taosdata/TDengine
synced 2026-05-24 10:09:01 +00:00
af7e34e189
* Feat: support use keep while create super table. * Test(keep): add test for create super table with keep option. * Feat(keep): Add tmsg for create keep. * Feat(keep): support alter table option keep. * Fix(keep): Add baisc test for alter table option. * Fix(keep): memory leek. * Feat(keep): add keep to metaEntry&metaCache and fix earliestTs with stn keep. * Test(keep): add some cases for select with stb keep. * Fix: fix ci core while alter stb. * Feat(keep): delete expired data in super table level. * Feat: remove get stb keep while query. * Fix : build error. * Revert "Fix : build error." This reverts commit0ed66e4e84. * Revert "Feat(keep): delete expired data in super table level." This reverts commit36330f6b4c. * Fix : build errors. * Feat : support restart taosd. * Fix : alter table comment problems. * Test : add tests for super table keep. * Fix: change sdb stb reserve size. * Test: add more tests. * Feat: Disable normal tables and sub tables from setting the keep parameter * Fix: add more checks to avoid unknown address. * Docs: Add docs for stable keep. * Fix: some review changes. * Fix: review errors.
226 lines
No EOL
10 KiB
Python
226 lines
No EOL
10 KiB
Python
###################################################################
|
|
# Copyright (c) 2016 by TAOS Technologies, Inc.
|
|
# All rights reserved.
|
|
#
|
|
# This file is proprietary and confidential to TAOS Technologies.
|
|
# No part of this file may be reproduced, stored, transmitted,
|
|
# disclosed or used in any form or by any means other than as
|
|
# expressly provided by the written permission from Jianhui Tao
|
|
#
|
|
###################################################################
|
|
|
|
# -*- coding: utf-8 -*-
|
|
|
|
import sys
|
|
import time
|
|
import datetime
|
|
import taos
|
|
import frame
|
|
import frame.etool
|
|
|
|
from frame.log import *
|
|
from frame.cases import *
|
|
from frame.sql import *
|
|
from frame.caseBase import *
|
|
from frame import *
|
|
|
|
|
|
class TDTestCase(TBase):
|
|
|
|
def prepare_database_and_data(self):
|
|
tdLog.info("===== Preparing database and tables for testing keep parameter =====")
|
|
|
|
# Create database with 7-day retention period
|
|
tdLog.info("Creating database with 7-day retention period")
|
|
tdSql.execute("DROP DATABASE IF EXISTS test_keep")
|
|
tdSql.execute("CREATE DATABASE test_keep DURATION 1 KEEP 7")
|
|
tdSql.execute("USE test_keep")
|
|
tdLog.info("Database created successfully")
|
|
|
|
# Create super table with 5-day retention period
|
|
tdLog.info("Creating super table with 5-day retention period")
|
|
tdSql.execute("CREATE STABLE stb_keep5 (ts TIMESTAMP, val INT) TAGS (t_id INT) KEEP 5d")
|
|
|
|
# Create super table with 2-day retention period
|
|
tdLog.info("Creating super table with 2-day retention period")
|
|
tdSql.execute("CREATE STABLE stb_keep2 (ts TIMESTAMP, val INT) TAGS (t_id INT) KEEP 2d")
|
|
|
|
# Create child tables
|
|
tdLog.info("Creating child tables")
|
|
tdSql.execute("CREATE TABLE tb_keep5_1 USING stb_keep5 TAGS(1)")
|
|
tdSql.execute("CREATE TABLE tb_keep2_1 USING stb_keep2 TAGS(1)")
|
|
|
|
# Get current timestamp
|
|
now = int(time.time() * 1000)
|
|
|
|
# Insert current data
|
|
tdLog.info("Inserting current data")
|
|
tdSql.execute(f"INSERT INTO tb_keep5_1 VALUES ({now}, 100)")
|
|
tdSql.execute(f"INSERT INTO tb_keep2_1 VALUES ({now}, 100)")
|
|
|
|
# Insert data from 1 day ago (relative to base_time)
|
|
day1_before = now - 24 * 3600 * 1000
|
|
tdLog.info("Inserting data from 1 day ago")
|
|
tdSql.execute(f"INSERT INTO tb_keep5_1 VALUES ({day1_before}, 90)")
|
|
tdSql.execute(f"INSERT INTO tb_keep2_1 VALUES ({day1_before}, 90)")
|
|
|
|
# Insert data from 3 days ago (relative to base_time)
|
|
day3_before = now - 3 * 24 * 3600 * 1000
|
|
tdLog.info("Inserting data from 3 days ago")
|
|
tdSql.execute(f"INSERT INTO tb_keep5_1 VALUES ({day3_before}, 70)")
|
|
tdSql.execute(f"INSERT INTO tb_keep2_1 VALUES ({day3_before}, 70)")
|
|
|
|
# Insert data from 6 days ago (relative to base_time)
|
|
day6_before = now - 6 * 24 * 3600 * 1000
|
|
tdLog.info("Inserting data from 6 days ago")
|
|
tdSql.execute(f"INSERT INTO tb_keep5_1 VALUES ({day6_before}, 40)")
|
|
tdSql.execute(f"INSERT INTO tb_keep2_1 VALUES ({day6_before}, 40)")
|
|
|
|
# Log the timestamps of inserted data points
|
|
current_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(now/1000))
|
|
day1_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(day1_before/1000))
|
|
day3_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(day3_before/1000))
|
|
day6_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(day6_before/1000))
|
|
|
|
tdLog.info(f"Inserted data points at: Base time ({current_time}), 1 day before ({day1_time}), 3 days before ({day3_time}), 6 days before ({day6_time})")
|
|
|
|
# Verify data was properly inserted
|
|
tdLog.info("Verifying all data was properly inserted (ignoring keep settings for now)")
|
|
tdSql.query("SELECT COUNT(*) FROM tb_keep5_1")
|
|
count1 = tdSql.getData(0, 0)
|
|
tdLog.info(f"tb_keep5_1 has {count1} rows inserted")
|
|
|
|
tdSql.query("SELECT COUNT(*) FROM tb_keep2_1")
|
|
count2 = tdSql.getData(0, 0)
|
|
tdLog.info(f"tb_keep2_1 has {count2} rows inserted")
|
|
|
|
def check_stb_keep_influences_query(self):
|
|
tdLog.info("===== Testing scenario 1: Super table keep parameter influences query results =====")
|
|
tdSql.execute("USE test_keep")
|
|
|
|
# Verify data visibility in stb_keep5
|
|
tdLog.info("Checking data visibility in stb_keep5 (keep=5d)")
|
|
tdSql.query("SELECT * FROM stb_keep5")
|
|
tdSql.checkRows(3)
|
|
|
|
tdLog.info("Checking oldest visible data in stb_keep5")
|
|
tdSql.query("SELECT val FROM stb_keep5 ORDER BY ts ASC")
|
|
tdSql.checkData(0, 0, 70)
|
|
|
|
# Verify data visibility in stb_keep2
|
|
tdLog.info("Checking data visibility in stb_keep2 (keep=2d)")
|
|
tdSql.query("SELECT * FROM stb_keep2")
|
|
tdSql.checkRows(2)
|
|
|
|
tdLog.info("Checking oldest visible data in stb_keep2")
|
|
tdSql.query("SELECT val FROM stb_keep2 ORDER BY ts ASC")
|
|
tdSql.checkData(0, 0, 90)
|
|
|
|
tdLog.info("Super table keep parameter successfully influences query results")
|
|
|
|
def prepare_db_keep_override_test(self):
|
|
tdLog.info("===== Preparing database and tables for testing DB keep parameter override =====")
|
|
|
|
# Create database with 1-day retention period
|
|
tdLog.info("Creating database with 1-day retention period")
|
|
tdSql.execute("DROP DATABASE IF EXISTS test_db_keep")
|
|
tdSql.execute("CREATE DATABASE test_db_keep DURATION 60m KEEP 1")
|
|
tdSql.execute("USE test_db_keep")
|
|
|
|
# Create super table with 7-day retention period
|
|
tdLog.info("Creating super table with 7-day retention period")
|
|
tdSql.execute("CREATE STABLE stb_keep7 (ts TIMESTAMP, val INT) TAGS (t_id INT) KEEP 7d")
|
|
|
|
# Create child table
|
|
tdLog.info("Creating child table")
|
|
tdSql.execute("CREATE TABLE tb_keep7_1 USING stb_keep7 TAGS(1)")
|
|
|
|
# Get current timestamp for data insertion
|
|
# We'll use the real current time to ensure we're within the keep window
|
|
now = int(time.time() * 1000)
|
|
|
|
# Insert current data
|
|
tdLog.info("Inserting current data")
|
|
tdSql.execute(f"INSERT INTO tb_keep7_1 VALUES ({now}, 100)")
|
|
|
|
# Insert data from 8 hours ago (safely within the 1-day keep period)
|
|
hours8_before = now - 8 * 3600 * 1000
|
|
tdLog.info("Inserting data from 8 hours ago (within DB keep=1d)")
|
|
tdSql.execute(f"INSERT INTO tb_keep7_1 VALUES ({hours8_before}, 90)")
|
|
|
|
# Log information about the timestamps
|
|
current_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(now/1000))
|
|
hours8_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(hours8_before/1000))
|
|
|
|
tdLog.info(f"Inserted data points at: Current time ({current_time}), 8 hours ago ({hours8_time})")
|
|
|
|
# Verify data was properly inserted
|
|
tdSql.query("SELECT COUNT(*) FROM tb_keep7_1")
|
|
count = tdSql.getData(0, 0)
|
|
tdLog.info(f"tb_keep7_1 has {count} rows inserted")
|
|
|
|
# For demonstration purposes, calculate what the 2-day timestamp would be
|
|
# (we can't insert it, but we'll use it for our test explanation)
|
|
day2_before = now - 2 * 24 * 3600 * 1000
|
|
day2_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(day2_before/1000))
|
|
tdLog.info(f"Note: Data from 2 days ago ({day2_time}) cannot be inserted because it exceeds the database keep=1d setting")
|
|
tdLog.info(f"We will verify the database keep value (1d) overrides the super table keep value (7d) in the next test")
|
|
|
|
def check_db_keep_overrides_stb_keep(self):
|
|
tdLog.info("===== Testing scenario 2: Database keep parameter overrides super table keep parameter =====")
|
|
tdSql.execute("USE test_db_keep")
|
|
|
|
# Verify the data we inserted is visible
|
|
tdLog.info("Checking data visibility in stb_keep7 (DB keep=1d, STB keep=7d)")
|
|
tdSql.query("SELECT COUNT(*) FROM stb_keep7")
|
|
count = tdSql.getData(0, 0)
|
|
tdLog.info(f"stb_keep7 returned {count} rows")
|
|
tdSql.checkEqual(count, 2) # Should see both recent records
|
|
|
|
# Attempt to demonstrate that DB keep overrides STB keep
|
|
tdLog.info("Verifying database keep (1d) overrides super table keep (7d):")
|
|
|
|
# Method 1: Check that trying to insert data beyond DB keep fails
|
|
tdLog.info("Method 1: Checking that inserting data beyond DB keep (1d) fails even though STB keep is 7d")
|
|
now = int(time.time() * 1000)
|
|
day2_before = now - 2 * 24 * 3600 * 1000
|
|
day2_query = f"INSERT INTO tb_keep7_1 VALUES ({day2_before}, 80)"
|
|
|
|
try:
|
|
# This should fail with "Timestamp data out of range" because it's beyond DB keep
|
|
tdSql.error(day2_query, expectErrInfo="Timestamp data out of range")
|
|
tdLog.info("Success: Database rejected data beyond keep period (1d) as expected")
|
|
except Exception as e:
|
|
tdLog.info(f"Test validation failed: {e}")
|
|
|
|
# Method 2: Verify we can't query data that would be valid under STB keep but invalid under DB keep
|
|
tdLog.info("Method 2: Verifying data from 2 days ago is not visible (if it existed)")
|
|
day2_time = time.strftime("%Y-%m-%d", time.localtime(day2_before/1000))
|
|
query = f"SELECT COUNT(*) FROM stb_keep7 WHERE ts <= '{day2_time} 23:59:59.999' AND ts >= '{day2_time} 00:00:00.000'"
|
|
tdSql.query(query)
|
|
count = tdSql.getData(0, 0)
|
|
tdLog.info(f"Found {count} rows for 2-day old data (expecting 0)")
|
|
tdSql.checkEqual(count, 0)
|
|
|
|
tdLog.info("Conclusion: Database keep parameter (1d) successfully overrides super table keep parameter (7d)")
|
|
|
|
# Run tests
|
|
def run(self):
|
|
tdLog.debug(f"Start to execute {__file__}")
|
|
|
|
# Prepare test data
|
|
self.prepare_database_and_data()
|
|
|
|
# Test scenario 1: Super table keep parameter influences query results
|
|
self.check_stb_keep_influences_query()
|
|
|
|
# Prepare test data for database keep override
|
|
self.prepare_db_keep_override_test()
|
|
|
|
# Test scenario 2: Database keep parameter overrides super table keep parameter
|
|
self.check_db_keep_overrides_stb_keep()
|
|
|
|
tdLog.success(f"{__file__} successfully executed")
|
|
|
|
tdCases.addLinux(__file__, TDTestCase())
|
|
tdCases.addWindows(__file__, TDTestCase()) |