mirror of
https://github.com/taosdata/TDengine
synced 2026-05-24 10:09:01 +00:00
436 lines
22 KiB
Python
436 lines
22 KiB
Python
from new_test_framework.utils import tdLog, tdSql
|
|
import taos
|
|
import sys
|
|
import time
|
|
import socket
|
|
import os
|
|
import threading
|
|
import platform
|
|
|
|
class TestDbCompact:
|
|
|
|
def setup_class(cls):
|
|
cls.default_compact_options = [ "0d", "0d,0d", "0h"]
|
|
cls.compact_options = [["db00", "0m", "-0d,0", "0", "0d", "0d,0d", "0h"],
|
|
["db01", "0m", "-2d,-1", "0", "0d", "-2d,-1d", "0h"],
|
|
["db02", "2880m", "-61d,-1", "0", "2d", "-61d,-1d", "0h"],
|
|
["db03", "48h", "-87840m,-60", "1h", "2d", "-61d,-60d", "1h"],
|
|
["db04", "2d", "-87840m,-1440h", "12", "2d", "-61d,-60d", "12h"],
|
|
["db05", "2", "-61,-1440h", "23h", "2d", "-61d,-60d", "23h"],
|
|
]
|
|
|
|
#
|
|
# ------------------- test_compact.py ----------------
|
|
#
|
|
def do_compact(self):
|
|
tdSql.query("CREATE DATABASE power KEEP 365 DURATION 10 BUFFER 16 WAL_LEVEL 1 vgroups 1 replica 1;")
|
|
|
|
tdSql.query("CREATE DATABASE power1 KEEP 365 DURATION 10 BUFFER 16 WAL_LEVEL 1 vgroups 1 replica 1;")
|
|
|
|
#first
|
|
tdSql.query("compact database power;")
|
|
|
|
tdLog.info("compact id:%d"%tdSql.queryResult[0][1])
|
|
|
|
tdSql.query("show compact %d;"%tdSql.queryResult[0][1])
|
|
|
|
tdLog.info("detail:%d"%tdSql.queryRows)
|
|
|
|
#second
|
|
tdSql.query("compact database power1;")
|
|
|
|
tdLog.info("compact id:%d"%tdSql.queryResult[0][1])
|
|
|
|
tdSql.query("show compact %d;"%tdSql.queryResult[0][1])
|
|
|
|
tdLog.info("detail:%d"%tdSql.queryRows)
|
|
|
|
|
|
#kill
|
|
tdSql.query("show compacts;")
|
|
number1 = tdSql.queryResult[0][0]
|
|
number2 = tdSql.queryResult[1][0]
|
|
|
|
#first
|
|
tdLog.info("kill compact %d;"%number1)
|
|
tdSql.query("kill compact %d;"%number1)
|
|
|
|
#second
|
|
tdLog.info("kill compact %d;"%number2)
|
|
tdSql.query("kill compact %d;"%number2)
|
|
|
|
|
|
#show
|
|
count = 0
|
|
tdLog.info("query progress")
|
|
while count < 50:
|
|
tdSql.query("show compact %d;"%number1)
|
|
|
|
row1 = tdSql.queryRows
|
|
|
|
tdSql.query("show compact %d;"%number2)
|
|
|
|
row2 = tdSql.queryRows
|
|
|
|
tdLog.info("compact%d:detail count:%d"%(number1, row1))
|
|
tdLog.info("compact%d:detail count:%d"%(number2, row2))
|
|
|
|
if row1 == 0 and row2 == 0 :
|
|
break
|
|
|
|
time.sleep(1)
|
|
|
|
count +=1
|
|
#tdLog.info("loop%d"%count)
|
|
|
|
if row1 != 0 or row2 != 0:
|
|
tdLog.exit("compact failed")
|
|
|
|
print("\ndo compact ............................ [passed]")
|
|
|
|
#
|
|
# ------------------- test_compact_auto.py ----------------
|
|
#
|
|
def create_db_compact(self):
|
|
tdLog.info("create db compact options")
|
|
for item in self.compact_options:
|
|
tdSql.execute(f'create database {item[0]} compact_interval {item[1]} compact_time_range {item[2]} compact_time_offset {item[3]} duration 1d')
|
|
tdSql.query(f'select * from information_schema.ins_databases where name = "{item[0]}"')
|
|
tdSql.checkEqual(tdSql.queryResult[0][35], item[4])
|
|
tdSql.checkEqual(tdSql.queryResult[0][36], item[5])
|
|
tdSql.checkEqual(tdSql.queryResult[0][37], item[6])
|
|
tdSql.query(f'show create database {item[0]}')
|
|
tdSql.checkEqual(tdSql.queryResult[0][0], item[0])
|
|
tdSql.checkEqual(True, f'COMPACT_INTERVAL {item[4]} COMPACT_TIME_RANGE {item[5]} COMPACT_TIME_OFFSET {item[6]}' in tdSql.queryResult[0][1])
|
|
tdSql.execute(f'drop database {item[0]}')
|
|
|
|
def checkShowCreateWithTimeout(self, db, expectResult, timeout=30):
|
|
result = False
|
|
for i in range(timeout):
|
|
tdSql.query(f'show create database `%s`' %(db))
|
|
tdSql.checkEqual(tdSql.queryResult[0][0], db)
|
|
if expectResult in tdSql.queryResult[0][1]:
|
|
result = True
|
|
break
|
|
time.sleep(1)
|
|
if result == False:
|
|
raise Exception(f"Unexpected result of 'show create database `{db}`':{tdSql.queryResult[0][1]}, expect:{expectResult}")
|
|
|
|
def alter_db_compact(self):
|
|
tdLog.info("alter db compact options together")
|
|
for item in self.compact_options:
|
|
tdSql.execute(f'create database {item[0]} duration 1d')
|
|
tdSql.query(f'select * from information_schema.ins_databases where name = "{item[0]}"')
|
|
tdSql.checkEqual(tdSql.queryResult[0][35], self.default_compact_options[0])
|
|
tdSql.checkEqual(tdSql.queryResult[0][36], self.default_compact_options[1])
|
|
tdSql.checkEqual(tdSql.queryResult[0][37], self.default_compact_options[2])
|
|
tdSql.query(f'show create database {item[0]}')
|
|
tdSql.checkEqual(tdSql.queryResult[0][0], item[0])
|
|
tdSql.checkEqual(True, f'COMPACT_INTERVAL {self.default_compact_options[0]} COMPACT_TIME_RANGE {self.default_compact_options[1]} COMPACT_TIME_OFFSET {self.default_compact_options[2]}' in tdSql.queryResult[0][1])
|
|
tdSql.execute(f'alter database {item[0]} compact_interval {item[1]} compact_time_range {item[2]} compact_time_offset {item[3]}')
|
|
tdSql.query(f'select * from information_schema.ins_databases where name = "{item[0]}"')
|
|
tdSql.checkEqual(tdSql.queryResult[0][35], item[4])
|
|
tdSql.checkEqual(tdSql.queryResult[0][36], item[5])
|
|
tdSql.checkEqual(tdSql.queryResult[0][37], item[6])
|
|
for item in self.compact_options:
|
|
self.checkShowCreateWithTimeout(item[0], f'COMPACT_INTERVAL {item[4]} COMPACT_TIME_RANGE {item[5]} COMPACT_TIME_OFFSET {item[6]}')
|
|
tdSql.execute(f'drop database {item[0]}')
|
|
|
|
tdLog.info("alter db compact options separately")
|
|
compact_separate_options = [["db100", "0m", "-0d,0", "0", "0d", "0d,0d", "0h"],
|
|
["db101", "10m", "-2d,-1", "1", "10m", "-2d,-1d", "1h"]]
|
|
for item in compact_separate_options:
|
|
tdSql.execute(f'create database {item[0]} duration 1d')
|
|
tdSql.query(f'select * from information_schema.ins_databases where name = "{item[0]}"')
|
|
tdSql.checkEqual(tdSql.queryResult[0][35], self.default_compact_options[0])
|
|
tdSql.checkEqual(tdSql.queryResult[0][36], self.default_compact_options[1])
|
|
tdSql.checkEqual(tdSql.queryResult[0][37], self.default_compact_options[2])
|
|
tdSql.query(f'show create database {item[0]}')
|
|
tdSql.checkEqual(tdSql.queryResult[0][0], item[0])
|
|
tdSql.checkEqual(True, f'COMPACT_INTERVAL {self.default_compact_options[0]} COMPACT_TIME_RANGE {self.default_compact_options[1]} COMPACT_TIME_OFFSET {self.default_compact_options[2]}' in tdSql.queryResult[0][1])
|
|
tdSql.execute(f'alter database {item[0]} compact_time_offset {item[3]}', queryTimes=10)
|
|
tdSql.query(f'select * from information_schema.ins_databases where name = "{item[0]}"')
|
|
tdSql.checkEqual(tdSql.queryResult[0][35], self.default_compact_options[0])
|
|
tdSql.checkEqual(tdSql.queryResult[0][36], self.default_compact_options[1])
|
|
tdSql.checkEqual(tdSql.queryResult[0][37], item[6])
|
|
self.checkShowCreateWithTimeout(item[0], f'COMPACT_INTERVAL {self.default_compact_options[0]} COMPACT_TIME_RANGE {self.default_compact_options[1]} COMPACT_TIME_OFFSET {item[6]}')
|
|
tdSql.execute(f'alter database {item[0]} compact_interval {item[1]}', queryTimes=10)
|
|
tdSql.query(f'select * from information_schema.ins_databases where name = "{item[0]}"')
|
|
tdSql.checkEqual(tdSql.queryResult[0][35], item[4])
|
|
tdSql.checkEqual(tdSql.queryResult[0][36], self.default_compact_options[1])
|
|
tdSql.checkEqual(tdSql.queryResult[0][37], item[6])
|
|
self.checkShowCreateWithTimeout(item[0], f'COMPACT_INTERVAL {item[4]} COMPACT_TIME_RANGE {self.default_compact_options[1]} COMPACT_TIME_OFFSET {item[6]}')
|
|
tdSql.execute(f'alter database {item[0]} compact_time_range {item[2]}', queryTimes=10)
|
|
tdSql.query(f'select * from information_schema.ins_databases where name = "{item[0]}"')
|
|
tdSql.checkEqual(tdSql.queryResult[0][35], item[4])
|
|
tdSql.checkEqual(tdSql.queryResult[0][36], item[5])
|
|
tdSql.checkEqual(tdSql.queryResult[0][37], item[6])
|
|
self.checkShowCreateWithTimeout(item[0], f'COMPACT_INTERVAL {item[4]} COMPACT_TIME_RANGE {item[5]} COMPACT_TIME_OFFSET {item[6]}')
|
|
tdSql.execute(f'alter database {item[0]} compact_time_offset {item[3]}', queryTimes=10)
|
|
tdSql.query(f'select * from information_schema.ins_databases where name = "{item[0]}"')
|
|
tdSql.checkEqual(tdSql.queryResult[0][35], item[4])
|
|
tdSql.checkEqual(tdSql.queryResult[0][36], item[5])
|
|
tdSql.checkEqual(tdSql.queryResult[0][37], item[6])
|
|
self.checkShowCreateWithTimeout(item[0], f'COMPACT_INTERVAL {item[4]} COMPACT_TIME_RANGE {item[5]} COMPACT_TIME_OFFSET {item[6]}')
|
|
for item in compact_separate_options:
|
|
tdSql.execute(f'drop database {item[0]}', queryTimes=10)
|
|
|
|
def compact_error(self):
|
|
compact_err_list = [["compact_time_range 86400m,61d", "Invalid option compact_time_range: 86400m, start time should be in range: [-5256000m, -14400m]"],
|
|
["compact_time_range 60,61", "Invalid option compact_time_range: 86400m, start time should be in range: [-5256000m, -14400m]"],
|
|
["compact_time_range 60d,61d", "Invalid option compact_time_range: 86400m, start time should be in range: [-5256000m, -14400m]"],
|
|
["compact_time_range -60,-60", "Invalid option compact_time_range: -86400m,-86400m, start time should be less than end time"],
|
|
["compact_time_range -60,-1440h", "Invalid option compact_time_range: -86400m,-86400m, start time should be less than end time"],
|
|
["compact_time_range -60d,-61d", "Invalid option compact_time_range: -86400m,-87840m, start time should be less than end time"],
|
|
["compact_time_range -5256001m,-1", "Invalid option compact_time_range: -5256001m, start time should be in range: [-5256000m, -14400m]"],
|
|
["compact_time_range -199999999999m,-199999999998m", "start time should be in range: [-5256000m, -14400m]"],
|
|
["compact_time_range -99999999999999999999m,-99999999999999999998m", "Invalid value type: -99999999999999999999"],
|
|
["compact_time_range -60d,-1", "Invalid option compact_time_range: -1440m, end time should be in range: [-5256000m, -14400m]"],
|
|
["compact_interval 24h compact_time_range -60,61", "Invalid option compact_time_range: 87840m, end time should be in range: [-5256000m, -14400m]"],
|
|
["compact_interval 100 compact_time_range -60d,61d", "Invalid option compact_time_range: 87840m, end time should be in range: [-5256000m, -14400m]"],
|
|
["compact_time_range -60d,87840m", "Invalid option compact_time_range: 87840m, end time should be in range: [-5256000m, -14400m]"],
|
|
["compact_time_range -100,-90s", "Invalid option compact_time_range end unit: s, only m, h, d allowed"],
|
|
["compact_interval 10m compact_time_range -120d,-14400m compact_time_offset -1", "syntax error near"],
|
|
["compact_time_range -100,-99d compact_interval -1", "syntax error near"],
|
|
["compact_time_range 0", "Invalid option compact_time_range, should have 2 value"],
|
|
["compact_time_range -100", "Invalid option compact_time_range, should have 2 value"],
|
|
["compact_time_range -100,-90,-80", "Invalid option compact_time_range, should have 2 value"],
|
|
["compact_time_range -100;-90", "Invalid option compact_time_range, should have 2 value"],
|
|
["compact_time_range -100:-90", "syntax error near"],
|
|
["compact_time_range -100 -90", "syntax error near"],
|
|
["compact_interval 1m", "Invalid option compact_interval: 1, valid range: [10, 5256000]"],
|
|
["compact_interval 199999999999m", "valid range: [10, 5256000]"],
|
|
["compact_interval 9999999999999m", "Invalid option compact_interval: 9999999999999, valid range: [10, 5256000]"],
|
|
["compact_interval 5256001m", "Invalid option compact_interval: 5256001, valid range: [10, 5256000]"],
|
|
["compact_interval 3651", "Invalid option compact_interval: 5257440, valid range: [10, 5256000]"],
|
|
["compact_interval 86400s", "Invalid option compact_interval unit: s, only m, h, d allowed"],
|
|
["compact_interval -1", "syntax error near"],
|
|
["compact_time_offset -1", "syntax error near"],
|
|
["compact_time_offset 3600s", "Invalid option compact_time_offset unit: s, only h allowed"],
|
|
["compact_time_offset 1d", "Invalid option compact_time_offset unit: d, only h allowed"],
|
|
["compact_time_offset 24", "Invalid option compact_time_offset: 24, valid range: [0, 23]"],
|
|
["compact_time_offset 24h", "Invalid option compact_time_offset: 24, valid range: [0, 23]"],
|
|
["compact_time_offset 9999999999999", "valid range: [0, 23]"],
|
|
["compact_time_offset 199999999999", "valid range: [0, 23]"],
|
|
["compact_time_offset 1d", "Invalid option compact_time_offset unit: d, only h allowed"],
|
|
["compact_interval 10m compact_time_range -120d,-60 compact_time_offset 1d", "Invalid option compact_time_offset unit: d, only h allowed"],
|
|
]
|
|
tdSql.execute('create database if not exists db')
|
|
for item in compact_err_list:
|
|
tdSql.error(f"create database db {item[0]}", expectErrInfo=item[1], fullMatched=False)
|
|
tdSql.error(f"alter database db {item[0]}", expectErrInfo=item[1], fullMatched=False)
|
|
tdSql.execute('drop database db')
|
|
|
|
def do_compact_auto(self):
|
|
self.create_db_compact()
|
|
self.alter_db_compact()
|
|
self.compact_error()
|
|
|
|
print("do compact auto ....................... [passed]")
|
|
|
|
#
|
|
# ------------------- test_compact_vgroups.py ----------------
|
|
#
|
|
# Test cases ======================
|
|
def run_compact_vgroups_error(self):
|
|
# invalid sql
|
|
sql = "compact vgroups;"
|
|
tdLog.info(f"expect error SQL: {sql}")
|
|
tdSql.error(sql)
|
|
|
|
# invalid sql
|
|
sql = "compact vgroups in"
|
|
tdLog.info(f"expect error SQL: {sql}")
|
|
tdSql.error(sql)
|
|
|
|
# invalid sql
|
|
sql = "compact vgroups in ()"
|
|
tdLog.info(f"expect error SQL: {sql}")
|
|
tdSql.error(sql)
|
|
|
|
# error without using database
|
|
sql = "compact vgroups in (2)"
|
|
tdLog.info(f"expect error SQL: {sql}")
|
|
tdSql.error(sql)
|
|
|
|
# error with duplicate vgroup
|
|
sql = "compact db1.vgroups in (2, 2)"
|
|
tdLog.info(f"expect error SQL: {sql}")
|
|
tdSql.error(sql)
|
|
|
|
# error with invalid vgroup id
|
|
sql = "compact db1.vgroups in (0, -1)"
|
|
tdLog.info(f"expect error SQL: {sql}")
|
|
tdSql.error(sql)
|
|
|
|
# error to compact vgroups not in the same dat
|
|
sql = "compact db1.vgroups in (7, 8)"
|
|
tdLog.info(f"expect error SQL: {sql}")
|
|
tdSql.error(sql)
|
|
|
|
# error to compact vgroups not in the same database
|
|
sql = "compact db1.vgroups in (2, 5, 8)"
|
|
tdLog.info(f"expect error SQL: {sql}")
|
|
tdSql.error(sql)
|
|
|
|
def waitCompactFinish(self):
|
|
while True:
|
|
sql = 'show compacts'
|
|
rows = tdSql.query(sql)
|
|
if rows == 0:
|
|
break
|
|
time.sleep(1)
|
|
|
|
def run_compact_vgroups_sql(self):
|
|
# make sure there is no compacts
|
|
sql = 'show compacts'
|
|
rows = tdSql.query(sql)
|
|
tdSql.checkEqual(rows, 0)
|
|
|
|
# use db1 and compact with db name should be ok
|
|
sql = 'use db1'
|
|
tdLog.info(f'expect success SQL: {sql}')
|
|
tdSql.execute(sql)
|
|
|
|
sql = 'compact vgroups in (2)'
|
|
tdLog.info(f'expect success SQL: {sql}')
|
|
tdSql.execute(sql)
|
|
|
|
# check there should be one row in compacts
|
|
sql = 'show compacts'
|
|
rows = tdSql.query(sql)
|
|
tdSql.checkEqual(rows, 1)
|
|
|
|
compactId = tdSql.getData(0, 0)
|
|
|
|
# query the compact status
|
|
sql = f'show compact {compactId}'
|
|
tdLog.info(f'expect success SQL: {sql}')
|
|
rows = tdSql.query(sql)
|
|
tdSql.checkEqual(rows, 1)
|
|
tdSql.checkEqual(tdSql.getData(0, 0), compactId) # compact_id
|
|
tdSql.checkEqual(tdSql.getData(0, 1), 2) # vgroup_id
|
|
|
|
# wait for compact finish
|
|
self.waitCompactFinish()
|
|
|
|
# start a new compact
|
|
sql = 'compact db2.vgroups in (7, 10)'
|
|
tdLog.info(f'expect success SQL: {sql}')
|
|
tdSql.execute(sql)
|
|
|
|
sql = 'show compacts'
|
|
rows = tdSql.query(sql)
|
|
tdSql.checkEqual(rows, 1)
|
|
|
|
compactId = tdSql.getData(0, 0)
|
|
sql = f'show compact {compactId}'
|
|
tdLog.info(f'expect success SQL: {sql}')
|
|
rows = tdSql.query(sql)
|
|
tdSql.checkEqual(rows, 2)
|
|
tdSql.checkEqual(tdSql.getData(0, 1) in (7, 10), True)
|
|
tdSql.checkEqual(tdSql.getData(1, 1) in (7, 10), True)
|
|
tdSql.checkEqual(tdSql.getData(0, 1) != tdSql.getData(1, 1), True)
|
|
|
|
# wait for compact finish
|
|
self.waitCompactFinish()
|
|
|
|
def do_compact_vgroups(self):
|
|
# create database db1
|
|
sql = "create database db1 vgroups 5"
|
|
tdLog.info(sql)
|
|
tdSql.execute(sql)
|
|
|
|
# create database db2
|
|
sql = "create database db2 vgroups 5"
|
|
tdLog.info(sql)
|
|
tdSql.execute(sql)
|
|
|
|
# error test
|
|
self.run_compact_vgroups_error()
|
|
|
|
# success to compact vgroups
|
|
self.run_compact_vgroups_sql()
|
|
print("do compact vgroups .................... [passed]")
|
|
|
|
#
|
|
# ------------------- main ----------------
|
|
#
|
|
def do_compact_col(self):
|
|
tdSql.execute("drop database if exists tbname_vgroup")
|
|
tdSql.execute("create database if not exists tbname_vgroup")
|
|
tdSql.execute('use tbname_vgroup')
|
|
tdSql.execute('drop database if exists dbvg')
|
|
tdSql.execute('create database dbvg vgroups 8;')
|
|
|
|
tdSql.execute('use dbvg;')
|
|
|
|
tdSql.execute('create table st(ts timestamp, f int) tags (t int);')
|
|
|
|
tables = []
|
|
tables.append("ct1 using st tags(1) values('2021-04-19 00:00:01', 1)")
|
|
tables.append("ct2 using st tags(2) values('2021-04-19 00:00:02', 2)")
|
|
tables.append("ct3 using st tags(3) values('2021-04-19 00:00:03', 3)")
|
|
tables.append("ct4 using st tags(4) values('2021-04-19 00:00:04', 4)")
|
|
|
|
sql = "insert into " + " ".join(tables)
|
|
tdSql.execute(sql)
|
|
|
|
tdSql.execute('create table st2(ts timestamp, f int) tags (t int);')
|
|
|
|
tables = []
|
|
tables.append("ct21 using st2 tags(1) values('2021-04-19 00:00:01', 1)")
|
|
tables.append("ct22 using st2 tags(2) values('2021-04-19 00:00:02', 2)")
|
|
tables.append("ct23 using st2 tags(3) values('2021-04-19 00:00:03', 3)")
|
|
tables.append("ct24 using st2 tags(4) values('2021-04-19 00:00:04', 4)")
|
|
|
|
sql = "insert into " + " ".join(tables)
|
|
tdSql.execute(sql)
|
|
|
|
col_names = tdSql.getColNameList("compact database tbname_vgroup")
|
|
if col_names[0] != "result":
|
|
raise Exception("first column name of compact result shall be result")
|
|
|
|
col_names = tdSql.getColNameList("show variables")
|
|
if col_names[0] != "name":
|
|
raise Exception("first column name of compact result shall be name")
|
|
|
|
tdSql.execute('drop database tbname_vgroup')
|
|
|
|
print("do compact cols ....................... [passed]")
|
|
|
|
#
|
|
# ------------------- main ----------------
|
|
#
|
|
def test_db_compact(self):
|
|
"""Database compact
|
|
|
|
1. Compact a empty database and verify no error occurs
|
|
2. Show compacts and get compact id
|
|
3. Show compact with the compact id and verify the result
|
|
4. Kill the compact operation and verify it is killed successfully
|
|
5. Create databases with different compact options
|
|
6. Verify the compact options are set correctly
|
|
7. Alter the compact options and verify the options are changed correctly
|
|
8. Verify error handling for invalid compact options
|
|
9. Create databases with vgroups
|
|
10. Compact specific vgroups and verify the operation is successful
|
|
11. Verify "compact database" command can return column names correctly
|
|
|
|
Since: v3.0.0.0
|
|
|
|
Labels: common,ci
|
|
|
|
Jira: None
|
|
|
|
History:
|
|
- 2025-11-03 Alex Duan Migrated from uncatalog/system-test/0-others/test_compact.py
|
|
- 2025-11-03 Alex Duan Migrated from uncatalog/system-test/0-others/test_compact_auto.py
|
|
- 2025-11-03 Alex Duan Migrated from uncatalog/system-test/0-others/test_compact_vgroups.py
|
|
- 2025-11-04 Alex Duan Migrated from uncatalog/system-test/2-query/test_compact_col.py
|
|
|
|
"""
|
|
self.do_compact_vgroups()
|
|
self.do_compact()
|
|
self.do_compact_auto()
|
|
self.do_compact_col()
|