TDengine/tests/script/tsim/db/create_all_options.sim
Bomin Zhang 876979b5f5
feat[ts-6107]: shared storage (#31552)
* add API to use s3 as shared storage

* support using local file system as shared storage

* upload file to shared storage

* support read, compact and drop

* finish basic mnode & vnode msg processing

* follower sync migration state

* implement mnode transaction, and improve log

* send migration progress msg to dnode to avoid deadlock

* implement following migration

* remove mcount

* avoid redo migration on startup

* avoid follower deadlock when leader is down

* trigger migrate by timer,  avoid compact after migration

* comment out the usage of 'tcs' functions in stream

* change config item prefix from s3 to ss

* change db option prefix from s3 to ss

* rename s3 data struct, function, file to ss

* rename s3 macro to ss

* update s3 sql to ss

* rename remaining s3 items to ss

* check ss configruation, improve s3 retry

* grant object storage -> shared storage,  check ssEnabled

* fix memory leaks

* update build options

* omit sensitive information when dump config

* fix backward compatibility issue

* fix issues found in ci-checks

* fix some failed test cases

* avoid follower timeout and improve log

* fix: follower timeout because migration status not updated

* refuse migration if there's an in progress one

* fix ss test case

* remove garbage files and other minor improvement

* fix failed test cases

* update unit test

* fix failed test case

* fix failed test case

* update document

* update document and fix failed test cases

* fix minor issues in code, test and document

* check new commit after migration task is scheduled

* fix several issus

1. migrate information cannot be dropped sometimes because progress response was put into read queue.
2. memory leak in rare cases
3. data corruption in rare cases
4. failed test case

* add shared storage upgrade tool

* fix compile error
2025-07-14 16:33:53 +08:00

475 lines
18 KiB
Text

system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/deploy.sh -n dnode2 -i 2
system sh/deploy.sh -n dnode3 -i 3
system sh/exec.sh -n dnode1 -s start
system sh/exec.sh -n dnode2 -s start
system sh/exec.sh -n dnode3 -s start
sql connect
sql create dnode $hostname port 7200
sql create dnode $hostname port 7300
$x = 0
step1:
$x = $x + 1
sleep 1000
if $x == 10 then
print ====> dnode not ready!
return -1
endi
sql select * from information_schema.ins_dnodes
print ===> $data00 $data01 $data02 $data03 $data04 $data05
print ===> $data10 $data11 $data12 $data13 $data14 $data15
print ===> $data20 $data21 $data22 $data23 $data24 $data25
print ===> $data30 $data31 $data32 $data33 $data34 $data35
if $rows != 3 then
return -1
endi
if $data(1)[4] != ready then
goto step1
endi
if $data(2)[4] != ready then
goto step1
endi
if $data(3)[4] != ready then
goto step1
endi
print ============= create database with all options
#database_option: {
# | BUFFER value [3~16384, default: 256]
# | PAGES value [64~16384, default: 256]
# | PAGESIZE value [1~16384, default: 4]
# | CACHEMODEL value ['node', 'last_row', 'last_value', 'both', default: 'node']
# | COMP [0 | 1 | 2, default: 2]
# | DURATION value [60m ~ min(3650d,keep), default: 10d, unit may be minut/hour/day]
# | WAL_FSYNC_PERIOD value [0 ~ 180000 ms, default: 3000]
# | MAXROWS value [200~10000, default: 4096]
# | MINROWS value [10~1000, default: 100]
# | KEEP value [max(1d ~ 365000d), default: 1d, unit may be minut/hour/day]
# | PRECISION ['ms' | 'us' | 'ns', default: ms]
# | REPLICA value [1 | 3, default: 1]
# | WAL_LEVEL value [0 | 1 | 2, default: 1]
# | VGROUPS value [default: 2]
# | SINGLE_STABLE [0 | 1, default: ]
#
#$data0_db : name
#$data1_db : create_time
#$data2_db : vgroups
#$data3_db : ntables
#$data4_db : replica
#$data6_db : duration
#$data7_db : keep
#$data10_db : minrows
#$data11_db : maxrows
#$data12_db : wal_level
#$data13_db : fsync
#$data14_db : comp
#$data15_db : cachelast
#$data16_db : precision
print ====> create database db, with default
sql create database db
sql select * from information_schema.ins_databases
print rows: $rows
print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
if $rows != 3 then
return -1
endi
if $data0_db != db then # name
return -1
endi
if $data2_db != 2 then # vgroups
return -1
endi
if $data3_db != 0 then # ntables
return -1
endi
if $data4_db != 1 then # replica
print expect 1, actual: $data4_db
return -1
endi
if $data5_db != on then # strict
return -1
endi
if $data6_db != 10d then # duration
return -1
endi
if $data7_db != 3650d,3650d,3650d then # keep
return -1
endi
if $data8_db != 256 then # buffer
return -1
endi
if $data9_db != 4 then # pagesize
return -1
endi
if $data10_db != 256 then # pages
return -1
endi
if $data11_db != 100 then # minrows
return -1
endi
if $data12_db != 4096 then # maxrows
return -1
endi
if $data13_db != 2 then # comp
return -1
endi
if $data14_db != ms then # precision
return -1
endi
if $data18_db != none then # cachelast
return -1
endi
if $data20_db != 1 then # wal_level
return -1
endi
if $data21_db != 3000 then # wal_fsync_period
return -1
endi
if $data30_db != 525600m then # ss_keeplocal
return -1
endi
if $data31_db != 1 then # ss_compact
return -1
endi
sql drop database db
#print ====> BLOCKS value [3~1000, default: 6]
#sql create database db BLOCKS 3
#sql select * from information_schema.ins_databases
#print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
#if $data9_db != 3 then
# return -1
#endi
#sql drop database db
#sql create database db BLOCKS 1000
#sql select * from information_schema.ins_databases
#print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
#if $data9_db != 1000 then
# return -1
#endi
#sql drop database db
#sql_error create database db BLOCKS 2
#sql_error create database db BLOCKS 0
#sql_error create database db BLOCKS -1
#print ====> CACHE value [default: 16]
#sql create database db CACHE 1
#sql select * from information_schema.ins_databases
#print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
#if $data8_db != 1 then
# return -1
#endi
#sql drop database db
#sql create database db CACHE 128
#sql select * from information_schema.ins_databases
#print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
#if $data8_db != 128 then
# return -1
#endi
#sql drop database db
print ====> CACHEMODEL value [0, 1, 2, 3, default: 0]
sql create database db CACHEMODEL 'last_row'
sql select * from information_schema.ins_databases
print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
if $data18_db != last_row then
return -1
endi
sql drop database db
sql create database db CACHEMODEL 'last_value'
sql select * from information_schema.ins_databases
print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
if $data18_db != last_value then
return -1
endi
sql drop database db
sql create database db CACHEMODEL 'both'
sql select * from information_schema.ins_databases
print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
if $data18_db != both then
return -1
endi
sql drop database db
sql_error create database db CACHEMODEL 'other'
sql_error create database db CACHEMODEL '-1'
print ====> COMP [0 | 1 | 2, default: 2]
sql create database db COMP 1
sql select * from information_schema.ins_databases
print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
if $data13_db != 1 then
return -1
endi
sql drop database db
sql create database db COMP 0
sql select * from information_schema.ins_databases
print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
if $data13_db != 0 then
return -1
endi
sql drop database db
sql_error create database db COMP 3
sql_error create database db COMP -1
#print ====> DURATION value [60m ~ min(3650d,keep), default: 10d, unit may be minut/hour/day]
#print ====> KEEP value [max(1d ~ 365000d), default: 1d, unit may be minut/hour/day]
#sql create database db DURATION 60m KEEP 60m
#sql select * from information_schema.ins_databases
#print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
#if $data6_db != 60 then
# return -1
#endi
#if $data7_db != 60,60,60 then
# return -1
#endi
#sql drop database db
#sql create database db DURATION 60m KEEP 1d
#sql select * from information_schema.ins_databases
#print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
#if $data6_db != 60 then
# return -1
#endi
#if $data7_db != 1440,1440,1440 then
# return -1
#endi
#sql create database db DURATION 3650d KEEP 365000d
#sql select * from information_schema.ins_databases
#print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
#if $data6_db != 5256000 then
# return -1
#endi
#if $data7_db != 525600000,525600000,525600000 then
# return -1
#endi
#sql drop database db
#sql_error create database db DURATION -59m
#sql_error create database db DURATION 59m
#sql_error create database db DURATION 5256001m
#sql_error create database db DURATION 3651d
#sql_error create database db KEEP -59m
#sql_error create database db KEEP 14399m
#sql_error create database db KEEP 525600001m
#sql_error create database db KEEP 365001d
print ====> WAL_FSYNC_PERIOD value [0 ~ 180000 ms, default: 3000]
sql create database db WAL_FSYNC_PERIOD 0
sql select * from information_schema.ins_databases
print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
if $data21_db != 0 then
return -1
endi
sql drop database db
sql create database db WAL_FSYNC_PERIOD 180000
sql select * from information_schema.ins_databases
print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
if $data21_db != 180000 then
return -1
endi
sql drop database db
sql_error create database db WAL_FSYNC_PERIOD 180001
sql_error create database db WAL_FSYNC_PERIOD -1
print ====> MAXROWS value [200~10000, default: 4096], MINROWS value [10~1000, default: 100]
sql create database db MAXROWS 10000 MINROWS 1000
sql select * from information_schema.ins_databases
print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
if $data11_db != 1000 then
return -1
endi
if $data12_db != 10000 then
return -1
endi
sql drop database db
sql create database db MAXROWS 200 MINROWS 10
sql select * from information_schema.ins_databases
print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
if $data11_db != 10 then
return -1
endi
if $data12_db != 200 then
return -1
endi
sql drop database db
sql_error create database db MAXROWS -1
sql_error create database db MAXROWS 0
sql_error create database db MAXROWS 199
sql_error create database db MAXROWS 10000001
sql_error create database db MINROWS -1
sql_error create database db MINROWS 0
sql_error create database db MINROWS 9
sql_error create database db MINROWS 1000001
sql_error create database db MAXROWS 500 MINROWS 1000
print ====> PRECISION ['ms' | 'us' | 'ns', default: ms]
sql create database db PRECISION 'us'
sql select * from information_schema.ins_databases
print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
if $data14_db != us then
return -1
endi
sql drop database db
sql create database db PRECISION 'ns'
sql select * from information_schema.ins_databases
print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
if $data14_db != ns then
return -1
endi
sql drop database db
sql_error create database db PRECISION 'as'
sql_error create database db PRECISION -1
print ====> QUORUM value [1 | 2, default: 1] 3.0 not support this item
#sql_error create database db QUORUM 2
#sql select * from information_schema.ins_databases
#print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
#if $data5_db != 2 then
# return -1
#endi
#sql drop database db
#sql create database db QUORUM 1
#sql select * from information_schema.ins_databases
#print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
#if $data5_db != 1 then
# return -1
#endi
#sql drop database db
sql_error create database db QUORUM 1
sql_error create database db QUORUM 2
sql_error create database db QUORUM 3
sql_error create database db QUORUM 0
sql_error create database db QUORUM -1
print ====> REPLICA value [1 | 3, default: 1]
sql create database db REPLICA 3
sql select * from information_schema.ins_databases
print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
if $data4_db != 3 then
return -1
endi
sql drop database db
sql create database db REPLICA 1
sql select * from information_schema.ins_databases
print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
if $data4_db != 1 then
return -1
endi
sql drop database db
#sql_error create database db REPLICA 2
sql_error create database db REPLICA 0
sql_error create database db REPLICA -1
sql_error create database db REPLICA 4
#print ====> TTL value [1d ~ , default: 1]
#sql create database db TTL 1
#sql select * from information_schema.ins_databases
#print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
#if $dataXX_db != 1 then
# return -1
#endi
#sql drop database db
#sql create database db TTL 10
#sql select * from information_schema.ins_databases
#print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
#if $dataXX_db != 10 then
# return -1
#endi
#sql drop database db
#sql_error create database db TTL 0
#sql_error create database db TTL -1
print ====> WAL_LEVEL value [1 | 2, default: 1]
sql create database db WAL_LEVEL 2
sql select * from information_schema.ins_databases
print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
if $data20_db != 2 then
return -1
endi
sql drop database db
sql create database db WAL_LEVEL 1
sql select * from information_schema.ins_databases
print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
if $data20_db != 1 then
return -1
endi
sql drop database db
sql_error create database db WAL_LEVEL 3
sql_error create database db WAL_LEVEL -1
#sql_error create database db WAL_LEVEL 0
print ====> VGROUPS value [1~4096, default: 2]
sql create database db VGROUPS 1
sql select * from information_schema.ins_databases
print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
if $data2_db != 1 then
return -1
endi
sql drop database db
sql create database db VGROUPS 16
sql select * from information_schema.ins_databases
print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
if $data2_db != 16 then
return -1
endi
sql drop database db
sql_error create database db VGROUPS 4097
sql_error create database db VGROUPS -1
sql_error create database db VGROUPS 0
print ====> SINGLE_STABLE [0 | 1, default: ]
sql create database db SINGLE_STABLE 1
sql select * from information_schema.ins_databases
print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
#if $dataXXXX_db != 1 then
# return -1
#endi
sql drop database db
sql create database db SINGLE_STABLE 0
sql select * from information_schema.ins_databases
print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
#if $dataXXXX_db != 0 then
# return -1
#endi
sql drop database db
sql_error create database db SINGLE_STABLE 2
sql_error create database db SINGLE_STABLE -1
#print ====> STREAM_MODE [0 | 1, default: ]
#sql create database db STREAM_MODE 1
#sql select * from information_schema.ins_databases
#print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
#if $dataXXX_db != 1 then
# return -1
#endi
#sql drop database db
#sql create database db STREAM_MODE 0
#sql select * from information_schema.ins_databases
#print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
#if $dataXXX_db != 0 then
# return -1
#endi
#sql drop database db
#sql_error create database db STREAM_MODE 2
#sql_error create database db STREAM_MODE -1
system sh/exec.sh -n dnode1 -s stop -x SIGINT
system sh/exec.sh -n dnode2 -s stop -x SIGINT
system sh/exec.sh -n dnode3 -s stop -x SIGINT