mirror of
https://github.com/taosdata/TDengine
synced 2026-05-24 10:09:01 +00:00
Merge remote-tracking branch 'origin/main' into merge/3.3.6tomain
This commit is contained in:
Simon Guan
commit
fb10a4afdf
3877 changed files with 820929 additions and 381998 deletions
114
.github/scripts/validate_tests.py
vendored
114
.github/scripts/validate_tests.py
vendored
|
|
@ -3,6 +3,57 @@ import os
|
|||
import ast
|
||||
import re
|
||||
|
||||
def check_required_field(lines, field, method_node, class_node, file_path, other_fields):
|
||||
"""
|
||||
检查必填字段（如 Since)，支持多行内容。
|
||||
"""
|
||||
index = next((i for i, line in enumerate(lines) if re.match(rf'^{field}\s*:\s*.*$', line, re.IGNORECASE)), None)
|
||||
has_errors = False
|
||||
if index is None:
|
||||
print(f"The docstring in method {method_node.name} in class {class_node.name} in {file_path} does not contain a valid '{field}:' section.")
|
||||
has_errors = True
|
||||
else:
|
||||
if index > 0 and lines[index - 1].strip():
|
||||
print(f"The docstring in method {method_node.name} in class {class_node.name} in {file_path} must have a blank line before '{field}'.")
|
||||
has_errors = True
|
||||
value_lines = []
|
||||
first_line_value = re.sub(rf'^{field}\s*:\s*', '', lines[index], flags=re.IGNORECASE).strip()
|
||||
if first_line_value:
|
||||
value_lines.append(first_line_value)
|
||||
for i in range(index + 1, len(lines)):
|
||||
if lines[i].strip() and not re.match(rf'^({"|".join(other_fields)})\s*:\s*.*$', lines[i], re.IGNORECASE):
|
||||
value_lines.append(lines[i].strip())
|
||||
else:
|
||||
break
|
||||
if not value_lines:
|
||||
print(f"The docstring in method {method_node.name} in class {class_node.name} in {file_path} does not contain a value for the '{field}:' field.")
|
||||
has_errors = True
|
||||
return has_errors
|
||||
|
||||
def check_optional_field(lines, field, method_node, class_node, file_path, other_fields):
|
||||
"""
|
||||
检查可选字段（如 Labels、Jira、History, Catalog)，支持多行内容。
|
||||
"""
|
||||
index = next((i for i, line in enumerate(lines) if re.match(rf'^{field}\s*:\s*.*$', line, re.IGNORECASE)), None)
|
||||
has_errors = False
|
||||
if index is not None:
|
||||
if index > 0 and lines[index - 1].strip():
|
||||
print(f"The docstring in method {method_node.name} in class {class_node.name} in {file_path} must have a blank line before '{field}'.")
|
||||
has_errors = True
|
||||
value_lines = []
|
||||
first_line_value = re.sub(rf'^{field}\s*:\s*', '', lines[index], flags=re.IGNORECASE).strip()
|
||||
if first_line_value:
|
||||
value_lines.append(first_line_value)
|
||||
for i in range(index + 1, len(lines)):
|
||||
if lines[i].strip() and not re.match(rf'^({"|".join(other_fields)})\s*:\s*.*$', lines[i], re.IGNORECASE):
|
||||
value_lines.append(lines[i].strip())
|
||||
else:
|
||||
break
|
||||
if not value_lines:
|
||||
print(f"The docstring in method {method_node.name} in class {class_node.name} in {file_path} does not contain a value for the '{field}' field.")
|
||||
has_errors = True
|
||||
return has_errors
|
||||
|
||||
def validate_test_file(file_path):
|
||||
with open(file_path, 'r', encoding='utf-8') as file:
|
||||
content = file.read()
|
||||
|
|
@ -61,66 +112,27 @@ def validate_test_file(file_path):
|
|||
else:
|
||||
# Check for description (can be multi-line)
|
||||
description_end = next((i for i, line in enumerate(lines[2:], start=2) if not line.strip()), len(lines))
|
||||
if description_end == 2 or re.match(r'^(Since|Labels|History|Jira)\s*:', lines[2].strip(), re.IGNORECASE):
|
||||
if description_end == 2 or re.match(r'^(Since|Labels|History|Jira|Catalog)\s*:', lines[2].strip(), re.IGNORECASE):
|
||||
print(f"The docstring in method {method_node.name} in class {class_node.name} in {file_path} does not contain a description.")
|
||||
has_errors = True
|
||||
|
||||
# Check for 'Since:' (required field, multi-line allowed)
|
||||
since_index = next((i for i, line in enumerate(lines) if re.match(r'^Since\s*:\s*.*$', line, re.IGNORECASE)), None)
|
||||
if since_index is None:
|
||||
print(f"The docstring in method {method_node.name} in class {class_node.name} in {file_path} does not contain a valid 'Since:' section.")
|
||||
has_errors = True
|
||||
else:
|
||||
if since_index > 0 and lines[since_index - 1].strip():
|
||||
print(f"The docstring in method {method_node.name} in class {class_node.name} in {file_path} must have a blank line before 'Since'.")
|
||||
has_errors = True
|
||||
|
||||
# Initialize since_value_lines with the value after the colon on the same line
|
||||
since_value_lines = []
|
||||
first_line_value = re.sub(r'^Since\s*:\s*', '', lines[since_index], flags=re.IGNORECASE).strip()
|
||||
if first_line_value:
|
||||
since_value_lines.append(first_line_value)
|
||||
# Collect multi-line values
|
||||
for i in range(since_index + 1, len(lines)):
|
||||
if lines[i].strip() and not re.match(r'^(Labels|Jira|History)\s*:\s*.*$', lines[i], re.IGNORECASE):
|
||||
since_value_lines.append(lines[i].strip())
|
||||
else:
|
||||
break
|
||||
# Ensure 'Since:' has at least one value
|
||||
if not since_value_lines:
|
||||
print(f"The docstring in method {method_node.name} in class {class_node.name} in {file_path} does not contain a value for the 'Since:' field.")
|
||||
# Required fields
|
||||
required_fields = ['Since']
|
||||
all_fields = ['Since', 'Catalog', 'Labels', 'Jira', 'History']
|
||||
for field in required_fields:
|
||||
if check_required_field(lines, field, method_node, class_node, file_path, [f for f in all_fields if f != field]):
|
||||
has_errors = True
|
||||
|
||||
# Optional fields: Labels, Jira, History (multi-line allowed)
|
||||
optional_fields = ['Labels', 'Jira', 'History']
|
||||
for i, field in enumerate(optional_fields):
|
||||
field_index = next((j for j, line in enumerate(lines) if re.match(rf'^{field.strip()}\s*:\s*.*$', line, re.IGNORECASE)), None)
|
||||
if field_index is not None:
|
||||
# Ensure there is a blank line before each optional field
|
||||
if field_index > 0 and lines[field_index - 1].strip():
|
||||
print(f"The docstring in method {method_node.name} in class {class_node.name} in {file_path} must have a blank line before '{field}'.")
|
||||
has_errors = True
|
||||
|
||||
# Initialize field_value_lines with the value after the colon on the same line
|
||||
field_value_lines = []
|
||||
first_line_value = re.sub(rf'^{field.strip()}\s*:\s*', '', lines[field_index], flags=re.IGNORECASE).strip()
|
||||
if first_line_value:
|
||||
field_value_lines.append(first_line_value)
|
||||
# Collect multi-line values
|
||||
for k in range(field_index + 1, len(lines)):
|
||||
if lines[k].strip() and not re.match(r'^(Labels|Jira|History|Since)\s*:\s*.*$', lines[k], re.IGNORECASE):
|
||||
field_value_lines.append(lines[k].strip())
|
||||
else:
|
||||
break
|
||||
# Ensure the field(Labels, Jira, History) has at least one value
|
||||
if not field_value_lines:
|
||||
print(f"The docstring in method {method_node.name} in class {class_node.name} in {file_path} does not contain a value for the '{field}' field.")
|
||||
has_errors = True
|
||||
# Optional fields
|
||||
optional_fields = ['Labels', 'Jira', 'History', 'Catalog']
|
||||
for field in optional_fields:
|
||||
if check_optional_field(lines, field, method_node, class_node, file_path, [f for f in all_fields if f != field]):
|
||||
has_errors = True
|
||||
|
||||
if has_errors:
|
||||
return False
|
||||
|
||||
print(f"File {file_path} passed validation.")
|
||||
#print(f"File {file_path} passed validation.")
|
||||
return True
|
||||
|
||||
|
||||
|
|
|
|||
34
.github/workflows/new-framework-test-for-stream-dev.yml
vendored
Normal file
34
.github/workflows/new-framework-test-for-stream-dev.yml
vendored
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
name: New Framework Test For Stream Dev
|
||||
|
||||
on:
|
||||
# push:
|
||||
# branches:
|
||||
# - 'feat/TS-6100-3.0'
|
||||
# paths-ignore:
|
||||
# - 'packaging/**'
|
||||
# - 'docs/**'
|
||||
pull_request:
|
||||
branches:
|
||||
- 'feat/TS-6100-3.0'
|
||||
paths-ignore:
|
||||
- 'packaging/**'
|
||||
- 'docs/**'
|
||||
- 'tools/tdgpt/**'
|
||||
- 'source/libs/executor/src/forecastoperator.c'
|
||||
- 'source/libs/executor/src/anomalywindowoperator.c'
|
||||
- 'source/dnode/mnode/impl/src/mndAnode.c'
|
||||
- 'include/common/tanalytics.h'
|
||||
- 'source/common/src/tanalytics.c'
|
||||
- 'tests/parallel/tdgpt_cases.task'
|
||||
- 'tests/script/tsim/analytics'
|
||||
- '**/*.md'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}-TDengineNewFramework
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
test-new-cases:
|
||||
uses: taosdata/.github/.github/workflows/new-framework-test-for-stream-dev.yml@feat/TS-6100-3.0
|
||||
with:
|
||||
tdinternal: false
|
||||
1
.github/workflows/tdengine-test.yml
vendored
1
.github/workflows/tdengine-test.yml
vendored
|
|
@ -49,6 +49,7 @@ jobs:
|
|||
run-tests-on-linux:
|
||||
# NOTE: using tomchon-patch-3 branch for the moment
|
||||
uses: taosdata/.github/.github/workflows/run-tests-on-linux.yml@main
|
||||
#uses: taosdata/.github/.github/workflows/run-tests-on-linux.yml@enh/remove-return-tests
|
||||
with:
|
||||
tdinternal: false
|
||||
specified_source_branch: ${{ github.event_name == 'pull_request' && 'unavailable' || inputs.specified_source_branch }}
|
||||
|
|
|
|||
8
.gitignore
vendored
8
.gitignore
vendored
|
|
@ -56,8 +56,16 @@ tests/**/__pycache__/
|
|||
sim/
|
||||
test/run/
|
||||
test/allure-results/
|
||||
test/screenlog.0
|
||||
test/output.txt
|
||||
test/env/ci_default.yaml
|
||||
|
||||
# generated by ide
|
||||
.idea/
|
||||
cmake-build-debug/
|
||||
cmake-build-release/
|
||||
test/allure-results
|
||||
test/env
|
||||
test/run
|
||||
test/screenlog*
|
||||
test/output.tmp
|
||||
|
|
|
|||
|
|
@ -48,7 +48,7 @@ psim/
|
|||
pysim/
|
||||
*.out
|
||||
*DS_Store
|
||||
tests/script/api/batchprepare
|
||||
test/new_test_framework/script/api/batchprepare
|
||||
taosadapter
|
||||
taosadapter-debug
|
||||
tools/taosws-rs/*
|
||||
|
|
|
|||
|
|
@ -130,7 +130,7 @@ IF(TD_WINDOWS)
|
|||
ELSE()
|
||||
MESSAGE("${Green} will build Debug version! ${ColourReset}")
|
||||
# NOTE: let cmake to choose default compile options
|
||||
# SET(COMMON_FLAGS "/w /D_WIN32 /DWIN32 /Zi /MDd")
|
||||
SET(COMMON_FLAGS "/w /D_WIN32 /DWIN32 /Zi /MDd")
|
||||
ENDIF()
|
||||
|
||||
SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /MANIFEST:NO /FORCE:MULTIPLE")
|
||||
|
|
|
|||
|
|
@ -39,7 +39,7 @@ macro(INIT_EXT name) # {
|
|||
set(${name}_inc_dir "")
|
||||
set(${name}_libs "")
|
||||
set(${name}_have_dev FALSE)
|
||||
set(${name}_build_contrib FALsE)
|
||||
set(${name}_build_contrib FALSE)
|
||||
|
||||
set(options)
|
||||
set(oneValueArgs INC_DIR)
|
||||
|
|
|
|||
|
|
@ -35,7 +35,7 @@ endif()
|
|||
# otherwise you might encounter the error like:
|
||||
# error: pathspec 'xxx' did not match any file(s) known to git
|
||||
if(NOT DEFINED TAOSADAPTER_GIT_TAG)
|
||||
set(TAOSADAPTER_GIT_TAG "3.3.6" CACHE STRING "which tag/branch/commit-sha1 to checkout for taosadapter.git" FORCE)
|
||||
set(TAOSADAPTER_GIT_TAG "main" CACHE STRING "which tag/branch/commit-sha1 to checkout for taosadapter.git" FORCE)
|
||||
endif()
|
||||
|
||||
# preprocess TAOSADAPTER_GIT_TAG
|
||||
|
|
@ -52,7 +52,7 @@ endif()
|
|||
# TAOSWS_GIT_TAG
|
||||
# eg.: main
|
||||
if(NOT DEFINED TAOSWS_GIT_TAG)
|
||||
set(TAOSWS_GIT_TAG "3.3.6" CACHE STRING "which tag/branch/commit-sha1 to checkout for taosws(rust connector)" FORCE)
|
||||
set(TAOSWS_GIT_TAG "main" CACHE STRING "which tag/branch/commit-sha1 to checkout for taosws(rust connector)" FORCE)
|
||||
endif()
|
||||
|
||||
# preprocess TAOSWS_GIT_TAG
|
||||
|
|
@ -229,8 +229,8 @@ ENDIF()
|
|||
IF(${TD_LINUX})
|
||||
|
||||
option(
|
||||
BUILD_S3
|
||||
"If build with s3"
|
||||
BUILD_SHARED_STORAGE
|
||||
"If build with shared storage"
|
||||
ON
|
||||
)
|
||||
|
||||
|
|
@ -263,8 +263,8 @@ option(
|
|||
|
||||
# NOTE: set option variable in this ways is not a good practice
|
||||
IF(NOT TD_ENTERPRISE)
|
||||
MESSAGE("switch s3 off with community version")
|
||||
set(BUILD_S3 OFF)
|
||||
MESSAGE("switch shared storage off with community version")
|
||||
set(BUILD_SHARED_STORAGE OFF)
|
||||
set(BUILD_WITH_S3 OFF)
|
||||
set(BUILD_WITH_COS OFF)
|
||||
set(BUILD_WITH_ANALYSIS OFF)
|
||||
|
|
@ -273,7 +273,7 @@ ENDIF ()
|
|||
# NOTE: set option variable in this ways is not a good practice
|
||||
IF(${BUILD_WITH_ANALYSIS})
|
||||
message("build with analysis")
|
||||
set(BUILD_S3 ON)
|
||||
set(BUILD_SHARED_STORAGE ON)
|
||||
set(BUILD_WITH_S3 ON)
|
||||
ENDIF()
|
||||
|
||||
|
|
@ -282,21 +282,18 @@ IF(${TD_LINUX})
|
|||
set(BUILD_WITH_ANALYSIS ON)
|
||||
ENDIF()
|
||||
|
||||
IF(${BUILD_S3})
|
||||
IF(${BUILD_SHARED_STORAGE})
|
||||
add_definitions(-DUSE_SHARED_STORAGE)
|
||||
|
||||
IF(${BUILD_WITH_S3})
|
||||
|
||||
add_definitions(-DUSE_S3)
|
||||
# NOTE: BUILD_WITH_S3 does NOT coexist with BUILD_WITH_COS?
|
||||
option(BUILD_WITH_COS "If build with cos" OFF)
|
||||
|
||||
ELSE ()
|
||||
|
||||
# NOTE: BUILD_WITH_S3 does NOT coexist with BUILD_WITH_COS?
|
||||
option(BUILD_WITH_COS "If build with cos" ON)
|
||||
|
||||
# NOTE1: BUILD_WITH_S3 does NOT coexist with BUILD_WITH_COS?
|
||||
# option(BUILD_WITH_COS "If build with cos" ON)
|
||||
MESSAGE("shared storage does not support COS at present, please use s3 instead")
|
||||
ENDIF ()
|
||||
|
||||
ELSE ()
|
||||
|
||||
option(BUILD_WITH_S3 "If build with s3" OFF)
|
||||
|
|
@ -386,7 +383,7 @@ option(
|
|||
OFF
|
||||
)
|
||||
|
||||
message(STATUS "BUILD_S3:${BUILD_S3}")
|
||||
message(STATUS "BUILD_SHARED_STORAGE:${BUILD_SHARED_STORAGE}")
|
||||
message(STATUS "BUILD_WITH_S3:${BUILD_WITH_S3}")
|
||||
message(STATUS "BUILD_WITH_COS:${BUILD_WITH_COS}")
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
IF (DEFINED VERNUMBER)
|
||||
SET(TD_VER_NUMBER ${VERNUMBER})
|
||||
ELSE ()
|
||||
SET(TD_VER_NUMBER "3.3.6.13.alpha")
|
||||
SET(TD_VER_NUMBER "3.3.7.0.alpha")
|
||||
ENDIF ()
|
||||
|
||||
IF (DEFINED VERCOMPATIBLE)
|
||||
|
|
|
|||
|
|
@ -1773,7 +1773,7 @@ int main(int argc, char **argv){
|
|||
}
|
||||
memset(&lem, 0, sizeof(lem));
|
||||
lem.errorcnt = 0;
|
||||
qsort(azDefine, nDefine, sizeof(azDefine[0]), defineCmp);
|
||||
if (azDefine) qsort(azDefine, nDefine, sizeof(azDefine[0]), defineCmp);
|
||||
|
||||
/* Initialize the machine */
|
||||
Strsafe_init();
|
||||
|
|
|
|||
|
|
@ -4,9 +4,9 @@ if(${BUILD_WITH_ROCKSDB})
|
|||
endif(${BUILD_WITH_ROCKSDB})
|
||||
|
||||
# cos
|
||||
if(${BUILD_WITH_COS})
|
||||
add_subdirectory(cos)
|
||||
endif(${BUILD_WITH_COS})
|
||||
#if(${BUILD_WITH_COS})
|
||||
# add_subdirectory(cos)
|
||||
#endif(${BUILD_WITH_COS})
|
||||
|
||||
if(${BUILD_WITH_LUCENE})
|
||||
add_subdirectory(lucene)
|
||||
|
|
@ -20,7 +20,7 @@ if(${BUILD_WITH_SQLITE})
|
|||
add_subdirectory(sqlite)
|
||||
endif(${BUILD_WITH_SQLITE})
|
||||
|
||||
# if(${BUILD_S3})
|
||||
# if(${BUILD_SHARED_STORAGE})
|
||||
# add_subdirectory(azure)
|
||||
# endif()
|
||||
|
||||
|
|
|
|||
|
|
@ -4,19 +4,13 @@
|
|||
{
|
||||
// Disable some built-in rules
|
||||
"config": {
|
||||
"MD001": true, // heading-increment 标题级别只能逐级递增
|
||||
"MD011": true, // no-reversed-link 禁止反向链接，link 必须用标准格式
|
||||
"MD012": true, // no-multiple-blank 禁止多个连续的空行。
|
||||
"MD001": false, // heading-increment 标题级别不需要逐级递增
|
||||
"MD013": false, // line-length 禁止行长度限制
|
||||
"MD018": true, // no-missing-space-atx 禁止 ATX # 标题前后缺少空格
|
||||
"MD019": true, // no-multiple-space-atx 禁止 ATX # 标题前后有多个空格
|
||||
"MD023": true, // no-heading-punctuation 标题必须顶格写，不能有前导空格
|
||||
"MD024": false, // no-duplicate-heading 禁止重复标题
|
||||
"MD029": false, // ol-prefix 列表前缀可以任意设置
|
||||
"MD025": false, // single-title 同一文档只能有一个一级标题
|
||||
"MD033": false, // no-inline-html 禁止行内HTML
|
||||
"MD041": false, // first-line-heading 禁止第一行必须是标题
|
||||
"MD042": true, // no-empty-links 禁止空链接
|
||||
"MD046": {"style": "fenced"}, // code-fence-style 代码块使用围栏样式
|
||||
"MD048": {"style": "backtick"}, // Code fence style 只允许用反引号 ``` 作为围栏
|
||||
"MD052": false // reference-links-image: 禁止引用链接和图片
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ slug: /get-started/deploy-enterprise-edition
|
|||
- Ensure that your servers are running Ubuntu 18.04, CentOS 7.9 or later.
|
||||
- Ensure that `systemd` is installed and enabled on all servers.
|
||||
- Confirm the location of all storage media that you want to use for tiered storage.
|
||||
- If you want to use S3, obtain the access key and bucket name for the desired S3 instance.
|
||||
- If you want to use shared storage, such as S3, prepare the access string for the desired storage device.
|
||||
- If you want to try TDengine Enterprise using Docker, please refer to [Get Started with TDengine Using Docker](../deploy-in-docker/) or [docker hub](https://hub.docker.com/r/tdengine/tdengine-ee).
|
||||
|
||||
## Prepare Your Environment
|
||||
|
|
@ -86,18 +86,35 @@ Before starting TDengine, open the `/etc/taos/taos.cfg` file and configure the f
|
|||
|
||||
Note that each cluster can have only one primary storage device, and the primary storage device must be on tier 0.
|
||||
|
||||
Then add the following parameters to the configuration file to enable S3 storage:
|
||||
TDengine also supports using cloud providers' object storage as storage device. However, since most object storage systems already have multiple replicas, enabling TDengine's multi-replica feature on top would lead to wasted storage space and increased costs. To address this, TDengine has further improved object storage into shared storage—where the storage device logically maintains only one copy of data that is shared across all nodes in the TDengine cluster. This approach significantly reduces both storage space requirements and costs.
|
||||
|
||||
To enable shared storage, please add the following parameters to the configuration file:
|
||||
|
||||
```conf
|
||||
s3EndPoint <your-endpoint>
|
||||
s3AccessKey <secret-id>:<secret-key>
|
||||
s3BucketName <your-s3-bucket>
|
||||
s3UploadDelaySec 10
|
||||
s3MigrateIntervalSec 600
|
||||
s3MigrateEnabled 1
|
||||
s3PageCacheSize 1
|
||||
ssEnabled 2
|
||||
ssAccessString s3:endpoint=s3.amazonaws.com;bucket=mybucket;uriStyle=path;protocol=https;accessKeyId=AKMYACCESSKEY;secretAccessKey=MYSECRETACCESSKEY;region=us-east-2;chunkSize=64;maxChunks=10000;maxRetry=3
|
||||
ssUploadDelaySec 10
|
||||
ssAutoMigrateIntervalSec 600
|
||||
ssPageCacheSize 1
|
||||
```
|
||||
|
||||
The possible values for `ssEnabled` are `0`, `1` and `2`. `0` is the default, which means shared storage is disabled; `1` means only enable manual migration(migrate local data to shared storage), and `2` means also enable auto migration.
|
||||
|
||||
The format of `ssAccessString` is `<device-type>:<option-name>=<option-value>;<option-name>=<option-value>;...`. Currently, TDengine supports using Amazon S3 compatible object storage as shared storage, its `device-type` is `s3`, and the following table list all possible options:
|
||||
|
||||
Name | Description
|
||||
----------------|----------------------------------------------
|
||||
endpoint | host name / ip address, and an optional port number of the object storage server.
|
||||
bucket | bucket name.
|
||||
protocol | `https` or `http`, `https` is the default.
|
||||
uriStyle | `virtualHost` or `path`, `virtualHost` is the default, but please note that some object storage provider only support `path`.
|
||||
region | object storage service region, optional.
|
||||
accessKeyId | your access key id.
|
||||
secretAccessKey | your secret access key.
|
||||
chunkSize | chunk size in MB, files larger than this size will use multipart upload, default is 64.
|
||||
maxChunks | max number of allowed chunks in a multipart upload, default is 10000.
|
||||
maxRetry | max retry times when encounter retryable errors, default is 3, negative value means unlimited retry.
|
||||
|
||||
3. (Optional) Modify the `/etc/taos/taosadapter.toml` file as follows to enable SSL on taosAdapter:
|
||||
|
||||
```toml
|
||||
|
|
|
|||
|
|
@ -1,143 +0,0 @@
|
|||
---
|
||||
title: Data Subscription
|
||||
slug: /advanced-features/data-subscription
|
||||
---
|
||||
|
||||
To meet the needs of applications to obtain data written to TDengine in real-time, or to process data in the order of event arrival, TDengine provides data subscription and consumption interfaces similar to those of message queue products. In many scenarios, by adopting TDengine's time-series big data platform, there is no need to integrate additional message queue products, thus simplifying application design and reducing maintenance costs.
|
||||
|
||||
Similar to Kafka, users need to define topics in TDengine. However, a topic in TDengine can be a database, a supertable, or based on existing supertables, subtables, or basic tables with specific query conditions, i.e., a query statement. Users can use SQL to filter by tags, table names, columns, expressions, etc., and perform scalar function and UDF computations (excluding data aggregation). Compared to other message queue tools, this is the biggest advantage of TDengine's data subscription feature. It offers greater flexibility; the granularity of the data is determined by the SQL defining the topic, and the filtering and preprocessing of data are automatically handled by TDengine, reducing the amount of data transmitted and simplifying application complexity.
|
||||
|
||||
After subscribing to a topic, consumers can receive the latest data in real-time. Multiple consumers can form a consumption group to share consumption progress, enabling multi-threaded, distributed data consumption to increase consumption speed. Consumers in different consumption groups do not share consumption progress even if they consume the same topic. A consumer can subscribe to multiple topics. If the topic corresponds to a supertable or database, the data may be distributed across multiple different nodes or data shards. When there are multiple consumers in a consumption group, consumption efficiency can be improved. TDengine's message queue provides an ACK (Acknowledgment) mechanism to ensure at least once consumption in complex environments such as crashes and restarts.
|
||||
|
||||
To implement the above functions, TDengine automatically creates indexes for Write-Ahead Logging (WAL) files to support fast random access and provides flexible and configurable file switching and retention mechanisms. Users can specify the retention time and size of WAL files according to their needs. Through these methods, WAL is transformed into a persistent storage engine that retains the order of event arrival. For queries created in the form of topics, TDengine reads data from WAL. During consumption, TDengine reads data directly from WAL based on the current consumption progress, performs filtering, transformation, and other operations using a unified query engine, and then pushes the data to consumers.
|
||||
|
||||
Starting from version 3.2.0.0, data subscription supports vnode migration and splitting. Due to the dependence of data subscription on wal files, wal does not synchronize during vnode migration and splitting. Therefore, after migration or splitting, wal data that has not been consumed before cannot be consumed. So please ensure that all data has been consumed before proceeding with vnode migration or splitting, otherwise data loss may occur during consumption.
|
||||
|
||||
## Topics
|
||||
|
||||
TDengine uses SQL to create three types of topics, which are introduced below.
|
||||
|
||||
### Query Topic
|
||||
|
||||
Subscribe to the results of an SQL query, essentially a continuous query, returning only the latest values each time, with the following creation syntax:
|
||||
|
||||
```sql
|
||||
CREATE TOPIC [IF NOT EXISTS] topic_name as subquery
|
||||
```
|
||||
|
||||
This SQL subscribes through a SELECT statement (including SELECT *, or specific query subscriptions like SELECT ts, c1, with condition filtering, scalar function computations, but does not support aggregate functions or time window aggregation). Note that:
|
||||
|
||||
1. Once this type of TOPIC is created, the structure of the subscribed data is fixed.
|
||||
2. Columns or tags that are subscribed to or used for calculations cannot be deleted (ALTER table DROP) or modified (ALTER table MODIFY).
|
||||
3. If table structure changes occur, newly added columns will not appear in the results.
|
||||
4. For select *, it subscribes to all columns at the time of creation (data columns for subtables and basic tables, data columns plus tag columns for supertables).
|
||||
|
||||
Suppose you need to subscribe to data where the voltage value in all smart meters is greater than 200, and only return the timestamp, current, and voltage (not phase), then you can create the topic power_topic with the following SQL.
|
||||
|
||||
```sql
|
||||
CREATE TOPIC power_topic AS SELECT ts, current, voltage FROM power.meters WHERE voltage > 200;
|
||||
```
|
||||
|
||||
### Supertable Topic
|
||||
|
||||
Subscribe to all data in a supertable, with the following syntax:
|
||||
|
||||
```sql
|
||||
CREATE TOPIC [IF NOT EXISTS] topic_name [with meta] AS STABLE stb_name [where_condition]
|
||||
```
|
||||
|
||||
The difference from subscribing using `SELECT * from stbName` is:
|
||||
|
||||
1. It does not restrict user table structure changes, i.e., both structure changes and new data after changes can be subscribed to.
|
||||
2. It returns unstructured data, and the structure of the returned data will change with the structure of the supertable.
|
||||
3. The with meta parameter is optional; when selected, it returns statements for creating supertables, subtables, etc., mainly used for supertable migration in taosx.
|
||||
4. The where_condition parameter is optional; when selected, it will be used to filter subtables that meet the conditions, subscribing to these subtables. The where condition cannot include ordinary columns, only tags or tbname, and functions can be used to filter tags, but not aggregate functions, as subtable tag values cannot be aggregated. It can also be a constant expression, such as 2 > 1 (subscribe to all subtables), or false (subscribe to 0 subtables).
|
||||
5. Returned data does not include tags.
|
||||
|
||||
### Database Topics
|
||||
|
||||
Subscribe to all data in a database, with the syntax as follows:
|
||||
|
||||
```sql
|
||||
CREATE TOPIC [IF NOT EXISTS] topic_name [with meta] AS DATABASE db_name;
|
||||
```
|
||||
|
||||
This statement creates a subscription that includes all table data in the database:
|
||||
|
||||
1. The `with meta` parameter is optional. When selected, it will return the creation, deletion, and modification statements of all supertables, subtables, and basic tables' metadata in the database, mainly used for database migration in taosx.
|
||||
2. Subscriptions to supertables and databases are advanced subscription modes and are prone to errors. If you really need to use them, please consult technical support personnel.
|
||||
|
||||
## Delete Topic
|
||||
|
||||
If you no longer need to subscribe to the data, you can delete the topic. If the current topic is subscribed to by a consumer, it can be forcibly deleted using the FORCE syntax. After the forced deletion, the subscribed consumer will consume data with errors (FORCE syntax supported from version 3.3.6.0).
|
||||
|
||||
```sql
|
||||
DROP TOPIC [IF EXISTS] [FORCE] topic_name;
|
||||
```
|
||||
|
||||
## View Topics
|
||||
|
||||
```sql
|
||||
SHOW TOPICS;
|
||||
```
|
||||
|
||||
The above SQL will display information about all topics under the current database.
|
||||
|
||||
## Consumers
|
||||
|
||||
### Creating Consumers
|
||||
|
||||
Consumers can only be created through the TDengine client driver or APIs provided by connectors. For details, refer to the development guide or reference manual.
|
||||
|
||||
### View Consumers
|
||||
|
||||
```sql
|
||||
SHOW CONSUMERS;
|
||||
```
|
||||
|
||||
Displays information about all consumers in the current database, including the consumer's status, creation time, etc.
|
||||
|
||||
### Delete Consumer Group
|
||||
|
||||
When creating a consumer, a consumer group is assigned to the consumer. Consumers cannot be explicitly deleted, but the consumer group can be deleted. If there are consumers in the current consumer group who are consuming, the FORCE syntax can be used to force deletion. After forced deletion, subscribed consumers will consume data with errors (FORCE syntax supported from version 3.3.6.0).
|
||||
|
||||
```sql
|
||||
DROP CONSUMER GROUP [IF EXISTS] [FORCE] cgroup_name ON topic_name;
|
||||
```
|
||||
|
||||
## Data Subscription
|
||||
|
||||
### View Subscription Information
|
||||
|
||||
```sql
|
||||
SHOW SUBSCRIPTIONS;
|
||||
```
|
||||
|
||||
Displays consumption information of the topic on different vgroups, useful for viewing consumption progress.
|
||||
|
||||
### Subscribe to Data
|
||||
|
||||
TDengine provides comprehensive and rich data subscription APIs, aimed at meeting data subscription needs under different programming languages and frameworks. These interfaces include but are not limited to creating consumers, subscribing to topics, unsubscribing, obtaining real-time data, submitting consumption progress, and getting and setting consumption progress. Currently, TDengine supports a variety of mainstream programming languages, including C, Java, Go, Rust, Python, and C#, enabling developers to easily use TDengine's data subscription features in various application scenarios.
|
||||
|
||||
It is worth mentioning that TDengine's data subscription APIs are highly consistent with the popular Kafka subscription APIs in the industry, making it easy for developers to get started and leverage their existing knowledge and experience. To facilitate user understanding and reference, TDengine's official documentation provides detailed descriptions and example codes of various APIs, which can be accessed in the connectors section of the TDengine official website. Through these APIs, developers can efficiently implement real-time data subscription and processing to meet data handling needs in various complex scenarios.
|
||||
|
||||
### Replay Feature
|
||||
|
||||
TDengine's data subscription feature supports a replay function, allowing users to replay the data stream in the actual order of data writing. This feature is based on TDengine's efficient WAL mechanism, ensuring data consistency and reliability.
|
||||
|
||||
To use the data subscription's replay feature, users can specify the time range in the query statement to precisely control the start and end times of the replay. This allows users to easily replay data within a specific time period, whether for troubleshooting, data analysis, or other purposes.
|
||||
|
||||
If the following 3 data entries were written, then during replay, the first entry is returned first, followed by the second entry after 5 seconds, and the third entry 3 seconds after obtaining the second entry.
|
||||
|
||||
```text
|
||||
2023/09/22 00:00:00.000
|
||||
2023/09/22 00:00:05.000
|
||||
2023/09/22 00:00:08.000
|
||||
```
|
||||
|
||||
When using the data subscription's replay feature, note the following:
|
||||
|
||||
- Enable replay function by configuring the consumption parameter enable.replay to true
|
||||
- The replay function of data subscription only supports data playback for query subscriptions; supertable and database subscriptions do not support playback.
|
||||
- Replay does not support progress saving.
|
||||
- Because data playback itself requires processing time, there is a precision error of several tens of milliseconds in playback.
|
||||
143
docs/en/06-advanced/01-subscription/01-mqtt.md
Normal file
143
docs/en/06-advanced/01-subscription/01-mqtt.md
Normal file
|
|
@ -0,0 +1,143 @@
|
|||
---
|
||||
title: MQTT Data Subscription
|
||||
slug: /advanced-features/data-subscription/mqtt-data-sub
|
||||
---
|
||||
|
||||
In addition to classic data subscription, TDengine supports subscription over MQTT. You can create a broker node (bnode) in TDengine and connect your MQTT client to it. The client can then subscribe to TDengine topics.
|
||||
|
||||
Key Features:
|
||||
|
||||
1. Protocol Support: MQTT 5.0
|
||||
2. Authentication: Uses TDengine native authentication
|
||||
3. Topic Management: Unlike the standard MQTT protocol, topics must be pre-created (as message publishing is not supported, topics cannot be dynamically created via message publishing)
|
||||
4. Shared Topics: Topics in the format $shared/group_id/topic_name are treated as shared subscriptions, suitable for scenarios requiring load balancing and high availability
|
||||
5. Subscription Position: Supports latest and earliest (WAL earliest position)
|
||||
6. Quality of Service: Supports QoS 0 and QoS 1
|
||||
|
||||
## Bnode Management
|
||||
|
||||
You manage bnodes through the taos CLI.
|
||||
|
||||
### Create a Bnode
|
||||
|
||||
Use the following SQL statement to create a bnode:
|
||||
|
||||
```sql
|
||||
CREATE BNODE ON DNODE <dnode_id>;
|
||||
```
|
||||
|
||||
You can create only one bnode on each dnode. Once the bnode is successfully created, a bnode subprocess named `taosmqtt` is automatically started to provide MQTT subscription services.
|
||||
|
||||
The `taosmqtt` service uses port 6083 by default. You can modify the `mqttPort` parameter in `taos.cfg` to provide MQTT subscription services on a different port.
|
||||
|
||||
### View Bnodes
|
||||
|
||||
Use the following SQL statement to display information about the bnodes in your cluster:
|
||||
|
||||
```sql
|
||||
SHOW BNODES;
|
||||
```
|
||||
|
||||
Information similar to the following is displayed:
|
||||
|
||||
```text
|
||||
taos> show bnodes;
|
||||
id | endpoint | protocol | create_time |
|
||||
======================================================================
|
||||
1 | 192.168.0.1:6083 | mqtt | 2024-11-28 18:44:27.089 |
|
||||
Query OK, 1 row(s) in set (0.037205s)
|
||||
```
|
||||
|
||||
### Delete a Bnode
|
||||
|
||||
Use the following SQL statement to delete a bnode:
|
||||
|
||||
```sql
|
||||
DROP BNODE ON DNODE <dnode_id>;
|
||||
```
|
||||
|
||||
Deleting a bnode removes it from the TDengine cluster and stops the `taosmqtt` service.
|
||||
|
||||
## MQTT Data Subscription Example
|
||||
|
||||
This example creates test data in TDengine and subscribes to the data. You can use any client that supports the MQTT v5.0 protocol to subscribe; this example uses the Python `paho-mqtt` library.
|
||||
|
||||
### Create Test Data
|
||||
|
||||
In the taos CLI, run the following SQL statements to set up your environment for this example.
|
||||
|
||||
```sql
|
||||
CREATE DATABASE db VGROUPS 1;
|
||||
CREATE TABLE db.meters (ts TIMESTAMP, f1 INT) TAGS (t1 INT);
|
||||
CREATE TOPIC topic_meters AS SELECT ts, tbname, f1, t1 FROM db.meters;
|
||||
INSERT INTO db.tb USING db.meters TAGS (1) VALUES (now, 1);
|
||||
CREATE BNODE ON DNODE 1;
|
||||
```
|
||||
|
||||
This creates a database, a supertable, and a topic. It then inserts a test data point and creates a bnode.
|
||||
|
||||
### Create a Consumer
|
||||
|
||||
Write a consumer in Python as follows:
|
||||
|
||||
```python
|
||||
import time
|
||||
import paho.mqtt
|
||||
import paho.mqtt.properties as p
|
||||
import paho.mqtt.packettypes as pt
|
||||
import paho.mqtt.client as mqttClient
|
||||
|
||||
def on_connect(client, userdata, flags, rc, properties=None):
|
||||
print("CONNACK received with code %s." % rc)
|
||||
sub_properties = p.Properties(pt.PacketTypes.SUBSCRIBE)
|
||||
sub_properties.UserProperty = ('sub-offset', 'earliest')
|
||||
client.subscribe("$share/g1/topic_meters", qos=1, properties=sub_properties)
|
||||
|
||||
def on_subscribe(client, userdata, mid, granted_qos, properties=None):
|
||||
print("Subscribed: " + str(mid) + " " + str(granted_qos))
|
||||
|
||||
def on_message(client, userdata, msg):
|
||||
print(msg.topic + " " + str(msg.qos) + " " + str(msg.payload))
|
||||
|
||||
if paho.mqtt.__version__[0] > '1':
|
||||
client = mqttClient.Client(mqttClient.CallbackAPIVersion.VERSION2, client_id="tmq_sub_cid", userdata=None, protocol=mqttClient.MQTTv5)
|
||||
else:
|
||||
client = mqttClient.Client(client_id="tmq_sub_cid", userdata=None, protocol=mqttClient.MQTTv5)
|
||||
|
||||
client.on_connect = on_connect
|
||||
client.username_pw_set("root", "taosdata")
|
||||
client.connect("127.0.1.1", 6083)
|
||||
|
||||
client.on_subscribe = on_subscribe
|
||||
client.on_message = on_message
|
||||
|
||||
client.loop_forever()
|
||||
```
|
||||
|
||||
Save this program as `sub.py`.
|
||||
|
||||
### Subscribe to the Topic
|
||||
|
||||
Run the following commands to subscribe to the data in `topic_meters`:
|
||||
|
||||
```shell
|
||||
python3 -m venv .test-env
|
||||
source .test-env/bin/activate
|
||||
pip3 install paho-mqtt==2.1.0
|
||||
python3 ./sub.py
|
||||
```
|
||||
|
||||
Once the subscription is successful, any new data written to the `topic_meters` topic will be automatically pushed to the client via MQTT.
|
||||
|
||||
## Message Format
|
||||
|
||||
With the example in the previous section, the following information shall be outputted:
|
||||
|
||||
```shell
|
||||
CONNACK received with code Success.
|
||||
Subscribed: 1 [ReasonCode(Suback, 'Granted QoS 1')]
|
||||
topic_meters 1 b'{"topic":"topic_meters","db":"db","vid":2,"rows":[{"ts":1753086482326,"tbname":"tb","f1":1,"t1":1}]}'
|
||||
```
|
||||
|
||||
In the third line, `topic_meters`, is the topic we subscribed to. 1 is the QoS value for this message, followed by a JSON message encoded in UTF-8, where `rows` is an array of data rows.
|
||||
|
||||
154
docs/en/06-advanced/01-subscription/index.md
Normal file
154
docs/en/06-advanced/01-subscription/index.md
Normal file
|
|
@ -0,0 +1,154 @@
|
|||
---
|
||||
title: Data Subscription
|
||||
slug: /advanced-features/data-subscription
|
||||
---
|
||||
|
||||
To meet the needs of applications to obtain data written to TDengine in real-time, or to process data in the order of event arrival, TDengine provides data subscription and consumption interfaces similar to those of message queue products. In many scenarios, by adopting TDengine's time-series big data platform, there is no need to integrate additional message queue products, thus simplifying application design and reducing maintenance costs.
|
||||
|
||||
Similar to Kafka, users need to define topics in TDengine. However, a topic in TDengine can be a database, a supertable, or based on existing supertables, subtables, or basic tables with specific query conditions, i.e., a query statement. Users can use SQL to filter by tags, table names, columns, expressions, etc., and perform scalar function and UDF computations (excluding data aggregation). Compared to other message queue tools, this is the biggest advantage of TDengine's data subscription feature. It offers greater flexibility; the granularity of the data is determined by the SQL defining the topic, and the filtering and preprocessing of data are automatically handled by TDengine, reducing the amount of data transmitted and simplifying application complexity.
|
||||
|
||||
After subscribing to a topic, consumers can receive the latest data in real-time. Multiple consumers can form a consumption group to share consumption progress, enabling multi-threaded, distributed data consumption to increase consumption speed. Consumers in different consumption groups do not share consumption progress even if they consume the same topic. A consumer can subscribe to multiple topics. If the topic corresponds to a supertable or database, the data may be distributed across multiple different nodes or data shards. When there are multiple consumers in a consumption group, consumption efficiency can be improved. TDengine's message queue provides an ACK (Acknowledgment) mechanism to ensure at least once consumption in complex environments such as crashes and restarts.
|
||||
|
||||
To implement the above functions, TDengine automatically creates indexes for Write-Ahead Logging (WAL) files to support fast random access and provides flexible and configurable file switching and retention mechanisms. Users can specify the retention time and size of WAL files according to their needs. Through these methods, WAL is transformed into a persistent storage engine that retains the order of event arrival. For queries created in the form of topics, TDengine reads data from WAL. During consumption, TDengine reads data directly from WAL based on the current consumption progress, performs filtering, transformation, and other operations using a unified query engine, and then pushes the data to consumers.
|
||||
|
||||
Starting from version 3.2.0.0, data subscription supports vnode migration and splitting. Due to the dependence of data subscription on wal files, wal does not synchronize during vnode migration and splitting. Therefore, after migration or splitting, wal data that has not been consumed before cannot be consumed. So please ensure that all data has been consumed before proceeding with vnode migration or splitting, otherwise data loss may occur during consumption.
|
||||
|
||||
## Topics
|
||||
|
||||
TDengine uses SQL to create three types of topics, which are introduced below.
|
||||
|
||||
### Query Topic
|
||||
|
||||
Subscribe to the results of an SQL query, essentially a continuous query, returning only the latest values each time, with the following creation syntax:
|
||||
|
||||
```sql
|
||||
CREATE TOPIC [IF NOT EXISTS] topic_name AS subquery;
|
||||
```
|
||||
|
||||
This SQL subscribes through a SELECT statement (including SELECT \*, or specific query subscriptions like SELECT ts, c1, with condition filtering, scalar function computations, but does not support aggregate functions or time window aggregation). Note that:
|
||||
|
||||
1. Once this type of TOPIC is created, the structure of the subscribed data is fixed.
|
||||
1. Columns or tags that are subscribed to or used for calculations cannot be deleted (ALTER table DROP) or modified (ALTER table MODIFY).
|
||||
1. If table structure changes occur, newly added columns will not appear in the results.
|
||||
1. For select \*, it subscribes to all columns at the time of creation (data columns for subtables and basic tables, data columns plus tag columns for supertables).
|
||||
|
||||
Suppose you need to subscribe to data where the voltage value in all smart meters is greater than 200, and only return the timestamp, current, and voltage (not phase), then you can create the topic power_topic with the following SQL.
|
||||
|
||||
```sql
|
||||
CREATE TOPIC power_topic AS SELECT ts, current, voltage FROM power.meters WHERE voltage > 200;
|
||||
```
|
||||
|
||||
### Supertable Topic
|
||||
|
||||
Subscribe to all data in a supertable, with the following syntax:
|
||||
|
||||
```sql
|
||||
CREATE TOPIC [IF NOT EXISTS] topic_name [WITH META] AS STABLE stb_name [where_condition];
|
||||
```
|
||||
|
||||
The difference from subscribing using `SELECT * from stbName` is:
|
||||
|
||||
1. It does not restrict user table structure changes, i.e., both structure changes and new data after changes can be subscribed to.
|
||||
1. It returns unstructured data, and the structure of the returned data will change with the structure of the supertable.
|
||||
1. The with meta parameter is optional; when selected, it returns statements for creating supertables, subtables, etc., mainly used for supertable migration in taosx.
|
||||
1. The where_condition parameter is optional; when selected, it will be used to filter subtables that meet the conditions, subscribing to these subtables. The where condition cannot include ordinary columns, only tags or tbname, and functions can be used to filter tags, but not aggregate functions, as subtable tag values cannot be aggregated. It can also be a constant expression, such as 2 > 1 (subscribe to all subtables), or false (subscribe to 0 subtables).
|
||||
1. Returned data does not include tags.
|
||||
|
||||
### Database Topics
|
||||
|
||||
Subscribe to all data in a database, with the syntax as follows:
|
||||
|
||||
```sql
|
||||
CREATE TOPIC [IF NOT EXISTS] topic_name [WITH META] AS DATABASE db_name;
|
||||
```
|
||||
|
||||
This statement creates a subscription that includes all table data in the database:
|
||||
|
||||
1. The `with meta` parameter is optional. When selected, it will return the creation, deletion, and modification statements of all supertables, subtables, and basic tables' metadata in the database, mainly used for database migration in taosx.
|
||||
2. Subscriptions to supertables and databases are advanced subscription modes and are prone to errors. If you really need to use them, please consult technical support personnel.
|
||||
|
||||
## Delete Topic
|
||||
|
||||
If you no longer need to subscribe to the data, you can delete the topic. If the current topic is subscribed to by a consumer, it can be forcibly deleted using the FORCE syntax. After the forced deletion, the subscribed consumer will consume data with errors (FORCE syntax supported from version 3.3.6.0).
|
||||
|
||||
```sql
|
||||
DROP TOPIC [IF EXISTS] [FORCE] topic_name;
|
||||
```
|
||||
|
||||
## View Topics
|
||||
|
||||
```sql
|
||||
SHOW TOPICS;
|
||||
```
|
||||
|
||||
The above SQL will display information about all topics under the current database.
|
||||
|
||||
## Consumers
|
||||
|
||||
### Creating Consumers
|
||||
|
||||
Consumers can only be created through the TDengine client driver or APIs provided by connectors. For details, refer to the development guide or reference manual.
|
||||
|
||||
### View Consumers
|
||||
|
||||
```sql
|
||||
SHOW CONSUMERS;
|
||||
```
|
||||
|
||||
Displays information about all consumers in the current database, including the consumer's status, creation time, etc.
|
||||
|
||||
### Delete Consumer Group
|
||||
|
||||
When creating a consumer, a consumer group is assigned to the consumer. Consumers cannot be explicitly deleted, but the consumer group can be deleted. If there are consumers in the current consumer group who are consuming, the FORCE syntax can be used to force deletion. After forced deletion, subscribed consumers will consume data with errors (FORCE syntax supported from version 3.3.6.0).
|
||||
|
||||
```sql
|
||||
DROP CONSUMER GROUP [IF EXISTS] [FORCE] cgroup_name ON topic_name;
|
||||
```
|
||||
|
||||
## Data Subscription
|
||||
|
||||
### View Subscription Information
|
||||
|
||||
```sql
|
||||
SHOW SUBSCRIPTIONS;
|
||||
```
|
||||
|
||||
Displays consumption information of the topic on different vgroups, useful for viewing consumption progress.
|
||||
|
||||
### Subscribe to Data
|
||||
|
||||
TDengine provides comprehensive and rich data subscription APIs, aimed at meeting data subscription needs under different programming languages and frameworks. These interfaces include but are not limited to creating consumers, subscribing to topics, unsubscribing, obtaining real-time data, submitting consumption progress, and getting and setting consumption progress. Currently, TDengine supports a variety of mainstream programming languages, including C, Java, Go, Rust, Python, and C#, enabling developers to easily use TDengine's data subscription features in various application scenarios.
|
||||
|
||||
It is worth mentioning that TDengine's data subscription APIs are highly consistent with the popular Kafka subscription APIs in the industry, making it easy for developers to get started and leverage their existing knowledge and experience. To facilitate user understanding and reference, TDengine's official documentation provides detailed descriptions and example codes of various APIs, which can be accessed in the connectors section of the TDengine official website. Through these APIs, developers can efficiently implement real-time data subscription and processing to meet data handling needs in various complex scenarios.
|
||||
|
||||
TDengine 3.3.7.0 includes MQTT-based data subscription, allowing data to be subscribed to directly via an MQTT client. For details, refer to the MQTT Data Subscription section.
|
||||
|
||||
### Replay Feature
|
||||
|
||||
TDengine's data subscription feature supports a replay function, allowing users to replay the data stream in the actual order of data writing. This feature is based on TDengine's efficient WAL mechanism, ensuring data consistency and reliability.
|
||||
|
||||
To use the data subscription's replay feature, users can specify the time range in the query statement to precisely control the start and end times of the replay. This allows users to easily replay data within a specific time period, whether for troubleshooting, data analysis, or other purposes.
|
||||
|
||||
If the following 3 data entries were written, then during replay, the first entry is returned first, followed by the second entry after 5 seconds, and the third entry 3 seconds after obtaining the second entry.
|
||||
|
||||
```text
|
||||
2023-09-22 00:00:00.000
|
||||
2023-09-22 00:00:05.000
|
||||
2023-09-22 00:00:08.000
|
||||
```
|
||||
|
||||
When using the data subscription's replay feature, note the following:
|
||||
|
||||
- Enable replay function by configuring the consumption parameter enable.replay to true
|
||||
- The replay function of data subscription only supports data playback for query subscriptions; supertable and database subscriptions do not support playback.
|
||||
- Replay does not support progress saving.
|
||||
- Because data playback itself requires processing time, there is a precision error of several tens of milliseconds in playback.
|
||||
- MQTT data subscription does not support replay.
|
||||
|
||||
|
||||
```mdx-code-block
|
||||
import DocCardList from '@theme/DocCardList';
|
||||
import {useCurrentSidebarCategory} from '@docusaurus/theme-common';
|
||||
|
||||
<DocCardList items={useCurrentSidebarCategory().items}/>
|
||||
```
|
||||
|
|
@ -145,7 +145,7 @@ If you are using Maven to manage your project, simply add the following dependen
|
|||
- Install a specific version
|
||||
|
||||
```shell
|
||||
pip3 install taospy==2.8.2
|
||||
pip3 install taospy==2.8.3
|
||||
```
|
||||
|
||||
- Install from GitHub
|
||||
|
|
@ -365,6 +365,12 @@ Complete DSN format:
|
|||
username:password@protocol(address)/dbname?param=value
|
||||
```
|
||||
|
||||
When using an IPv6 address (supported in v3.7.1 and above), the address needs to be enclosed in square brackets, for example:
|
||||
|
||||
```text
|
||||
root:taosdata@ws([::1]:6041)/testdb
|
||||
```
|
||||
|
||||
Supported DSN parameters are as follows:
|
||||
|
||||
Native connection:
|
||||
|
|
|
|||
|
|
@ -14,7 +14,9 @@ This section introduces the multi-tier storage feature unique to TDengine Enterp
|
|||
- Easy maintenance -- After configuring the storage mount points for each level, system data migration and other tasks do not require manual intervention; storage expansion is more flexible and convenient
|
||||
- Transparent to SQL -- Regardless of whether the queried data spans multiple levels, a single SQL query can return all data, simple and efficient
|
||||
|
||||
The storage media involved in multi-tier storage are all local storage devices. In addition to local storage devices, TDengine Enterprise also supports using object storage (S3) to save the coldest batch of data on the cheapest media to further reduce storage costs, and still allows querying when necessary, and where the data is stored is also transparent to SQL. Support for object storage was first released in version 3.3.0.0, and it is recommended to use the latest version.
|
||||
The storage media involved in multi-tier storage are all local storage devices. In addition to local storage devices, TDengine also supports using object storage as storage device. However, since most object storage systems already have multiple replicas, enabling TDengine's multi-replica feature on top would lead to wasted storage space and increased costs. To address this, TDengine has further improved object storage into shared storage—where the storage device logically maintains only one copy of data that is shared across all nodes in the TDengine cluster. This approach significantly reduces both storage space requirements and costs, and still allows querying when necessary, and where the data is stored is also transparent to SQL.
|
||||
|
||||
Shared storage was first released in version 3.3.7.0, it is an enhancement of the object storage (S3) support released in version 3.3.0.0. However, version 3.3.7.0 is not compatible with previous versions, if you have object storage (S3) feature enabled before, upgrading to version 3.3.7.0 and above will require manual operations.
|
||||
|
||||
## Multi-Tier Storage
|
||||
|
||||
|
|
@ -22,7 +24,7 @@ The storage media involved in multi-tier storage are all local storage devices.
|
|||
|
||||
Multi-tier storage supports 3 levels, with up to 128 mount points per level.
|
||||
|
||||
**Tips** Typical configuration schemes include: Level 0 configured with multiple mount points, each corresponding to a single SAS hard drive; Level 1 configured with multiple mount points, each corresponding to a single or multiple SATA hard drives; Level 2 can be configured with S3 storage or other inexpensive network storage.
|
||||
**Tips** Typical configuration schemes include: Level 0 configured with multiple mount points, each corresponding to a single SAS hard drive; Level 1 configured with multiple mount points, each corresponding to a single or multiple SATA hard drives.
|
||||
|
||||
The configuration method for TDengine multi-tier storage is as follows (in the configuration file /etc/taos/taos.cfg):
|
||||
|
||||
|
|
@ -63,75 +65,103 @@ To address this issue, starting from 3.1.1.0, a new configuration minDiskFreeSiz
|
|||
|
||||
Starting from version 3.3.2.0, a new configuration `disable_create_new_file` has been introduced to control the prohibition of generating new files on a certain mount point. The default value is `false`, which means new files can be generated on each mount point by default.
|
||||
|
||||
## Object Storage
|
||||
## Shared Storage
|
||||
|
||||
This section describes how to use S3 object storage in TDengine Enterprise. This feature is based on the generic S3 SDK and has been adapted for compatibility with various S3 platforms, allowing access to object storage services such as MinIO, Tencent Cloud COS, Amazon S3, etc. By configuring the appropriate parameters, most of the colder time-series data can be stored in S3 services.
|
||||
This section describes how to use shared storage in TDengine Enterprise. This feature is currently based on the generic S3 SDK and has been adapted for various S3 compatible platforms, allowing access to object storage services such as MinIO, Tencent Cloud COS, Amazon S3, etc. By configuring the appropriate parameters, most of the colder time-series data can be stored in S3 compatible services.
|
||||
|
||||
**Note** When used in conjunction with multi-tier storage, data saved on each storage medium may be backed up to remote object storage and local data files deleted according to rules.
|
||||
**Note** When used in conjunction with multi-tier storage, data saved on each storage medium may be migrated to shared storage and local data files deleted according to rules.
|
||||
|
||||
### Configuration Method
|
||||
|
||||
In the configuration file `/etc/taos/taos.cfg`, add parameters for S3 access:
|
||||
In the configuration file `/etc/taos/taos.cfg`, add parameters for shared storage access:
|
||||
|
||||
| Parameter Name | Description |
|
||||
|:-------------|:-----------------------------------------------|
|
||||
| s3EndPoint | The COS service domain name in the user's region, supports http and https, the bucket's region must match the endpoint's, otherwise access is not possible. |
|
||||
| s3AccessKey | Colon-separated user SecretId:SecretKey. For example: AKIDsQmwsfKxTo2A6nGVXZN0UlofKn6JRRSJ:lIdoy99ygEacU7iHfogaN2Xq0yumSm1E |
|
||||
| s3BucketName | Bucket name, the hyphen is followed by the AppId registered for the COS service. AppId is unique to COS, AWS and Alibaba Cloud do not have it, it needs to be part of the bucket name, separated by a hyphen. Parameter values are all string types, but do not need quotes. For example: test0711-1309024725 |
|
||||
| s3UploadDelaySec | How long a data file remains unchanged before being uploaded to S3, in seconds. Minimum: 1; Maximum: 2592000 (30 days), default value 60 seconds |
|
||||
| s3PageCacheSize | Number of s3 page cache pages, in pages. Minimum: 4; Maximum: 1024*1024*1024, default value 4096 |
|
||||
| s3MigrateIntervalSec | The trigger cycle for automatic upload of local data files to S3, in seconds. Minimum: 600; Maximum: 100000. Default value 3600 |
|
||||
| s3MigrateEnabled | Whether to automatically perform S3 migration, default value is 0, which means auto S3 migration is off, can be set to 1. |
|
||||
| ssEnabled | Whether to enable shared storage or not, allowed values are `0`, `1` and `2`. `0` is the default, which means shared storage is disabled; `1` means only enable manual migration, and `2` means also enable auto migation.
|
||||
| ssAccessString | A string which contains various options for accessing the shared storage, the format is `<device-type>:<option-name>=<option-value>;<option-name>=<option-value>;...`, currently, only Amazon S3 compatible object storage providers are supported, so the `device-type` should be `s3`, and please refer the next table for the possible options. |
|
||||
| ssUploadDelaySec | How long a data file remains unchanged before being uploaded to shared storage, in seconds. Minimum: 1; Maximum: 2592000 (30 days), default value 60 seconds |
|
||||
| ssPageCacheSize | Number of shared storage page cache pages, in pages. Minimum: 4; Maximum: 1024 * 1024 * 1024, default value 4096 |
|
||||
| ssAutoMigrateIntervalSec | The trigger cycle for automatic upload of local data files to shared storage, in seconds. Minimum: 600; Maximum: 100000. Default value 3600 |
|
||||
|
||||
|
||||
`ssAccessString` options for S3 compatible object storage providers:
|
||||
|
||||
Name | Description
|
||||
----------------|----------------------------------------------
|
||||
endpoint | host name / ip address, and optional port number of the object storage server.
|
||||
bucket | bucket name.
|
||||
protocol | `https` or `http`, `https` is the default.
|
||||
uriStyle | `virtualHost` or `path`, `virtualHost` is the default, but please note that some object storage providers only support one of them.
|
||||
region | object storage service region, optional.
|
||||
accessKeyId | your access key id.
|
||||
secretAccessKey | your secret access key.
|
||||
chunkSize | chunk size in MB, files larger than this size will use multipart upload, default is 64.
|
||||
maxChunks | max number of allowed chunks in a multipart upload, default is 10000.
|
||||
maxRetry | max retry times when encounter retryable errors, default is 3, negative value means unlimited retry.
|
||||
|
||||
For example:
|
||||
|
||||
```
|
||||
ssAccessString s3:endpoint=s3.amazonaws.com;bucket=mybucket;uriStyle=path;protocol=https;accessKeyId=AKMYACCESSKEY;secretAccessKey=MYSECRETACCESSKEY;region=us-east-2;chunkSize=64;maxChunks=10000;maxRetry=3
|
||||
```
|
||||
|
||||
### Check Configuration Parameter Availability
|
||||
|
||||
After configuring s3 in `taos.cfg`, the availability of the configured S3 service can be checked using the `taosd` command with the `checks3` parameter:
|
||||
After configuring shared storage in `taos.cfg`, the availability of the configured shared storage service can be checked using the `taosd` command with the `checkss` parameter:
|
||||
|
||||
```shell
|
||||
taosd --checks3
|
||||
taosd --checkss
|
||||
```
|
||||
|
||||
If the configured S3 service is inaccessible, this command will output the corresponding error information during execution.
|
||||
If the configured shared storage service is inaccessible, this command will output the corresponding error information during execution.
|
||||
|
||||
### Create a DB Using S3
|
||||
### Create a DB Using Shared Storage
|
||||
|
||||
After configuration, you can start the TDengine cluster and create a database using S3, for example:
|
||||
After configuration, you can start the TDengine cluster and create a database using shared storage, for example:
|
||||
|
||||
```sql
|
||||
create database demo_db duration 1d s3_keeplocal 3d;
|
||||
create database demo_db duration 1d ss_keeplocal 3d;
|
||||
```
|
||||
|
||||
After writing time-series data into the database `demo_db`, time-series data older than 3 days will automatically be segmented and stored in S3 storage.
|
||||
After writing time-series data into the database `demo_db`, time-series data older than 3 days will automatically be segmented and migrated into shared storage.
|
||||
|
||||
By default, mnode issues S3 data migration check commands every hour. If there is time-series data that needs to be uploaded, it will automatically be segmented and stored in S3 storage. You can also manually trigger this operation using SQL commands, initiated by the user, with the following syntax:
|
||||
If the value of `ssEnabled` is `2`, mnode issues shared storage data migration check commands every hour. If there is time-series data that needs to be uploaded, it will automatically be segmented and migrated into shared storage. This process can also be manually trigger using SQL commands, with the following syntax:
|
||||
|
||||
```sql
|
||||
s3migrate database <db_name>;
|
||||
ssmigrate database <db_name>;
|
||||
```
|
||||
|
||||
Note that if the value of `ssEnabled` is `1`, mnode will never trigger the migration process automatically and user must trigger it manually if a migration is desired.
|
||||
|
||||
Detailed DB parameters are shown in the table below:
|
||||
|
||||
| # | Parameter | Default | Min | Max | Description |
|
||||
| :--- | :----------- | :----- | :----- | :------ | :----------------------------------------------------------- |
|
||||
| 1 | s3_keeplocal | 365 | 1 | 365000 | The number of days data is kept locally, i.e., how long data files are retained on local disks before they can be uploaded to S3. Default unit: days, supports m (minutes), h (hours), and d (days) |
|
||||
| 2 | s3_chunkpages | 131072 | 131072 | 1048576 | The size threshold for uploading objects, same as the tsdb_pagesize parameter, unmodifiable, in TSDB pages |
|
||||
| 3 | s3_compact | 1 | 0 | 1 | Whether to automatically perform compact operation when TSDB files are first uploaded to S3. |
|
||||
| 1 | ss_keeplocal | 365 | 1 | 365000 | The number of days data is kept locally, i.e., how long data files are retained on local disks before they can be migrated to shared storage. Default unit: days, supports m (minutes), h (hours), and d (days) |
|
||||
| 2 | ss_chunkpages | 131072 | 131072 | 1048576 | The size threshold for migrating objects, same as the tsdb_pagesize parameter, unmodifiable, in TSDB pages |
|
||||
| 3 | ss_compact | 1 | 0 | 1 | Whether to automatically perform compact operation when before the first migration of TSDB files |
|
||||
|
||||
### Estimation of Read and Write Operations for Object Storage
|
||||
|
||||
The cost of using object storage services is related to the amount of data stored and the number of requests. Below, we discuss the processes of data upload and download separately.
|
||||
|
||||
#### Data Upload
|
||||
#### Data Migration
|
||||
|
||||
When the TSDB time-series data exceeds the time specified by the `s3_keeplocal` parameter, the related data files will be split into multiple file blocks, each with a default size of 512 MB (`s3_chunkpages * tsdb_pagesize`). Except for the last file block, which is retained on the local file system, the rest of the file blocks are uploaded to the object storage service.
|
||||
When the TSDB time-series data exceeds the time specified by the `ss_keeplocal` parameter, the leader vnode will split the related data files into multiple file blocks, each with a default size of 512 MB (`ss_chunkpages * tsdb_pagesize`), except the last file block. All of the file blocks are uploaded to the shared storage, and all of the file blocks except the last one are removed from local disk.
|
||||
|
||||
When creating a database, you can adjust the size of each file block through the `ss_chunkpages` parameter, thereby controlling the number of uploads for each data file.
|
||||
|
||||
Other types of files such as head, stt, sma, etc., are also uploaded to the shared storage, but retained on the local file system to speed up pre-computed related queries.
|
||||
|
||||
The leader vnode also creats/updates a manifests file in the shared storage to save the migration information after the migration process finishes successfully.
|
||||
|
||||
The follower vnodes will then download the last block of the data file and all other files to local from the shared storage to complete the overall migration process, they rely on the manifests file to decide which specific files should be downloaded.
|
||||
|
||||
That's, the read/write count during data migration is about:
|
||||
|
||||
```text
|
||||
Upload Count = Data File Size / (s3_chunkpages * tsdb_pagesize) - 1
|
||||
```
|
||||
|
||||
When creating a database, you can adjust the size of each file block through the `s3_chunkpages` parameter, thereby controlling the number of uploads for each data file.
|
||||
|
||||
Other types of files such as head, stt, sma, etc., are retained on the local file system to speed up pre-computed related queries.
|
||||
Read/Write Count = Number of Data Blocks + (Number of Other Files + 2) x Number of vnodes
|
||||
```
|
||||
|
||||
#### Data Download
|
||||
|
||||
|
|
@ -143,10 +173,14 @@ Adjacent multiple data pages are downloaded as a single data block from object s
|
|||
Download Count = Number of Data Blocks Needed for Query - Number of Cached Data Blocks
|
||||
```
|
||||
|
||||
The page cache is a memory cache, and data needs to be re-downloaded after a node restart. The cache uses an LRU (Least Recently Used) strategy, and when there is not enough cache space, the least recently used data will be evicted. The size of the cache can be adjusted through the `s3PageCacheSize` parameter; generally, the larger the cache, the fewer the downloads.
|
||||
The page cache is a memory cache, and data needs to be re-downloaded after a node restart. The cache uses an LRU (Least Recently Used) strategy, and when there is not enough cache space, the least recently used data will be evicted. The size of the cache can be adjusted through the `ssPageCacheSize` parameter; generally, the larger the cache, the fewer the downloads.
|
||||
|
||||
## Azure Blob Storage
|
||||
|
||||
Support for Azure Blob object storage is temporarily disabled, if you are using this service before the release of TDengine version 3.3.7.0, please wait for the new version.
|
||||
|
||||
<!--
|
||||
|
||||
This section describes how to use Microsoft Azure Blob object storage in TDengine Enterprise. This feature is an extension of the 'Object Storage' feature discussed in the previous section and depends additionally on the Flexify service's S3 gateway. With proper parameter configuration, most of the colder time-series data can be stored in the Azure Blob service.
|
||||
|
||||
### Flexify Service
|
||||
|
|
@ -182,3 +216,5 @@ The user interface is the same as S3, but the configuration of the following thr
|
|||
| 3 | s3BucketName | test-container | Container name |
|
||||
|
||||
The `fd2d01c73` is the account ID; Microsoft Blob storage service only supports the Https protocol, not Http.
|
||||
|
||||
-->
|
||||
|
|
|
|||
|
|
@ -228,3 +228,7 @@ taosd -y {encryptKey}
|
|||
```
|
||||
|
||||
Updating the key configuration requires stopping taosd first, and using the exact same key, meaning the key cannot be changed after the database is created.
|
||||
|
||||
### encrypt user password
|
||||
|
||||
The user password is stored as an MD5 string by default. This behavior can be changed by configuring the `encryptPassAlgorithm` parameter. By default, `encryptPassAlgorithm` is unset. When `encryptPassAlgorithm` is set to `SM4`, user passwords are stored as encrypted strings using the SM4 encryption algorithm. The encryption key must be configured before setting `encryptPassAlgorithm`.
|
||||
|
|
@ -4,7 +4,7 @@ slug: /operations-and-maintenance/active-active-deployment
|
|||
---
|
||||
|
||||
import Image from '@theme/IdealImage';
|
||||
import imgDual from '../assets/active-standby-deployment-01.png';
|
||||
import imgDual from '../assets/active-active-01.png';
|
||||
|
||||
import Enterprise from '../assets/resources/_enterprise.mdx';
|
||||
|
||||
|
|
|
|||
2
docs/en/10-third-party/05-bi/15-spark.md
vendored
2
docs/en/10-third-party/05-bi/15-spark.md
vendored
|
|
@ -151,7 +151,7 @@ Data reading is achieved through table mapping and is completed in four steps:
|
|||
Dataset<Row> df = reader.option("dbtable", dbtable).load();
|
||||
// show
|
||||
df.show(Integer.MAX_VALUE, 40, false);
|
||||
df.close()
|
||||
df.close();
|
||||
```
|
||||
|
||||
4. Close the Interaction.
|
||||
|
|
|
|||
|
|
@ -166,21 +166,19 @@ The effective value of charset is UTF-8.
|
|||
|
||||
### Storage Related
|
||||
|
||||
| Parameter Name | Supported Version | Dynamic Modification | Description |
|
||||
| ---------------------- | ----------------- | ---------------------------------- | ------------------------------------------------------------ |
|
||||
| dataDir | | Not supported | Directory for data files, all data files are written to this directory, default value /var/lib/taos |
|
||||
| tempDir | | Not supported | Specifies the directory for generating temporary files during system operation, default value /tmp |
|
||||
| minimalDataDirGB | | Not supported | Minimum space to be reserved in the time-series data storage directory specified by dataDir, in GB, default value 2 |
|
||||
| minimalTmpDirGB | | Not supported | Minimum space to be reserved in the temporary file directory specified by tempDir, in GB, default value 1 |
|
||||
| minDiskFreeSize | After 3.1.1.0 | Supported, effective immediately | When the available space on a disk is less than or equal to this threshold, the disk will no longer be selected for generating new data files, unit is bytes, range 52428800-2199023255552, default value 52428800; Enterprise parameter |
|
||||
| s3MigrateIntervalSec | After 3.3.4.3 | Supported, effective immediately | Trigger cycle for automatic upload of local data files to S3, in seconds. Minimum: 600; Maximum: 100000. Default value 3600; Enterprise parameter |
|
||||
| s3MigrateEnabled | After 3.3.4.3 | Supported, effective immediately | Whether to automatically perform S3 migration, default value is 0, which means auto S3 migration is off, can be set to 1; Enterprise parameter |
|
||||
| s3Accesskey | After 3.3.4.3 | Supported, effective after restart | Colon-separated user SecretId:SecretKey, for example AKIDsQmwsfKxTo2A6nGVXZN0UlofKn6JRRSJ:lIdoy99ygEacU7iHfogaN2Xq0yumSm1E; Enterprise parameter |
|
||||
| s3Endpoint | After 3.3.4.3 | Supported, effective after restart | COS service domain name in the user's region, supports http and https, the region of the bucket must match the endpoint, otherwise it cannot be accessed; Enterprise parameter |
|
||||
| s3BucketName | After 3.3.4.3 | Supported, effective after restart | Bucket name, followed by a hyphen and the AppId of the user registered COS service, where AppId is unique to COS, not present in AWS and Alibaba Cloud, needs to be part of the bucket name, separated by a hyphen; parameter values are string type, but do not need quotes; for example test0711-1309024725; Enterprise parameter |
|
||||
| s3PageCacheSize | After 3.3.4.3 | Supported, effective after restart | Number of S3 page cache pages, range 4-1048576, unit is pages, default value 4096; Enterprise parameter |
|
||||
| s3UploadDelaySec | After 3.3.4.3 | Supported, effective immediately | How long a data file remains unchanged before being uploaded to S3, range 1-2592000 (30 days), in seconds, default value 60; Enterprise parameter |
|
||||
| cacheLazyLoadThreshold | | Supported, effective immediately | Internal parameter, cache loading strategy |
|
||||
| Parameter Name | Supported Version | Dynamic Modification | Description |
|
||||
| ------------------------ | ----------------- | ---------------------------------- | ------------------------------------------------------------ |
|
||||
| dataDir | | Not supported | Directory for data files, all data files are written to this directory, default value /var/lib/taos |
|
||||
| tempDir | | Not supported | Specifies the directory for generating temporary files during system operation, default value /tmp |
|
||||
| minimalDataDirGB | | Not supported | Minimum space to be reserved in the time-series data storage directory specified by dataDir, in GB, default value 2 |
|
||||
| minimalTmpDirGB | | Not supported | Minimum space to be reserved in the temporary file directory specified by tempDir, in GB, default value 1 |
|
||||
| minDiskFreeSize | After 3.1.1.0 | Supported, effective immediately | When the available space on a disk is less than or equal to this threshold, the disk will no longer be selected for generating new data files, unit is bytes, range 52428800-2199023255552, default value 52428800; Enterprise parameter |
|
||||
| ssAutoMigrateIntervalSec | After 3.3.7.0 | Supported, effective immediately | Trigger cycle for automatic upload of local data files to shared storage, in seconds. Minimum: 600; Maximum: 100000. Default value 3600; Enterprise parameter |
|
||||
| ssEnabled | After 3.3.7.0 | Supported, effective after restart | Whether to enable shared storage, default value is 0, which means disabled, 1 means only enable manual shared storage migration, 2 means enable auto shared storage migration |
|
||||
| ssAccessString | After 3.3.7.0 | Supported, effective after restart | A string which contains various options for accessing the shared storage, the format is `<device-type>:<option-name>=<option-value>;<option-name>=<option-value>;...`, the possible options vary from shared storage providers, please refer related document for details |
|
||||
| ssPageCacheSize | After 3.3.7.0 | Supported, effective after restart | Number of shared storage page cache pages, range 4-1048576, unit is pages, default value 4096; Enterprise parameter |
|
||||
| ssUploadDelaySec | After 3.3.7.0 | Supported, effective immediately | How long a data file remains unchanged before being uploaded to S3, range 1-2592000 (30 days), in seconds, default value 60; Enterprise parameter |
|
||||
| cacheLazyLoadThreshold | | Supported, effective immediately | Internal parameter, cache loading strategy |
|
||||
|
||||
### Cluster Related
|
||||
|
||||
|
|
@ -210,6 +208,7 @@ The effective value of charset is UTF-8.
|
|||
| auditCreateTable | | Supported, effective immediately | Whether to enable audit feature for creating subtables; Enterprise parameter |
|
||||
| encryptAlgorithm | | Not supported | Data encryption algorithm; Enterprise parameter |
|
||||
| encryptScope | | Not supported | Encryption scope; Enterprise parameter |
|
||||
| encryptPassAlgorithm |v3.3.7.0 |Supported, effective immediately |Switch for saving user password as encrypted string |
|
||||
| enableWhiteList | | Supported, effective immediately | Switch for whitelist feature; Enterprise parameter |
|
||||
| syncLogBufferMemoryAllowed | | Supported, effective immediately | Maximum memory allowed for sync log cache messages for a dnode, in bytes, range 104857600-INT64_MAX, default value is 1/10 of server memory, effective from versions 3.1.3.2/3.3.2.13 |
|
||||
| syncElectInterval | | Not supported | Internal parameter, for debugging synchronization module |
|
||||
|
|
|
|||
|
|
@ -41,10 +41,12 @@ The taosAdapter provides the following features:
|
|||
icinga2 is a software for collecting check results metrics and performance data. Visit [https://icinga.com/docs/icinga-2/latest/doc/14-features/#opentsdb-writer](https://icinga.com/docs/icinga-2/latest/doc/14-features/#opentsdb-writer) for more information.
|
||||
- TCollector data writing:
|
||||
TCollector is a client process that collects data from local collectors and pushes it to OpenTSDB. Visit [http://opentsdb.net/docs/build/html/user_guide/utilities/tcollector.html](http://opentsdb.net/docs/build/html/user_guide/utilities/tcollector.html) for more information.
|
||||
- node_exporter data collection and writing:
|
||||
node_exporter is an exporter of machine metrics. Visit [https://github.com/prometheus/node_exporter](https://github.com/prometheus/node_exporter) for more information.
|
||||
- OpenMetrics data collection and writing:
|
||||
OpenMetrics is an emerging standard in the field of cloud-native monitoring. It extends and standardizes the Prometheus metric format and has become the de facto standard for modern monitoring tools. Visit [OpenMetrics Specification](https://github.com/prometheus/OpenMetrics/blob/main/specification/OpenMetrics.md) for more information.
|
||||
- Supports Prometheus remote_read and remote_write:
|
||||
remote_read and remote_write are Prometheus's data read-write separation cluster solutions. Visit [https://prometheus.io/blog/2019/10/10/remote-read-meets-streaming/#remote-apis](https://prometheus.io/blog/2019/10/10/remote-read-meets-streaming/#remote-apis) for more information.
|
||||
- node_exporter data collection and writing:
|
||||
node_exporter is an exporter of machine metrics. Visit [https://github.com/prometheus/node_exporter](https://github.com/prometheus/node_exporter) for more information.
|
||||
- RESTful API:
|
||||
[RESTful API](../../client-libraries/rest-api/)
|
||||
|
||||
|
|
@ -63,6 +65,7 @@ Supported InfluxDB parameters are as follows:
|
|||
- `u` TDengine username
|
||||
- `p` TDengine password
|
||||
- `ttl` the lifespan of automatically created subtables, determined by the TTL parameter of the first data entry in the subtable, which cannot be updated. For more information, please refer to the TTL parameter in the [table creation document](../../sql-manual/manage-tables/).
|
||||
- `table_name_key` the custom tag key for subtable names. If set, the subtable name will use the value of this tag key
|
||||
|
||||
Note: Currently, InfluxDB's token authentication method is not supported, only Basic authentication and query parameter verification are supported.
|
||||
Example: `curl --request POST http://127.0.0.1:6041/influxdb/v1/write?db=test --user "root:taosdata" --data-binary "measurement,host=host1 field1=2i,field2=2.0 1577836800000000000"`
|
||||
|
|
@ -92,18 +95,34 @@ You can use any client that supports the HTTP protocol to write data in OpenTSDB
|
|||
|
||||
<TCollector />
|
||||
|
||||
### OpenMetrics data collection and writing
|
||||
|
||||
OpenMetrics is an open standard supported by CNCF (Cloud Native Computing Foundation) that focuses on standardizing the collection and transmission of metric data.
|
||||
It serves as one of the core specifications for monitoring and observability systems in the cloud-native ecosystem.
|
||||
|
||||
Starting from version **3.3.7.0**, taosAdapter supports OpenMetrics v1.0.0 data collection and writing,
|
||||
while maintaining compatibility with Prometheus 0.0.4 protocol to ensure seamless integration with the Prometheus ecosystem.
|
||||
|
||||
To enable OpenMetrics data collection and writing, follow these steps:
|
||||
|
||||
1. Enable taosAdapter configuration by setting `open_metrics.enable`
|
||||
2. Configure OpenMetrics related settings
|
||||
3. Restart taosAdapter service
|
||||
|
||||
### Supports Prometheus remote_read and remote_write
|
||||
|
||||
<Prometheus />
|
||||
|
||||
### node_exporter data collection and writing
|
||||
|
||||
Starting with version **3.3.7.0**, you can use the OpenMetrics plugin as a replacement for node_exporter to perform data collection and writing.
|
||||
|
||||
An exporter used by Prometheus that exposes hardware and operating system metrics from \*NIX kernels
|
||||
|
||||
- Enable configuration of taosAdapter node_exporter.enable
|
||||
- Set the relevant configuration for node_exporter
|
||||
- Restart taosAdapter
|
||||
|
||||
### Supports Prometheus remote_read and remote_write
|
||||
|
||||
<Prometheus />
|
||||
|
||||
### RESTful API
|
||||
|
||||
You can use any client that supports the HTTP protocol to write data to TDengine or query data from TDengine by accessing the RESTful interface URL `http://<fqdn>:6041/rest/sql`. For details, please refer to the [REST API documentation](../../client-libraries/rest-api/).
|
||||
|
|
@ -179,6 +198,7 @@ taosAdapter uses a connection pool to manage connections to TDengine, improving
|
|||
- collectd data writing
|
||||
- StatsD data writing
|
||||
- node_exporter data collection writing
|
||||
- OpenMetrics data collection and writing
|
||||
- Prometheus remote_read and remote_write
|
||||
|
||||
The configuration parameters for the connection pool are as follows:
|
||||
|
|
@ -263,6 +283,7 @@ The `smlAutoCreateDB` parameter only affects the following interfaces:
|
|||
- collectd data writing
|
||||
- StatsD data writing
|
||||
- node_exporter data writing
|
||||
- OpenMetrics data collection and writing
|
||||
|
||||
#### Parameter Description
|
||||
|
||||
|
|
@ -386,68 +407,6 @@ curl --location --request PUT 'http://127.0.0.1:6041/config' \
|
|||
|
||||
Enable/disable InfluxDB protocol support (Default: `true`)
|
||||
|
||||
#### Node Exporter Configuration
|
||||
|
||||
- **`node_exporter.enable`**
|
||||
|
||||
Enable node_exporter data collection (Default: `false`)
|
||||
|
||||
- **`node_exporter.db`**
|
||||
|
||||
Target database name (Default: `"node_exporter"`)
|
||||
|
||||
- **`node_exporter.urls`**
|
||||
|
||||
Service endpoints (Default: `["http://localhost:9100"]`)
|
||||
|
||||
- **`node_exporter.gatherDuration`**
|
||||
|
||||
Collection interval (Default: `5s`)
|
||||
|
||||
- **`node_exporter.responseTimeout`**
|
||||
|
||||
Request timeout (Default: `5s`)
|
||||
|
||||
- **`node_exporter.user`**
|
||||
|
||||
Database username (Default: `"root"`)
|
||||
|
||||
- **`node_exporter.password`**
|
||||
|
||||
Database password (Default: `"taosdata"`)
|
||||
|
||||
- **`node_exporter.ttl`**
|
||||
|
||||
Data TTL (Default: `0`)
|
||||
|
||||
- **`node_exporter.httpUsername`**
|
||||
|
||||
HTTP Basic Auth username (Optional)
|
||||
|
||||
- **`node_exporter.httpPassword`**
|
||||
|
||||
HTTP Basic Auth password (Optional)
|
||||
|
||||
- **`node_exporter.httpBearerTokenString`**
|
||||
|
||||
HTTP Bearer Token (Optional)
|
||||
|
||||
- **`node_exporter.insecureSkipVerify`**
|
||||
|
||||
Skip SSL verification (Default: `true`)
|
||||
|
||||
- **`node_exporter.certFile`**
|
||||
|
||||
Client certificate path (Optional)
|
||||
|
||||
- **`node_exporter.keyFile`**
|
||||
|
||||
Client key path (Optional)
|
||||
|
||||
- **`node_exporter.caCertFile`**
|
||||
|
||||
CA certificate path (Optional)
|
||||
|
||||
#### OpenTSDB Configuration
|
||||
|
||||
- **`opentsdb.enable`**
|
||||
|
|
@ -566,6 +525,134 @@ curl --location --request PUT 'http://127.0.0.1:6041/config' \
|
|||
|
||||
Enable Prometheus protocol (Default: `true`)
|
||||
|
||||
#### OpenMetrics Configuration
|
||||
|
||||
- **`open_metrics.enable`**
|
||||
|
||||
Enable/disable OpenMetrics data collection (Default: `false`).
|
||||
|
||||
- **`open_metrics.user`**
|
||||
|
||||
Username for TDengine connection (Default: `"root"`).
|
||||
|
||||
- **`open_metrics.password`**
|
||||
|
||||
Password for TDengine connection (Default: `"taosdata"`).
|
||||
|
||||
- **`open_metrics.urls`**
|
||||
|
||||
List of OpenMetrics data collection endpoints (Default: `["http://localhost:9100"]`, automatically appends `/metrics` if no route specified).
|
||||
|
||||
- **`open_metrics.dbs`**
|
||||
|
||||
Target databases for data writing (Default: `["open_metrics"]`, must match number of collection URLs).
|
||||
|
||||
- **`open_metrics.responseTimeoutSeconds`**
|
||||
|
||||
Collection timeout in seconds (Default: `[5]`, must match number of collection URLs).
|
||||
|
||||
- **`open_metrics.httpUsernames`**
|
||||
|
||||
Basic authentication usernames (If enabled, must match number of collection URLs, Default: empty).
|
||||
|
||||
- **`open_metrics.httpPasswords`**
|
||||
|
||||
Basic authentication passwords (If enabled, must match number of collection URLs, Default: empty).
|
||||
|
||||
- **`open_metrics.httpBearerTokenStrings`**
|
||||
|
||||
Bearer token authentication strings (If enabled, must match number of collection URLs, Default: empty).
|
||||
|
||||
- **`open_metrics.caCertFiles`**
|
||||
|
||||
Root certificate file paths (If enabled, must match number of collection URLs, Default: empty).
|
||||
|
||||
- **`open_metrics.certFiles`**
|
||||
|
||||
Client certificate file paths (If enabled, must match number of collection URLs, Default: empty).
|
||||
|
||||
- **`open_metrics.keyFiles`**
|
||||
|
||||
Client certificate key file paths (If enabled, must match number of collection URLs, Default: empty).
|
||||
|
||||
- **`open_metrics.insecureSkipVerify`**
|
||||
|
||||
Skip HTTPS certificate verification (Default: `true`).
|
||||
|
||||
- **`open_metrics.gatherDurationSeconds`**
|
||||
|
||||
Collection interval in seconds (Default: `[5]`, must match number of collection URLs).
|
||||
|
||||
- **`open_metrics.ttl`**
|
||||
|
||||
Table Time-To-Live in seconds (`0` means no expiration, if enabled must match number of collection URLs, Default: empty).
|
||||
|
||||
- **`open_metrics.ignoreTimestamp`**
|
||||
|
||||
Ignore timestamps in collected data (uses collection time if ignored, Default: `false`).
|
||||
|
||||
#### Node Exporter Configuration
|
||||
|
||||
- **`node_exporter.enable`**
|
||||
|
||||
Enable node_exporter data collection (Default: `false`)
|
||||
|
||||
- **`node_exporter.db`**
|
||||
|
||||
Target database name (Default: `"node_exporter"`)
|
||||
|
||||
- **`node_exporter.urls`**
|
||||
|
||||
Service endpoints (Default: `["http://localhost:9100"]`)
|
||||
|
||||
- **`node_exporter.gatherDuration`**
|
||||
|
||||
Collection interval (Default: `5s`)
|
||||
|
||||
- **`node_exporter.responseTimeout`**
|
||||
|
||||
Request timeout (Default: `5s`)
|
||||
|
||||
- **`node_exporter.user`**
|
||||
|
||||
Database username (Default: `"root"`)
|
||||
|
||||
- **`node_exporter.password`**
|
||||
|
||||
Database password (Default: `"taosdata"`)
|
||||
|
||||
- **`node_exporter.ttl`**
|
||||
|
||||
Data TTL (Default: `0`)
|
||||
|
||||
- **`node_exporter.httpUsername`**
|
||||
|
||||
HTTP Basic Auth username (Optional)
|
||||
|
||||
- **`node_exporter.httpPassword`**
|
||||
|
||||
HTTP Basic Auth password (Optional)
|
||||
|
||||
- **`node_exporter.httpBearerTokenString`**
|
||||
|
||||
HTTP Bearer Token (Optional)
|
||||
|
||||
- **`node_exporter.insecureSkipVerify`**
|
||||
|
||||
Skip SSL verification (Default: `true`)
|
||||
|
||||
- **`node_exporter.certFile`**
|
||||
|
||||
Client certificate path (Optional)
|
||||
|
||||
- **`node_exporter.keyFile`**
|
||||
|
||||
Client key path (Optional)
|
||||
|
||||
- **`node_exporter.caCertFile`**
|
||||
|
||||
CA certificate path (Optional)
|
||||
|
||||
### Metrics Reporting Configuration
|
||||
|
||||
taosAdapter reports metrics to taosKeeper with these parameters:
|
||||
|
|
@ -656,6 +743,22 @@ Configuration Parameters and their corresponding environment variables:
|
|||
| `node_exporter.ttl` | `TAOS_ADAPTER_NODE_EXPORTER_TTL` |
|
||||
| `node_exporter.urls` | `TAOS_ADAPTER_NODE_EXPORTER_URLS` |
|
||||
| `node_exporter.user` | `TAOS_ADAPTER_NODE_EXPORTER_USER` |
|
||||
| `open_metrics.enable` | `TAOS_ADAPTER_OPEN_METRICS_ENABLE` |
|
||||
| `open_metrics.user` | `TAOS_ADAPTER_OPEN_METRICS_USER` |
|
||||
| `open_metrics.password` | `TAOS_ADAPTER_OPEN_METRICS_PASSWORD` |
|
||||
| `open_metrics.urls` | `TAOS_ADAPTER_OPEN_METRICS_URLS` |
|
||||
| `open_metrics.dbs` | `TAOS_ADAPTER_OPEN_METRICS_DBS` |
|
||||
| `open_metrics.responseTimeoutSeconds` | `TAOS_ADAPTER_OPEN_METRICS_RESPONSE_TIMEOUT_SECONDS` |
|
||||
| `open_metrics.httpUsernames` | `TAOS_ADAPTER_OPEN_METRICS_HTTP_USERNAMES` |
|
||||
| `open_metrics.httpPasswords` | `TAOS_ADAPTER_OPEN_METRICS_HTTP_PASSWORDS` |
|
||||
| `open_metrics.httpBearerTokenStrings` | `TAOS_ADAPTER_OPEN_METRICS_HTTP_BEARER_TOKEN_STRINGS` |
|
||||
| `open_metrics.caCertFiles` | `TAOS_ADAPTER_OPEN_METRICS_CA_CERT_FILES` |
|
||||
| `open_metrics.certFiles` | `TAOS_ADAPTER_OPEN_METRICS_CERT_FILES` |
|
||||
| `open_metrics.keyFiles` | `TAOS_ADAPTER_OPEN_METRICS_KEY_FILES` |
|
||||
| `open_metrics.insecureSkipVerify` | `TAOS_ADAPTER_OPEN_METRICS_INSECURE_SKIP_VERIFY` |
|
||||
| `open_metrics.gatherDurationSeconds` | `TAOS_ADAPTER_OPEN_METRICS_GATHER_DURATION_SECONDS` |
|
||||
| `open_metrics.ignoreTimestamp` | `TAOS_ADAPTER_OPEN_METRICS_IGNORE_TIMESTAMP` |
|
||||
| `open_metrics.ttl` | `TAOS_ADAPTER_OPEN_METRICS_TTL` |
|
||||
| `opentsdb.enable` | `TAOS_ADAPTER_OPENTSDB_ENABLE` |
|
||||
| `opentsdb_telnet.batchSize` | `TAOS_ADAPTER_OPENTSDB_TELNET_BATCH_SIZE` |
|
||||
| `opentsdb_telnet.dbs` | `TAOS_ADAPTER_OPENTSDB_TELNET_DBS` |
|
||||
|
|
|
|||
|
|
@ -84,6 +84,9 @@ taosBenchmark -f <json file>
|
|||
- **-o/--output \<file>** :
|
||||
Path of the output file for results, default value is ./output.txt.
|
||||
|
||||
- **-j/--output-json-file \<file>** :
|
||||
Path of the output JSON file for results.
|
||||
|
||||
- **-T/--thread \<threadNum>** :
|
||||
Number of threads for inserting data, default is 8.
|
||||
|
||||
|
|
@ -99,7 +102,7 @@ taosBenchmark -f <json file>
|
|||
- **-t/--tables \<tableNum>** :
|
||||
Specifies the number of subtables, default is 10000.
|
||||
|
||||
-**-s/ --start-timestamp \<NUMBER>**:
|
||||
- **-s/--start-timestamp \<NUMBER>**:
|
||||
Specify start timestamp to insert data for each child table
|
||||
|
||||
- **-S/--timestampstep \<stepLength>** :
|
||||
|
|
@ -204,6 +207,8 @@ The parameters listed in this section apply to all functional modes.
|
|||
|
||||
- **password** : Password for connecting to the TDengine server, default value is taosdata.
|
||||
|
||||
- **result_json_file**：The path to the result output JSON file. If not configured, the file will not be output.
|
||||
|
||||
### Insertion Configuration Parameters
|
||||
|
||||
In insertion scenarios, `filetype` must be set to `insert`. For this parameter and other common parameters, see Common Configuration Parameters.
|
||||
|
|
@ -211,7 +216,7 @@ In insertion scenarios, `filetype` must be set to `insert`. For this parameter a
|
|||
- **keep_trying**: Number of retries after failure, default is no retry. Requires version v3.0.9 or above.
|
||||
|
||||
- **trying_interval**: Interval between retries in milliseconds, effective only when retries are specified in keep_trying. Requires version v3.0.9 or above.
|
||||
- **childtable_from and childtable_to**: Specifies the range of child tables to write to, the interval is [childtable_from, childtable_to).
|
||||
- **childtable_from and childtable_to**: Specifies the range of child tables to write to, the interval is [childtable_from, childtable_to].
|
||||
|
||||
- **continue_if_fail**: Allows users to define behavior after failure
|
||||
|
||||
|
|
@ -283,6 +288,10 @@ Parameters related to supertable creation are configured in the `super_tables` s
|
|||
|
||||
- **tags_file** : Effective only when insert_mode is taosc, rest. The final value of the tag is related to childtable_count, if the tag data rows in the csv file are less than the given number of subtables, then the csv file data will be read in a loop until the childtable_count specified subtable number is generated; otherwise, only childtable_count rows of tag data will be read. Thus, the final number of subtables generated is the smaller of the two.
|
||||
|
||||
- **use_tag_table_name**：When set to yes, the first column in the tag data within the CSV file will be the name of the subtable to be created; otherwise, the system will automatically generate subtable names for creation.
|
||||
|
||||
- **primary_key_name**：Specify the name of the primary key for the supertable. If not specified, the default is ts.
|
||||
|
||||
- **primary_key** : Specifies whether the supertable has a composite primary key, values are 1 and 0, composite primary key columns can only be the second column of the supertable, after specifying the generation of composite primary keys, ensure that the second column meets the data type of composite primary keys, otherwise an error will occur
|
||||
- **repeat_ts_min** : Numeric type, when composite primary key is enabled, specifies the minimum number of records with the same timestamp to be generated, the number of records with the same timestamp is a random value within the range [repeat_ts_min, repeat_ts_max], when the minimum value equals the maximum value, it is a fixed number
|
||||
- **repeat_ts_max** : Numeric type, when composite primary key is enabled, specifies the maximum number of records with the same timestamp to be generated
|
||||
|
|
@ -361,8 +370,6 @@ Specify the configuration parameters for tag and data columns in `super_tables`
|
|||
|
||||
- **result_file** : The path to the result output file, default is ./output.txt.
|
||||
|
||||
- **result_json_file**：The path to the result output JSON file. If not configured, the file will not be output.
|
||||
|
||||
- **confirm_parameter_prompt** : A toggle parameter that requires user confirmation after a prompt to continue. The value can be "yes" or "no", by default "no".
|
||||
|
||||
- **interlace_rows** : Enables interleaved insertion mode and specifies the number of rows to insert into each subtable at a time. Interleaved insertion mode refers to inserting the specified number of rows into each subtable in sequence and repeating this process until all subtable data has been inserted. The default is 0, meaning data is inserted into one subtable completely before moving to the next.
|
||||
|
|
@ -493,6 +500,7 @@ For the following parameters, see the description of [Subscription](../../../adv
|
|||
| 17 | GEOMETRY | geometry
|
||||
| 18 | JSON | json
|
||||
| 19 | DECIMAL | decimal
|
||||
| 20 | BLOB | blob
|
||||
|
||||
Note: Data types in the taosBenchmark configuration file must be in lowercase to be recognized.
|
||||
|
||||
|
|
|
|||
|
|
@ -43,7 +43,8 @@ In TDengine, the following data types can be used in the data model of basic tab
|
|||
| 16 | VARCHAR | Custom | Alias for BINARY type |
|
||||
| 17 | GEOMETRY | Custom | Geometry type, supported starting from version 3.1.0.0 |
|
||||
| 18 | VARBINARY | Custom | Variable-length binary data, supported starting from version 3.1.1.0 |
|
||||
| 19 | DECIMAL | 8 or 16 | High-precision numeric type. The range of values depends on the precision and scale specified in the type. Supported starting from version 3.3.6. See the description below. |
|
||||
| 19 | DECIMAL | 8 or 16 | High-precision numeric type. The range of values depends on the precision and scale specified in the type. Supported starting from version 3.3.6. See the description below. |
|
||||
| 20 | BLOB | 4M | Variable-length binary data, supported starting from version 3.3.7.0 |
|
||||
|
||||
:::note
|
||||
|
||||
|
|
@ -61,6 +62,7 @@ In TDengine, the following data types can be used in the data model of basic tab
|
|||
- In SQL statements, the type of numerical values will be determined based on the presence of a decimal point or the use of scientific notation, so care must be taken to avoid type overflow. For example, 9999999999999999999 will be considered to exceed the upper boundary of long integers and overflow, while 9999999999999999999.0 will be considered a valid floating point number.
|
||||
- VARBINARY is a data type for storing binary data, with a maximum length of 65,517 bytes for data columns and 16,382 bytes for label columns. Binary data can be written via SQL or schemaless methods (needs to be converted to a string starting with \x), or through stmt methods (can use binary directly). Displayed as hexadecimal starting with \x.
|
||||
|
||||
- BLOB is a data type for storing binary data, with a maximum length of 419,430,465 bytes for data columns. BLOB data can be written via SQL or schemaless methods (needs to be converted to a string starting with \x), or through stmt methods (can use binary directly). Displayed as hexadecimal starting with \x.
|
||||
:::
|
||||
|
||||
### DECIMAL Data Type
|
||||
|
|
@ -75,6 +77,13 @@ When performing operations between integer types and the `DECIMAL` type, the int
|
|||
|
||||
When querying `DECIMAL` type expressions, if the intermediate result of the calculation exceeds the maximum value that the current type can represent, a `DECIMAL_OVERFLOW` error is reported.
|
||||
|
||||
### BLOB Data type
|
||||
The BLOB data type is used for storing binary data, with a maximum length of 4,194,304 bytes. Binary data can be written via SQL or stmt2 by converting it to a string that starts with \x, or directly as binary data using the stmt interface. When displayed, BLOB data is shown in hexadecimal format starting with \x
|
||||
`Limitations`
|
||||
Only one BLOB column is allowed per table.
|
||||
BLOB columns are not supported as tag columns.
|
||||
Currently, BLOB is not supported in virtual tables or stream computing.
|
||||
Conditional filtering on BLOB columns is not supported.
|
||||
## Constants
|
||||
|
||||
TDengine supports multiple types of constants, details as shown in the table below:
|
||||
|
|
|
|||
|
|
@ -104,3 +104,8 @@ SHOW SUBSCRIPTIONS;
|
|||
```
|
||||
|
||||
Displays the allocation relationship and consumption information between consumers and vgroups.
|
||||
|
||||
## MQTT Data Subscription
|
||||
|
||||
TDengine 3.3.7.0 includes MQTT-based data subscription, allowing data to be subscribed to directly via an MQTT client. For details, refer to the MQTT Data Subscription section.
|
||||
|
||||
|
|
|
|||
|
|
@ -330,6 +330,7 @@ notification_definition:
|
|||
event_type:
|
||||
'WINDOW_OPEN'
|
||||
| 'WINDOW_CLOSE'
|
||||
| 'ON_TIME'
|
||||
|
||||
notification_options: {
|
||||
NOTIFY_HISTORY [0|1]
|
||||
|
|
@ -364,7 +365,7 @@ When the specified events are triggered, taosd will send a POST request to the g
|
|||
|
||||
The details of the event information depend on the type of window:
|
||||
|
||||
1. Time Window: At the opening, the start time is sent; at the closing, the start time, end time, and computation result are sent.
|
||||
1. Interval Window: At the opening, the start time is sent; at the closing, the start time, end time, and computation result are sent.
|
||||
2. State Window: At the opening, the start time, previous window's state, and current window's state are sent; at closing, the start time, end time, computation result, current window state, and next window state are sent.
|
||||
3. Session Window: At the opening, the start time is sent; at the closing, the start time, end time, and computation result are sent.
|
||||
4. Event Window: At the opening, the start time along with the data values and corresponding condition index that triggered the window opening are sent; at the closing, the start time, end time, computation result, and the triggering data value and condition index for window closure are sent.
|
||||
|
|
@ -384,8 +385,8 @@ An example structure for the notification message is shown below:
|
|||
"tableName": "t_a667a16127d3b5a18988e32f3e76cd30",
|
||||
"eventType": "WINDOW_OPEN",
|
||||
"eventTime": 1733284887097,
|
||||
"windowId": "window-id-67890",
|
||||
"windowType": "Time",
|
||||
"triggerId": "window-id-67890",
|
||||
"triggerType": "Interval",
|
||||
"groupId": "2650968222368530754",
|
||||
"windowStart": 1733284800000
|
||||
},
|
||||
|
|
@ -393,8 +394,8 @@ An example structure for the notification message is shown below:
|
|||
"tableName": "t_a667a16127d3b5a18988e32f3e76cd30",
|
||||
"eventType": "WINDOW_CLOSE",
|
||||
"eventTime": 1733284887197,
|
||||
"windowId": "window-id-67890",
|
||||
"windowType": "Time",
|
||||
"triggerId": "window-id-67890",
|
||||
"triggerType": "Interval",
|
||||
"groupId": "2650968222368530754",
|
||||
"windowStart": 1733284800000,
|
||||
"windowEnd": 1733284860000,
|
||||
|
|
@ -412,8 +413,8 @@ An example structure for the notification message is shown below:
|
|||
"tableName": "t_96f62b752f36e9b16dc969fe45363748",
|
||||
"eventType": "WINDOW_OPEN",
|
||||
"eventTime": 1733284887231,
|
||||
"windowId": "window-id-13579",
|
||||
"windowType": "Event",
|
||||
"triggerId": "window-id-13579",
|
||||
"triggerType": "Event",
|
||||
"groupId": "7533998559487590581",
|
||||
"windowStart": 1733284800000,
|
||||
"triggerCondition": {
|
||||
|
|
@ -428,8 +429,8 @@ An example structure for the notification message is shown below:
|
|||
"tableName": "t_96f62b752f36e9b16dc969fe45363748",
|
||||
"eventType": "WINDOW_CLOSE",
|
||||
"eventTime": 1733284887231,
|
||||
"windowId": "window-id-13579",
|
||||
"windowType": "Event",
|
||||
"triggerId": "window-id-13579",
|
||||
"triggerType": "Event",
|
||||
"groupId": "7533998559487590581",
|
||||
"windowStart": 1733284800000,
|
||||
"windowEnd": 1733284810000,
|
||||
|
|
@ -473,13 +474,28 @@ These fields are common to all event objects.
|
|||
1. "tableName": A string indicating the name of the target subtable.
|
||||
1. "eventType": A string representing the event type ("WINDOW_OPEN", "WINDOW_CLOSE", or "WINDOW_INVALIDATION").
|
||||
1. "eventTime": A long integer timestamp that indicates when the event was generated, accurate to the millisecond (i.e., the number of milliseconds since '00:00, Jan 1 1970 UTC').
|
||||
1. "windowId": A string representing the unique identifier for the window. This ID ensures that the open and close events for the same window can be correlated. In the case that taosd restarts due to a fault, some events may be sent repeatedly, but the windowId remains constant for the same window.
|
||||
1. "windowType": A string that indicates the window type ("Time", "State", "Session", "Event", or "Count").
|
||||
1. "triggerId": A string representing the unique identifier for the window. This ID ensures that the open and close events for the same window can be correlated. In the case that taosd restarts due to a fault, some events may be sent repeatedly, but the triggerId remains constant for the same window.
|
||||
1. "triggerType": A string that indicates the window type ("Time", "State", "Session", "Event", or "Count").
|
||||
1. "groupId": A string that uniquely identifies the corresponding group. If stream is partitioned by table, it matches the uid of that table.
|
||||
|
||||
#### Fields for Time Windows
|
||||
|
||||
These fields are present only when "windowType" is "Time".
|
||||
#### Fields for Period Trigger
|
||||
|
||||
These fields are relevant when triggerType is set to Period in the event object.
|
||||
|
||||
1. eventType is fixed as ON_TIME, the following field is included:
|
||||
1. "result": An object containing key-value pairs of the computed result columns and their corresponding values.
|
||||
|
||||
#### Fields for Sliding Trigger
|
||||
|
||||
These fields are relevant when triggerType is set to Sliding in the event object.
|
||||
|
||||
1. eventType is fixed as ON_TIME, the following field is included:
|
||||
1. "result": An object containing key-value pairs of the computed result columns and their corresponding values.
|
||||
|
||||
#### Fields for Interval Windows
|
||||
|
||||
These fields are present only when "triggerType" is "Interval".
|
||||
|
||||
1. When "eventType" is "WINDOW_OPEN", the following field is included:
|
||||
1. "windowStart": A long integer timestamp representing the start time of the window, matching the time precision of the result table.
|
||||
|
|
@ -490,7 +506,7 @@ These fields are present only when "windowType" is "Time".
|
|||
|
||||
#### Fields for State Windows
|
||||
|
||||
These fields are present only when "windowType" is "State".
|
||||
These fields are present only when "triggerType" is "State".
|
||||
|
||||
1. When "eventType" is "WINDOW_OPEN", the following fields are included:
|
||||
1. "windowStart": A long integer timestamp representing the start time of the window.
|
||||
|
|
@ -505,7 +521,7 @@ These fields are present only when "windowType" is "State".
|
|||
|
||||
#### Fields for Session Windows
|
||||
|
||||
These fields are present only when "windowType" is "Session".
|
||||
These fields are present only when "triggerType" is "Session".
|
||||
|
||||
1. When "eventType" is "WINDOW_OPEN", the following field is included:
|
||||
1. "windowStart": A long integer timestamp representing the start time of the window.
|
||||
|
|
@ -516,7 +532,7 @@ These fields are present only when "windowType" is "Session".
|
|||
|
||||
#### Fields for Event Windows
|
||||
|
||||
These fields are present only when "windowType" is "Event".
|
||||
These fields are present only when "triggerType" is "Event".
|
||||
|
||||
1. When "eventType" is "WINDOW_OPEN", the following fields are included:
|
||||
1. "windowStart": A long integer timestamp representing the start time of the window.
|
||||
|
|
@ -533,7 +549,7 @@ These fields are present only when "windowType" is "Event".
|
|||
|
||||
#### Fields for Count Windows
|
||||
|
||||
These fields are present only when "windowType" is "Count".
|
||||
These fields are present only when "triggerType" is "Count".
|
||||
|
||||
1. When "eventType" is "WINDOW_OPEN", the following field is included:
|
||||
1. "windowStart": A long integer timestamp representing the start time of the window.
|
||||
|
|
|
|||
|
|
@ -384,9 +384,6 @@ The list of keywords is as follows:
|
|||
|
||||
|Keyword|Description|
|
||||
|----------------------|-|
|
||||
| S3_CHUNKPAGES | |
|
||||
| S3_COMPACT | |
|
||||
| S3_KEEPLOCAL | |
|
||||
| SCHEMALESS | |
|
||||
| SCORES | |
|
||||
| SELECT | |
|
||||
|
|
@ -407,6 +404,10 @@ The list of keywords is as follows:
|
|||
| SNODES | |
|
||||
| SOFFSET | |
|
||||
| SPLIT | |
|
||||
| SS_CHUNKPAGES | |
|
||||
| SS_COMPACT | |
|
||||
| SS_KEEPLOCAL | |
|
||||
| SSMIGRATE | |
|
||||
| STABLE | |
|
||||
| STABLES | |
|
||||
| STAR | |
|
||||
|
|
|
|||
|
|
@ -114,6 +114,30 @@ DROP QNODE ON DNODE dnode_id;
|
|||
|
||||
Delete the QNODE on the DNODE with ID dnode_id, but this does not affect the status of that dnode.
|
||||
|
||||
## Create Subscription Node
|
||||
|
||||
```sql
|
||||
CREATE BNODE ON DNODE dnode_id PROTOCOL protocol;
|
||||
```
|
||||
|
||||
By default, there are no BNODEs when the system starts. Users can create BNODEs to start subscription services. Only one BNODE can be created on a DNODE. The `PROTOCOL` is optional, and the default is “mqtt”, if not provided; Other protocols will be added later. After bnode is created successfully, dnode will start the subprocess `taosmqtt` to provide subscription services.
|
||||
|
||||
## View Subscription Nodes
|
||||
|
||||
```sql
|
||||
SHOW BNODES;
|
||||
```
|
||||
|
||||
List all subscription nodes in the cluster, including their ID and the DNODE they are on.
|
||||
|
||||
## Delete Subscription Node
|
||||
|
||||
```sql
|
||||
DROP BNODE ON DNODE dnode_id;
|
||||
```
|
||||
|
||||
Delete the BNODE on the DNODE with ID dnode_id, and the `taosmqtt` subprocess on this dnode will exit to stop the subscription service.
|
||||
|
||||
## Query Cluster Status
|
||||
|
||||
```sql
|
||||
|
|
|
|||
|
|
@ -337,7 +337,7 @@ Note: Users with SYSINFO property set to 0 cannot view this table.
|
|||
| 6 | data3 | BIGINT | Data file size on tertiary storage, in KB |
|
||||
| 7 | cache_rdb | BIGINT | Size of last/last_row files, in KB |
|
||||
| 8 | table_meta | BIGINT | Size of meta files, in KB |
|
||||
| 9 | s3 | BIGINT | Size occupied on S3, in KB |
|
||||
| 9 | ss | BIGINT | Size occupied on shared storage, in KB |
|
||||
| 10 | raw_data | BIGINT | Estimated size of raw data, in KB |
|
||||
|
||||
note:
|
||||
|
|
|
|||
|
|
@ -653,7 +653,9 @@ This section showcases example code for common access methods to the TDengine cl
|
|||
|
||||
- Asynchronous query example: [Asynchronous Query](https://github.com/taosdata/TDengine/tree/main/docs/examples/c/asyncdemo.c)
|
||||
|
||||
- Parameter binding example: [Parameter Binding](https://github.com/taosdata/TDengine/tree/main/docs/examples/c/prepare.c)
|
||||
- Parameter binding example: [Parameter Binding](https://github.com/taosdata/TDengine/tree/main/docs/examples/c/stmt2_insert_demo.c)
|
||||
|
||||
- Parameter binding(old) example: [Parameter Binding](https://github.com/taosdata/TDengine/tree/main/docs/examples/c/prepare.c)
|
||||
|
||||
- Schemaless write example: [Schemaless Write](https://github.com/taosdata/TDengine/tree/main/docs/examples/c/schemaless.c)
|
||||
|
||||
|
|
@ -892,6 +894,99 @@ TDengine's asynchronous APIs all use a non-blocking call mode. Applications can
|
|||
|
||||
In addition to directly calling `taos_query()` for queries, TDengine also offers a Prepare API that supports parameter binding, similar in style to MySQL, currently only supporting the use of a question mark `?` to represent the parameter to be bound.
|
||||
|
||||
Starting from version 3.3.5.0, TDengine has significantly simplified the usage interface of the old parameter binding. This avoids the resource consumption of SQL syntax parsing when writing data through the parameter binding interface, and through batch binding, significantly improves writing performance in most cases. The typical operation steps are as follows:
|
||||
|
||||
1. Call `taos_stmt2_init()` to create a parameter binding object;
|
||||
2. Call `taos_stmt2_prepare()` to parse INSERT or SELECT statements;
|
||||
3. Call `taos_stmt2_bind_param()` to bind multiple subtables to a single supertable, with each subtable able to bind multiple rows of data;
|
||||
4. Call `taos_stmt2_exec()` to execute the prepared batch processing command;
|
||||
5. Steps 3 to 4 can be repeated to write more data rows without re-parsing SQL;
|
||||
6. After execution is complete, call `taos_stmt2_close()` to release all resources.
|
||||
|
||||
Note: If `taos_stmt2_exec()` executes successfully and there is no need to change the SQL statement, then it is possible to reuse the parsing result of `taos_stmt2_prepare()` and directly proceed to steps 3 to 4 to bind new data. However, if there is an error in execution, it is not recommended to continue working in the current context. Instead, it is advisable to release resources and start over from the `taos_stmt2_init()` step. You can check the specific error reason through `taos_stmt2_error`.
|
||||
|
||||
The difference between stmt2 and stmt is:
|
||||
- stmt2 supports batch binding of data in multiple tables, while stmt only supports binding data in a single table.
|
||||
- stmt2 supports asynchronous execution, while stmt only supports synchronous execution.
|
||||
- stmt2 supports efficient write mode and automatic table creation, while stmt does not support it.
|
||||
- stmt2 supports `insert into stb(...tbname,...)values(?,?,?)` syntax, while stmt does not support it.
|
||||
- stmt2 supports some labels/columns as fixed values, while stmt requires all columns to be `?`.
|
||||
|
||||
stmt upgrade stmt2 changes:
|
||||
1. Change `taos_stmt_init()` to `taos_stmt2_init()`, add `TAOS_STMT2_OPTION`.
|
||||
2. Change `taos_stmt_prepare()` to `taos_stmt2_prepare()`.
|
||||
3. Change `taos_stmt_set_tbname_tags`, `taos_stmt_bind_param()` and `taos_stmt_add_batch` to `taos_stmt2_bind_param()`, change `TAOS_MULTI_BIND` to `TAOS_STMT2_BINDV`.
|
||||
4. Change `taos_stmt_execute()` to `taos_stmt2_exec()`, add `affected_rows` parameter.
|
||||
5. Change `taos_stmt_close()` to `taos_stmt2_close()`.
|
||||
|
||||
The specific functions related to the interface are as follows (you can also refer to the [stmt2_insert_demo.c](https://github.com/taosdata/TDengine/tree/main/docs/examples/c/stmt2_insert_demo.c) file for how to use the corresponding functions):
|
||||
|
||||
- `TAOS_STMT2 *taos_stmt2_init(TAOS *taos, TAOS_STMT2_OPTION *option)`
|
||||
- **Interface Description**: Initializes a precompiled SQL statement object.
|
||||
- **Parameter Description**:
|
||||
- taos: [Input] Pointer to the database connection, which is established through the `taos_connect()` function.
|
||||
- option: [Input] Creation configuration. When selecting high-efficiency write mode, `singleStbInsert` and `singleTableBindOnce` need to be set to `true`; when selecting asynchronous execution, the callback function `asyncExecFn` and parameter `userdata` need to be set.
|
||||
- **Return Value**: Non-`NULL`: Success, returns a pointer to a TAOS_STMT2 structure representing the precompiled SQL statement object. `NULL`: Failure, please call taos_stmt_errstr() function for error details.
|
||||
|
||||
- `int taos_stmt2_prepare(TAOS_STMT2 *stmt, const char *sql, unsigned long length)`
|
||||
- **Interface Description**: Parses a precompiled SQL statement and binds the parsing results and parameter information to stmt.
|
||||
- **Parameter Description**:
|
||||
- stmt: [Input] Pointer to a valid precompiled SQL statement object.
|
||||
- sql: [Input] SQL statement to be parsed.
|
||||
- length: [Input] Length of the sql parameter. If the length is greater than 0, this parameter will be used as the length of the SQL statement; if it is 0, the length of the SQL statement will be automatically determined.
|
||||
- **Return Value**: `0`: Success. Non-`0`: Failure, please refer to the error code page for details.
|
||||
|
||||
- `int taos_stmt2_bind_param(TAOS_STMT2 *stmt, TAOS_STMT2_BINDV *bindv, int32_t col_idx)`
|
||||
- **Interface Description**: Binds a batch of parameters to a precompiled SQL statement.
|
||||
- **Parameter Description**:
|
||||
- stmt: [Input] Pointer to a valid precompiled SQL statement object.
|
||||
- bindv: [Input] Pointer to a valid TAOS_STMT2_BINDV structure, which contains the table names, tags, and data to be bound. `count` represents the number of tables to be bound, one `tbnames` can correspond to one set of `tags` and multiple sets of `bind_cols`; if it is a `SELECT` statement, only `bind_cols` needs to be bound.
|
||||
- col_idx: [Input] Represents the position of the specified column to be bound, `-1` means full column binding.
|
||||
- **Return Value**: `0`: Success. Non-`0`: Failure, please refer to the error code page for details.
|
||||
|
||||
- `int taos_stmt2_exec(TAOS_STMT2 *stmt, int *affected_rows)`
|
||||
- **Interface Description**: Executes the SQL with bound data, can be synchronous or asynchronous, determined by option.
|
||||
- **Parameter Description**:
|
||||
- stmt: [Input] Pointer to a valid precompiled SQL statement object.
|
||||
- affected_rows: [Output] If synchronous execution, represents the number of rows affected by this execution.
|
||||
- **Return Value**: `0`: Success. Non-`0`: Failure, please refer to the error code page for details.
|
||||
|
||||
- `int taos_stmt2_close(TAOS_STMT2 *stmt)`
|
||||
- **Interface Description**: After execution, releases all resources.
|
||||
- **Parameter Description**:
|
||||
- stmt: [Input] Pointer to a valid precompiled SQL statement object.
|
||||
- **Return Value**: `0`: Success. Non-`0`: Failure, please refer to the error code page for details.
|
||||
|
||||
- `int taos_stmt2_get_fields(TAOS_STMT2 *stmt, int *count, TAOS_FIELD_ALL **fields)`
|
||||
- **Interface Description**: Gets an array of column data attributes (column name, column data type, column length, column schema type) corresponding to the `?` order.
|
||||
- **Parameter Description**:
|
||||
- stmt: [Input] Pointer to a valid precompiled SQL statement object.
|
||||
- count: [Output] Returns the number of `?` in the bound SQL.
|
||||
- fields: [Output] Returns the column data attributes corresponding to the `?` order. If it is a `SELECT` statement, this structure returns `NULL`.
|
||||
- **Return Value**: `0`: Success. Non-`0`: Failure, please refer to the error code page for details.
|
||||
|
||||
- `void taos_stmt2_free_fields(TAOS_STMT2 *stmt, TAOS_FIELD_ALL *fields)`
|
||||
- **Interface Description**: Releases the memory of TAOS_FIELD_ALL return value, generally used after taos_stmt2_get_fields.
|
||||
- **Parameter Description**:
|
||||
- stmt: [Input] Pointer to a valid precompiled SQL statement object.
|
||||
- fields: [Input] Pointer to the resource to be released.
|
||||
- **Return Value**: None.
|
||||
|
||||
- `TAOS_RES *taos_stmt2_result(TAOS_STMT2 *stmt)`
|
||||
- **Interface Description**: Gets the result returned after executing SQL.
|
||||
- **Parameter Description**:
|
||||
- stmt: [Input] Pointer to a valid precompiled SQL statement object.
|
||||
- **Return Value**: Returns a pointer to a TAOS_RES structure, which contains the results of the insert operation. The returned TAOS_RES must be managed by the caller to avoid memory leaks.
|
||||
|
||||
- `char *taos_stmt2_error(TAOS_STMT2 *stmt)`
|
||||
- **Interface Description**: Used to obtain error information when other STMT2 APIs return an error (return error code or null pointer).
|
||||
- **Parameter Description**:
|
||||
- stmt: [Input] Pointer to a valid precompiled SQL statement object.
|
||||
- **Return Value**: Returns a pointer to a string containing error information.
|
||||
|
||||
<details>
|
||||
<summary>Parameter Binding(old)</summary>
|
||||
|
||||
Starting from versions 2.1.1.0 and 2.1.2.0, TDengine has significantly improved the parameter binding interface support for data writing (INSERT) scenarios. This avoids the resource consumption of SQL syntax parsing when writing data through the parameter binding interface, thereby significantly improving writing performance in most cases. The typical operation steps are as follows:
|
||||
|
||||
1. Call `taos_stmt_init()` to create a parameter binding object;
|
||||
|
|
@ -985,6 +1080,7 @@ The specific functions related to the interface are as follows (you can also ref
|
|||
- **Interface Description**: (Added in version 2.1.3.0) Used to obtain error information when other STMT APIs return an error (return error code or null pointer).
|
||||
- stmt: [Input] Points to a valid pointer to a precompiled SQL statement object.
|
||||
- **Return Value**: Returns a pointer to a string containing error information.
|
||||
</details>
|
||||
|
||||
#### Schemaless Insert
|
||||
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ The JDBC driver implementation for TDengine strives to be consistent with relati
|
|||
|
||||
| taos-jdbcdriver Version | Major Changes | TDengine Version |
|
||||
| ----------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------ |
|
||||
| 3.7.0 | 1. Replace Java-WebSocket library with Netty to enhance small query performance. <br/> 2. Add IPv6 protocol compatibility. <br/> 3. Implement BLOB (Binary Large Object) data type support. <br/> 4. Enable TDengine version compatibility checks. <br/> 5. Support `varcharAsString` in connection property. <br/> 6. Optimize memory utilization in WebSocket query operations. <br/> 7. Fix timezone handling in WebSocket connections. <br/> |-| | - |
|
||||
| 3.6.3 | Fixed data type conversion bug in database or super table subscription. | - |
|
||||
| 3.6.2 | 1. Supports data subscription for databases and super tables (subscription meta not supported). <br/> 1. Resolved the bug in cloud service subscription. <br/> 1. Improved the implement of setQueryTimeout with param 0. | - |
|
||||
| 3.6.1 | Fixed the performance issue of small queries in WebSocket connection. | - |
|
||||
|
|
@ -108,6 +109,12 @@ Please refer to the specific error codes:
|
|||
| 0x231c | httpEntity is null, sql: | An exception occurred in REST connection execution. |
|
||||
| 0x231d | can't create connection with server within | Increase the httpConnectTimeout parameter to extend the connection time, or check the connection with taosAdapter. |
|
||||
| 0x231e | failed to complete the task within the specified time | Increase the messageWaitTimeout parameter to extend the execution time, or check the connection with taosAdapter. |
|
||||
| 0x231f | RESTful client query exception | HTTP request error. Check details for more information. |
|
||||
| 0x2320 | Type conversion exception | Verify correct data types are being used. |
|
||||
| 0x2321 | TDengine version incompatible | TDengine version mismatch. Upgrade to the required version. |
|
||||
| 0x2322 | Resource has been freed | Resource has been released. Confirm operation validity. |
|
||||
| 0x2323 | BLOB unsupported on server | BLOB type is not supported by the server. Server upgrade required. |
|
||||
| 0x2324 | Line bind mode unsupported | Line binding mode is not supported by the server. Server upgrade required. |
|
||||
| 0x2350 | unknown error | Unknown exception, please provide feedback to the developers on github. |
|
||||
| 0x2352 | Unsupported encoding | An unsupported character encoding set was specified in the local connection. |
|
||||
| 0x2353 | internal error of database, please see taoslog for more details | An error occurred while executing prepareStatement in local connection, check taos log for troubleshooting. |
|
||||
|
|
@ -153,7 +160,7 @@ TDengine currently supports timestamp, numeric, character, boolean types, and th
|
|||
| JSON | java.lang.String | only supported in tags |
|
||||
| VARBINARY | byte[] | |
|
||||
| GEOMETRY | byte[] | |
|
||||
| BLOB | byte[] | only supported in columns |
|
||||
| BLOB | byte[] | only supported in columns |
|
||||
| DECIMAL | java.math.BigDecimal | only supported in WebSocket connections |
|
||||
|
||||
**Note**: Due to historical reasons, the BINARY type in TDengine is not truly binary data and is no longer recommended. Please use VARBINARY type instead.
|
||||
|
|
@ -223,7 +230,8 @@ taos-jdbcdriver implements the JDBC standard Driver interface, providing 3 imple
|
|||
The JDBC URL format for TDengine is:
|
||||
`jdbc:[TAOS|TAOS-WS|TAOS-RS]://[host_name]:[port]/[database_name]?[user={user}|&password={password}|&charset={charset}|&cfgdir={config_dir}|&locale={locale}|&timezone={timezone}|&batchfetch={batchfetch}]`
|
||||
|
||||
The host_name parameter supports valid domain names or IP addresses. The taos-jdbcdriver supports both IPv4 and IPv6 formats. For IPv6 addresses, square brackets must be used (e.g., `[::1]` or `[2001:db8:1234:5678::1]`) to avoid port number parsing conflicts.
|
||||
The host_name parameter supports valid domain names or IP addresses. The taos-jdbcdriver supports both IPv4 and IPv6 formats. For IPv6 addresses, square brackets must be used (e.g., `[::1]` or `[2001:db8:1234:5678::1]`) to avoid port number parsing conflicts.
|
||||
All properties in **Properties** are supported in the JDBC URL. For details, please refer to the **Properties** section below.
|
||||
|
||||
**Native Connection**
|
||||
`jdbc:TAOS://taosdemo.com:6030/power?user=root&password=taosdata`, using the TSDBDriver for native JDBC connection, establishes a connection to the hostname taosdemo.com, port 6030 (TDengine's default port), and database name power. This URL specifies the username (user) as root and the password (password) as taosdata.
|
||||
|
|
@ -303,46 +311,48 @@ For REST connections, the configuration parameters in the URL are as follows:
|
|||
|
||||
#### Properties
|
||||
|
||||
In addition to obtaining a connection through a specified URL, you can also use Properties to specify parameters when establishing a connection.
|
||||
In addition to obtaining a connection through a specified URL, you can also use Properties to specify parameters when establishing a connection.
|
||||
All configuration parameters in Properties can also be specified in the JDBC URL. The parameter names in square brackets can be used in the JDBC URL (e.g., TSDBDriver.PROPERTY_KEY_USER[`user`] can be set in the JDBC URL as `user=root` to specify the username).
|
||||
|
||||
> **Note**: The client parameter set in the application is at the process level, meaning if you need to update the client's parameters, you must restart the application. This is because the client parameter is a global parameter and only takes effect the first time it is set in the application.
|
||||
|
||||
The configuration parameters in properties are as follows:
|
||||
|
||||
- TSDBDriver.PROPERTY_KEY_USER: Login username for TDengine, default value 'root'.
|
||||
- TSDBDriver.PROPERTY_KEY_PASSWORD: User login password, default value 'taosdata'.
|
||||
- TSDBDriver.PROPERTY_KEY_BATCH_LOAD: true: Fetch result sets in batches during query execution; false: Fetch result sets row by row. The default value is false. For historical reasons, when using a REST connection, setting this parameter to true will switch to a WebSocket connection.
|
||||
- TSDBDriver.PROPERTY_KEY_BATCH_ERROR_IGNORE: true: Continue executing subsequent SQLs when one SQL fails during the execution of Statement's executeBatch; false: Do not execute any statements after a failed SQL. The default value is false.
|
||||
- TSDBDriver.PROPERTY_KEY_CONFIG_DIR: Effective only when using native JDBC connections. Client configuration file directory path, default value on Linux OS is `/etc/taos`, on Windows OS is `C:/TDengine/cfg`.
|
||||
- TSDBDriver.PROPERTY_KEY_CHARSET: Character set used by the client, default value is the system character set.
|
||||
- TSDBDriver.PROPERTY_KEY_LOCALE: Effective only when using native JDBC connections. Client locale, default value is the current system locale.
|
||||
- TSDBDriver.PROPERTY_KEY_TIME_ZONE:
|
||||
- TSDBDriver.PROPERTY_KEY_USER [`user`]: Login username for TDengine, default value 'root'.
|
||||
- TSDBDriver.PROPERTY_KEY_PASSWORD [`password`]: User login password, default value 'taosdata'.
|
||||
- TSDBDriver.PROPERTY_KEY_BATCH_LOAD [`batchfetch`]: true: Fetch result sets in batches during query execution; false: Fetch result sets row by row. The default value is false. For historical reasons, when using a REST connection, setting this parameter to true will switch to a WebSocket connection.
|
||||
- TSDBDriver.PROPERTY_KEY_BATCH_ERROR_IGNORE [`batchErrorIgnore`]: true: Continue executing subsequent SQLs when one SQL fails during the execution of Statement's executeBatch; false: Do not execute any statements after a failed SQL. The default value is false.
|
||||
- TSDBDriver.PROPERTY_KEY_CONFIG_DIR [`cfgdir`]: Effective only when using native JDBC connections. Client configuration file directory path, default value on Linux OS is `/etc/taos`, on Windows OS is `C:/TDengine/cfg`.
|
||||
- TSDBDriver.PROPERTY_KEY_CHARSET [`charset`]: Character set used by the client, default value is the system character set.
|
||||
- TSDBDriver.PROPERTY_KEY_LOCALE [`locale`]: Effective only when using native JDBC connections. Client locale, default value is the current system locale.
|
||||
- TSDBDriver.PROPERTY_KEY_TIME_ZONE [`timezone`]:
|
||||
- Native connections: Client time zone, default value is the current system time zone. Effective globally. Due to historical reasons, we only support part of the POSIX standard, such as UTC-8 (representing Shanghai, China), GMT-8, Asia/Shanghai.
|
||||
- WebSocket connections. Client time zone, default value is the current system time zone. Effective on the connection. Only IANA time zones are supported, such as Asia/Shanghai. It is recommended not to set this parameter, as using the system time zone provides better performance.
|
||||
- TSDBDriver.HTTP_CONNECT_TIMEOUT: Connection timeout, in ms, default value is 60000. Effective only in REST connections.
|
||||
- TSDBDriver.HTTP_SOCKET_TIMEOUT: Socket timeout, in ms, default value is 60000. Effective only in REST connections and when batchfetch is set to false.
|
||||
- TSDBDriver.PROPERTY_KEY_MESSAGE_WAIT_TIMEOUT: Message timeout, in ms, default value is 60000. Effective only under WebSocket connections.
|
||||
- TSDBDriver.PROPERTY_KEY_USE_SSL: Whether to use SSL in the connection. Effective only in WebSocket/REST connections.
|
||||
- TSDBDriver.HTTP_POOL_SIZE: REST concurrent request size, default 20.
|
||||
- TSDBDriver.PROPERTY_KEY_ENABLE_COMPRESSION: Whether to enable compression during transmission. Effective only when using REST/WebSocket connections. true: enabled, false: not enabled. Default is false.
|
||||
- TSDBDriver.PROPERTY_KEY_ENABLE_AUTO_RECONNECT: Whether to enable auto-reconnect. Effective only when using WebSocket connections. true: enabled, false: not enabled. Default is false.
|
||||
|
||||
- WebSocket connections: Client time zone, default value is the current system time zone. Effective on the connection. Only IANA time zones are supported, such as Asia/Shanghai. It is recommended not to set this parameter, as using the system time zone provides better performance.
|
||||
- TSDBDriver.HTTP_CONNECT_TIMEOUT [`httpConnectTimeout`]: Connection timeout, in ms, default value is 60000. Effective only in REST connections.
|
||||
- TSDBDriver.HTTP_SOCKET_TIMEOUT [`httpSocketTimeout`]: Socket timeout, in ms, default value is 60000. Effective only in REST connections and when batchfetch is set to false.
|
||||
- TSDBDriver.PROPERTY_KEY_MESSAGE_WAIT_TIMEOUT [`messageWaitTimeout`]: Message timeout, in ms, default value is 60000. Effective only under WebSocket connections.
|
||||
- TSDBDriver.PROPERTY_KEY_USE_SSL [`useSSL`]: Whether to use SSL in the connection. Effective only in WebSocket/REST connections.
|
||||
- TSDBDriver.HTTP_POOL_SIZE [`httpPoolSize`]: REST concurrent request size, default 20.
|
||||
- TSDBDriver.PROPERTY_KEY_ENABLE_COMPRESSION [`enableCompression`]: Whether to enable compression during transmission. Effective only when using REST/WebSocket connections. true: enabled, false: not enabled. Default is false.
|
||||
- TSDBDriver.PROPERTY_KEY_ENABLE_AUTO_RECONNECT [`enableAutoReconnect`]: Whether to enable auto-reconnect. Effective only when using WebSocket connections. true: enabled, false: not enabled. Default is false.
|
||||
> **Note**: Enabling auto-reconnect is only effective for simple SQL execution, schema-less writing, and data subscription. It is ineffective for parameter binding. Auto-reconnect is only effective for connections established through parameters specifying the database, and ineffective for later `use db` statements to switch databases.
|
||||
|
||||
- TSDBDriver.PROPERTY_KEY_RECONNECT_INTERVAL_MS: Auto-reconnect retry interval, in milliseconds, default value 2000. Effective only when PROPERTY_KEY_ENABLE_AUTO_RECONNECT is true.
|
||||
- TSDBDriver.PROPERTY_KEY_RECONNECT_RETRY_COUNT: Auto-reconnect retry count, default value 3, effective only when PROPERTY_KEY_ENABLE_AUTO_RECONNECT is true.
|
||||
- TSDBDriver.PROPERTY_KEY_DISABLE_SSL_CERT_VALIDATION: Disable SSL certificate validation. Effective only when using WebSocket connections. true: enabled, false: not enabled. Default is false.
|
||||
- TSDBDriver.PROPERTY_KEY_VARCHAR_AS_STRING: Maps VARCHAR/BINARY types to String. Effective only when using WebSocket connections. Default value is false.
|
||||
- TSDBDriver.PROPERTY_KEY_APP_NAME: App name, can be used for display in the `show connections` query result. Effective only when using WebSocket connections. Default value is java.
|
||||
- TSDBDriver.PROPERTY_KEY_APP_IP: App IP, can be used for display in the `show connections` query result. Effective only when using WebSocket connections. Default value is empty.
|
||||
- TSDBDriver.PROPERTY_KEY_RECONNECT_INTERVAL_MS [`reconnectIntervalMs`]: Auto-reconnect retry interval, in milliseconds, default value 2000. Effective only when PROPERTY_KEY_ENABLE_AUTO_RECONNECT is true.
|
||||
- TSDBDriver.PROPERTY_KEY_RECONNECT_RETRY_COUNT [`reconnectRetryCount`]: Auto-reconnect retry count, default value 3, effective only when PROPERTY_KEY_ENABLE_AUTO_RECONNECT is true.
|
||||
- TSDBDriver.PROPERTY_KEY_DISABLE_SSL_CERT_VALIDATION [`disableSSLCertValidation`]: Disable SSL certificate validation. Effective only when using WebSocket connections. true: enabled, false: not enabled. Default is false.
|
||||
- TSDBDriver.PROPERTY_KEY_VARCHAR_AS_STRING [`varcharAsString`]: Maps VARCHAR/BINARY types to String. Effective only when using WebSocket connections. Default value is false.
|
||||
- TSDBDriver.PROPERTY_KEY_APP_NAME [`app_name`]: App name, can be used for display in the `show connections` query result. Effective only when using WebSocket connections. Default value is java.
|
||||
- TSDBDriver.PROPERTY_KEY_APP_IP [`app_ip`]: App IP, can be used for display in the `show connections` query result. Effective only when using WebSocket connections. Default value is empty.
|
||||
|
||||
- TSDBDriver.PROPERTY_KEY_ASYNC_WRITE: Efficient Writing mode. Currently, only the `stmt` method is supported. Effective only when using WebSocket connections. DeDefault value is empty, meaning Efficient Writing mode is not enabled.
|
||||
- TSDBDriver.PROPERTY_KEY_BACKEND_WRITE_THREAD_NUM: In Efficient Writing mode, this refers to the number of background write threads. Effective only when using WebSocket connections. Default value is 10.
|
||||
- TSDBDriver.PROPERTY_KEY_BATCH_SIZE_BY_ROW: In Efficient Writing mode, this is the batch size for writing data, measured in rows. Effective only when using WebSocket connections. Default value is 1000.
|
||||
- TSDBDriver.PROPERTY_KEY_CACHE_SIZE_BY_ROW: In Efficient Writing mode, this is the cache size, measured in rows. Effective only when using WebSocket connections. Default value is 10000.
|
||||
- TSDBDriver.PROPERTY_KEY_COPY_DATA: In Efficient Writing mode, this determines Whether to copy the binary data passed by the application through the `addBatch` method. Effective only when using WebSocket connections. Default value is false.
|
||||
- TSDBDriver.PROPERTY_KEY_STRICT_CHECK: In Efficient Writing mode, this determines whether to validate the length of table names and variable-length data types. Effective only when using WebSocket connections. Default value is false.
|
||||
- TSDBDriver.PROPERTY_KEY_RETRY_TIMES: In Efficient Writing mode, this is the number of retry attempts for failed write operations. Effective only when using WebSocket connections. Default value is 3.
|
||||
- TSDBDriver.PROPERTY_KEY_ASYNC_WRITE [`asyncWrite`]: Efficient Writing mode. Currently, only the `stmt` method is supported. Effective only when using WebSocket connections. Default value is empty, meaning Efficient Writing mode is not enabled.
|
||||
- TSDBDriver.PROPERTY_KEY_BACKEND_WRITE_THREAD_NUM [`backendWriteThreadNum`]: In Efficient Writing mode, this refers to the number of background write threads. Effective only when using WebSocket connections. Default value is 10.
|
||||
- TSDBDriver.PROPERTY_KEY_BATCH_SIZE_BY_ROW [`batchSizeByRow`]: In Efficient Writing mode, this is the batch size for writing data, measured in rows. Effective only when using WebSocket connections. Default value is 1000.
|
||||
- TSDBDriver.PROPERTY_KEY_CACHE_SIZE_BY_ROW [`cacheSizeByRow`]: In Efficient Writing mode, this is the cache size, measured in rows. Effective only when using WebSocket connections. Default value is 10000.
|
||||
- TSDBDriver.PROPERTY_KEY_COPY_DATA [`copyData`]: In Efficient Writing mode, this determines whether to copy the binary data passed by the application through the `addBatch` method. Effective only when using WebSocket connections. Default value is false.
|
||||
- TSDBDriver.PROPERTY_KEY_STRICT_CHECK [`strictCheck`]: In Efficient Writing mode, this determines whether to validate the length of table names and variable-length data types. Effective only when using WebSocket connections. Default value is false.
|
||||
- TSDBDriver.PROPERTY_KEY_RETRY_TIMES [`retryTimes`]: In Efficient Writing mode, this is the number of retry attempts for failed write operations. Effective only when using WebSocket connections. Default value is 3.
|
||||
|
||||
- TSDBDriver.PROPERTY_KEY_PBS_MODE [`pbsMode`]: Parameter binding serialization mode, currently an experimental feature, only supports `line` mode, which can improve performance when each subtable has only one piece of data in a batch of bound data. Effective only when using WebSocket connections, and not supported in Efficient Writing mode. Default value is empty.
|
||||
|
||||
Priority of Configuration Parameters:
|
||||
|
||||
|
|
|
|||
|
|
@ -24,23 +24,26 @@ Supports Go 1.14 and above.
|
|||
|
||||
| driver-go Version | Major Changes | TDengine Version |
|
||||
|-------------------|-------------------------------------------------------------------------------------------------|--------------------|
|
||||
| v3.7.3 | Fix crash when WebSocket connection STMT query results contain decimal data. | - |
|
||||
| v3.7.2 | support BLOB type. | - |
|
||||
| v3.7.1 | support IPv6 connection. | - |
|
||||
| v3.7.0 | support decimal type. | 3.3.6.0 and higher |
|
||||
| v3.6.0 | stmt2 native interface, DSN supports passwords containing special characters (url.QueryEscape). | 3.3.5.0 and higher |
|
||||
| v3.5.8 | Fixed null pointer exception. | - |
|
||||
| v3.5.7 | taosWS and taosRestful support passing request id. | - |
|
||||
| v3.5.6 | Improved websocket query and insert performance. | 3.3.2.0 and higher |
|
||||
| v3.5.6 | Improved WebSocket query and insert performance. | 3.3.2.0 and higher |
|
||||
| v3.5.5 | Restful supports skipping SSL certificate check. | - |
|
||||
| v3.5.4 | Compatible with TDengine 3.3.0.0 tmq raw data. | - |
|
||||
| v3.5.3 | Refactored taosWS. | - |
|
||||
| v3.5.2 | Websocket compression and optimized tmq subscription performance. | 3.2.3.0 and higher |
|
||||
| v3.5.2 | WebSocket compression and optimized tmq subscription performance. | 3.2.3.0 and higher |
|
||||
| v3.5.1 | Native stmt query and geometry type support. | 3.2.1.0 and higher |
|
||||
| v3.5.0 | Support tmq get assignment and seek offset. | 3.0.5.0 and higher |
|
||||
| v3.3.1 | Schemaless protocol insert based on websocket. | 3.0.4.1 and higher |
|
||||
| v3.3.1 | Schemaless protocol insert based on WebSocket. | 3.0.4.1 and higher |
|
||||
| v3.1.0 | Provided Kafka-like subscription API. | - |
|
||||
| v3.0.4 | Added request id related interfaces. | 3.0.2.2 and higher |
|
||||
| v3.0.3 | Websocket-based statement insert. | - |
|
||||
| v3.0.2 | Websocket-based data query and insert. | 3.0.1.5 and higher |
|
||||
| v3.0.1 | Websocket-based message subscription. | - |
|
||||
| v3.0.3 | WebSocket-based statement insert. | - |
|
||||
| v3.0.2 | WebSocket-based data query and insert. | 3.0.1.5 and higher |
|
||||
| v3.0.1 | WebSocket-based message subscription. | - |
|
||||
| v3.0.0 | Adapted to TDengine 3.0 query and insert. | 3.0.0.0 and higher |
|
||||
|
||||
## Exceptions and Error Codes
|
||||
|
|
@ -84,6 +87,7 @@ For errors in other TDengine modules, please refer to [Error Codes](../../error-
|
|||
| GEOMETRY | []byte |
|
||||
| VARBINARY | []byte |
|
||||
| DECIMAL | string |
|
||||
| BLOB | []byte |
|
||||
|
||||
**Note**: The JSON type is only supported in tags.
|
||||
The GEOMETRY type is binary data in little endian byte order, conforming to the WKB standard. For more details, please refer to [Data Types](../../sql-manual/data-types/)
|
||||
|
|
@ -145,6 +149,12 @@ username:password@protocol(address)/dbname?param=value
|
|||
|
||||
When the password contains special characters, it needs to be escaped using url.QueryEscape.
|
||||
|
||||
When using an IPv6 address (supported in v3.7.1 and above), the address needs to be enclosed in square brackets, for example:
|
||||
|
||||
```text
|
||||
root:taosdata@ws([::1]:6041)/testdb
|
||||
```
|
||||
|
||||
##### Native Connection
|
||||
|
||||
Import the driver:
|
||||
|
|
|
|||
|
|
@ -57,6 +57,7 @@ Python Connector historical versions (it is recommended to use the latest versio
|
|||
|
||||
|Python Connector Version | Major Changes | TDengine Version|
|
||||
| --------- | ----------------------------------------------------------------------------------------------------- | ----------------- |
|
||||
|2.8.3 | Support BLOB data type. | - |
|
||||
|2.8.2 | The connection parameter settings support cross-platform compatibility. | - |
|
||||
|2.8.1 | Add two functions to set the connect property | - |
|
||||
|2.8.0 | Remove Apache Superset Driver | - |
|
||||
|
|
@ -141,7 +142,7 @@ TDengine currently supports timestamp, numeric, character, boolean types, and th
|
|||
| GEOMETRY | bytearray |
|
||||
| VARBINARY | bytearray |
|
||||
| DECIMAL | Decimal |
|
||||
|
||||
| BLOB | bytearray |
|
||||
## Example Programs Summary
|
||||
|
||||
| Example Program Link | Example Program Content |
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ import RequestId from "../../assets/resources/_request_id.mdx";
|
|||
|
||||
| Connector Version | Major Changes | TDengine Version |
|
||||
|-------------------|------------------------------------------------------------|--------------------|
|
||||
| 3.1.7 | Support IPv6 connections and DECIMAL data type. | 3.3.6.0 and higher |
|
||||
| 3.1.6 | Optimize WebSocket connection message handling. | - |
|
||||
| 3.1.5 | Fix WebSocket encoding error for Chinese character length. | - |
|
||||
| 3.1.4 | Improved WebSocket query and insert performance. | 3.3.2.0 and higher |
|
||||
|
|
@ -59,10 +60,20 @@ For error reporting in other TDengine modules, please refer to [Error Codes](../
|
|||
| JSON | byte[] |
|
||||
| VARBINARY | byte[] |
|
||||
| GEOMETRY | byte[] |
|
||||
| DECIMAL | decimal |
|
||||
|
||||
**Note**: JSON type is only supported in tags.
|
||||
The GEOMETRY type is binary data in little endian byte order, conforming to the WKB standard. For more details, please refer to [Data Types](../../sql-manual/data-types/)
|
||||
**Note**:
|
||||
|
||||
- JSON type is only supported in tags.
|
||||
- The GEOMETRY type is binary data in little endian byte order, conforming to the WKB standard. For more details, please refer to [Data Types](../../sql-manual/data-types/)
|
||||
For WKB standard, please refer to [Well-Known Binary (WKB)](https://libgeos.org/specifications/wkb/)
|
||||
- The DECIMAL type in C# is represented using the `decimal` type, which supports high-precision decimal numbers.
|
||||
Since C#'s `decimal` type differs from TDengine's DECIMAL type in precision and range,
|
||||
the C#'s `decimal` has a maximum precision of 29 digits, while TDengine's DECIMAL type supports up to 38 digits of precision.
|
||||
The following should be noted when using it:
|
||||
- When the value does not exceed the range of C#'s `decimal` type, you can use `GetDecimal` or `GetValue` to retrieve it.
|
||||
- When the value exceeds the range of C#'s `decimal` type, the `GetDecimal` and `GetValue` methods will throw an `OverflowException`.
|
||||
In such cases, you can use the `GetString` method to obtain the string representation.
|
||||
|
||||
## Summary of Example Programs
|
||||
|
||||
|
|
@ -330,6 +341,15 @@ The `DbDataReader` interface provides the following methods to retrieve the resu
|
|||
- **Return Value**: Date and time value.
|
||||
- **Exception**: Throws `InvalidCastException` if the type does not correspond.
|
||||
|
||||
- `public decimal GetDecimal(int ordinal)`
|
||||
- **Interface Description**: Gets the decimal value of the specified column.
|
||||
- **Parameter Description**:
|
||||
- `ordinal`: Column index.
|
||||
- **Return Value**: Decimal value.
|
||||
- **Exception**:
|
||||
- Throws `InvalidCastException` if the type does not correspond.
|
||||
- Throws `OverflowException` if the value exceeds the range of C#'s `decimal` type.
|
||||
|
||||
- `public double GetDouble(int ordinal)`
|
||||
- **Interface Description**: Gets the double precision floating-point value of the specified column.
|
||||
- **Parameter Description**:
|
||||
|
|
|
|||
|
|
@ -4,6 +4,10 @@ title: PHP Client Library
|
|||
slug: /tdengine-reference/client-libraries/php
|
||||
---
|
||||
|
||||
import CommunityLibrary from '../../assets/resources/_community-library.mdx';
|
||||
|
||||
<CommunityLibrary/>
|
||||
|
||||
`php-tdengine` is a PHP connector extension contributed by the community, which also specifically supports Swoole coroutine.
|
||||
|
||||
The PHP connector depends on the TDengine client driver.
|
||||
|
|
|
|||
|
|
@ -274,16 +274,19 @@ Column types use the following strings:
|
|||
- "JSON"
|
||||
- "VARBINARY"
|
||||
- "GEOMETRY"
|
||||
- "DECIMAL(precision, scale)"
|
||||
- "BLOB"
|
||||
|
||||
`VARBINARY` and `GEOMETRY` types return data as Hex strings, example:
|
||||
For the `DECIMAL` data type, `precision` refers to the maximum number of significant digits supported, and `scale` refers to the maximum number of decimal places. For example, `DECIMAL(8, 4)` represents a range of `[-9999.9999, 9999.9999]`.
|
||||
|
||||
`VARBINARY`, `GEOMETRY` and `BLOB` types return data as Hex strings, example:
|
||||
|
||||
Prepare data
|
||||
|
||||
```shell
|
||||
create database demo
|
||||
use demo
|
||||
create table t(ts timestamp,c1 varbinary(20),c2 geometry(100))
|
||||
insert into t values(now,'\x7f8290','point(100 100)')
|
||||
create database demo;
|
||||
create table demo.t(ts timestamp,c1 varbinary(20),c2 geometry(100),c3 blob);
|
||||
insert into demo.t values(now,'\x7f8290','point(100 100)','\x010203ddff');
|
||||
```
|
||||
|
||||
Execute query
|
||||
|
|
@ -315,13 +318,19 @@ Return result
|
|||
"c2",
|
||||
"GEOMETRY",
|
||||
100
|
||||
],
|
||||
[
|
||||
"c3",
|
||||
"BLOB",
|
||||
4194304
|
||||
]
|
||||
],
|
||||
"data": [
|
||||
[
|
||||
"2023-11-01T06:28:15.210Z",
|
||||
"2025-07-22T05:58:41.798Z",
|
||||
"7f8290",
|
||||
"010100000000000000000059400000000000005940"
|
||||
"010100000000000000000059400000000000005940",
|
||||
"010203ddff"
|
||||
]
|
||||
],
|
||||
"rows": 1
|
||||
|
|
|
|||
|
|
@ -79,28 +79,28 @@ This document details the server error codes that may be encountered when using
|
|||
|
||||
## tsc
|
||||
|
||||
| Error Code | Error Description | Possible Error Scenarios or Reasons | Recommended Actions for Users |
|
||||
| ---------- | --------------------------- | ----------------------------------------------- | ------------------------------------------------------------ |
|
||||
| 0x80000207 | Invalid user name | Invalid database username | Check if the database username is correct |
|
||||
| 0x80000208 | Invalid password | Invalid database password | Check if the database password is correct |
|
||||
| 0x80000209 | Database name too long | Invalid database name | Check if the database name is correct |
|
||||
| 0x8000020A | Table name too long | Invalid table name | Check if the table name is correct |
|
||||
| 0x8000020F | Query terminated | Query was terminated | Check if the query was terminated by a user |
|
||||
| 0x80000213 | Disconnected from server | Connection was interrupted | Check if the connection was interrupted by someone or if the client is exiting |
|
||||
| 0x80000216 | Syntax error in SQL | SQL syntax error | Check and correct the SQL statement |
|
||||
| 0x80000219 | SQL statement too long | SQL length exceeds limit | Check and correct the SQL statement |
|
||||
| 0x8000021A | File is empty | File content is empty | Check the content of the input file |
|
||||
| 0x8000021F | Invalid column length | Incorrect column length | Preserve the scene and logs, report issue on GitHub |
|
||||
| 0x80000222 | Invalid JSON data type | Incorrect JSON data type | Check the JSON content input |
|
||||
| 0x80000224 | Value out of range | Data size exceeds type range | Check the data value input |
|
||||
| 0x80000229 | Invalid tsc input | API input error | Check the parameters passed when calling the API from the application |
|
||||
| 0x8000022A | Stmt API usage error | Incorrect usage of STMT API | Check the order of STMT API calls, applicable scenarios, and error handling |
|
||||
| 0x8000022B | Stmt table name not set | STMT table name not set correctly | Check if the table name setting interface was called |
|
||||
| 0x8000022D | Query killed | Query was terminated | Check if the query was terminated by a user |
|
||||
| 0x8000022E | No available execution node | No available query execution node | Check the current query policy configuration, ensure available Qnode if needed |
|
||||
| 0x8000022F | Table is not a supertable | Table name in the statement is not a supertable | Check if the table name used in the statement is a supertable |
|
||||
| 0x80000230 | Stmt cache error | STMT internal cache error | Preserve the scene and logs, report issue on GitHub |
|
||||
| 0x80000231 | Tsc internal error | TSC internal error | Preserve the scene and logs, report issue on GitHub |
|
||||
| Error Code | Error Description | Possible Error Scenarios or Reasons | Recommended Actions for Users |
|
||||
| ---------- | --------------------------------- | ----------------------------------------------- | --------------------------------------------------------------------------------- |
|
||||
| 0x80000207 | Invalid user name | Invalid database username | Check if the database username is correct |
|
||||
| 0x80000208 | Invalid password | Invalid database password | Check if the database password is correct |
|
||||
| 0x80000209 | Database name too long | Invalid database name | Check if the database name is correct |
|
||||
| 0x8000020A | Table name too long | Invalid table name | Check if the table name is correct |
|
||||
| 0x8000020F | Query terminated | Query was terminated | Check if the query was terminated by a user |
|
||||
| 0x80000213 | Disconnected from server | Connection was interrupted | Check if the connection was interrupted by someone or if the client is exiting |
|
||||
| 0x80000216 | Syntax error in SQL | SQL syntax error | Check and correct the SQL statement |
|
||||
| 0x80000219 | SQL statement too long | SQL length exceeds limit | Check and correct the SQL statement |
|
||||
| 0x8000021A | File is empty | File content is empty | Check the content of the input file |
|
||||
| 0x8000021F | Invalid column length | Incorrect column length | Preserve the scene and logs, report issue on GitHub |
|
||||
| 0x80000222 | Invalid JSON data type | Incorrect JSON data type | Check the JSON content input |
|
||||
| 0x80000224 | Value out of range | Data size exceeds type range | Check the data value input |
|
||||
| 0x80000229 | Invalid tsc input | API input error | Check the parameters passed when calling the API from the application |
|
||||
| 0x8000022A | Stmt API usage error | Incorrect usage of STMT/STMT2 API | Check the order of STMT/STMT2 API calls, applicable scenarios, and error handling |
|
||||
| 0x8000022B | Stmt table name not set correctly | STMT/STMT2 table name not set correctly | Check if the STMT/STMT2 tbname bind is legal |
|
||||
| 0x8000022D | Query killed | Query was terminated | Check if the query was terminated by a user |
|
||||
| 0x8000022E | No available execution node | No available query execution node | Check the current query policy configuration, ensure available Qnode if needed |
|
||||
| 0x8000022F | Table is not a supertable | Table name in the statement is not a supertable | Check if the table name used in the statement is a supertable |
|
||||
| 0x80000230 | Stmt cache error | STMT/STMT2 internal cache error | Preserve the scene and logs, report issue on GitHub |
|
||||
| 0x80000231 | Tsc internal error | TSC internal error | Preserve the scene and logs, report issue on GitHub |
|
||||
|
||||
## mnode
|
||||
|
||||
|
|
@ -227,6 +227,18 @@ This document details the server error codes that may be encountered when using
|
|||
| 0x80000483 | index already exists | Already exists | Confirm if the operation is correct |
|
||||
| 0x80000484 | index not exist | Does not exist | Confirm if the operation is correct |
|
||||
|
||||
## Bnode
|
||||
|
||||
| Error Code | Description | Possible Error Scenarios or Reasons | Recommended Actions |
|
||||
| ---------- | -------------------------- | ----------------------------------- | ----------------------------------------- |
|
||||
| 0x80000450 | Bnode already exists | Already created | Check node status |
|
||||
| 0x80000451 | Bnode already deployed | Already deployed | Confirm if correct |
|
||||
| 0x80000452 | Bnode not deployed | Internal error | Report issue |
|
||||
| 0x80000453 | Bnode not there | Offline | Confirm if correct |
|
||||
| 0x80000454 | Bnode not found | Internal error | Report issue |
|
||||
| 0x80000455 | Bnode exec launch failed | Internal error | Report issue |
|
||||
| 0x8000261C | Invalid Bnode option | Illegal Bnode option value | Check and correct the Bnode option values |
|
||||
|
||||
## dnode
|
||||
|
||||
| Error Code | Description | Possible Error Scenarios or Reasons | Recommended Actions |
|
||||
|
|
|
|||
BIN
docs/en/assets/active-active-01.png
Normal file
BIN
docs/en/assets/active-active-01.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 206 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 161 KiB |
5
docs/en/assets/resources/_community-library.mdx
Normal file
5
docs/en/assets/resources/_community-library.mdx
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
:::warning[Community Library]
|
||||
|
||||
This client library is developed by our open-source community. It is **not tested or supported by TDengine.** If you encounter any issues with this library, contact the developer.
|
||||
|
||||
:::
|
||||
|
|
@ -19,7 +19,7 @@
|
|||
<dependency>
|
||||
<groupId>com.taosdata.jdbc</groupId>
|
||||
<artifactId>taos-jdbcdriver</artifactId>
|
||||
<version>3.6.3</version>
|
||||
<version>3.7.0</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.locationtech.jts</groupId>
|
||||
|
|
|
|||
|
|
@ -47,7 +47,7 @@
|
|||
<dependency>
|
||||
<groupId>com.taosdata.jdbc</groupId>
|
||||
<artifactId>taos-jdbcdriver</artifactId>
|
||||
<version>3.6.3</version>
|
||||
<version>3.7.0</version>
|
||||
</dependency>
|
||||
|
||||
</dependencies>
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@
|
|||
<dependency>
|
||||
<groupId>com.taosdata.jdbc</groupId>
|
||||
<artifactId>taos-jdbcdriver</artifactId>
|
||||
<version>3.6.3</version>
|
||||
<version>3.7.0</version>
|
||||
</dependency>
|
||||
<!-- druid -->
|
||||
<dependency>
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ public class DruidDemo {
|
|||
dataSource.setMinIdle(10);
|
||||
dataSource.setMaxActive(10);
|
||||
dataSource.setMaxWait(30000);
|
||||
dataSource.setValidationQuery("SELECT SERVER_VERSION()");
|
||||
dataSource.setValidationQuery("SELECT 1");
|
||||
|
||||
Connection connection = dataSource.getConnection(); // get connection
|
||||
Statement statement = connection.createStatement(); // get statement
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ public class HikariDemo {
|
|||
config.setConnectionTimeout(30000); // maximum wait milliseconds for get connection from pool
|
||||
config.setMaxLifetime(0); // maximum life time for each connection
|
||||
config.setIdleTimeout(0); // max idle time for recycle idle connection
|
||||
config.setConnectionTestQuery("SELECT SERVER_VERSION()"); // validation query
|
||||
config.setConnectionTestQuery("SELECT 1"); // validation query
|
||||
|
||||
HikariDataSource dataSource = new HikariDataSource(config); // create datasource
|
||||
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@
|
|||
<dependency>
|
||||
<groupId>com.taosdata.jdbc</groupId>
|
||||
<artifactId>taos-jdbcdriver</artifactId>
|
||||
<version>3.6.3</version>
|
||||
<version>3.7.0</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.google.guava</groupId>
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@
|
|||
<dependency>
|
||||
<groupId>com.taosdata.jdbc</groupId>
|
||||
<artifactId>taos-jdbcdriver</artifactId>
|
||||
<version>3.6.3</version>
|
||||
<version>3.7.0</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>commons-cli</groupId>
|
||||
|
|
|
|||
|
|
@ -83,7 +83,7 @@
|
|||
<dependency>
|
||||
<groupId>com.taosdata.jdbc</groupId>
|
||||
<artifactId>taos-jdbcdriver</artifactId>
|
||||
<version>3.6.3</version>
|
||||
<version>3.7.0</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
|
|
|
|||
|
|
@ -112,7 +112,7 @@
|
|||
<dependency>
|
||||
<groupId>com.taosdata.jdbc</groupId>
|
||||
<artifactId>taos-jdbcdriver</artifactId>
|
||||
<version>3.6.3</version>
|
||||
<version>3.7.0</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
|
|
|
|||
|
|
@ -67,7 +67,7 @@
|
|||
<dependency>
|
||||
<groupId>com.taosdata.jdbc</groupId>
|
||||
<artifactId>taos-jdbcdriver</artifactId>
|
||||
<version>3.6.3</version>
|
||||
<version>3.7.0</version>
|
||||
<!-- <scope>system</scope>-->
|
||||
<!-- <systemPath>${project.basedir}/src/main/resources/lib/taos-jdbcdriver-2.0.15-dist.jar</systemPath>-->
|
||||
</dependency>
|
||||
|
|
|
|||
|
|
@ -22,8 +22,8 @@
|
|||
#include <sys/time.h>
|
||||
#include "taos.h"
|
||||
|
||||
#define NUM_OF_SUB_TABLES 10
|
||||
#define NUM_OF_ROWS 10
|
||||
#define NUM_OF_SUB_TABLES 1
|
||||
#define NUM_OF_ROWS 1
|
||||
|
||||
/**
|
||||
* @brief Executes an SQL query and checks for errors.
|
||||
|
|
@ -70,6 +70,7 @@ void prepareBindData(char ***table_name, TAOS_STMT2_BIND ***tags, TAOS_STMT2_BIN
|
|||
*tags = (TAOS_STMT2_BIND **)malloc(NUM_OF_SUB_TABLES * sizeof(TAOS_STMT2_BIND *));
|
||||
*params = (TAOS_STMT2_BIND **)malloc(NUM_OF_SUB_TABLES * sizeof(TAOS_STMT2_BIND *));
|
||||
|
||||
int32_t len = 10 * 1024;
|
||||
for (int i = 0; i < NUM_OF_SUB_TABLES; i++) {
|
||||
// Allocate and assign table name
|
||||
(*table_name)[i] = (char *)malloc(20 * sizeof(char));
|
||||
|
|
@ -90,21 +91,29 @@ void prepareBindData(char ***table_name, TAOS_STMT2_BIND ***tags, TAOS_STMT2_BIN
|
|||
(*tags)[i][1] = (TAOS_STMT2_BIND){TSDB_DATA_TYPE_BINARY, location, location_len, NULL, 1};
|
||||
|
||||
// Allocate memory for columns data
|
||||
(*params)[i] = (TAOS_STMT2_BIND *)malloc(4 * sizeof(TAOS_STMT2_BIND));
|
||||
(*params)[i] = (TAOS_STMT2_BIND *)malloc(5 * sizeof(TAOS_STMT2_BIND));
|
||||
|
||||
int64_t *ts = (int64_t *)malloc(NUM_OF_ROWS * sizeof(int64_t));
|
||||
float *current = (float *)malloc(NUM_OF_ROWS * sizeof(float));
|
||||
int *voltage = (int *)malloc(NUM_OF_ROWS * sizeof(int));
|
||||
float *phase = (float *)malloc(NUM_OF_ROWS * sizeof(float));
|
||||
char **phase = (char **)malloc(NUM_OF_ROWS * sizeof(char *));
|
||||
int32_t *ts_len = (int32_t *)malloc(NUM_OF_ROWS * sizeof(int32_t));
|
||||
int32_t *current_len = (int32_t *)malloc(NUM_OF_ROWS * sizeof(int32_t));
|
||||
for (int j = 0; j < NUM_OF_ROWS; j++) {
|
||||
phase[j] = (char *)malloc(sizeof(char) * 20); // Allocate memory for phase
|
||||
sprintf(phase[j], "phase_%d", j); // Assign a value to phase
|
||||
}
|
||||
|
||||
int32_t *voltage_len = (int32_t *)malloc(NUM_OF_ROWS * sizeof(int32_t));
|
||||
int32_t *phase_len = (int32_t *)malloc(NUM_OF_ROWS * sizeof(int32_t));
|
||||
for (int j = 0; j < NUM_OF_ROWS; j++) {
|
||||
phase_len[j] = strlen(phase[j]); // Set length for phase
|
||||
}
|
||||
|
||||
(*params)[i][0] = (TAOS_STMT2_BIND){TSDB_DATA_TYPE_TIMESTAMP, ts, ts_len, NULL, NUM_OF_ROWS};
|
||||
(*params)[i][1] = (TAOS_STMT2_BIND){TSDB_DATA_TYPE_FLOAT, current, current_len, NULL, NUM_OF_ROWS};
|
||||
(*params)[i][2] = (TAOS_STMT2_BIND){TSDB_DATA_TYPE_INT, voltage, voltage_len, NULL, NUM_OF_ROWS};
|
||||
(*params)[i][3] = (TAOS_STMT2_BIND){TSDB_DATA_TYPE_FLOAT, phase, phase_len, NULL, NUM_OF_ROWS};
|
||||
(*params)[i][3] = (TAOS_STMT2_BIND){TSDB_DATA_TYPE_BINARY, phase, phase_len, NULL, NUM_OF_ROWS};
|
||||
|
||||
for (int j = 0; j < NUM_OF_ROWS; j++) {
|
||||
struct timeval tv;
|
||||
|
|
@ -112,12 +121,12 @@ void prepareBindData(char ***table_name, TAOS_STMT2_BIND ***tags, TAOS_STMT2_BIN
|
|||
ts[j] = tv.tv_sec * 1000LL + tv.tv_usec / 1000 + j;
|
||||
current[j] = (float)rand() / RAND_MAX * 30;
|
||||
voltage[j] = rand() % 300;
|
||||
phase[j] = (float)rand() / RAND_MAX;
|
||||
// phase[j] = (char *)malloc(20 * sizeof(char));
|
||||
|
||||
ts_len[j] = sizeof(int64_t);
|
||||
current_len[j] = sizeof(float);
|
||||
voltage_len[j] = sizeof(int);
|
||||
phase_len[j] = sizeof(float);
|
||||
// phase_len[j] = sizeof(float);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -195,9 +204,10 @@ int main() {
|
|||
// create database and table
|
||||
executeSQL(taos, "CREATE DATABASE IF NOT EXISTS power");
|
||||
executeSQL(taos, "USE power");
|
||||
executeSQL(taos,
|
||||
"CREATE STABLE IF NOT EXISTS power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS "
|
||||
"(groupId INT, location BINARY(24))");
|
||||
executeSQL(
|
||||
taos,
|
||||
"CREATE STABLE IF NOT EXISTS power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase binary(12)) TAGS "
|
||||
"(groupId INT, location BINARY(24))");
|
||||
insertData(taos);
|
||||
taos_close(taos);
|
||||
taos_cleanup();
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@
|
|||
<dependency>
|
||||
<groupId>com.taosdata.jdbc</groupId>
|
||||
<artifactId>taos-jdbcdriver</artifactId>
|
||||
<version>3.6.3</version>
|
||||
<version>3.7.0</version>
|
||||
</dependency>
|
||||
<!-- ANCHOR_END: dep-->
|
||||
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ public class DruidDemo {
|
|||
dataSource.setMinIdle(10);
|
||||
dataSource.setMaxActive(10);
|
||||
dataSource.setMaxWait(30000);
|
||||
dataSource.setValidationQuery("SELECT SERVER_VERSION()");
|
||||
dataSource.setValidationQuery("SELECT 1");
|
||||
|
||||
Connection connection = dataSource.getConnection(); // get connection
|
||||
Statement statement = connection.createStatement(); // get statement
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ public class HikariDemo {
|
|||
config.setConnectionTimeout(30000); // maximum wait milliseconds for get connection from pool
|
||||
config.setMaxLifetime(0); // maximum life time for each connection
|
||||
config.setIdleTimeout(0); // max idle time for recycle idle connection
|
||||
config.setConnectionTestQuery("SELECT SERVER_VERSION()"); // validation query
|
||||
config.setConnectionTestQuery("SELECT 1"); // validation query
|
||||
|
||||
HikariDataSource dataSource = new HikariDataSource(config); // create datasource
|
||||
|
||||
|
|
|
|||
|
|
@ -1,135 +0,0 @@
|
|||
---
|
||||
sidebar_label: 数据订阅
|
||||
title: 数据订阅
|
||||
toc_max_heading_level: 4
|
||||
---
|
||||
|
||||
为了满足应用程序实时获取 TDengine 写入的数据的需求，或以事件到达顺序处理数据，TDengine 提供了类似于消息队列产品的数据订阅和消费接口。在许多场景中，采用 TDengine 的时序大数据平台，无须再集成消息队列产品，从而简化应用程序设计并降低运维成本。
|
||||
|
||||
与 Kafka 类似，用户需要在 TDengine 中定义主题（topic）。然而，TDengine 的主题可以是一个数据库、一张超级表，或者基于现有超级表、子表或普通表的查询条件，即一条查询语句。用户可以利用 SQL 对标签、表名、列、表达式等条件进行过滤，并对数据进行标量函数与 UDF 计算（不包括数据聚合）。与其他消息队列工具相比，这是 TDengine 数据订阅功能的最大优势。它提供了更高的灵活性，数据的粒度由定义主题的 SQL 决定，而且数据的过滤与预处理由 TDengine 自动完成，从而减少传输的数据量并降低应用程序的复杂度。
|
||||
|
||||
消费者订阅主题后，可以实时接收最新的数据。多个消费者可以组成一个消费组，共享消费进度，实现多线程、分布式地消费数据，提高消费速度。不同消费组的消费者即使消费同一个主题，也不共享消费进度。一个消费者可以订阅多个主题。如果主题对应的是超级表或库，数据可能会分布在多个不同的节点或数据分片上。当一个消费组中有多个消费者时，可以提高消费效率。TDengine 的消息队列提供了消息的 ACK（Acknowledgment，确认，也译作收到）机制，确保在宕机、重启等复杂环境下实现至少一次（at least once）消费。
|
||||
|
||||
为实现上述功能，TDengine 会为预写数据日志（Write-Ahead Logging，WAL）文件自动创建索引，以支持快速随机访问，并提供了灵活可配置的文件切换与保留机制。用户可以根据需求指定 WAL 文件的保留时间和大小。通过这些方法，WAL 被改造成一个保留事件到达顺序的、可持久化的存储引擎。对于以主题形式创建的查询，TDengine 将从 WAL 读取数据。在消费过程中，TDengine 根据当前消费进度从 WAL 直接读取数据，并使用统一的查询引擎实现过滤、变换等操作，然后将数据推送给消费者。
|
||||
|
||||
从 3.2.0.0 版本开始，数据订阅支持 vnode 迁移和分裂。由于数据订阅依赖 wal 文件，而在 vnode 迁移和分裂的过程中，wal 文件并不会进行同步。因此，在迁移或分裂操作完成后，您将无法继续消费之前尚未消费完 wal 数据。请务必在执行 vnode 迁移或分裂之前，将所有 wal 数据消费完毕。
|
||||
|
||||
## 主题类型
|
||||
|
||||
TDengine 使用 SQL 创建的主题共有 3 种类型，下面分别介绍。
|
||||
|
||||
### 查询主题
|
||||
|
||||
订阅一条 SQL 查询的结果，本质上是连续查询，每次查询仅返回最新值，创建语法如下：
|
||||
|
||||
```sql
|
||||
CREATE TOPIC [IF NOT EXISTS] topic_name as subquery
|
||||
```
|
||||
该 SQL 通过 SELECT 语句订阅（包括 SELECT *，或 SELECT ts, c1 等指定查询订阅，可以带条件过滤、标量函数计算，但不支持聚合函数、不支持时间窗口聚合）。需要注意的是：
|
||||
1. 该类型 TOPIC 一旦创建则订阅数据的结构确定。
|
||||
2. 被订阅或用于计算的列或标签不可被删除（ALTER table DROP）、修改（ALTER table MODIFY）。
|
||||
3. 若发生表结构变更，新增的列不出现在结果中。
|
||||
4. 对于 select *，则订阅展开为创建时所有的列（子表、普通表为数据列，超级表为数据列加标签列）
|
||||
|
||||
假设需要订阅所有智能电表中电压值大于 200 的数据，且仅仅返回时间戳、电流、电压 3 个采集量（不返回相位），那么可以通过下面的 SQL 创建 power_topic 这个主题。
|
||||
```sql
|
||||
CREATE TOPIC power_topic AS SELECT ts, current, voltage FROM power.meters WHERE voltage > 200;
|
||||
```
|
||||
|
||||
### 超级表主题
|
||||
|
||||
订阅一个超级表中的所有数据，语法如下：
|
||||
```sql
|
||||
CREATE TOPIC [IF NOT EXISTS] topic_name [with meta] AS STABLE stb_name [where_condition]
|
||||
```
|
||||
|
||||
与使用 `SELECT * from stbName` 订阅的区别是：
|
||||
|
||||
1. 不会限制用户的表结构变更，即表结构变更以及变更后的新数据都能够订阅到。
|
||||
2. 返回的是非结构化的数据，返回数据的结构会随着超级表的表结构变化而变化。
|
||||
3. with meta 参数可选，选择时将返回创建超级表，子表等语句，主要用于 taosX 做超级表迁移。
|
||||
4. where_condition 参数可选，选择时将用来过滤符合条件的子表，订阅这些子表。where 条件里不能有普通列，只能是 tag 或 tbname，where 条件里可以用函数，用来过滤 tag，但是不能是聚合函数，因为子表 tag 值无法做聚合。可以是常量表达式，比如 2 > 1（订阅全部子表），或者 false（订阅 0 个子表）。
|
||||
5. 返回数据不包含标签。
|
||||
|
||||
### 数据库主题
|
||||
|
||||
订阅一个数据库里所有数据，其语法如下：
|
||||
```sql
|
||||
CREATE TOPIC [IF NOT EXISTS] topic_name [with meta] AS DATABASE db_name;
|
||||
```
|
||||
|
||||
通过该语句可创建一个包含数据库所有表数据的订阅：
|
||||
1. with meta 参数可选，选择时将返回数据库里所有超级表，子表、普通表的元数据创建、删除、修改语句，主要用于 taosX 做数据库迁移。
|
||||
2. 超级表订阅和库订阅属于高级订阅模式，容易出错，如确实要使用，请咨询技术支持人员。
|
||||
|
||||
## 删除主题
|
||||
|
||||
如果不再需要订阅数据，可以删除 topic，如果当前 topic 被消费者订阅，通过 FORCE 语法可强制删除，强制删除后订阅的消费者会消费数据会出错（FORCE 语法从 v3.3.6.0 开始支持）。
|
||||
|
||||
```sql
|
||||
DROP TOPIC [IF EXISTS] [FORCE] topic_name;
|
||||
```
|
||||
|
||||
## 查看主题
|
||||
|
||||
```sql
|
||||
SHOW TOPICS;
|
||||
```
|
||||
|
||||
上面的 SQL 会显示当前数据库下的所有主题的信息。
|
||||
|
||||
## 消费者
|
||||
|
||||
### 创建消费者
|
||||
|
||||
消费者的创建只能通过 TDengine 客户端驱动或者连接器所提供的 API 创建，详情可以参考开发指南或参考手册。
|
||||
|
||||
### 查看消费者
|
||||
|
||||
```sql
|
||||
SHOW CONSUMERS;
|
||||
```
|
||||
|
||||
显示当前数据库下所有消费者的信息，会显示消费者的状态，创建时间等信息。
|
||||
|
||||
### 删除消费组
|
||||
|
||||
消费者创建的时候，会给消费者指定一个消费者组，消费者不能显式的删除，但是可以删除消费者组。如果当前消费者组里有消费者在消费，通过 FORCE 语法可强制删除，强制删除后订阅的消费者会消费数据会出错（FORCE 语法从 v3.3.6.0 开始支持）。
|
||||
```sql
|
||||
DROP CONSUMER GROUP [IF EXISTS] [FORCE] cgroup_name ON topic_name;
|
||||
```
|
||||
|
||||
## 数据订阅
|
||||
|
||||
### 查看订阅信息
|
||||
|
||||
```sql
|
||||
SHOW SUBSCRIPTIONS;
|
||||
```
|
||||
显示 topic 在不同 vgroup 上的消费信息，可用于查看消费进度。
|
||||
|
||||
### 订阅数据
|
||||
|
||||
TDengine 提供了全面且丰富的数据订阅 API，旨在满足不同编程语言和框架下的数据订阅需求。这些接口包括但不限于创建消费者、订阅主题、取消订阅、获取实时数据、提交消费进度以及获取和设置消费进度等功能。目前，TDengine 支持多种主流编程语言，包括 C、Java、Go、Rust、Python 和 C# 等，使得开发者能够轻松地在各种应用场景中使用 TDengine 的数据订阅功能。
|
||||
|
||||
值得一提的是，TDengine 的数据订阅 API 与业界流行的 Kafka 订阅 API 保持了高度的一致性，以便于开发者能够快速上手并利用现有的知识经验。为了方便用户了解和参考，TDengine 的官方文档详细介绍了各种 API 的使用方法和示例代码，具体内容可访问 TDengine 官方网站的连接器部分。通过这些 API，开发者可以高效地实现数据的实时订阅和处理，从而满足各种复杂场景下的数据处理需求。
|
||||
|
||||
### 回放功能
|
||||
|
||||
TDengine 的数据订阅功能支持回放（replay）功能，允许用户按照数据的实际写入时间顺序重新播放数据流。这一功能基于 TDengine 的高效 WAL 机制实现，确保了数据的一致性和可靠性。
|
||||
|
||||
要使用数据订阅的回放功能，用户可以在查询语句中指定时间范围，从而精确控制回放的起始时间和结束时间。这使得用户能够轻松地重放特定时间段内的数据，无论是为了故障排查、数据分析还是其他目的。
|
||||
|
||||
如果写入了如下 3 条数据，那么回放时则先返回第 1 条数据，5s 后返回第 2 条数据，在获取第 2 条数据 3s 后返回第 3 条数据。
|
||||
|
||||
```text
|
||||
2023/09/22 00:00:00.000
|
||||
2023/09/22 00:00:05.000
|
||||
2023/09/22 00:00:08.000
|
||||
```
|
||||
|
||||
使用数据订阅的回放功能时需要注意如下几项：
|
||||
- 通过配置消费参数 enable.replay 为 true 开启回放功能。
|
||||
- 数据订阅的回放功能仅查询订阅支持数据回放，超级表和库订阅不支持回放。
|
||||
- 回放不支持进度保存。
|
||||
- 因为数据回放本身需要处理时间，所以回放的精度存在几十毫秒的误差。
|
||||
126
docs/zh/06-advanced/01-subscription/01-mqtt.md
Normal file
126
docs/zh/06-advanced/01-subscription/01-mqtt.md
Normal file
|
|
@ -0,0 +1,126 @@
|
|||
---
|
||||
sidebar_label: MQTT 数据订阅
|
||||
title: MQTT 数据订阅
|
||||
toc_max_heading_level: 4
|
||||
---
|
||||
|
||||
TDengine v3.3.7.0 版本开始提供 MQTT 订阅功能，通过 MQTT 客户端连接 TDengine Bnode 服务，可直接订阅系统中已有主题的数据。
|
||||
|
||||
主要特性：
|
||||
|
||||
1. 协议支持：MQTT 5.0
|
||||
2. 身份验证：使用 TDengine 原生验证
|
||||
3. 主题管理：与标准 MQTT 协议不同，主题必须预先创建（因不支持消息发布，无法通过发布消息动态创建）
|
||||
4. 共享主题：形如 $shared/group_id/topic_name 的主题被视为共享订阅，适用于需要负载均衡和高可用场景
|
||||
5. 订阅位置：支持 latest，earliest (WAL 最早位置)
|
||||
6. 服务质量：支持 QoS 0，QoS 1
|
||||
|
||||
## Bnode 节点管理
|
||||
|
||||
用户可通过 TDengine 的命令行工具 taos 进行 Bnode 的管理。执行下述命令都需要确保命令行工具 taos 工作正常。
|
||||
|
||||
### 创建 Bnode
|
||||
|
||||
```sql
|
||||
CREATE BNODE ON DNODE {dnode_id}
|
||||
```
|
||||
|
||||
一个 dnode 上只能创建一个 bnode。bnode 创建成功后，会自动启动 bnode 子进程 `taosmqtt`，默认在 6083 端口对外提供 MQTT 订阅服务，端口可在文件 taos.cfg 中通过参数 `mqttPort` 配置。例如：`create bnode on dnode 1`。
|
||||
|
||||
### 查看 Bnode
|
||||
|
||||
列出集群中所有的数据订阅节点，包括其 `id`, `endpoint`, `create_time`等属性。
|
||||
|
||||
```sql
|
||||
SHOW BNODES;
|
||||
|
||||
taos> show bnodes;
|
||||
id | endpoint | protocol | create_time |
|
||||
======================================================================
|
||||
1 | 192.168.0.1:6083 | mqtt | 2024-11-28 18:44:27.089 |
|
||||
Query OK, 1 row(s) in set (0.037205s)
|
||||
```
|
||||
|
||||
### 删除 Bnode
|
||||
|
||||
```sql
|
||||
DROP BNODE ON DNODE {dnode_id}
|
||||
```
|
||||
|
||||
删除 bnode 将把 bnode 从 TDengine 集群中移除，同时停止 taosmqtt 服务。
|
||||
|
||||
## 订阅数据示例
|
||||
|
||||
### 环境准备
|
||||
|
||||
```sql
|
||||
create database db vgroups 1;
|
||||
create table db.meters (ts timestamp, f1 int) tags(t1 int);
|
||||
create topic topic_meters as select ts, tbname, f1, t1 from db.meters;
|
||||
insert into db.tb using db.meters tags(1) values(now, 1);
|
||||
create bnode on dnode 1;
|
||||
```
|
||||
|
||||
在命令行工具 taos 中执行上面的 SQL 语句，创建数据库，超级表，主题 `topic_meters` ，bnode 节点，写入一条数据供下一步订阅使用。
|
||||
|
||||
### 客户端订阅
|
||||
|
||||
可以使用兼容 MQTT 协议 v5.0 版本的客户端来订阅前一步环境中的数据，这里使用 Python paho-mqtt 来举例说明：
|
||||
|
||||
在操作系统命令行界面中依次执行下面这些命令，便可以订阅到上一步中写入的数据；订阅成功后，如果 `topic_meters` 主题中有新增的写入数据，则会自动通过 MQTT 协议推送到客户端。
|
||||
|
||||
```shell
|
||||
python3 -m venv .test-env
|
||||
source .test-env/bin/activate
|
||||
pip3 install paho-mqtt==2.1.0
|
||||
python3 ./sub.py
|
||||
```
|
||||
|
||||
其中 sub.py 文件的内容如下：
|
||||
|
||||
```python
|
||||
import time
|
||||
import paho.mqtt
|
||||
import paho.mqtt.properties as p
|
||||
import paho.mqtt.packettypes as pt
|
||||
import paho.mqtt.client as mqttClient
|
||||
|
||||
def on_connect(client, userdata, flags, rc, properties=None):
|
||||
print("CONNACK received with code %s." % rc)
|
||||
sub_properties = p.Properties(pt.PacketTypes.SUBSCRIBE)
|
||||
sub_properties.UserProperty = ('sub-offset', 'earliest')
|
||||
client.subscribe("$share/g1/topic_meters", qos=1, properties=sub_properties)
|
||||
|
||||
def on_subscribe(client, userdata, mid, granted_qos, properties=None):
|
||||
print("Subscribed: " + str(mid) + " " + str(granted_qos))
|
||||
|
||||
def on_message(client, userdata, msg):
|
||||
print(msg.topic + " " + str(msg.qos) + " " + str(msg.payload))
|
||||
|
||||
if paho.mqtt.__version__[0] > '1':
|
||||
client = mqttClient.Client(mqttClient.CallbackAPIVersion.VERSION2, client_id="tmq_sub_cid", userdata=None, protocol=mqttClient.MQTTv5)
|
||||
else:
|
||||
client = mqttClient.Client(client_id="tmq_sub_cid", userdata=None, protocol=mqttClient.MQTTv5)
|
||||
|
||||
client.on_connect = on_connect
|
||||
client.username_pw_set("root", "taosdata")
|
||||
client.connect("127.0.1.1", 6083)
|
||||
|
||||
client.on_subscribe = on_subscribe
|
||||
client.on_message = on_message
|
||||
|
||||
client.loop_forever()
|
||||
```
|
||||
|
||||
## 消息格式
|
||||
|
||||
上一节的示例中，会输出下面的信息：
|
||||
|
||||
```shell
|
||||
CONNACK received with code Success.
|
||||
Subscribed: 1 [ReasonCode(Suback, 'Granted QoS 1')]
|
||||
topic_meters 1 b'{"topic":"topic_meters","db":"db","vid":2,"rows":[{"ts":1753086482326,"tbname":"tb","f1":1,"t1":1}]}'
|
||||
```
|
||||
|
||||
其中第三行 `topic_meters` 是我们订阅的主题，1 是这一条消息的 QoS 值，后面是一个 utf-8 编码的 JSON 消息，其中 `rows` 是数据行的数组。
|
||||
|
||||
158
docs/zh/06-advanced/01-subscription/index.md
Normal file
158
docs/zh/06-advanced/01-subscription/index.md
Normal file
|
|
@ -0,0 +1,158 @@
|
|||
---
|
||||
sidebar_label: 数据订阅
|
||||
title: 数据订阅
|
||||
toc_max_heading_level: 4
|
||||
---
|
||||
|
||||
为了满足应用程序实时获取 TDengine 写入的数据的需求，或以事件到达顺序处理数据，TDengine 提供了类似于消息队列产品的数据订阅和消费接口。在许多场景中，采用 TDengine 的时序大数据平台，无须再集成消息队列产品，从而简化应用程序设计并降低运维成本。
|
||||
|
||||
与 Kafka 类似，用户需要在 TDengine 中定义主题（topic）。然而，TDengine 的主题可以是一个数据库、一张超级表，或者基于现有超级表、子表或普通表的查询条件，即一条查询语句。用户可以利用 SQL 对标签、表名、列、表达式等条件进行过滤，并对数据进行标量函数与 UDF 计算（不包括数据聚合）。与其他消息队列工具相比，这是 TDengine 数据订阅功能的最大优势。它提供了更高的灵活性，数据的粒度由定义主题的 SQL 决定，而且数据的过滤与预处理由 TDengine 自动完成，从而减少传输的数据量并降低应用程序的复杂度。
|
||||
|
||||
消费者订阅主题后，可以实时接收最新的数据。多个消费者可以组成一个消费组，共享消费进度，实现多线程、分布式地消费数据，提高消费速度。不同消费组的消费者即使消费同一个主题，也不共享消费进度。一个消费者可以订阅多个主题。如果主题对应的是超级表或库，数据可能会分布在多个不同的节点或数据分片上。当一个消费组中有多个消费者时，可以提高消费效率。TDengine 的消息队列提供了消息的 ACK（Acknowledgment，确认，也译作收到）机制，确保在宕机、重启等复杂环境下实现至少一次（at least once）消费。
|
||||
|
||||
为实现上述功能，TDengine 会为预写数据日志（Write-Ahead Logging，WAL）文件自动创建索引，以支持快速随机访问，并提供了灵活可配置的文件切换与保留机制。用户可以根据需求指定 WAL 文件的保留时间和大小。通过这些方法，WAL 被改造成一个保留事件到达顺序的、可持久化的存储引擎。对于以主题形式创建的查询，TDengine 将从 WAL 读取数据。在消费过程中，TDengine 根据当前消费进度从 WAL 直接读取数据，并使用统一的查询引擎实现过滤、变换等操作，然后将数据推送给消费者。
|
||||
|
||||
从 3.2.0.0 版本开始，数据订阅支持 vnode 迁移和分裂。由于数据订阅依赖 wal 文件，而在 vnode 迁移和分裂的过程中，wal 文件并不会进行同步。因此，在迁移或分裂操作完成后，您将无法继续消费之前尚未消费完 wal 数据。请务必在执行 vnode 迁移或分裂之前，将所有 wal 数据消费完毕。
|
||||
|
||||
## 主题类型
|
||||
|
||||
TDengine 使用 SQL 创建的主题共有 3 种类型，下面分别介绍。
|
||||
|
||||
### 查询主题
|
||||
|
||||
订阅一条 SQL 查询的结果，本质上是连续查询，每次查询仅返回最新值，创建语法如下：
|
||||
|
||||
```sql
|
||||
CREATE TOPIC [IF NOT EXISTS] topic_name as subquery
|
||||
```
|
||||
|
||||
该 SQL 通过 SELECT 语句订阅（包括 SELECT *，或 SELECT ts, c1 等指定查询订阅，可以带条件过滤、标量函数计算，但不支持聚合函数、不支持时间窗口聚合）。需要注意的是：
|
||||
|
||||
1. 该类型 TOPIC 一旦创建则订阅数据的结构确定。
|
||||
2. 被订阅或用于计算的列或标签不可被删除（ALTER table DROP）、修改（ALTER table MODIFY）。
|
||||
3. 若发生表结构变更，新增的列不出现在结果中。
|
||||
4. 对于 select *，则订阅展开为创建时所有的列（子表、普通表为数据列，超级表为数据列加标签列）
|
||||
|
||||
假设需要订阅所有智能电表中电压值大于 200 的数据，且仅仅返回时间戳、电流、电压 3 个采集量（不返回相位），那么可以通过下面的 SQL 创建 power_topic 这个主题。
|
||||
|
||||
```sql
|
||||
CREATE TOPIC power_topic AS SELECT ts, current, voltage FROM power.meters WHERE voltage > 200;
|
||||
```
|
||||
|
||||
### 超级表主题
|
||||
|
||||
订阅一个超级表中的所有数据，语法如下：
|
||||
|
||||
```sql
|
||||
CREATE TOPIC [IF NOT EXISTS] topic_name [with meta] AS STABLE stb_name [where_condition]
|
||||
```
|
||||
|
||||
与使用 `SELECT * from stbName` 订阅的区别是：
|
||||
|
||||
1. 不会限制用户的表结构变更，即表结构变更以及变更后的新数据都能够订阅到。
|
||||
2. 返回的是非结构化的数据，返回数据的结构会随着超级表的表结构变化而变化。
|
||||
3. with meta 参数可选，选择时将返回创建超级表，子表等语句，主要用于 taosX 做超级表迁移。
|
||||
4. where_condition 参数可选，选择时将用来过滤符合条件的子表，订阅这些子表。where 条件里不能有普通列，只能是 tag 或 tbname，where 条件里可以用函数，用来过滤 tag，但是不能是聚合函数，因为子表 tag 值无法做聚合。可以是常量表达式，比如 2 > 1（订阅全部子表），或者 false（订阅 0 个子表）。
|
||||
5. 返回数据不包含标签。
|
||||
|
||||
### 数据库主题
|
||||
|
||||
订阅一个数据库里所有数据，其语法如下：
|
||||
|
||||
```sql
|
||||
CREATE TOPIC [IF NOT EXISTS] topic_name [with meta] AS DATABASE db_name;
|
||||
```
|
||||
|
||||
通过该语句可创建一个包含数据库所有表数据的订阅：
|
||||
|
||||
1. with meta 参数可选，选择时将返回数据库里所有超级表，子表、普通表的元数据创建、删除、修改语句，主要用于 taosX 做数据库迁移。
|
||||
2. 超级表订阅和库订阅属于高级订阅模式，容易出错，如确实要使用，请咨询技术支持人员。
|
||||
|
||||
## 删除主题
|
||||
|
||||
如果不再需要订阅数据，可以删除 topic，如果当前 topic 被消费者订阅，通过 FORCE 语法可强制删除，强制删除后订阅的消费者会消费数据会出错（FORCE 语法从 v3.3.6.0 开始支持）。
|
||||
|
||||
```sql
|
||||
DROP TOPIC [IF EXISTS] [FORCE] topic_name;
|
||||
```
|
||||
|
||||
## 查看主题
|
||||
|
||||
```sql
|
||||
SHOW TOPICS;
|
||||
```
|
||||
|
||||
上面的 SQL 会显示当前数据库下的所有主题的信息。
|
||||
|
||||
## 消费者
|
||||
|
||||
### 创建消费者
|
||||
|
||||
消费者的创建只能通过 TDengine 客户端驱动或者连接器所提供的 API 创建，详情可以参考开发指南或参考手册。
|
||||
|
||||
### 查看消费者
|
||||
|
||||
```sql
|
||||
SHOW CONSUMERS;
|
||||
```
|
||||
|
||||
显示当前数据库下所有消费者的信息，会显示消费者的状态，创建时间等信息。
|
||||
|
||||
### 删除消费组
|
||||
|
||||
消费者创建的时候，会给消费者指定一个消费者组，消费者不能显式的删除，但是可以删除消费者组。如果当前消费者组里有消费者在消费，通过 FORCE 语法可强制删除，强制删除后订阅的消费者会消费数据会出错（FORCE 语法从 v3.3.6.0 开始支持）。
|
||||
|
||||
```sql
|
||||
DROP CONSUMER GROUP [IF EXISTS] [FORCE] cgroup_name ON topic_name;
|
||||
```
|
||||
|
||||
## 数据订阅
|
||||
|
||||
### 查看订阅信息
|
||||
|
||||
```sql
|
||||
SHOW SUBSCRIPTIONS;
|
||||
```
|
||||
|
||||
显示 topic 在不同 vgroup 上的消费信息，可用于查看消费进度。
|
||||
|
||||
### 订阅数据
|
||||
|
||||
TDengine 提供了全面且丰富的数据订阅 API，旨在满足不同编程语言和框架下的数据订阅需求。这些接口包括但不限于创建消费者、订阅主题、取消订阅、获取实时数据、提交消费进度以及获取和设置消费进度等功能。目前，TDengine 支持多种主流编程语言，包括 C、Java、Go、Rust、Python 和 C# 等，使得开发者能够轻松地在各种应用场景中使用 TDengine 的数据订阅功能。
|
||||
|
||||
值得一提的是，TDengine 的数据订阅 API 与业界流行的 Kafka 订阅 API 保持了高度的一致性，以便于开发者能够快速上手并利用现有的知识经验。为了方便用户了解和参考，TDengine 的官方文档详细介绍了各种 API 的使用方法和示例代码，具体内容可访问 TDengine 官方网站的连接器部分。通过这些 API，开发者可以高效地实现数据的实时订阅和处理，从而满足各种复杂场景下的数据处理需求。
|
||||
|
||||
TDengine v3.3.7.0 版本提供了 MQTT 订阅功能，可以通过 MQTT 客户端直接订阅数据，具体内容请参考 MQTT 数据订阅部分。
|
||||
|
||||
### 回放功能
|
||||
|
||||
TDengine 的数据订阅功能支持回放（replay）功能，允许用户按照数据的实际写入时间顺序重新播放数据流。这一功能基于 TDengine 的高效 WAL 机制实现，确保了数据的一致性和可靠性。
|
||||
|
||||
要使用数据订阅的回放功能，用户可以在查询语句中指定时间范围，从而精确控制回放的起始时间和结束时间。这使得用户能够轻松地重放特定时间段内的数据，无论是为了故障排查、数据分析还是其他目的。
|
||||
|
||||
如果写入了如下 3 条数据，那么回放时则先返回第 1 条数据，5s 后返回第 2 条数据，在获取第 2 条数据 3s 后返回第 3 条数据。
|
||||
|
||||
```text
|
||||
2023/09/22 00:00:00.000
|
||||
2023/09/22 00:00:05.000
|
||||
2023/09/22 00:00:08.000
|
||||
```
|
||||
|
||||
使用数据订阅的回放功能时需要注意如下几项：
|
||||
|
||||
- 通过配置消费参数 enable.replay 为 true 开启回放功能。
|
||||
|
||||
- 数据订阅的回放功能仅查询订阅支持数据回放，超级表和库订阅不支持回放。
|
||||
|
||||
- 回放不支持进度保存。
|
||||
|
||||
- 因为数据回放本身需要处理时间，所以回放的精度存在几十毫秒的误差。
|
||||
|
||||
|
||||
|
||||
```mdx-code-block
|
||||
import DocCardList from '@theme/DocCardList';
|
||||
import {useCurrentSidebarCategory} from '@docusaurus/theme-common';
|
||||
|
||||
<DocCardList items={useCurrentSidebarCategory().items}/>
|
||||
```
|
||||
|
|
@ -4,286 +4,189 @@ title: 流计算
|
|||
toc_max_heading_level: 4
|
||||
---
|
||||
|
||||
在时序数据的处理中，经常要对原始数据进行清洗、预处理，再使用时序数据库进行长久的储存，而且经常还需要使用原始的时序数据通过计算生成新的时序数据。在传统的时序数据解决方案中，常常需要部署 Kafka、Flink 等流处理系统，而流处理系统的复杂性，带来了高昂的开发与运维成本。
|
||||
在时序数据的处理中，存在大量的流计算需求，例如：
|
||||
|
||||
TDengine 的流计算引擎提供了实时处理写入的数据流的能力，使用 SQL 定义实时流变换，当数据被写入流的源表后，数据会被以定义的方式自动处理，并根据定义的触发模式向目的表推送结果。它提供了替代复杂流处理系统的轻量级解决方案，并能够在高吞吐的数据写入的情况下，提供毫秒级的计算结果延迟。
|
||||
- **数据分级存储与智能降采样**：工业设备每秒生成数万条原始数据，若全量存储，则存储成本激增，查询效率低下，历史趋势分析响应时间长
|
||||
- **预计算加速实时决策**：用户查询全量数据时，可能需扫描百亿级别数据，很难实时获取查询结果，大屏/报表产生卡顿
|
||||
- **异常检测和低延迟告警**：异常检测、监控报警，需要根据规则低延迟地获取特定数据，传统批处理的延迟通常在分钟级别
|
||||
|
||||
流计算可以包含数据过滤，标量函数计算（含 UDF），以及窗口聚合（支持滑动窗口、会话窗口与状态窗口），能够以超级表、子表、普通表为源表，写入到目的超级表。在创建流时，目的超级表将被自动创建，随后新插入的数据会被流定义的方式处理并写入其中，通过 partition by 子句，可以以表名或标签划分 partition，不同的 partition 将写入到目的超级表的不同子表。
|
||||
在传统的时序数据解决方案中，常常需要部署 Kafka、Flink 等流处理系统，而流处理系统的复杂性，带来了高昂的开发与运维成本。TDengine 的流计算引擎提供了实时处理写入的数据流的能力，使用 SQL 定义实时流变换，当数据被写入流的源表后，数据会被以定义的方式自动处理，并根据定义的触发模式向目的表推送结果。它提供了替代复杂流处理系统的轻量级解决方案，并能够在高吞吐的数据写入的情况下，提供毫秒级的计算结果延迟。与传统的流计算相比，TDengine 的流计算采用的是触发与计算分离的策略，处理的依然是持续的无界的数据流，但是进行了以下几个方面的扩展：
|
||||
|
||||
TDengine 的流计算能够支持分布在多个节点中的超级表聚合，能够处理乱序数据的写入。它提供 watermark 机制以度量容忍数据乱序的程度，并提供了 ignore expired 配置项以决定乱序数据的处理策略 —— 丢弃或者重新计算。
|
||||
- **处理对象的扩展**：传统流计算的事件驱动对象与计算对象往往是统一的，根据同一份数据产生事件和计算。TDengine 的流计算支持触发（事件驱动）与计算的分离，也就意味着触发对象可以与计算对象进行分离。触发表与计算的数据源表可以不相同，甚至可以不需要触发表，处理的数据集合无论是列、时间范围都可以不相同。
|
||||
- **触发方式的扩展**：除了数据写入触发方式外，TDengine 的流计算支持更多触发方式的扩展。通过支持窗口触发，用户可以灵活的定义和使用各种方式的窗口来产生触发事件，可以选择在开窗、关窗以及开关窗同时进行触发。除了与触发表关联的事件时间驱动外，还支持与事件时间无关的驱动，即定时触发。在事件触发之前，还支持对触发数据进行预先过滤处理，只有符合条件的数据才会进入触发判断。
|
||||
- **计算的扩展**：既可以对触发表进行计算，也可以对其他库、表进行计算。计算类型不受限制，支持任何查询语句。计算结果的应用可根据需要进行选择，支持发送通知、写入输出表，也可以两者同时使用。
|
||||
|
||||
注意：windows 平台不支持流计算。
|
||||
TDengine 的流计算引擎还提供了其他使用上的便利。针对结果延迟的不同需求，支持用户在结果时效性与资源负载之间进行平衡。针对非正常顺序写入场景的不同需求，支持用户灵活选择适合的处理方式与策略。它提供了替代复杂流处理系统的轻量级解决方案，并能够在高吞吐的数据写入的情况下，提供毫秒级的计算结果延迟。
|
||||
|
||||
下面详细介绍流计算使用的具体方法。
|
||||
流计算的使用方法如下，详细内容参见 [SQL 手册](../../reference/taos-sql/stream)。
|
||||
|
||||
## 创建流计算
|
||||
|
||||
语法如下：
|
||||
## 流式计算的创建
|
||||
|
||||
```sql
|
||||
CREATE STREAM [IF NOT EXISTS] stream_name [stream_options] INTO stb_name
|
||||
[(field1_name, ...)] [TAGS (column_definition [, column_definition] ...)]
|
||||
SUBTABLE(expression) AS subquery
|
||||
CREATE STREAM [IF NOT EXISTS] [db_name.]stream_name options [INTO [db_name.]table_name] [OUTPUT_SUBTABLE(tbname_expr)] [(column_name1, column_name2 [COMPOSITE KEY][, ...])] [TAGS (tag_definition [, ...])] [AS subquery]
|
||||
|
||||
stream_options: {
|
||||
TRIGGER [AT_ONCE | WINDOW_CLOSE | MAX_DELAY time | FORCE_WINDOW_CLOSE | CONTINUOUS_WINDOW_CLOSE [recalculate rec_time_val] ]
|
||||
WATERMARK time
|
||||
IGNORE EXPIRED [0|1]
|
||||
DELETE_MARK time
|
||||
FILL_HISTORY [0|1] [ASYNC]
|
||||
IGNORE UPDATE [0|1]
|
||||
options: {
|
||||
trigger_type [FROM [db_name.]table_name] [PARTITION BY col1 [, ...]] [STREAM_OPTIONS(stream_option [|...])] [notification_definition]
|
||||
}
|
||||
|
||||
column_definition:
|
||||
col_name col_type [COMMENT 'string_value']
|
||||
```
|
||||
|
||||
其中 subquery 是 select 普通查询语法的子集。
|
||||
|
||||
```sql
|
||||
subquery: SELECT select_list
|
||||
from_clause
|
||||
[WHERE condition]
|
||||
[PARTITION BY tag_list]
|
||||
[window_clause]
|
||||
|
||||
window_cluse: {
|
||||
SESSION(ts_col, tol_val)
|
||||
| STATE_WINDOW(col)
|
||||
| INTERVAL(interval_val [, interval_offset]) [SLIDING (sliding_val)]
|
||||
| EVENT_WINDOW START WITH start_trigger_condition END WITH end_trigger_condition
|
||||
| COUNT_WINDOW(count_val[, sliding_val])
|
||||
trigger_type: {
|
||||
PERIOD(period_time[, offset_time])
|
||||
| [INTERVAL(interval_val[, interval_offset])] SLIDING(sliding_val[, offset_time])
|
||||
| SESSION(ts_col, session_val)
|
||||
| STATE_WINDOW(col) [TRUE_FOR(duration_time)]
|
||||
| EVENT_WINDOW(START WITH start_condition END WITH end_condition) [TRUE_FOR(duration_time)]
|
||||
| COUNT_WINDOW(count_val[, sliding_val][, col1[, ...]])
|
||||
}
|
||||
|
||||
stream_option: {WATERMARK(duration_time) | EXPIRED_TIME(exp_time) | IGNORE_DISORDER | DELETE_RECALC | DELETE_OUTPUT_TABLE | FILL_HISTORY[(start_time)] | FILL_HISTORY_FIRST[(start_time)] | CALC_NOTIFY_ONLY | LOW_LATENCY_CALC | PRE_FILTER(expr) | FORCE_OUTPUT | MAX_DELAY(delay_time) | EVENT_TYPE(event_types)}
|
||||
|
||||
notification_definition:
|
||||
NOTIFY(url [, ...]) [ON (event_types)] [WHERE condition] [NOTIFY_OPTIONS(notify_option[|notify_option])]
|
||||
|
||||
notify_option: [NOTIFY_HISTORY | ON_FAILURE_PAUSE]
|
||||
|
||||
event_types:
|
||||
event_type [|event_type]
|
||||
|
||||
event_type: {WINDOW_OPEN | WINDOW_CLOSE}
|
||||
|
||||
tag_definition:
|
||||
tag_name type_name [COMMENT 'string_value'] AS expr
|
||||
```
|
||||
|
||||
subquery 支持会话窗口、状态窗口、时间窗口、事件窗口与计数窗口。其中，状态窗口、事件窗口与计数窗口搭配超级表时必须与 partition by tbname 一起使用。
|
||||
### 触发方式
|
||||
|
||||
1. 其中，SESSION 是会话窗口，tol_val 是时间间隔的最大范围。在 tol_val 时间间隔范围内的数据都属于同一个窗口，如果连续的两条数据的时间间隔超过 tol_val，则自动开启下一个窗口。
|
||||
- **定时触发**：通过系统时间的固定间隔来驱动，以建流当天系统时间的零点作为基准时间点，然后根据间隔来确定下次触发的时间点，可以通过指定时间偏移来改变基准时间点。
|
||||
- **滑动触发**：对触发表的写入数据按照事件时间的固定间隔来驱动的触发。可以有 INTERVAL 窗口，也可以没有。
|
||||
- **会话窗口触发**：对触发表的写入数据按照会话窗口的方式进行窗口划分，当窗口启动和（或）关闭时进行触发。
|
||||
- **状态窗口触发**：对触发表的写入数据按照状态窗口的方式进行窗口划分，当窗口启动和（或）关闭时进行触发。
|
||||
- **事件窗口触发**：对触发表的写入数据按照事件窗口的方式进行窗口划分，当窗口启动和（或）关闭时进行的触发。
|
||||
- **计数窗口触发**：对触发表的写入数据按照计数窗口的方式进行窗口划分，当窗口启动和（或）关闭时进行的触发。支持列的触发，当指定的列有数据写入时才触发。
|
||||
|
||||
2. STATE_WINDOW 是状态窗口，col 用来标识状态量，相同的状态量数值则归属于同一个状态窗口，col 数值改变后则当前窗口结束，自动开启下一个窗口。
|
||||
### 触发动作
|
||||
|
||||
3. INTERVAL 是时间窗口，又可分为滑动时间窗口和翻转时间窗口。INTERVAL 子句用于指定窗口相等时间周期，SLIDING 字句用于指定窗口向前滑动的时间。当 interval_val 与 sliding_val 相等的时候，时间窗口即为翻转时间窗口，否则为滑动时间窗口，注意：sliding_val 必须小于等于 interval_val。
|
||||
触发后可以根据需要执行不同的动作，比如发送[事件通知](../../reference/taos-sql/stream/#流式计算的通知机制)、[执行计算](../../reference/taos-sql/stream/#流式计算的计算任务)，或者两者同时进行。
|
||||
|
||||
4. EVENT_WINDOW 是事件窗口，根据开始条件和结束条件来划定窗口。当 start_trigger_condition 满足时则窗口开始，直到 end_trigger_condition 满足时窗口关闭。start_trigger_condition 和 end_trigger_condition 可以是任意 TDengine 支持的条件表达式，且可以包含不同的列。
|
||||
- 只通知不计算：通过 `WebSocket` 方式向外部应用发送事件通知。
|
||||
- 只计算不通知：执行任意一个查询并保存结果到流计算的输出表中。
|
||||
- 既通知又计算：执行任意一个查询，同时发送计算结果或事件通知给外部应用。
|
||||
|
||||
5. COUNT_WINDOW 是计数窗口，按固定的数据行数来划分窗口。count_val 是常量，是正整数，必须大于等于 2，小于 2147483648。count_val 表示每个 COUNT_WINDOW 包含的最大数据行数，总数据行数不能整除 count_val 时，最后一个窗口的行数会小于 count_val。sliding_val 是常量，表示窗口滑动的数量，类似于 INTERVAL 的 SLIDING。
|
||||
### 触发表与分组
|
||||
|
||||
窗口的定义与时序数据窗口查询中的定义完全相同，具体可参考 TDengine 窗口函数部分。
|
||||
通常意义来说，一个流计算只对应一个计算，比如根据一个子表触发和产生一个计算，结果保存到一张表中。根据 TDengine **一个设备一张表**的设计理念，如果需要对所有设备分别计算，那就需要为每个子表创建一个流计算，这会造成使用的不便和处理效率的降低。为了解决这个问题，TDengine 的流计算支持触发分组，分组是流计算的最小执行单元，从逻辑上可以认为每个分组对应一个单独的流计算，每个分组对应一个输出表和单独的事件通知。
|
||||
|
||||
如下 SQL 将创建一个流计算，执行后 TDengine 会自动创建名为 avg_vol 的超级表，此流计算以 1min 为时间窗口、30s 为前向增量统计这些智能电表的平均电压，并将来自 meters 的数据的计算结果写入 avg_vol，不同分区的数据会分别创建子表并写入不同子表。
|
||||
**总结来说，一个流计算输出表（子表或普通表）的个数与触发表的分组个数相同，未指定分组时只产生一个输出表（普通表）。**
|
||||
|
||||
### 计算任务
|
||||
|
||||
计算任务是流在事件触发后执行的计算动作，可以是**任意类型的查询语句**，既可以对触发表进行计算，也可以对其他库表进行计算。计算时如需使用触发时的关联信息，可在 SQL 语句中使用占位符，占位符在每次计算时会被作为常量替换到 SQL 语句中。包括：
|
||||
|
||||
- `_tprev_ts`：上一次触发的事件时间
|
||||
- `_tcurrent_ts`：本次触发的事件时间
|
||||
- `_tnext_ts`：下一次触发的事件时间
|
||||
- `_twstart`：本次触发窗口的起始时间戳
|
||||
- `_twend`：本次触发窗口的结束时间戳
|
||||
- `_twduration`：本次触发窗口的持续时间
|
||||
- `_twrownum`：本次触发窗口的记录条数
|
||||
- `_tprev_localtime`：上一次触发时刻的系统时间
|
||||
- `_tnext_localtime`：下一次触发时刻的系统时间
|
||||
- `_tgrpid`：触发分组的 ID 值
|
||||
- `_tlocaltime`：本次触发时刻的系统时间
|
||||
- `%%n`：触发分组列的引用，n 为分组列的下标
|
||||
- `%%tbname`：触发表每个分组表名的引用，可作为查询表名使用（`FROM %%tbname`）
|
||||
- `%%trows`：触发表每个分组的触发数据集（满足本次触发的数据集）的引用
|
||||
|
||||
### 通知机制
|
||||
|
||||
事件通知是流在事件触发后可选的执行动作，支持通过 `WebSocket` 协议发送事件通知到应用。用户可以指定需要通知的事件，以及用于接收通知消息的目标地址。通知内容可以包含计算结果，也可以在没有计算结果时只通知事件相关信息。
|
||||
|
||||
## 流式计算的示例
|
||||
|
||||
### 计数窗口触发
|
||||
|
||||
- 表 tb1 每写入 1 行数据时，计算表 tb2 在同一时刻前 5 分钟内 col1 的平均值，计算结果写入表 tb3。
|
||||
|
||||
```SQL
|
||||
CREATE STREAM sm1 COUNT_WINDOW(1) FROM tb1
|
||||
INTO tb3 AS
|
||||
SELECT _twstart, avg(col1) FROM tb2
|
||||
WHERE _c0 >= _twend - 5m AND _c0 <= _twend;
|
||||
```
|
||||
|
||||
- 表 tb1 每写入 10 行大于 0 的 col1 列数据时，计算这 10 条数据 col1 列的平均值，计算结果不需要保存，需要通知到 `ws://localhost:8080/notify`。
|
||||
|
||||
```SQL
|
||||
CREATE STREAM sm2 COUNT_WINDOW(10, 1, col1) FROM tb1
|
||||
STREAM_OPTIONS(CALC_ONTIFY_ONLY | PRE_FILTER(col1 > 0))
|
||||
NOTIFY("ws://localhost:8080/notify") ON (WINDOW_CLOSE)
|
||||
AS
|
||||
SELECT avg(col1) FROM %%trows;
|
||||
```
|
||||
|
||||
### 滑动触发
|
||||
|
||||
- 超级表 stb1 的每个子表在每 5 分钟的时间窗口结束后，计算这 5 分钟的 col1 的平均值，每个子表的计算结果分别写入超级表 stb2 的不同子表中。
|
||||
|
||||
```SQL
|
||||
CREATE STREAM sm1 INTERVAL(5m) SLIDING(5m) FROM stb1 PARTITION BY tbname
|
||||
INTO stb2 AS
|
||||
SELECT _twstart, avg(col1) FROM %%tbname
|
||||
WHERE _c0 >= _twstart AND _c0 <= _twend;
|
||||
```
|
||||
|
||||
> 上面 SQL 中的 `from %%tbname where _c0 >= _twstart and _c0 <= _twend` 与 `from %%trows` 的含义是不完全相同的。前者表示计算使用触发分组对应的表中在触发窗口时间段内的数据，窗口内的数据在计算时与 `%%trows` 相比较是有可能有变化的，后者则表示只使用触发时读取到的窗口数据。
|
||||
|
||||
- 超级表 stb1 的每个子表从最早的数据开始，在每 5 分钟的时间窗口结束后或从窗口启动 1 分钟后窗口仍然未关闭时，计算窗口内的 col1 的平均值，每个子表的计算结果分别写入超级表 stb2 的不同子表中。
|
||||
|
||||
```SQL
|
||||
CREATE STREAM sm2 INTERVAL(5m) SLIDING(5m) FROM stb1 PARTITION BY tbname
|
||||
STREAM_OPTIONS(MAX_DELAY(1m) | FILL_HISTORY_FIRST)
|
||||
INTO stb2 AS
|
||||
SELECT _twstart, avg(col1) FROM %%tbname WHERE _c0 >= _twstart AND _c0 <= _twend;
|
||||
```
|
||||
|
||||
- 计算电表电流的每分钟平均值，并在窗口打开、关闭时向两个通知地址发送通知，计算历史数据时不发送通知，不允许在通知发送失败时丢弃通知。
|
||||
|
||||
```sql
|
||||
CREATE STREAM avg_vol_s INTO avg_vol AS
|
||||
SELECT _wstart, count(*), avg(voltage) FROM power.meters PARTITION BY tbname INTERVAL(1m) SLIDING(30s);
|
||||
CREATE STREAM avg_stream INTERVAL(1m) SLIDING(1m) FROM meters
|
||||
NOTIFY ('ws://localhost:8080/notify', 'wss://192.168.1.1:8080/notify?key=foo') ON ('WINDOW_OPEN', 'WINDOW_CLOSE') NOTIFY_OPTIONS(NOTIFY_HISTORY | ON_FAILURE_PAUSE)
|
||||
INTO avg_stb
|
||||
AS SELECT _twstart, _twend, AVG(current) FROM %%trows;
|
||||
```
|
||||
|
||||
本节涉及的相关参数的说明如下。
|
||||
### 定时触发
|
||||
|
||||
- stb_name 是保存计算结果的超级表的表名，如果该超级表不存在，则会自动创建；如果已存在，则检查列的 schema 信息。
|
||||
- tags 子句定义了流计算中创建标签的规则。通过 tags 字段可以为每个分区对应的子表生成自定义的标签值。
|
||||
- 每过 1 小时计算表 tb1 中总的数据量，计算结果写入表 tb2 (毫秒库)。
|
||||
|
||||
## 流式计算的规则和策略
|
||||
|
||||
### 流计算的分区
|
||||
|
||||
在 TDengine 中，我们可以利用 partition by 子句结合 tbname、标签列、普通列或表达式，对一个流进行多分区的计算。每个分区都拥有独立的时间线和时间窗口，它们会分别进行数据聚合，并将结果写入目的表的不同子表中。如果不使用 partition by 子句，所有数据将默认写入同一张子表中。
|
||||
|
||||
特别地，partition by + tbname 是一种非常实用的操作，它表示对每张子表进行流计算。这样做的好处是可以针对每张子表的特点进行定制化处理，以提高计算效率。
|
||||
|
||||
在创建流时，如果不使用 substable 子句，流计算所创建的超级表将包含一个唯一的标签列 groupId。每个分区将被分配一个唯一的 groupId，并通过 MD5 算法计算相应的子表名称。TDengine 将自动创建这些子表，以便存储各个分区的计算结果。这种机制使得数据管理更加灵活和高效，同时也方便后续的数据查询和分析。
|
||||
|
||||
若创建流的语句中包含 substable 子句，用户可以为每个分区对应的子表生成自定义的表名。示例如下。
|
||||
|
||||
```sql
|
||||
CREATE STREAM avg_vol_s INTO avg_vol SUBTABLE(CONCAT('new-', tname)) AS SELECT _wstart, count(*), avg(voltage) FROM meters PARTITION BY tbname tname INTERVAL(1m);
|
||||
```SQL
|
||||
CREATE STREAM sm1 PERIOD(1h)
|
||||
INTO tb2 AS
|
||||
SELECT cast(_tlocaltime/1000000 AS TIMESTAMP), count(*) FROM tb1;
|
||||
```
|
||||
|
||||
PARTITION 子句中，为 tbname 定义了一个别名 tname，在 PARTITION 子句中的别名可以用于 SUBTABLE 子句中的表达式计算，在上述示例中，流新创建的子表规则为 `new- + 子表名 + _超级表名 + _groupId`。
|
||||
|
||||
**注意**：子表名的长度若超过 TDengine 的限制，将被截断。若要生成的子表名已经存在于另一超级表，由于 TDengine 的子表名是唯一的，因此对应新子表的创建以及数据的写入将会失败。
|
||||
|
||||
### 流计算处理历史数据
|
||||
|
||||
在正常情况下，流计算任务不会处理那些在流创建之前已经写入源表的数据。这是因为流计算的触发是基于新写入的数据，而非已有数据。然而，如果我们需要处理这些已有的历史数据，可以在创建流时设置 fill_history 选项为 1。
|
||||
|
||||
通过启用 fill_history 选项，创建的流计算任务将具备处理创建前、创建过程中以及创建后写入的数据的能力。这意味着，无论数据是在流创建之前还是之后写入的，都将纳入流计算的范围，从而确保数据的完整性和一致性。这一设置为用户提供了更大的灵活性，使其能够根据实际需求灵活处理历史数据和新数据。
|
||||
|
||||
注意：
|
||||
|
||||
- 开启 fill_history 时，创建流需要找到历史数据的分界点，如果历史数据很多，可能会导致创建流任务耗时较长，此时可以通过 fill_history 1 async（v3.3.6.0 开始支持）语法将创建流的任务放在后台处理，创建流的语句可立即返回，不阻塞后面的操作。async 只对 fill_history 1 起效，fill_history 0 时建流很快，不需要异步处理。
|
||||
|
||||
- 通过 show streams 可查看后台建流的进度（ready 状态表示成功，init 状态表示正在建流，failed 状态表示建流失败，失败时 message 列可以查看原因。对于建流失败的情况可以删除流重新建立）。
|
||||
|
||||
- 另外，不要同时异步创建多个流，可能由于事务冲突导致后面创建的流失败。
|
||||
比如，创建一个流，统计所有智能电表每 10s 产生的数据条数，并且计算历史数据。SQL 如下：
|
||||
|
||||
```sql
|
||||
|
||||
create stream if not exists count_history_s fill_history 1 into count_history as select count(*) from power.meters interval(10s)
|
||||
- 每过 1 小时通知 `ws://localhost:8080/notify` 当前系统时间。
|
||||
|
||||
```SQL
|
||||
CREATE STREAM sm1 PERIOD(1h)
|
||||
NOTIFY("ws://localhost:8080/notify");
|
||||
```
|
||||
|
||||
如果该流任务已经彻底过期，并且不再想让它检测或处理数据，您可以手动删除它，被计算出的数据仍会被保留。
|
||||
## 流式计算的其他特性
|
||||
|
||||
### 流计算的触发模式
|
||||
### 高可用
|
||||
|
||||
在创建流时，可以通过 TRIGGER 指令指定流计算的触发模式。对于非窗口计算，流计算的触发是实时的，对于窗口计算，目前提供 4 种触发模式，默认为 WINDOW_CLOSE。
|
||||
流式计算从架构上支持流的存算分离，在部署时要求系统中必须部署 snode，除数据读取外，所有流计算功能都只在 snode 上运行。
|
||||
|
||||
1. AT_ONCE：写入立即触发。
|
||||
2. WINDOW_CLOSE：窗口关闭时触发（窗口关闭由事件时间决定，可配合 watermark 使用）。
|
||||
3. MAX_DELAY time：若窗口关闭，则触发计算。若窗口未关闭，且未关闭时长超过 max delay 指定的时间，则触发计算。
|
||||
4. FORCE_WINDOW_CLOSE：以操作系统当前时间为准，只计算当前关闭窗口的结果，并推送出去。窗口只会在被关闭的时刻计算一次，后续不会再重复计算。该模式当前只支持 INTERVAL 窗口（支持滑动）；该模式时，FILL_HISTORY 自动设置为 0，IGNORE EXPIRED 自动设置为 1，IGNORE UPDATE 自动设置为 1；FILL 只支持 PREV、NULL、NONE、VALUE。
|
||||
- 该模式可用于实现连续查询，比如，创建一个流，每隔 1s 查询一次过去 10s 窗口内的数据条数。SQL 如下：
|
||||
- snode 是负责运行流计算计算任务的节点，在一个集群中可以部署 1 或多个 snode。
|
||||
- snode 部署在单独的 dnode 上时，可以保证资源隔离，不会对写入、查询等业务造成太大干扰。
|
||||
- 为了保证流计算的高可用，可在一个集群的多个物理节点中同时部署多个 snode：
|
||||
- 流计算在多个 snode 间进行负载均衡。
|
||||
- 每两个 snode 间互为副本，负责存储流的状态和进度等信息。
|
||||
|
||||
```sql
|
||||
create stream if not exists continuous_query_s trigger force_window_close into continuous_query as select count(*) from power.meters interval(10s) sliding(1s)
|
||||
```
|
||||
### 重新计算
|
||||
|
||||
5. CONTINUOUS_WINDOW_CLOSE：窗口关闭时输出结果。修改、删除数据，并不会立即触发重算，每等待 rec_time_val 时长，会进行周期性重算。如果不指定 rec_time_val，那么重算周期是 60 分钟。如果重算的时间长度超过 rec_time_val，在本次重算后，自动开启下一次重算。该模式当前只支持 INTERVAL 窗口。如果使用 FILL，需要配置 adapter 的相关信息：adapterFqdn、adapterPort、adapterToken。adapterToken 为 `{username}:{password}` 经过 Base64 编码之后的字符串，例如 `root:taosdata` 编码后为 `cm9vdDp0YW9zZGF0YQ==`。
|
||||
支持使用 `WATERMARK` 来解决一定程度的乱序、更新、删除场景带来的问题。`WATERMARK` 是用户可以指定的基于事件时间的时长，它代表的是事件时间在流计算中的进展，体现了用户对于乱序数据的容忍程度。`当前处理的最新事件时间 - WATERMARK 指定的固定间隔` 即为当前水位线，只有写入数据的事件时间早于当前水位线才会进入触发判断，只有窗口或其他触发的时间条件早于当前水位线才会启动触发。
|
||||
|
||||
窗口关闭是由事件时间决定的，如事件流中断、或持续延迟，此时事件时间无法更新，可能导致无法得到最新的计算结果。
|
||||
对于超出 `WATERMARK` 的乱序、更新、删除场景，使用重新计算的方式来保证最终结果的正确性，重新计算意味着对于乱序、更新和删除的数据覆盖区间重新进行触发和运算。为了保证这种方式的有效性，用户需要确保其计算语句和数据源表是与处理时间无关的，也就是说同一个触发即使执行多次其结果依然是有效的。
|
||||
|
||||
因此，流计算提供了以事件时间结合处理时间计算的 MAX_DELAY 触发模式：MAX_DELAY 模式在窗口关闭时会立即触发计算，它的单位可以自行指定，具体单位：a（毫秒）、s（秒）、m（分）、h（小时）、d（天）、w（周）。此外，当数据写入后，计算触发的时间超过 MAX_DELAY 指定的时间，则立即触发计算。
|
||||
|
||||
### 流计算的窗口关闭
|
||||
|
||||
流计算的核心在于以事件时间（即写入记录中的时间戳主键）为基准来计算窗口的关闭时间，而不是依赖于 TDengine 服务器的时间。采用事件时间作为基准可以有效地规避客户端与服务器时间不一致所带来的问题，并且能够妥善解决数据乱序写入等挑战。
|
||||
|
||||
为了进一步控制数据乱序的容忍程度，流计算引入了 watermark 机制。在创建流时，用户可以通过 stream_option 参数指定 watermark 的值，该值定义了数据乱序的容忍上界，默认情况下为 0。
|
||||
|
||||
假设 T= 最新事件时间－ watermark，那么每次写入新数据时，系统都会根据这个公式更新窗口的关闭时间。具体而言，系统会将窗口结束时间小于 T 的所有打开的窗口关闭。如果触发模式设置为 window_close 或 max_delay，则会推送窗口聚合的结果。下图展示了流计算的窗口关闭流程。
|
||||
|
||||
|
||||
|
||||
在上图中，纵轴表示时刻，横轴上的圆点表示已经收到的数据。相关流程说明如下。
|
||||
|
||||
1. T1 时刻，第 7 个数据点到达，根据 T = Latest event - watermark，算出的时间在第二个窗口内，所以第二个窗口没有关闭。
|
||||
2. T2 时刻，第 6 和第 8 个数据点延迟到达 TDengine，由于此时的 Latest event 没变，T 也没变，乱序数据进入的第二个窗口还未被关闭，因此可以被正确处理。
|
||||
3. T3 时刻，第 10 个数据点到达，T 向后推移超过了第二个窗口关闭的时间，该窗口被关闭，乱序数据被正确处理。
|
||||
|
||||
在 window_close 或 max_delay 模式下，窗口关闭直接影响推送结果。在 at_once 模式下，窗口关闭只与内存占用有关。
|
||||
|
||||
### 过期数据处理策略
|
||||
|
||||
对于已关闭的窗口，再次落入该窗口中的数据被标记为过期数据。TDengine 对于过期数据提供两种处理方式，由 IGNORE EXPIRED 选项指定。
|
||||
|
||||
1. 重新计算，即 IGNORE EXPIRED 0：从 TSDB 中重新查找对应窗口的所有数据并重新计算得到最新结果
|
||||
2. 直接丢弃，即 IGNORE EXPIRED 1：默认配置，忽略过期数据
|
||||
|
||||
无论在哪种模式下，watermark 都应该被妥善设置，来得到正确结果（直接丢弃模式）或避免频繁触发重算带来的性能开销（重新计算模式）
|
||||
|
||||
### 数据更新的处理策略
|
||||
|
||||
TDengine 对于修改数据提供两种处理方式，由 IGNORE UPDATE 选项指定。
|
||||
|
||||
1. 检查数据是否被修改，即 IGNORE UPDATE 0：默认配置，如果被修改，则重新计算对应窗口。
|
||||
2. 不检查数据是否被修改，全部按增量数据计算，即 IGNORE UPDATE 1。
|
||||
|
||||
## 流计算的其它策略
|
||||
|
||||
### 写入已存在的超级表
|
||||
|
||||
当流计算结果需要写入已存在的超级表时，应确保 stb_name 列与 subquery 输出结果之间的对应关系正确。如果 stb_name 列与 subquery 输出结果的位置、数量完全匹配，那么不需要显式指定对应关系；如果数据类型不匹配，系统会自动将 subquery 输出结果的类型转换为对应的 stb_name 列的类型。创建流计算时不能指定 stb_name 的列和 TAG 的数据类型，否则会报错。
|
||||
|
||||
对于已经存在的超级表，系统会检查列的 schema 信息，确保它们与 subquery 输出结果相匹配。以下是一些关键点。
|
||||
|
||||
1. 检查列的 schema 信息是否匹配，对于不匹配的，则自动进行类型转换，当前只有数据长度大于 4096 bytes 时才报错，其余场景都能进行类型转换。
|
||||
2. 检查列的个数是否相同，如果不同，需要显示的指定超级表与 subquery 的列的对应关系，否则报错。如果相同，可以指定对应关系，也可以不指定，不指定则按位置顺序对应。
|
||||
|
||||
**注意** 虽然流计算可以将结果写入已经存在的超级表，但不能让两个已经存在的流计算向同一张（超级）表中写入结果数据。这是为了避免数据冲突和不一致，确保数据的完整性和准确性。在实际应用中，应根据实际需求和数据结构合理设置列的对应关系，以实现高效、准确的数据处理。
|
||||
|
||||
### 自定义目标表的标签
|
||||
|
||||
用户可以为每个 partition 对应的子表生成自定义的 TAG 值，如下创建流的语句，
|
||||
|
||||
```sql
|
||||
CREATE STREAM output_tag trigger at_once INTO output_tag_s TAGS(alias_tag varchar(100)) as select _wstart, count(*) from power.meters partition by concat("tag-", tbname) as alias_tag interval(10s);
|
||||
```
|
||||
|
||||
在 PARTITION 子句中，为 concat（"tag-"，tbname）定义了一个别名 alias_tag，对应超级表 output_tag_s 的自定义 TAG 的名字。在上述示例中，流新创建的子表的 TAG 将以前缀 'tag-' 连接原表名作为 TAG 的值。会对 TAG 信息进行如下检查。
|
||||
|
||||
1. 检查 tag 的 schema 信息是否匹配，对于不匹配的，则自动进行数据类型转换，当前只有数据长度大于 4096 bytes 时才报错，其余场景都能进行类型转换。
|
||||
2. 检查 tag 的 个数是否相同，如果不同，需要显示的指定超级表与 subquery 的 tag 的对应关系，否则报错。如果相同，可以指定对应关系，也可以不指定，不指定则按位置顺序对应。
|
||||
|
||||
### 清理流计算的中间状态
|
||||
|
||||
```sql
|
||||
DELETE_MARK time
|
||||
```
|
||||
|
||||
DELETE_MARK 用于删除缓存的窗口状态，也就是删除流计算的中间结果。缓存的窗口状态主要用于过期数据导致的窗口结果更新操作。如果不设置，默认值是 10 年。
|
||||
|
||||
## 流计算的具体操作
|
||||
|
||||
### 删除流计算
|
||||
|
||||
仅删除流计算任务，由流计算写入的数据不会被删除，SQL 如下：
|
||||
|
||||
```sql
|
||||
DROP STREAM [IF EXISTS] stream_name;
|
||||
```
|
||||
|
||||
### 展示流计算
|
||||
|
||||
查看流计算任务的 SQL 如下：
|
||||
|
||||
```sql
|
||||
SHOW STREAMS;
|
||||
```
|
||||
|
||||
若要展示更详细的信息，可以使用
|
||||
|
||||
```sql
|
||||
SELECT * from information_schema.`ins_streams`;
|
||||
```
|
||||
|
||||
### 暂停流计算任务
|
||||
|
||||
暂停流计算任务的 SQL 如下：
|
||||
|
||||
```sql
|
||||
PAUSE STREAM [IF EXISTS] stream_name;
|
||||
```
|
||||
|
||||
没有指定 IF EXISTS，如果该 stream 不存在，则报错。如果存在，则暂停流计算。指定了 IF EXISTS，如果该 stream 不存在，则返回成功。如果存在，则暂停流计算。
|
||||
|
||||
### 恢复流计算任务
|
||||
|
||||
恢复流计算任务的 SQL 如下。如果指定了 ignore expired，则恢复流计算任务时，忽略流计算任务暂停期间写入的数据。
|
||||
|
||||
```sql
|
||||
RESUME STREAM [IF EXISTS] [IGNORE UNTREATED] stream_name;
|
||||
```
|
||||
|
||||
没有指定 IF EXISTS，如果该 stream 不存在，则报错。如果存在，则恢复流计算。指定了 IF EXISTS，如果 stream 不存在，则返回成功。如果存在，则恢复流计算。如果指定 IGNORE UNTREATED，则恢复流计算时，忽略流计算暂停期间写入的数据。
|
||||
|
||||
### 流计算升级故障恢复
|
||||
|
||||
升级 TDengine 后，如果流计算不兼容，需要删除流计算，然后重新创建流计算。步骤如下：
|
||||
|
||||
1.修改 taos.cfg，添加 disableStream 1
|
||||
|
||||
2.重启 taosd。如果启动失败，修改 stream 目录的名称，避免 taosd 启动的时候尝试加载 stream 目录下的流计算数据信息。不使用删除操作避免误操作导致的风险。需要修改的文件夹：$dataDir/vnode/vnode*/tq/stream，$dataDir 指 TDengine 存储数据的目录，在 $dataDir/vnode/ 目录下会有多个类似 vnode1、vnode2...vnode* 的目录，全部需要修改里面的 tq/stream 目录的名字，改为 tq/stream.bk
|
||||
|
||||
3.启动 taos
|
||||
|
||||
```sql
|
||||
drop stream xxxx; ---- xxx 指 stream name
|
||||
flush database stream_source_db; ---- 流计算读取数据的超级表所在的 database
|
||||
flush database stream_dest_db; ---- 流计算写入数据的超级表所在的 database
|
||||
```
|
||||
|
||||
举例：
|
||||
|
||||
```sql
|
||||
create stream streams1 into test1.streamst as select _wstart, count(a) c1 from test.st interval(1s) ;
|
||||
drop stream streams1;
|
||||
flush database test;
|
||||
flush database test1;
|
||||
```
|
||||
|
||||
4.关闭 taosd
|
||||
|
||||
5.修改 taos.cfg，去掉 disableStream 1，或将 disableStream 改为 0
|
||||
|
||||
6.启动 taosd
|
||||
重新计算可以分为自动重新计算与手动重新计算，如果用户不需要自动重新计算，可以通过选项关闭。
|
||||
|
|
|
|||
175
docs/zh/06-advanced/05-data-in/18-sparkplugb.mdx
Normal file
175
docs/zh/06-advanced/05-data-in/18-sparkplugb.mdx
Normal file
|
|
@ -0,0 +1,175 @@
|
|||
---
|
||||
title: "SparkplugB"
|
||||
sidebar_label: "SparkplugB"
|
||||
---
|
||||
|
||||
本节讲述如何通过 Explorer 界面创建数据写入任务，从 SparkplugB 读取数据写入到当前 TDengine 集群。
|
||||
|
||||
## 功能概述
|
||||
|
||||
SparkplugB 是一种开放消息规范，专为工业物联网 (IIoT) 应用设计，基于 MQTT 协议。
|
||||
|
||||
TDengine 可以通过 SparkplugB 连接器从 MQTT 代理订阅数据并将其写入 TDengine，以实现实时数据流入库。
|
||||
|
||||
## 创建任务
|
||||
|
||||
### 1. 新增数据源
|
||||
|
||||
在数据写入页面中，点击 **+新增数据源** 按钮，进入新增数据源页面。
|
||||
|
||||
|
||||
|
||||
### 2. 配置基本信息
|
||||
|
||||
在 **名称** 中输入任务名称，如：“test_spb”；
|
||||
|
||||
在 **类型** 下拉列表中选择 **SparkplugB**。
|
||||
|
||||
**代理** 是非必填项，如有需要，可以在下拉框中选择指定的代理，也可以先点击右侧的 **+创建新的代理** 按钮
|
||||
|
||||
在 **目标数据库** 下拉列表中选择一个目标数据库，也可以先点击右侧的 **+创建数据库** 按钮
|
||||
|
||||
|
||||
|
||||
### 3. 配置连接和认证信息
|
||||
|
||||
在 **Brokers** 中填写 MQTT 代理的地址，例如：`localhost:1883`, 可以填写多个，用 ',' 分隔，用于连接多个 broker。
|
||||
|
||||
在 **MQTT 协议** 中选择使用的 MQTT 协议版本，默认 5.0 版本。
|
||||
|
||||
在 **客户端 ID** 中填写连接到每个 broker 所使用的客户端标识符。
|
||||
|
||||
在 **Keep Alive** 中输入保持活动间隔。如果代理在保持活动间隔内没有收到来自客户端的任何消息，它将假定客户端已断开连接，并关闭连接。
|
||||
保持活动间隔是指客户端和代理之间协商的时间间隔，用于检测客户端是否活动。如果客户端在保持活动间隔内没有向代理发送消息，则代理将断开连接。
|
||||
|
||||
在 **用户** 中填写 MQTT 代理的用户名。
|
||||
|
||||
在 **密码** 中填写 MQTT 代理的密码。
|
||||
|
||||
在 **TLS 校验** 中选择 TLS 证书的校验方式
|
||||
|
||||
1. 不开启：表示不进行 TLS 证书认证。在连接 MQTT 时，会先进行 TCP 连接，如果连接失败，会进行无证书认证模式的 TLS 连接。
|
||||
|
||||
2. 单向认证：开启 TLS 连接，并验证服务端证书，此时需要上传 CA 证书。
|
||||
|
||||
3. 双向认证：开启 TLS 连接，并与服务端进行双向认证，此时需要上传 CA 证书，客户端证书以及客户端密钥。
|
||||
|
||||
点击 **检查连通性** 按钮，检查数据源是否可用。
|
||||
|
||||
|
||||
|
||||
## 4. 订阅配置
|
||||
|
||||
在 **Group ID** 中填写 SparkplugB 规定的 group id 字段，通常一个 group id 代表一个集团/公司/工厂/流水线 等概念。
|
||||
|
||||
在 **节点/设备列表** 中填写需要订阅的节点和设备的列表，以逗号分隔，其中节点直接填写 ID 即可，设备需要按照 节点 ID/设备 ID 格式填写。
|
||||
|
||||
在 **消息类型** 中填写需要订阅的 SparkplugB 消息类型，以逗号分隔，有 NBIRTH/NDEATH/NDATA/NCMD/DBIRTH/DDEATH/DDATA/DCMD/STATE。在订阅时，NBIRTH/NDEATH/NDATA/NCMD 类型的消息只会匹配 "节点/设备列表" 中的节点，而 DBIRTH/DDEATH/DDATA/DCMD 只会匹配 "节点/设备列表" 中的设备。
|
||||
|
||||
当 **下发 REBIRTH 命令** 开启后，taosX 会自动下发 NCMD 中的 "Node Control/Rebirth" 命令，获取节点和设备的所有 metric 信息，从而可以获取 metric name 和 metric alias 的对应关系。如果节点/设备在上报数据时不使用 alias 别名机制，可以不开启此选项。
|
||||
|
||||
|
||||
|
||||
### 5. 配置 Payload 转换
|
||||
|
||||
在 **Payload 解析** 区域填写 Payload 解析相关的配置参数。
|
||||
|
||||
#### 5.1 解析
|
||||
|
||||
有三种获取示例数据的方法：
|
||||
|
||||
点击 **从服务器检索** 按钮，从 MQTT 获取示例数据。
|
||||
|
||||
点击 **文件上传** 按钮，上传 CSV 文件，获取示例数据。
|
||||
|
||||
在 **消息体** 中填写 MQTT 消息体中的示例数据，由于 SparkplugB 消息使用 protobuf 进行编码，因此从服务器检索的数据是经过编码为 json 格式的数据。
|
||||
|
||||
json 数据支持 JSONObject 或者 JSONArray，可以用于解析 SparkplugB 中的 metadata 和 properties 等 json 格式的字段。
|
||||
|
||||
|
||||
|
||||
点击 **放大镜图标** 可查看预览解析结果。
|
||||
|
||||
|
||||
|
||||
#### 5.2 字段拆分
|
||||
|
||||
在 **从列中提取或拆分** 中填写从消息体中提取或拆分的字段，例如：将 `datatype_str` 字段的值转换为 TDengine 类型，在 **rule** 输入框中填写如下 json 值，在 **name** 中填写 `td_datatype`
|
||||
|
||||
```json
|
||||
{
|
||||
"Int8": "TINYINT",
|
||||
"UInt8": "TINYINT UNSIGNED",
|
||||
"Int16": "SMALLINT",
|
||||
"UInt16": "SMALLINT UNSIGNED",
|
||||
"Int32": "INT",
|
||||
"UInt32": "INT UNSIGNED",
|
||||
"Int64": "BIGINT",
|
||||
"UInt64": "BIGINT UNSIGNED",
|
||||
"Float": "FLOAT",
|
||||
"DOUBLE": "DOUBLE",
|
||||
"Boolean": "BOOL",
|
||||
"String": "VARCHAR(128)",
|
||||
"DateTime": "TIMESTAMP"
|
||||
}
|
||||
```
|
||||
|
||||
会将 `datatype_str` 列的字段的值如 "Int8" 转换为对应的 "TINYINT"，新的列命名为 `td_datatype`
|
||||
|
||||
|
||||
|
||||
点击 **删除**，可以删除当前提取规则。
|
||||
|
||||
点击 **新增**，可以添加更多提取规则。
|
||||
|
||||
点击 **放大镜图标** 可查看预览提取/拆分结果。
|
||||
|
||||
|
||||
|
||||
#### 5.3 数据过滤
|
||||
|
||||
在 **过滤** 中，填写过滤条件，例如：填写`datatype_str != "Int8"`，则只有 datatype_str 不为 `Int8` 的数据才会被写入 TDengine。
|
||||
|
||||
|
||||
|
||||
点击 **删除**，可以删除当前过滤规则。
|
||||
|
||||
点击 **放大镜图标** 可查看预览过滤结果。
|
||||
|
||||
|
||||
|
||||
#### 5.4 表映射
|
||||
|
||||
在 **目标超级表** 的下拉列表中选择一个目标超级表，也可以先点击右侧的 **创建超级表** 按钮创建新的超级表。
|
||||
|
||||
当超级表需要根据消息动态生成时，可以选择 **创建模板**。其中，超级表名称，列名，列类型等均可以使用模板变量，当接收到数据后，程序会自动计算模板变量并生成对应的超级表模板，当数据库中超级表不存在时，会使用此模板创建超级表；对于已创建的超级表，如果缺少通过模板变量计算得到的列，也会自动创建对应列。
|
||||
|
||||
|
||||
|
||||
在 **映射** 中，填写目标超级表中的子表名称，例如：`t_{id}`。根据需求填写映射规则，其中 mapping 支持设置缺省值。
|
||||
|
||||
|
||||
|
||||
点击 **预览**，可以查看映射的结果。
|
||||
|
||||
|
||||
|
||||
### 6. 高级选项
|
||||
|
||||
在 **处理批次上限** 中填写可以同时进行数据处理流程的批次数量，当到达此上限后，不再从消息缓存队列中获取消息，会导致缓存队列的消息积压，最小值为 1。
|
||||
|
||||
在 **批次大小** 中填写每次发送给数据处理流程的消息数量，和 **批次延时** 配合使用，当读取的 MQTT 消息数量达到批次大小时，就算 **批次延时** 没有到达也立即向数据处理流程发送数据，最小值为 1。
|
||||
|
||||
在 **批次延时** 中填写每次生成批次消息的超时时间（单位：毫秒），从每批次接收到的第一个消息开始算起，和 **批次大小** 配合使用，当读取消息到达超时时间时，就算 **批次大小** 不满足数量也立即向数据处理流程发送数据，最小值为 1。
|
||||
|
||||
|
||||
|
||||
### 7. 异常处理策略
|
||||
|
||||
import Contributing from './_03-exception-handling-strategy.mdx'
|
||||
|
||||
<Contributing />
|
||||
|
||||
### 8. 创建完成
|
||||
|
||||
点击 **提交** 按钮，完成创建 MQTT 到 TDengine 的数据同步任务，回到**数据源列表**页面可查看任务执行情况。
|
||||
|
|
@ -348,3 +348,4 @@ import {useCurrentSidebarCategory} from '@docusaurus/theme-common';
|
|||
|
||||
<DocCardList items={useCurrentSidebarCategory().items}/>
|
||||
```
|
||||
|
||||
|
|
|
|||
BIN
docs/zh/06-advanced/05-data-in/spb-01.png
Normal file
BIN
docs/zh/06-advanced/05-data-in/spb-01.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 88 KiB |
BIN
docs/zh/06-advanced/05-data-in/spb-02.png
Normal file
BIN
docs/zh/06-advanced/05-data-in/spb-02.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 27 KiB |
BIN
docs/zh/06-advanced/05-data-in/spb-03.png
Normal file
BIN
docs/zh/06-advanced/05-data-in/spb-03.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 38 KiB |
BIN
docs/zh/06-advanced/05-data-in/spb-04.png
Normal file
BIN
docs/zh/06-advanced/05-data-in/spb-04.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 23 KiB |
BIN
docs/zh/06-advanced/05-data-in/spb-05.png
Normal file
BIN
docs/zh/06-advanced/05-data-in/spb-05.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 83 KiB |
BIN
docs/zh/06-advanced/05-data-in/spb-06.png
Normal file
BIN
docs/zh/06-advanced/05-data-in/spb-06.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 48 KiB |
BIN
docs/zh/06-advanced/05-data-in/spb-07.png
Normal file
BIN
docs/zh/06-advanced/05-data-in/spb-07.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 34 KiB |
BIN
docs/zh/06-advanced/05-data-in/spb-08.png
Normal file
BIN
docs/zh/06-advanced/05-data-in/spb-08.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 51 KiB |
BIN
docs/zh/06-advanced/05-data-in/spb-09.png
Normal file
BIN
docs/zh/06-advanced/05-data-in/spb-09.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 11 KiB |
BIN
docs/zh/06-advanced/05-data-in/spb-10.png
Normal file
BIN
docs/zh/06-advanced/05-data-in/spb-10.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 35 KiB |
BIN
docs/zh/06-advanced/05-data-in/spb-11.png
Normal file
BIN
docs/zh/06-advanced/05-data-in/spb-11.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 44 KiB |
BIN
docs/zh/06-advanced/05-data-in/spb-12.png
Normal file
BIN
docs/zh/06-advanced/05-data-in/spb-12.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 66 KiB |
BIN
docs/zh/06-advanced/05-data-in/spb-13.png
Normal file
BIN
docs/zh/06-advanced/05-data-in/spb-13.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 64 KiB |
BIN
docs/zh/06-advanced/05-data-in/spb-14.png
Normal file
BIN
docs/zh/06-advanced/05-data-in/spb-14.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 58 KiB |
|
|
@ -89,7 +89,7 @@ TDengine 提供了丰富的应用程序开发接口，为了便于用户快速
|
|||
<dependency>
|
||||
<groupId>com.taosdata.jdbc</groupId>
|
||||
<artifactId>taos-jdbcdriver</artifactId>
|
||||
<version>3.6.3</version>
|
||||
<version>3.7.0</version>
|
||||
</dependency>
|
||||
```
|
||||
|
||||
|
|
@ -115,7 +115,7 @@ TDengine 提供了丰富的应用程序开发接口，为了便于用户快速
|
|||
```
|
||||
- 指定某个特定版本安装
|
||||
```
|
||||
pip3 install taospy==2.8.2
|
||||
pip3 install taospy==2.8.3
|
||||
```
|
||||
- 从 GitHub 安装
|
||||
```
|
||||
|
|
@ -300,6 +300,12 @@ dotnet add package TDengine.Connector
|
|||
username:password@protocol(address)/dbname?param=value
|
||||
```
|
||||
|
||||
当使用 IPv6 地址时（v3.7.1 及以上版本支持），地址需要用方括号括起来，例如：
|
||||
|
||||
```text
|
||||
root:taosdata@ws([::1]:6041)/testdb
|
||||
```
|
||||
|
||||
支持的 DSN 参数如下
|
||||
|
||||
原生连接：
|
||||
|
|
|
|||
|
|
@ -13,6 +13,8 @@ import TabItem from "@theme/TabItem";
|
|||
- 预编译：当使用参数绑定时，SQL 语句可以被预编译并缓存，后续使用不同的参数值执行时，可以直接使用预编译的版本，提高执行效率。
|
||||
- 减少网络开销：参数绑定还可以减少发送到数据库的数据量，因为只需要发送参数值而不是完整的 SQL 语句，特别是在执行大量相似的插入或更新操作时，这种差异尤为明显。
|
||||
|
||||
参数绑定支持多种语言 API [连接器](../../reference/connector/)
|
||||
|
||||
**Tips: 数据写入推荐使用参数绑定方式**
|
||||
|
||||
:::note
|
||||
|
|
@ -61,7 +63,7 @@ import TabItem from "@theme/TabItem";
|
|||
{{#include docs/examples/python/stmt2_ws.py}}
|
||||
```
|
||||
|
||||
stmt 绑定参数的示例代码如下：
|
||||
stmt 绑定参数的示例代码如下（TDengine v3.3.5.0 已停止维护）：
|
||||
|
||||
```python
|
||||
{{#include docs/examples/python/stmt_ws.py}}
|
||||
|
|
@ -125,7 +127,7 @@ stmt2 绑定参数的示例代码如下（go 连接器 v3.6.0 及以上，TDengi
|
|||
{{#include docs/examples/go/stmt2/native/main.go}}
|
||||
```
|
||||
|
||||
stmt 绑定参数的示例代码如下：
|
||||
stmt 绑定参数的示例代码如下（TDengine v3.3.5.0 已停止维护）：
|
||||
|
||||
```go
|
||||
{{#include docs/examples/go/stmt/native/main.go}}
|
||||
|
|
@ -156,7 +158,7 @@ stmt2 绑定参数的示例代码如下（需要 TDengine v3.3.5.0 及以上）
|
|||
{{#include docs/examples/c/stmt2_insert_demo.c}}
|
||||
```
|
||||
|
||||
stmt 绑定参数的示例代码如下：
|
||||
stmt 绑定参数的示例代码如下（TDengine v3.3.5.0 已停止维护）：
|
||||
|
||||
```c
|
||||
{{#include docs/examples/c/stmt_insert_demo.c}}
|
||||
|
|
|
|||
|
|
@ -10,7 +10,11 @@ toc_max_heading_level: 4
|
|||
- **方便维护**：配置好各级存储挂载点后，系统数据迁移等工作，无需人工干预；存储扩容更灵活、方便
|
||||
- **对 SQL 透明**：无论查询的数据是否跨级，一条 SQL 可返回所有数据，简单高效
|
||||
|
||||
多级存储所涉及的各层存储介质都是本地存储设备。除了本地存储设备之外，TDengine Enterprise 还支持使用对象存储 (S3)，将最冷的一批数据保存在最廉价的介质上，以进一步降低存储成本，并在必要时仍然可以进行查询，且数据存储在哪里也对 SQL 透明。支持对象存储在 3.3.0.0 版本中首次发布，建议使用最新版本。
|
||||
多级存储所涉及的各层存储介质都是本地存储设备。除了本地存储设备之外，TDengine Enterprise 还支持使用对象存储，将最冷的一批数据保存在最廉价的介质上，以进一步降低存储成本，并在必要时仍然可以进行查询，且数据存储在哪里也对 SQL 透明。
|
||||
|
||||
由于大多数对象存储本身已经是多副本，如果 TDengine 在其基础上再叠加一个多副本，无疑将会浪费大量的存储资源并增加存储成本，所以，TDengine 进一步将对象存储优化为了共享存储，共享存储中的数据在逻辑上只有一份，并在 TDengine 集群节点之间共享。
|
||||
|
||||
共享存储在 3.3.7.0 版本中首次发布，它是对 3.3.0.0 版本开始支持的对象存储（S3）功能的增强。但需要注意，二者并不兼容，如果已经使用了之前版本的对象存储功能，升级到 3.3.7.0 时需要进行一些手工处理。
|
||||
|
||||
## 多级存储
|
||||
|
||||
|
|
@ -61,15 +65,15 @@ dataDir /mnt/data6 2 0
|
|||
从 3.3.2.0 版本开始，引入了一个新的配置 disable_create_new_file，用于控制在某个挂载点上禁止生成新文件，其缺省值为 false，即每个挂载点上默认都可以生成新文件。
|
||||
|
||||
|
||||
## 对象存储
|
||||
## 共享存储
|
||||
|
||||
本节介绍在 TDengine Enterprise 版本中如何使用对象存储功能，如 Amazon S3、Azure Blob Storage、华为 OBS、腾讯云 COS、阿里云 OSS、MinIO 等对象存储服务。
|
||||
本节介绍在 TDengine Enterprise 版本中如何使用共享存储功能，如 Amazon S3、Azure Blob Storage、华为 OBS、腾讯云 COS、阿里云 OSS、MinIO 等对象存储服务。
|
||||
|
||||
**注意** 在配合多级存储使用时，每一级存储介质上保存的数据都有可能被按规则备份到远程对象存储中并删除本地数据文件。
|
||||
**注意** 在配合多级存储使用时，每一级存储介质上保存的数据都有可能被按规则备份到远程共享存储中并删除本地数据文件。
|
||||
|
||||
### S3 对象存储
|
||||
|
||||
本功能基于通用 S3 SDK 实现，对各个 S3 平台的访问参数进行了兼容适配，通过适当的参数配置，可以把大部分较冷的时序数据存储到 S3 服务中。
|
||||
本功能基于通用 S3 SDK 实现，对各个 S3 兼容平台的访问参数进行了兼容适配，通过适当的参数配置，可以把大部分较冷的时序数据存储到 S3 兼容服务中。
|
||||
|
||||
#### 配置方式
|
||||
|
||||
|
|
@ -77,64 +81,86 @@ dataDir /mnt/data6 2 0
|
|||
|
||||
| 参数名称 | 参数含义 |
|
||||
|:---------------------|:-----------------------------------------------|
|
||||
| s3EndPoint | 用户所在地域的 COS 服务域名，支持 http 和 https，bucket 的区域需要与 endpoint 的保持一致，否则无法访问 |
|
||||
| s3AccessKey | 冒号分隔的用户 SecretId:SecretKey。例如：AKIDsQmwsfKxTo2A6nGVXZN0UlofKn6JRRSJ:lIdoy99ygEacU7iHfogaN2Xq0yumSm1E |
|
||||
| s3BucketName | 存储桶名称，减号后面是用户注册 COS 服务的 AppId。其中 AppId 是 COS 特有，AWS 和阿里云都没有，配置时需要作为 bucket name 的一部分，使用减号分隔。参数值均为字符串类型，但不需要引号。例如：test0711-1309024725 |
|
||||
| s3UploadDelaySec | data 文件持续多长时间不再变动后上传至 s3，单位：秒。最小值：1；最大值：2592000（30 天），默认值 60 秒 |
|
||||
| s3PageCacheSize | S3 page cache 缓存页数目，单位：页。最小值：4；最大值：1024*1024*1024。 ，默认值 4096|
|
||||
| s3MigrateIntervalSec | 本地数据文件自动上传 S3 的触发周期，单位为秒。最小值：600；最大值：100000。默认值 3600 |
|
||||
| s3MigrateEnabled | 是否自动进行 S3 迁移，默认值为 0，表示关闭自动 S3 迁移，可配置为 1 |
|
||||
| ssEnabled | 是否启用共享存储，默认值为 0，表示禁用共享存储。<br/>1 表示启用共享存储，但仅支持数据的手动迁移。<br/>2 表示启用共享存储且支持自动迁移。|
|
||||
| ssAccessString | 一个包含各种共享存储访问选项的字符串，格式为 `<device-type>:<option-name>=<option-value>;<option-name>=<option-value>;...`<br/>当前版本仅支持 S3 兼容的对象存储服务，下表列出了具体选项的详细信息。|
|
||||
| ssUploadDelaySec | data 文件持续多长时间不再变动后上传至共享存储，单位：秒。<br/>最小值：1；最大值：2592000（30 天），默认值 60 秒。|
|
||||
| ssPageCacheSize | 共享存储 page cache 缓存页数目，单位：页。<br/>最小值：4；最大值：1024*1024*1024。 ，默认值 4096。|
|
||||
| ssAutoMigrateIntervalSec | 本地数据文件自动上传共享存储的触发周期，单位为秒。<br/>最小值：600；最大值：100000。默认值 3600。|
|
||||
|
||||
`ssAccessString` 针对 S3 兼容的对象存储的选项如下：
|
||||
|
||||
| 名称 | 含义 |
|
||||
| ----------------|----------------------------------------------|
|
||||
| endpoint | 对象存储服务的机器名或 IP 地址，可以包含可选的端口号。|
|
||||
| bucket | 存储桶的名字。|
|
||||
| protocol | `https` 或 `http`，默认是 `https`。|
|
||||
| uriStyle | `virtualHost` 或 `path`，默认是 `virtualHost`。注意，部分对象存储仅支持其中之一。|
|
||||
| region | 对象存储服务所在区域，此参数可选。|
|
||||
| accessKeyId | 用于访问对象存储的 `access key id`。|
|
||||
| secretAccessKey | 上述 `access key id` 对应的密钥。|
|
||||
| chunkSize | 以 MB 为单位的数据片大小，默认值是 64，超过此大小的文件，将使用 multipart 方式上传。|
|
||||
| maxChunks | 单个数据文件的最大分片数量，默认值为 10000。|
|
||||
| maxRetry | 访问对象存储时出现可重试错误时的最大重试次数，默认值是 3，负值表示一直重试直到成功为止。|
|
||||
|
||||
|
||||
#### 检查配置参数可用性
|
||||
|
||||
在 taos.cfg 中完成对 S3 的配置后，通过 taosd 命令的 checks3 参数可以检查所配置的 S3 服务是否可用：
|
||||
在 taos.cfg 中完成对共享存储的配置后，通过 taosd 命令的 checkss 参数可以检查所配置的共享存储服务是否可用：
|
||||
|
||||
```
|
||||
taosd --checks3
|
||||
taosd --checkss
|
||||
```
|
||||
|
||||
如果配置的 S3 服务无法访问，此命令会在运行过程中输出相应的错误信息。
|
||||
如果配置的共享存储服务无法访问，此命令会在运行过程中输出相应的错误信息。
|
||||
|
||||
#### 创建使用 S3 的 DB
|
||||
#### 创建使用共享存储的 DB
|
||||
|
||||
完成配置后，即可启动 TDengine 集群，创建使用 S3 的数据库，比如：
|
||||
完成配置后，即可启动 TDengine 集群，创建使用共享存储的数据库，比如：
|
||||
|
||||
```sql
|
||||
create database demo_db duration 1d s3_keeplocal 3d;
|
||||
create database demo_db duration 1d ss_keeplocal 3d;
|
||||
```
|
||||
|
||||
数据库 demo_db 中写入时序数据后，3 天之前的时序数据会自动分块存放到 S3 存储中。
|
||||
数据库 demo_db 中写入时序数据后，3 天之前的时序数据会自动分块存放到共享存储中。
|
||||
|
||||
默认情况下，mnode 每小时会下发 S3 数据迁移检查的指令，如果有时序数据需要上传，则自动分块存放到 S3 存储中，也可以使用 SQL 命令手动触发，由用户触发这一操作，语法如下：
|
||||
如果 `ssEnabled` 是 `2`，mnode 每小时会下发数据迁移检查的指令，如果有时序数据需要迁移，则自动分块存放到共享存储中，也可以使用 SQL 命令手动触发，由用户触发这一操作，语法如下：
|
||||
|
||||
```sql
|
||||
s3migrate database <db_name>;
|
||||
ssmigrate database <db_name>;
|
||||
```
|
||||
|
||||
如果 `ssEnabled` 是 `1`，mnode 不会自动下发迁移指令，用户必须自己手动触发迁移。
|
||||
|
||||
详细的 DB 参数见下表：
|
||||
|
||||
| # | 参数 | 默认值 | 最小值 | 最大值 | 描述 |
|
||||
|:--|:--------------|:-------|:------ |:------- | :----------------------------------------------------------- |
|
||||
| 1 | s3_keeplocal | 365 | 1 | 365000 | 数据在本地保留的天数，即 data 文件在本地磁盘保留多长时间后可以上传到 S3。默认单位：天，支持 m（分钟）、h（小时）和 d（天）三个单位 |
|
||||
| 2 | s3_chunkpages | 131072 | 131072 | 1048576 | 上传对象的大小阈值，与 tsdb_pagesize 参数一样，不可修改，单位为 TSDB 页 |
|
||||
| 3 | s3_compact | 1 | 0 | 1 | TSDB 文件组首次上传 S3 时，是否自动进行 compact 操作 |
|
||||
| 1 | ss_keeplocal | 365 | 1 | 365000 | 数据在本地保留的天数，即 data 文件在本地磁盘保留多长时间后可以上传到共享存储。默认单位：天，支持 m（分钟）、h（小时）和 d（天）三个单位 |
|
||||
| 2 | ss_chunkpages | 131072 | 131072 | 1048576 | 上传对象的大小阈值，与 tsdb_pagesize 参数一样，不可修改，单位为 TSDB 页 |
|
||||
| 3 | ss_compact | 1 | 0 | 1 | TSDB 文件组首次上传共享存储时，是否自动进行 compact 操作 |
|
||||
|
||||
#### 对象存储读写次数估算
|
||||
|
||||
对象存储服务的使用成本与存储的数据量及请求次数相关，下面分别介绍数据的上传及下载过程。
|
||||
|
||||
##### 数据上传
|
||||
##### 数据迁移
|
||||
|
||||
当 TSDB 时序数据超过 `s3_keeplocal` 参数指定的时间，相关的数据文件会被切分成多个文件块，每个文件块的默认大小是 512M 字节 (`s3_chunkpages * tsdb_pagesize`)。除了最后一个文件块保留在本地文件系统外，其余的文件块会被上传到对象存储服务。
|
||||
当 TSDB 时序数据超过 `ss_keeplocal` 参数指定的时间，主节点（vnode）会将数据文件切分成多个文件块，除了最后一块，每个文件块的默认大小都是 512M 字节 (`ss_chunkpages * tsdb_pagesize`)。所有文件快都会被上传到共享存储，除了最后一个文件块，其他文件上传完成后都会从本地文件系统删除。
|
||||
|
||||
在创建数据库时，可以通过 `ss_chunkpages` 参数调整每个文件块的大小，从而控制每个数据文件的上传次数。
|
||||
|
||||
其它类型的文件如 head, stt, sma 等也会被上传到共享存储，但会保留本地文件以加速预计算相关查询。
|
||||
|
||||
所有文件上传完成后，主节点还会在共享存储中自动创建一个 manifests 文件，用于记录上传信息。
|
||||
|
||||
之后，从节点会读取 manifests 文件，然后根据其中的信息，将数据文件的最后一块和所有其他文件下载到本地。
|
||||
|
||||
所以，迁移过程中的读写次数约为：
|
||||
|
||||
```text
|
||||
上传次数 = 数据文件大小 / (s3_chunkpages * tsdb_pagesize) - 1
|
||||
读写次数 = 数据文件总块数 + （其他文件的总数 + 2）✖ 集群节点数
|
||||
```
|
||||
|
||||
在创建数据库时，可以通过 `s3_chunkpages` 参数调整每个文件块的大小，从而控制每个数据文件的上传次数。
|
||||
|
||||
其它类型的文件如 head, stt, sma 等，保留在本地文件系统，以加速预计算相关查询。
|
||||
|
||||
##### 数据下载
|
||||
|
||||
在查询操作中，如果需要访问对象存储中的数据，TSDB 不会下载整个数据文件，而是计算所需数据在文件中的位置，只下载相应的数据到 TSDB 页缓存中，然后将数据返回给查询执行引擎。后续查询首先检查页缓存，查看数据是否已被缓存。如果数据已缓存，则直接使用缓存中的数据，而无需重复从对象存储下载，从而有效降低从对象存储下载数据的次数。
|
||||
|
|
@ -145,9 +171,14 @@ s3migrate database <db_name>;
|
|||
下载次数 = 查询需要的数据块数量 - 已缓存的数据块数量
|
||||
```
|
||||
|
||||
页缓存是内存缓存，节点重启后，再次查询需要重新下载数据。缓存采用 LRU (Least Recently Used) 策略，当缓存空间不足时，最近最少使用的数据将被淘汰。缓存的大小可以通过 `s3PageCacheSize` 参数进行调整，通常来说，缓存越大，下载次数越少。
|
||||
页缓存是内存缓存，节点重启后，再次查询需要重新下载数据。缓存采用 LRU (Least Recently Used) 策略，当缓存空间不足时，最近最少使用的数据将被淘汰。缓存的大小可以通过 `ssPageCacheSize` 参数进行调整，通常来说，缓存越大，下载次数越少。
|
||||
|
||||
### Azure Blob 存储
|
||||
|
||||
共享存储对 Azure Blob 的支持将在后续版本提供，如果已经在 TDengine 3.3.7.0 之前的版本中使用了 Azure Blob，请暂缓升级。
|
||||
|
||||
<!--
|
||||
|
||||
本节介绍在 TDengine Enterprise 版本中如何使用微软 Azure Blob 存储。本功能可以通过两个方式使用：利用 Flexify 服务提供的 S3 网关功能和不依赖 Flexify 服务。通过配置参数，可以把大部分较冷的时序数据存储到 Azure Blob 服务中。
|
||||
|
||||
#### Flexify 服务
|
||||
|
|
@ -182,3 +213,5 @@ s3BucketName td-test
|
|||
| 3 | s3BucketName | test-container | Container name |
|
||||
|
||||
其中 fd2d01c73 是账户 ID；微软 Blob 存储服务只支持 Https 协议，不支持 Http。
|
||||
|
||||
-->
|
||||
|
|
@ -210,4 +210,7 @@ key_status 有三种取值：
|
|||
```shell
|
||||
taosd -y {encryptKey}
|
||||
```
|
||||
更新密钥配置，需要先停止 taosd，并且使用完全相同的密钥，也即密钥在数据库创建后不能修改。
|
||||
更新密钥配置，需要先停止 taosd，并且使用完全相同的密钥，也即密钥在数据库创建后不能修改。
|
||||
|
||||
### 加密用户密码
|
||||
默认的情况下，用户的密码会以 MD5 的形式进行存储。可以通过参数 encryptPassAlgorithm 将用户密码进行加密储存。encryptPassAlgorithm 默认是未设置的状态，在未设置时，不对用户密码进行加密，也即只以 MD5 的形式存储。当 encryptPassAlgorithm 设置为 sm4 时（目前只支持 sm4 加密算法），对用户密码进行加密存储。设置 encryptPassAlgorithm 参数前，同样按照前面的步骤配置密钥。
|
||||
File diff suppressed because it is too large
Load diff
|
|
@ -36,10 +36,12 @@ taosAdapter 提供了以下功能：
|
|||
icinga2 是一个收集检查结果指标和性能数据的软件。请访问 [https://icinga.com/docs/icinga-2/latest/doc/14-features/#opentsdb-writer](https://icinga.com/docs/icinga-2/latest/doc/14-features/#opentsdb-writer) 了解更多信息。
|
||||
- TCollector 数据写入：
|
||||
TCollector 是一个客户端进程，从本地收集器收集数据，并将数据推送到 OpenTSDB。请访问 [http://opentsdb.net/docs/build/html/user_guide/utilities/tcollector.html](http://opentsdb.net/docs/build/html/user_guide/utilities/tcollector.html) 了解更多信息。
|
||||
- node_exporter 采集写入：
|
||||
node_export 是一个机器指标的导出器。请访问 [https://github.com/prometheus/node_exporter](https://github.com/prometheus/node_exporter) 了解更多信息。
|
||||
- OpenMetrics 采集写入：
|
||||
OpenMetrics 是云原生监控领域的新兴标准，扩展并规范了 Prometheus 的指标格式，已成为现代监控工具的事实标准。请访问 [https://github.com/prometheus/OpenMetrics/blob/main/specification/OpenMetrics.md](https://github.com/prometheus/OpenMetrics/blob/main/specification/OpenMetrics.md) 了解更多信息。
|
||||
- Prometheus remote_read 和 remote_write：
|
||||
remote_read 和 remote_write 是 Prometheus 数据读写分离的集群方案。请访问 [https://prometheus.io/blog/2019/10/10/remote-read-meets-streaming/#remote-apis](https://prometheus.io/blog/2019/10/10/remote-read-meets-streaming/#remote-apis) 了解更多信息。
|
||||
- node_exporter 采集写入：
|
||||
node_export 是一个机器指标的导出器。请访问 [https://github.com/prometheus/node_exporter](https://github.com/prometheus/node_exporter) 了解更多信息。
|
||||
- RESTful 接口：
|
||||
[RESTful API](../../connector/rest-api)
|
||||
|
||||
|
|
@ -58,6 +60,7 @@ taosAdapter 提供了以下功能：
|
|||
- `u` TDengine 用户名
|
||||
- `p` TDengine 密码
|
||||
- `ttl` 自动创建的子表生命周期，以子表的第一条数据的 TTL 参数为准，不可更新。更多信息请参考 [创建表文档](../../taos-sql/table/#创建表)的 TTL 参数。
|
||||
- `table_name_key` 自定义子表名使用的标签名，如果设置了该参数，则子表名将使用该标签对应的值。
|
||||
|
||||
注意：目前不支持 InfluxDB 的 token 验证方式，仅支持 Basic 验证和查询参数验证。
|
||||
示例：
|
||||
|
|
@ -91,18 +94,32 @@ curl --request POST http://127.0.0.1:6041/influxdb/v1/write?db=test --user "root
|
|||
|
||||
<TCollector />
|
||||
|
||||
### node_exporter 采集写入
|
||||
### OpenMetrics 采集写入
|
||||
|
||||
Prometheus 使用的由 \*NIX 内核暴露的硬件和操作系统指标的输出器
|
||||
OpenMetrics 是一种由 CNCF（云原生计算基金会）支持的开放标准，专注于规范指标数据的采集和传输，是云原生生态中监控和可观测性系统的核心规范之一。
|
||||
|
||||
- 启用 taosAdapter 的配置 node_exporter.enable
|
||||
- 设置 node_exporter 的相关配置
|
||||
从 **3.3.7.0** 版本开始，taosAdapter 支持 OpenMetrics v1.0.0 数据采集与写入，同时兼容 Prometheus 0.0.4 协议，确保与 Prometheus 生态的无缝集成。
|
||||
|
||||
启用 OpenMetrics 数据采集写入需要以下步骤：
|
||||
|
||||
- 启用 taosAdapter 的配置 `open_metrics.enable`
|
||||
- 设置 OpenMetrics 的相关配置
|
||||
- 重新启动 taosAdapter
|
||||
|
||||
### Prometheus remote_read 和 remote_write
|
||||
|
||||
<Prometheus />
|
||||
|
||||
### node_exporter 采集写入
|
||||
|
||||
从 **3.3.7.0** 版本开始，可以使用 OpenMetrics 插件替代 node_exporter 进行数据采集和写入。
|
||||
|
||||
Prometheus 使用的由 \*NIX 内核暴露的硬件和操作系统指标的输出器。
|
||||
|
||||
- 启用 taosAdapter 的配置 node_exporter.enable
|
||||
- 设置 node_exporter 的相关配置
|
||||
- 重新启动 taosAdapter
|
||||
|
||||
### RESTful 接口
|
||||
|
||||
您可以使用任何支持 HTTP 协议的客户端通过访问 RESTful 接口地址 `http://<fqdn>:6041/rest/sql` 来写入数据到 TDengine 或从 TDengine 中查询数据。细节请参考 [REST API 文档](../../connector/rest-api/)。
|
||||
|
|
@ -174,7 +191,8 @@ taosAdapter 使用连接池管理与 TDengine 的连接，以提高并发性能
|
|||
- Telegraf 数据写入
|
||||
- collectd 数据写入
|
||||
- StatsD 数据写入
|
||||
- 采集 node_exporter 数据写入
|
||||
- node_exporter 数据写入
|
||||
- OpenMetrics 数据写入
|
||||
- Prometheus remote_read 和 remote_write
|
||||
|
||||
连接池的配置参数如下：
|
||||
|
|
@ -257,6 +275,7 @@ taosAdapter 将监测自身运行过程中内存使用率并通过两个阈值
|
|||
- collectd 数据写入
|
||||
- StatsD 数据写入
|
||||
- node_exporter 数据写入
|
||||
- OpenMetrics 数据写入
|
||||
|
||||
**参数说明**
|
||||
|
||||
|
|
@ -380,68 +399,6 @@ curl --location --request PUT 'http://127.0.0.1:6041/config' \
|
|||
|
||||
启用或禁用 InfluxDB 协议支持（布尔值，默认值：`true`）。
|
||||
|
||||
#### node_exporter 配置参数
|
||||
|
||||
- **`node_exporter.enable`**
|
||||
|
||||
是否启用 node_exporter 数据采集（默认值：`false`）。
|
||||
|
||||
- **`node_exporter.db`**
|
||||
|
||||
指定 node_exporter 数据写入的数据库名称（默认值：`"node_exporter"`）。
|
||||
|
||||
- **`node_exporter.urls`**
|
||||
|
||||
配置 node_exporter 服务地址（默认值：`["http://localhost:9100"]`）。
|
||||
|
||||
- **`node_exporter.gatherDuration`**
|
||||
|
||||
设置数据采集间隔时间（默认值：`5s`）。
|
||||
|
||||
- **`node_exporter.responseTimeout`**
|
||||
|
||||
配置请求超时时间（默认值：`5s`）。
|
||||
|
||||
- **`node_exporter.user`**
|
||||
|
||||
设置数据库连接用户名（默认值：`"root"`）。
|
||||
|
||||
- **`node_exporter.password`**
|
||||
|
||||
设置数据库连接密码（默认值：`"taosdata"`）。
|
||||
|
||||
- **`node_exporter.ttl`**
|
||||
|
||||
配置采集数据的生存时间（默认值：`0`，表示无超时）。
|
||||
|
||||
- **`node_exporter.httpUsername`**
|
||||
|
||||
配置 HTTP 基本认证用户名（可选）。
|
||||
|
||||
- **`node_exporter.httpPassword`**
|
||||
|
||||
配置 HTTP 基本认证密码（可选）。
|
||||
|
||||
- **`node_exporter.httpBearerTokenString`**
|
||||
|
||||
配置 HTTP Bearer Token 认证（可选）。
|
||||
|
||||
- **`node_exporter.insecureSkipVerify`**
|
||||
|
||||
是否跳过 SSL 证书验证（默认值：`true`）。
|
||||
|
||||
- **`node_exporter.certFile`**
|
||||
|
||||
指定客户端证书文件路径（可选）。
|
||||
|
||||
- **`node_exporter.keyFile`**
|
||||
|
||||
指定客户端证书密钥文件路径（可选）。
|
||||
|
||||
- **`node_exporter.caCertFile`**
|
||||
|
||||
指定 CA 证书文件路径（可选）。
|
||||
|
||||
#### OpenTSDB 配置参数
|
||||
|
||||
- **`opentsdb.enable`**
|
||||
|
|
@ -560,6 +517,134 @@ curl --location --request PUT 'http://127.0.0.1:6041/config' \
|
|||
|
||||
是否启用 Prometheus 协议支持（默认值：`true`）。
|
||||
|
||||
#### OpenMetrics 配置参数
|
||||
|
||||
- **`open_metrics.enable`**
|
||||
|
||||
启用或禁用 OpenMetrics 数据采集功能（默认值：`false`）。
|
||||
|
||||
- **`open_metrics.user`**
|
||||
|
||||
配置连接 TDengine 的用户名（默认值：`"root"`）。
|
||||
|
||||
- **`open_metrics.password`**
|
||||
|
||||
设置连接 TDengine 的密码（默认值：`"taosdata"`）。
|
||||
|
||||
- **`open_metrics.urls`**
|
||||
|
||||
指定 OpenMetrics 数据采集地址列表（默认值：`["http://localhost:9100"]`，未指定路由时会自动追加 `/metrics`）。
|
||||
|
||||
- **`open_metrics.dbs`**
|
||||
|
||||
设置数据写入的目标数据库列表（默认值：`["open_metrics"]`，需与采集地址数量相同）。
|
||||
|
||||
- **`open_metrics.responseTimeoutSeconds`**
|
||||
|
||||
配置采集超时时间（秒）（默认值：`[5]`，需与采集地址数量相同）。
|
||||
|
||||
- **`open_metrics.httpUsernames`**
|
||||
|
||||
设置 Basic 认证用户名列表（若启用需与采集地址数量相同，默认值：空）。
|
||||
|
||||
- **`open_metrics.httpPasswords`**
|
||||
|
||||
设置 Basic 认证密码列表（若启用需与采集地址数量相同，默认值：空）。
|
||||
|
||||
- **`open_metrics.httpBearerTokenStrings`**
|
||||
|
||||
配置 Bearer Token 认证列表（若启用需与采集地址数量相同，默认值：空）。
|
||||
|
||||
- **`open_metrics.caCertFiles`**
|
||||
|
||||
指定根证书文件路径列表（若启用需与采集地址数量相同，默认值：空）。
|
||||
|
||||
- **`open_metrics.certFiles`**
|
||||
|
||||
设置客户端证书文件路径列表（若启用需与采集地址数量相同，默认值：空）。
|
||||
|
||||
- **`open_metrics.keyFiles`**
|
||||
|
||||
配置客户端证书密钥文件路径列表（若启用需与采集地址数量相同，默认值：空）。
|
||||
|
||||
- **`open_metrics.insecureSkipVerify`**
|
||||
|
||||
是否跳过 HTTPS 证书验证（默认值：`true`）。
|
||||
|
||||
- **`open_metrics.gatherDurationSeconds`**
|
||||
|
||||
设置采集间隔时间（秒）（默认值：`[5]`，需与采集地址数量相同）。
|
||||
|
||||
- **`open_metrics.ttl`**
|
||||
|
||||
定义数据表的生存时间（秒）（`0` 表示无超时，若启用需与采集地址数量相同，默认值：空）。
|
||||
|
||||
- **`open_metrics.ignoreTimestamp`**
|
||||
|
||||
是否忽略采集数据中的时间戳（若忽略则使用采集时刻时间戳，默认值：`false`）。
|
||||
|
||||
#### node_exporter 配置参数
|
||||
|
||||
- **`node_exporter.enable`**
|
||||
|
||||
是否启用 node_exporter 数据采集（默认值：`false`）。
|
||||
|
||||
- **`node_exporter.db`**
|
||||
|
||||
指定 node_exporter 数据写入的数据库名称（默认值：`"node_exporter"`）。
|
||||
|
||||
- **`node_exporter.urls`**
|
||||
|
||||
配置 node_exporter 服务地址（默认值：`["http://localhost:9100"]`）。
|
||||
|
||||
- **`node_exporter.gatherDuration`**
|
||||
|
||||
设置数据采集间隔时间（默认值：`5s`）。
|
||||
|
||||
- **`node_exporter.responseTimeout`**
|
||||
|
||||
配置请求超时时间（默认值：`5s`）。
|
||||
|
||||
- **`node_exporter.user`**
|
||||
|
||||
设置数据库连接用户名（默认值：`"root"`）。
|
||||
|
||||
- **`node_exporter.password`**
|
||||
|
||||
设置数据库连接密码（默认值：`"taosdata"`）。
|
||||
|
||||
- **`node_exporter.ttl`**
|
||||
|
||||
配置采集数据的生存时间（默认值：`0`，表示无超时）。
|
||||
|
||||
- **`node_exporter.httpUsername`**
|
||||
|
||||
配置 HTTP 基本认证用户名（可选）。
|
||||
|
||||
- **`node_exporter.httpPassword`**
|
||||
|
||||
配置 HTTP 基本认证密码（可选）。
|
||||
|
||||
- **`node_exporter.httpBearerTokenString`**
|
||||
|
||||
配置 HTTP Bearer Token 认证（可选）。
|
||||
|
||||
- **`node_exporter.insecureSkipVerify`**
|
||||
|
||||
是否跳过 SSL 证书验证（默认值：`true`）。
|
||||
|
||||
- **`node_exporter.certFile`**
|
||||
|
||||
指定客户端证书文件路径（可选）。
|
||||
|
||||
- **`node_exporter.keyFile`**
|
||||
|
||||
指定客户端证书密钥文件路径（可选）。
|
||||
|
||||
- **`node_exporter.caCertFile`**
|
||||
|
||||
指定 CA 证书文件路径（可选）。
|
||||
|
||||
### 上报指标配置
|
||||
|
||||
taosAdapter 将指标上报到 taosKeeper 进行统一管理，参数如下：
|
||||
|
|
@ -650,6 +735,22 @@ taosAdapter 将指标上报到 taosKeeper 进行统一管理，参数如下：
|
|||
| `node_exporter.ttl` | `TAOS_ADAPTER_NODE_EXPORTER_TTL` |
|
||||
| `node_exporter.urls` | `TAOS_ADAPTER_NODE_EXPORTER_URLS` |
|
||||
| `node_exporter.user` | `TAOS_ADAPTER_NODE_EXPORTER_USER` |
|
||||
| `open_metrics.enable` | `TAOS_ADAPTER_OPEN_METRICS_ENABLE` |
|
||||
| `open_metrics.user` | `TAOS_ADAPTER_OPEN_METRICS_USER` |
|
||||
| `open_metrics.password` | `TAOS_ADAPTER_OPEN_METRICS_PASSWORD` |
|
||||
| `open_metrics.urls` | `TAOS_ADAPTER_OPEN_METRICS_URLS` |
|
||||
| `open_metrics.dbs` | `TAOS_ADAPTER_OPEN_METRICS_DBS` |
|
||||
| `open_metrics.responseTimeoutSeconds` | `TAOS_ADAPTER_OPEN_METRICS_RESPONSE_TIMEOUT_SECONDS` |
|
||||
| `open_metrics.httpUsernames` | `TAOS_ADAPTER_OPEN_METRICS_HTTP_USERNAMES` |
|
||||
| `open_metrics.httpPasswords` | `TAOS_ADAPTER_OPEN_METRICS_HTTP_PASSWORDS` |
|
||||
| `open_metrics.httpBearerTokenStrings` | `TAOS_ADAPTER_OPEN_METRICS_HTTP_BEARER_TOKEN_STRINGS` |
|
||||
| `open_metrics.caCertFiles` | `TAOS_ADAPTER_OPEN_METRICS_CA_CERT_FILES` |
|
||||
| `open_metrics.certFiles` | `TAOS_ADAPTER_OPEN_METRICS_CERT_FILES` |
|
||||
| `open_metrics.keyFiles` | `TAOS_ADAPTER_OPEN_METRICS_KEY_FILES` |
|
||||
| `open_metrics.insecureSkipVerify` | `TAOS_ADAPTER_OPEN_METRICS_INSECURE_SKIP_VERIFY` |
|
||||
| `open_metrics.gatherDurationSeconds` | `TAOS_ADAPTER_OPEN_METRICS_GATHER_DURATION_SECONDS` |
|
||||
| `open_metrics.ignoreTimestamp` | `TAOS_ADAPTER_OPEN_METRICS_IGNORE_TIMESTAMP` |
|
||||
| `open_metrics.ttl` | `TAOS_ADAPTER_OPEN_METRICS_TTL` |
|
||||
| `opentsdb.enable` | `TAOS_ADAPTER_OPENTSDB_ENABLE` |
|
||||
| `opentsdb_telnet.batchSize` | `TAOS_ADAPTER_OPENTSDB_TELNET_BATCH_SIZE` |
|
||||
| `opentsdb_telnet.dbs` | `TAOS_ADAPTER_OPENTSDB_TELNET_DBS` |
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ taosx -f <from-DSN> -t <to-DSN> <其它参数>
|
|||
```
|
||||
|
||||
taosX 的命令行参数分为三个主要部分：
|
||||
|
||||
- `-f` 指定数据源，即 Source DSN
|
||||
- `-t` 指定写入目标，即 Sink DSN
|
||||
- 其它参数
|
||||
|
|
@ -35,6 +36,7 @@ taosX 命令行模式使用 DSN 来表示一个数据源（来源或目的源）
|
|||
// url 示例
|
||||
tmq+ws://root:taosdata@localhost:6030/db1?timeout=never
|
||||
```
|
||||
|
||||
[] 中的数据都为可选参数。
|
||||
|
||||
1. 不同的驱动 (driver) 拥有不同的参数。driver 包含如下选项：
|
||||
|
|
@ -50,6 +52,7 @@ tmq+ws://root:taosdata@localhost:6030/db1?timeout=never
|
|||
- csv：从 CSV 文件解析数据
|
||||
|
||||
2. +protocol 包含如下选项：
|
||||
|
||||
- +ws：当 driver 取值为 taos 或 tmq 时使用，表示使用 rest 获取数据。不使用 +ws 则表示使用原生连接获取数据，此时需要 taosx 所在的服务器安装 taosc。
|
||||
- +ua：当 driver 取值为 opc 时使用，表示采集的数据的 opc-server 为 opc-ua
|
||||
- +da：当 driver 取值为 opc 时使用，表示采集的数据的 opc-server 为 opc-da
|
||||
|
|
@ -62,7 +65,7 @@ tmq+ws://root:taosdata@localhost:6030/db1?timeout=never
|
|||
### 其它参数
|
||||
|
||||
1. --jobs `<number>` 指定任务并发数，仅支持 tmq 任务
|
||||
2. -v 用于指定 taosx 的日志级别，-v 表示启用 info 级别日志，-vv 对应 debug，-vvv 对应 trace
|
||||
2. -v 用于指定 taosx 的日志级别，-v 表示启用 info 级别日志，-vv 对应 debug，-vvv 对应 trace
|
||||
|
||||
### 使用举例
|
||||
|
||||
|
|
@ -147,7 +150,7 @@ taosx run -f 'taos:///db1?mode=all' -t 'taos:///db2' -v
|
|||
```
|
||||
|
||||
5. 通过 --transform 或 -T 配置数据同步（仅支持 2.6 到 3.0 以及 3.0 之间同步）过程中对于表名及表字段的一些操作。暂无法通过 Explorer 进行设置。配置说明如下：
|
||||
|
||||
|
||||
```shell
|
||||
1.AddTag，为表添加 TAG。设置示例：-T add-tag:<tag1>=<value1>。
|
||||
2.表重命名：
|
||||
|
|
@ -174,7 +177,7 @@ taosx run -f 'taos:///db1?mode=all' -t 'taos:///db2' -v
|
|||
3. 使用 CSV 映射文件进行表重命名：以下示例使用 map.csv 文件进行表重命名
|
||||
|
||||
`-T rename-child-table:map:@./map.csv`
|
||||
|
||||
|
||||
CSV 文件 `./map.csv` 格式如下：
|
||||
|
||||
name1,newname1
|
||||
|
|
@ -229,9 +232,54 @@ d4,2017-07-14T10:40:00.006+08:00,-2.740636,10,-0.893545,7,California.LosAngles
|
|||
|
||||
它将从 `./meters/meters.csv.gz`（一个 gzip 压缩的 CSV 文件）导入数据到超级表 `meters`，每一行都插入到指定的表名 - `${tbname}` 使用 CSV 内容中的 `tbname` 列作为表名（即在 JSON 解析器中的 `.model.name`）。
|
||||
|
||||
#### TMQ 订阅数据发布到 MQTT
|
||||
|
||||
基本用法
|
||||
|
||||
```bash
|
||||
taosx run -f "tmq+ws://username:password@ip:port/topic?param=value..." -t "mqtt://ip:port?param=value..."
|
||||
```
|
||||
|
||||
其中 `-f` 指定 TMQ 订阅的 DSN，`-t` 指定 MQTT broker 的 DSN。
|
||||
|
||||
TMQ DSN 参数：
|
||||
|
||||
- `username`: 数据库用户名。
|
||||
- `password`: 数据库密码。
|
||||
- `ip` 和 `port`: 数据库连接使用的 IP 和端口。
|
||||
- `topic`: TMQ 订阅的 topic 名称。
|
||||
- `concurrency`: 任务运行时启动的消费者个数，默认为当前系统 CPU 核数。
|
||||
- `with_meta`: 是否同步元数据，如创建表，删除表，修改表，删除数据等操作，默认不同步。
|
||||
- `group_id`: TMQ 订阅参数，必填项，订阅的组 ID。
|
||||
- `client_id`: TMQ 订阅参数，选填项，订阅的客户端 ID。当指定了此 ID 且 `concurrency` 选项大于 1 时，会自动添加后缀用于区分不同的客户端。
|
||||
- `auto.offset.reset`: TMQ 订阅参数，订阅的起始位置。
|
||||
- `experimental.snapshot.enable`: TMQ 订阅参数，如启用，可以同步已经落盘到 TSDB 时序数据存储文件中（即不在 WAL 中）的数据。如关闭，则只同步尚未落盘（即保存在 WAL 中）的数据。
|
||||
|
||||
更多 TMQ 订阅时的所有参数，详见 [数据订阅](../..//07-develop/07-tmq.md)
|
||||
|
||||
MQTT DSN 参数：
|
||||
|
||||
- `ip`: MQTT broker 的 IP 地址。
|
||||
- `port`: MQTT broker 的 端口。
|
||||
- `version`: MQTT 协议版本，必填项，可选值 3.1/3.1.1/5.0。
|
||||
- `qos`: MQTT QoS 服务质量，默认值 0。
|
||||
- `client_id`: MQTT 客户端 ID，必填项，不同客户端的客户端 ID 必须不同。
|
||||
- `topic`: 数据发布的 MQTT 目标 Topic，必填项。
|
||||
- `meta_topic`: 元数据发布的目标 Topic，如果不指定，默认使用数据发布的 Topic。
|
||||
|
||||
在 MQTT 的 `topic` 和 `meta_topic` 中支持使用如下变量：
|
||||
|
||||
- `database`: 消息来源的数据库名，元数据和数据均包含。
|
||||
- `tmq_topic`: 消息来源的 TMQ 主题名，元数据和数据均包含。
|
||||
- `vgroup_id`: 消息来源 vgroup ID，元数据和数据均包含。
|
||||
- `stable`: 消息来源的超级表名称，仅创建超级表，创建子表，删除超级表的元数据消息包含。
|
||||
- `table`: 消息来源的表名/子表名，仅创建子表/普通表，修改表，删除子表/普通表以及数据类型的消息包含。
|
||||
|
||||
**注意**: 如果 topic 中包含了消息中不存在的变量，则当前消息不会被处理，即不会被发布到 MQTT broker。
|
||||
|
||||
## 服务模式
|
||||
|
||||
本节讲述如何以服务模式部署 `taosX`。以服务模式运行的 taosX，其各项功能需要通过 taosExplorer 上的图形界面来使用。
|
||||
本节讲述如何以服务模式部署 `taosX`。以服务模式运行的 taosX，其各项功能需要通过 taosExplorer 上的图形界面来使用。
|
||||
|
||||
### 配置
|
||||
|
||||
|
|
@ -304,8 +352,8 @@ d4,2017-07-14T10:40:00.006+08:00,-2.740636,10,-0.893545,7,California.LosAngles
|
|||
# When use this in explorer, please set explorer grpc configuration to **Public** IP or
|
||||
# FQDN with correct port, which might be changed exposing to Public network.
|
||||
#
|
||||
# - Example 1: "http://192.168.111.111:6055"
|
||||
# - Example 2: "http://node1.company.domain:6055"
|
||||
# - Example 1: "http://192.168.111.111:6055"
|
||||
# - Example 2: "http://node1.company.domain:6055"
|
||||
#
|
||||
# Please also make sure the above address is not blocked if firewall is enabled.
|
||||
#
|
||||
|
|
@ -389,6 +437,7 @@ sc.exe start taosx
|
|||
1. 修改 `taosX` 日志级别
|
||||
|
||||
`taosX` 的默认日志级别为 `info`，要指定不同的级别，请修改配置文件，或使用以下命令行参数：
|
||||
|
||||
- `error`：`taosx serve -qq`
|
||||
- `debug`：`taosx serve -q`
|
||||
- `info`：`taosx serve -v`
|
||||
|
|
@ -419,7 +468,7 @@ mv /var/lib/taos/taosx/taosx.db* /path/to/data/
|
|||
|
||||
## taosX 监控指标
|
||||
|
||||
taosX 会将监控指标上报给 taosKeeper，这些监控指标会被 taosKeeper 写入监控数据库，默认是 `log` 库，可以在 taoskeeper 配置文件中修改。以下是这些监控指标的详细介绍。
|
||||
taosX 会将监控指标上报给 taosKeeper，这些监控指标会被 taosKeeper 写入监控数据库，默认是 `log` 库，可以在 taoskeeper 配置文件中修改。以下是这些监控指标的详细介绍。
|
||||
|
||||
### taosX 服务
|
||||
|
||||
|
|
@ -439,7 +488,6 @@ taosX 会将监控指标上报给 taosKeeper，这些监控指标会被 taosKeep
|
|||
| process_disk_read_bytes | taosX 进程在一个监控周期（比如 10s）内从硬盘读取的字节数的平均值，单位 bytes/s |
|
||||
| process_disk_written_bytes | taosX 进程在一个监控周期（比如 10s）内写到硬盘的字节数的平均值，单位 bytres/s |
|
||||
|
||||
|
||||
### Agent
|
||||
|
||||
| 字段 | 描述 |
|
||||
|
|
@ -514,7 +562,7 @@ taosX 会将监控指标上报给 taosKeeper，这些监控指标会被 taosKeep
|
|||
|
||||
### taosX 其他数据源 任务
|
||||
|
||||
这些数据源包括：InfluxDB，OpenTSDB，OPC UA，OPC DA，PI，CSV，MQTT，AVEVA Historian 和 Kafka。
|
||||
这些数据源包括：InfluxDB，OpenTSDB，OPC UA，OPC DA，PI，CSV，MQTT，AVEVA Historian 和 Kafka。
|
||||
|
||||
| 字段 | 描述 |
|
||||
| ----------------------- | ----------------------------------------------------------- |
|
||||
|
|
@ -620,7 +668,7 @@ const char* parser_mutate(
|
|||
void* parser,
|
||||
const uint8_t* in_ptr, uint32_t in_len,
|
||||
const void* uint8_t* out_ptr, uint32_t* out_len
|
||||
);
|
||||
);
|
||||
```
|
||||
|
||||
`void* parser`：parser_new 生成的对象指针;
|
||||
|
|
|
|||
|
|
@ -57,12 +57,13 @@ taosBenchmark -f <json file>
|
|||
| -U/--supplement-insert | 写入数据而不提前建数据库和表，默认关闭 |
|
||||
| -p/--password \<passwd> | 用于连接 TDengine 服务端的密码，默认值为 taosdata |
|
||||
| -o/--output \<file> | 结果输出文件的路径，默认值为 ./output.txt |
|
||||
| -j/--output-json-file \<file>| 结果输出的 JSON 文件的路径 |
|
||||
| -T/--thread \<threadNum> | 插入数据的线程数量，默认为 8 |
|
||||
| -B/--interlace-rows \<rowNum> |启用交错插入模式并同时指定向每个子表每次插入的数据行数。交错插入模式是指依次向每张子表插入由本参数所指定的行数并重复这个过程，直到所有子表的数据都插入完成。默认值为 0，即向一张子表完成数据插入后才会向下一张子表进行数据插入 |
|
||||
| -i/--insert-interval \<timeInterval> | 指定交错插入模式的插入间隔，单位为 ms，默认值为 0。只有当 `-B/--interlace-rows` 大于 0 时才起作用 |意味着数据插入线程在为每个子表插入隔行扫描记录后，会等待该值指定的时间间隔后再进行下一轮写入 |
|
||||
| -r/--rec-per-req \<rowNum> | 每次向 TDengine 请求写入的数据行数，默认值为 30000 |
|
||||
| -t/--tables \<tableNum> | 指定子表的数量，默认为 10000 |
|
||||
| -s/ --start-timestamp \<NUMBER> | 每个子表数据开始时间，默认值为 1500000000000 |
|
||||
| -s/--start-timestamp \<NUMBER> | 每个子表数据开始时间，默认值为 1500000000000 |
|
||||
| -S/--timestampstep \<stepLength> | 每个子表中插入数据的时间戳步长，单位是 ms，默认值是 1 |
|
||||
| -n/--records \<recordNum> | 每个子表插入的记录数，默认值为 10000 |
|
||||
| -d/--database \<dbName> | 所使用的数据库的名称，默认值为 test |
|
||||
|
|
@ -82,8 +83,8 @@ taosBenchmark -f <json file>
|
|||
| -R/--disorder-range \<timeRange> | 指定乱序数据的时间戳回退范围。所生成的乱序时间戳为非乱序情况下应该使用的时间戳减去这个范围内的一个随机值。仅在 `-O/--disorder` 指定的乱序数据百分比大于 0 时有效|
|
||||
| -F/--prepare_rand \<Num> | 生成的随机数据中唯一值的数量。若为 1 则表示所有数据都相同。默认值为 10000 |
|
||||
| -a/--replica \<replicaNum> | 创建数据库时指定其副本数，默认值为 1 |
|
||||
| -k/--keep-trying \<NUMBER> | 失败后进行重试的次数，默认不重试。需使用 v3.0.9 以上版本|
|
||||
| -z/--trying-interval \<NUMBER> | 失败重试间隔时间，单位为毫秒，仅在 -k 指定重试后有效。需使用 v3.0.9 以上版本 |
|
||||
| -k/--keep-trying \<NUMBER> | 失败后进行重试的次数，默认不重试。需使用 v3.0.9 以上版本|
|
||||
| -z/--trying-interval \<NUMBER> | 失败重试间隔时间，单位为毫秒，仅在 -k 指定重试后有效。需使用 v3.0.9 以上版本 |
|
||||
| -v/--vgroups \<NUMBER> | 创建数据库时指定 vgroups 数，仅对 TDengine v3.0+ 有效|
|
||||
| -V/--version | 显示版本信息并退出。不能与其它参数混用|
|
||||
| -?/--help | 显示帮助信息并退出。不能与其它参数混用|
|
||||
|
|
@ -109,6 +110,8 @@ taosBenchmark -f <json file>
|
|||
|
||||
- **password**：用于连接 TDengine 服务端的密码，默认值为 taosdata。
|
||||
|
||||
- **result_json_file**：指定结果输出的 JSON 文件路径，若未配置则不输出该文件。
|
||||
|
||||
### 写入配置参数
|
||||
|
||||
写入场景下 `filetype` 必须设置为 `insert`，该参数及其它通用参数详见 [通用配置参数](#通用配置参数)
|
||||
|
|
@ -116,7 +119,7 @@ taosBenchmark -f <json file>
|
|||
- **keep_trying**：失败后进行重试的次数，默认不重试。需使用 v3.0.9 以上版本。
|
||||
|
||||
- **trying_interval**：失败重试间隔时间，单位为毫秒，仅在 keep_trying 指定重试后有效。需使用 v3.0.9 以上版本。
|
||||
- **childtable_from 和 childtable_to**：指定写入子表范围，开闭区间为 [childtable_from, childtable_to) 。
|
||||
- **childtable_from 和 childtable_to**：指定写入子表范围，开闭区间为 [childtable_from, childtable_to] 。
|
||||
|
||||
- **continue_if_fail**：允许用户定义失败后行为。
|
||||
|
||||
|
|
@ -188,8 +191,12 @@ taosBenchmark -f <json file>
|
|||
|
||||
- **tags_file**：仅当 insert_mode 为 taosc，rest 的模式下生效。最终的 tag 的数值与 childtable_count 有关，如果 csv 文件内的 tag 数据行小于给定的子表数量，那么会循环读取 csv 文件数据直到生成 childtable_count 指定的子表数量；否则则只会读取 childtable_count 行 tag 数据。也即最终生成的子表数量为二者取小。
|
||||
|
||||
- **use_tag_table_name**：当设置为 yes 时，csv 文件内的 tag 数据中的第一列为需要创建的子表名称，反之有系统自动生成子表名称进行创建。
|
||||
|
||||
- **primary_key**：指定超级表是否有复合主键，取值 1 和 0，复合主键列只能是超级表的第二列，指定生成复合主键后要确保第二列符合复合主键的数据类型，否则会报错。
|
||||
|
||||
- **primary_key_name**：指定超级表主键的名称，如果不指定默认为 `ts`。
|
||||
|
||||
- **repeat_ts_min**：数值类型，复合主键开启情况下指定生成相同时间戳记录的最小个数，生成相同时间戳记录的个数是在范围[repeat_ts_min, repeat_ts_max] 内的随机值，最小值等于最大值时为固定个数。
|
||||
|
||||
- **repeat_ts_max**：数值类型，复合主键开启情况下指定生成相同时间戳记录的最大个数。
|
||||
|
|
@ -269,8 +276,6 @@ taosBenchmark -f <json file>
|
|||
|
||||
- **result_file**：结果输出文件的路径，默认值为 ./output.txt。
|
||||
|
||||
- **result_json_file**：结果输出的 JSON 文件路径，若未配置则不输出该文件。
|
||||
|
||||
- **confirm_parameter_prompt**：开关参数，要求用户在提示后确认才能继续，可取值 "yes" or "no"。默认值为 "no" 。
|
||||
|
||||
- **interlace_rows**：启用交错插入模式并同时指定向每个子表每次插入的数据行数。交错插入模式是指依次向每张子表插入由本参数所指定的行数并重复这个过程，直到所有子表的数据都插入完成。默认值为 0，即向一张子表完成数据插入后才会向下一张子表进行数据插入。
|
||||
|
|
|
|||
|
|
@ -45,6 +45,7 @@ CREATE DATABASE db_name PRECISION 'ns';
|
|||
| 17 | GEOMETRY | 自定义 | 几何类型，v3.1.0.0 开始支持
|
||||
| 18 | VARBINARY | 自定义 | 可变长的二进制数据，v3.1.1.0 开始支持|
|
||||
| 19 | DECIMAL | 8 或 16 | 高精度数值类型，取值范围取决于类型中指定的 precision 和 scale，自 v3.3.6.0 开始支持，见下文描述|
|
||||
| 19 | BLOB | 最大长度 4M | 可变长的二进制数据，v3.3.7.0 开始支持|
|
||||
|
||||
:::note
|
||||
|
||||
|
|
@ -75,6 +76,17 @@ DECIMAL 类型仅支持普通列，暂不支持 tag 列。DECIMAL 类型只支
|
|||
|
||||
查询 DECIMAL 类型表达式时，若计算的中间结果超出当前类型可表示的最大值时，报 DECIMAL OVERFLOW 错误。
|
||||
|
||||
### BLOB 数据类型
|
||||
`BLOB`是一种存储二进制数据的数据类型，最大长度为 4,194,304 字节，可以通过 SQL 或 STMT2 方式写入二进制数据（也可以转换为 `\x` 开头的字符串写入）。
|
||||
|
||||
通过 SHELL 查询数据时，显示为 16 进制的字符串，以 `\x` 开头。
|
||||
|
||||
限制：
|
||||
- 仅支持在普通数据列中使用 BLOB 类型，BLOB 列数目不能超过 1 个。
|
||||
- 不支持 BLOB 列的条件过滤
|
||||
|
||||
短期限制：
|
||||
- 不支持虚拟表/流计算等功能
|
||||
|
||||
## 常量
|
||||
|
||||
|
|
|
|||
|
|
@ -104,4 +104,9 @@ SHOW CONSUMERS;
|
|||
SHOW SUBSCRIPTIONS;
|
||||
```
|
||||
|
||||
显示 consumer 与 vgroup 之间的分配关系和消费信息
|
||||
显示 consumer 与 vgroup 之间的分配关系和消费信息
|
||||
|
||||
## MQTT 数据订阅
|
||||
|
||||
TDengine v3.3.7.0 版本提供了 MQTT 订阅功能，可以通过 MQTT 客户端直接订阅数据，具体内容请参考 MQTT 数据订阅部分。
|
||||
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -364,9 +364,6 @@ description: TDengine 保留关键字的详细列表
|
|||
### S
|
||||
|关键字 | 说明|
|
||||
|----------------------|-|
|
||||
| S3_CHUNKPAGES | |
|
||||
| S3_COMPACT | |
|
||||
| S3_KEEPLOCAL | |
|
||||
| SCHEMALESS | |
|
||||
| SCORES | |
|
||||
| SELECT | |
|
||||
|
|
@ -387,6 +384,10 @@ description: TDengine 保留关键字的详细列表
|
|||
| SNODES | |
|
||||
| SOFFSET | |
|
||||
| SPLIT | |
|
||||
| SS_CHUNKPAGES | |
|
||||
| SS_COMPACT | |
|
||||
| SS_KEEPLOCAL | |
|
||||
| SSMIGRATE | |
|
||||
| STABLE | |
|
||||
| STABLES | |
|
||||
| STAR | |
|
||||
|
|
|
|||
|
|
@ -114,6 +114,30 @@ DROP QNODE ON DNODE dnode_id;
|
|||
|
||||
删除 ID 为 dnode_id 的 dnode 上的 qnode，但并不会影响该 dnode 的状态。
|
||||
|
||||
## 创建订阅节点
|
||||
|
||||
```sql
|
||||
CREATE BNODE ON DNODE dnode_id [PROTOCOL protocol];
|
||||
```
|
||||
|
||||
系统启动默认没有 BNODE，用户可以创建 BNODE 来启动订阅服务。一个 dnode 上只能创建一个 BNODE。protocol 为可选配置项，不提供时，默认为“mqtt”，后续会扩展其它协议。bnode 创建成功后，dnode 会启动子进程 taosmqtt，对外提供订阅服务。
|
||||
|
||||
## 查看订阅节点
|
||||
|
||||
```sql
|
||||
SHOW BNODES;
|
||||
```
|
||||
|
||||
列出集群中所有订阅节点，包括 ID，protocol，创建时间，及所在 dnode。
|
||||
|
||||
## 删除订阅节点
|
||||
|
||||
```sql
|
||||
DROP BNODE ON DNODE dnode_id;
|
||||
```
|
||||
|
||||
删除 ID 为 dnode_id 的 dnode 上的 bnode，此 dnode 上的 taosmqtt 子进程会退出，停止订阅服务。
|
||||
|
||||
## 查询集群状态
|
||||
|
||||
```sql
|
||||
|
|
|
|||
|
|
@ -341,7 +341,7 @@ TDengine 内置了一个名为 `INFORMATION_SCHEMA` 的数据库，提供对数
|
|||
| 6 | data3 | BIGINT | 三级存储上数据文件的大小，单位为 KB |
|
||||
| 7 | cache_rdb | BIGINT | last/last_row 文件的大小，单位为 KB |
|
||||
| 8 | table_meta | BIGINT | meta 文件的大小，单位为 KB |
|
||||
| 9 | s3 | BIGINT | s3 上占用的大小，单位为 KB |
|
||||
| 9 | ss | BIGINT | 共享存储上占用的大小，单位为 KB |
|
||||
| 10 | raw_data | BIGINT | 预估的原始数据的大小，单位为 KB |
|
||||
|
||||
|
||||
|
|
|
|||
110
docs/zh/14-reference/03-taos-sql/34-mount.md
Normal file
110
docs/zh/14-reference/03-taos-sql/34-mount.md
Normal file
|
|
@ -0,0 +1,110 @@
|
|||
---
|
||||
toc_max_heading_level: 4
|
||||
title: "数据挂载"
|
||||
sidebar_label: "数据挂载"
|
||||
---
|
||||
|
||||
自 3.3.7.0 版本起，TDengine 企业版提供数据挂载功能。该功能支持用户在宿主集群中，通过 SQL 命令将源集群（包括飞行器、车载设备、边缘设备、同机房其他集群、跨数据中心集群、历史备份数据等）挂载到宿主集群，使宿主集群能够直接识别并访问源集群的数据库、表及数据。挂载后无需数据迁移，即可在宿主集群内直接查询挂载的源集群数据，让集群间的数据访问更加便捷高效。为防止源集群的数据被篡改，当前仅支持只读挂载模式。
|
||||
|
||||
## 语法
|
||||
|
||||
### 创建挂载
|
||||
|
||||
```sql
|
||||
CREATE MOUNT mountName ON DNODE dnodeId FROM TDenginePath;
|
||||
```
|
||||
|
||||
说明
|
||||
|
||||
- mountName：挂载名称，不能包含下划线 `_`，挂载后的数据库名为 mountName_`<dbname`>。
|
||||
- dnodeId：源集群数据目录所在宿主集群的 dnode 节点标识。
|
||||
- TDenginePath：源集群数据目录的绝对路径，需用英文单引号或双引号括起。
|
||||
|
||||
使用限制
|
||||
|
||||
- 同一宿主集群中，mountName 不能重复。
|
||||
- mountName 不能与宿主集群中已有数据库重名。
|
||||
- mountName 和 TDenginePath 一一对应。
|
||||
- 目前仅支持在单个 dnode 节点进行挂载（如果源集群包含多级存储，目前需要手工调整 local.json 中非主挂载点的路径，非多级存储不需要调整）。
|
||||
- 同一源集群同一时刻只能被一个宿主集群挂载。
|
||||
- 宿主集群不能挂载其自身的数据目录。
|
||||
- 源集群的数据库唯一标识 dbid 不能与宿主集群中现有 dbid 相同。
|
||||
- 宿主集群中存在挂载数据库时，不允许创建新数据库（防止名称冲突）。
|
||||
- 挂载时：暂不支持源集群多副本；暂不支持源集群加密数据库；暂不支持源集群无数据库。
|
||||
- 宿主集群中，针对挂载库的操作：仅支持时序数据查询，不支持写入、元数据更新；不支持虚拟表时序数据查询，支持虚拟表元数据查询；不支持流计算、订阅、视图等功能；不支持 compact/trim/redistribute/split/restore 等管理操作。
|
||||
- 只有超级管理员拥有操作权限。
|
||||
|
||||
示例
|
||||
|
||||
- 执行以下命令，可在宿主集群的 dnode 1 节点上，将源集群的 /var/lib/taos_1 数据目录以 mount1 为名称挂载到本地，挂载后宿主集群可通过 mount1_`<dbname`> 访问源集群的数据库：
|
||||
|
||||
```sql
|
||||
CREATE MOUNT mount1 ON DNODE 1 FROM "/var/lib/taos_1";
|
||||
```
|
||||
|
||||
### 查看挂载
|
||||
|
||||
```sql
|
||||
SHOW MOUNTS;
|
||||
```
|
||||
|
||||
示例
|
||||
|
||||
```text
|
||||
taos> show mounts;
|
||||
name | dnode | create_time | path |
|
||||
=======================================================================================================
|
||||
mount1 | 1 | 2025-07-17 18:06:16.298 | /var/lib/taos_1 |
|
||||
Query OK, 1 row(s) in set (0.027046s)
|
||||
```
|
||||
|
||||
说明
|
||||
|
||||
- name：挂载名称。
|
||||
- dnode：宿主集群中执行挂载的 dnode 节点。
|
||||
- create_time：挂载在宿主集群的创建时间。
|
||||
- path：源集群数据目录绝对路径。
|
||||
- 只有拥有系统信息查看权限的用户可以查看。
|
||||
|
||||
### 删除挂载
|
||||
|
||||
```sql
|
||||
DROP MOUNT mountName;
|
||||
```
|
||||
|
||||
说明
|
||||
|
||||
- 清空宿主集群中与该挂载相关的元数据。
|
||||
- 恢复源集群数据目录的配置信息，解除宿主集群对源集群的访问关联。
|
||||
- 不会删除源集群的实际数据文件。
|
||||
- 只有超级管理员拥有操作权限。
|
||||
|
||||
## 技术特点
|
||||
|
||||
### 快速数据访问
|
||||
|
||||
- 无需数据迁移，通过挂载将源集群数据 "接入" 宿主集群，直接在宿主集群内查询。
|
||||
- 保持宿主集群查询操作的便捷性。
|
||||
|
||||
### 轻量级实现
|
||||
|
||||
- 通过元数据映射建立宿主集群与源集群的访问通道，源集群数据文件无需复制到宿主集群，仅在宿主集群记录挂载关联信息，实现轻量级跨集群访问。
|
||||
|
||||
### 实时查询
|
||||
|
||||
- 宿主集群直接访问源集群原始数据文件，避免数据迁移的延迟。
|
||||
|
||||
### 安全保障
|
||||
|
||||
- 宿主集群对挂载数据的只读限制，保护源集群的原始数据不被篡改。
|
||||
- 挂载操作仅建立宿主集群与源集群的逻辑访问关系，配合只读限制和唯一性校验，确保数据安全与系统稳定。
|
||||
- 卸载后自动恢复源集群配置，确保原始数据完整性不受挂载操作影响。
|
||||
|
||||
## 典型应用场景
|
||||
|
||||
此功能特别适用于需要频繁访问多个源集群数据但又不希望进行数据迁移的场景，为用户提供高效、安全的跨集群数据访问能力。
|
||||
|
||||
- 设备数据地面分析：源集群采集的数据无需导出迁移，通过挂载接入宿主分析集群，操作人员可在分析集群中直接查询设备数据。
|
||||
- 跨集群数据比对：将多个业务集群数据挂载到统一分析集群（宿主集群），在分析集群内直接对比不同集群的同源数据。
|
||||
- 历史数据查询：将离线存储的历史数据集群挂载到在线业务集群（宿主集群），无需回迁即可查询历史数据。
|
||||
- 数据备份验证：挂载备份存储的集群到验证集群（宿主集群），直接在验证集群中查询验证备份数据的完整性。
|
||||
|
|
@ -651,7 +651,9 @@ TDengine 客户端驱动的版本号与 TDengine 服务端的版本号是一一
|
|||
|
||||
- 异步查询示例：[异步查询](https://github.com/taosdata/TDengine/tree/main/docs/examples/c/asyncdemo.c)
|
||||
|
||||
- 参数绑定示例：[参数绑定](https://github.com/taosdata/TDengine/tree/main/docs/examples/c/prepare.c)
|
||||
- 参数绑定示例：[参数绑定](https://github.com/taosdata/TDengine/tree/main/docs/examples/c/stmt2_insert_demo.c)
|
||||
|
||||
- 参数绑定（旧）示例：[参数绑定（旧）](https://github.com/taosdata/TDengine/tree/main/docs/examples/c/prepare.c)
|
||||
|
||||
- 无模式写入示例：[无模式写入](https://github.com/taosdata/TDengine/tree/main/docs/examples/c/schemaless.c)
|
||||
|
||||
|
|
@ -888,6 +890,100 @@ TDengine 的异步 API 均采用非阻塞调用模式。应用程序可以用多
|
|||
|
||||
除了直接调用 `taos_query()` 进行查询，TDengine 也提供了支持参数绑定的 Prepare API，风格与 MySQL 类似，目前也仅支持用问号 `?` 来代表待绑定的参数。
|
||||
|
||||
从 3.3.5.0 版本开始，TDengine 大幅简化了旧版参数绑定的使用接口。这样在通过参数绑定接口写入数据时，可以避免了 SQL 语法解析的资源消耗，通过批量绑定的方式，在绝大多数情况下显著提升写入性能。此时的典型操作步骤如下：
|
||||
|
||||
1. 调用 `taos_stmt2_init()` 创建参数绑定对象；
|
||||
2. 调用 `taos_stmt2_prepare()` 解析 INSERT 或者 SELECT 语句；
|
||||
3. 调用 `taos_stmt2_bind_param()` 可以向单个超级表绑定多个子表，每个子表可以绑定多行数据；
|
||||
4. 调用 `taos_stmt2_exec()` 执行已经准备好的批处理指令；、
|
||||
5. 可以重复第 3 ～ 4 步，为写入更多的数据行，而无需再次解析 SQL；
|
||||
6. 执行完毕，调用 `taos_stmt2_close()` 释放所有资源。
|
||||
|
||||
- `char *taos_stmt2_error(TAOS_STMT2 *stmt)`
|
||||
说明：如果 `taos_stmt2_exec()` 执行成功，假如不需要改变 SQL 语句的话，那么是可以复用 `taos_stmt2_prepare()` 的解析结果，直接进行第 3 ～ 4 步绑定新数据的。但如果执行出错，那么并不建议继续在当前的环境上下文下继续工作，而是建议释放资源，然后从 `taos_stmt2_init()` 步骤重新开始，可以通过 `taos_stmt2_error` 查看具体错误原因。
|
||||
|
||||
stmt2 和 stmt 的区别在于：
|
||||
- stmt2 支持多表批量绑定数据，stmt 只支持单表绑定数据。
|
||||
- stmt2 支持异步执行，stmt 只支持同步执行。
|
||||
- stmt2 支持高效写入模式以及自动建表，stmt 不支持。
|
||||
- stmt2 支持 `insert into stb(...tbname,...)values(?,?,?)` 语法，stmt 不支持。
|
||||
- stmt2 支持部分标签/列为固定值，stmt 必须所有列为 `?` 。
|
||||
|
||||
stmt 升级到 stmt2 的改动：
|
||||
1. 将 `taos_stmt_init()` 改为 `taos_stmt2_init()`，并增加了 `TAOS_STMT2_OPTION` 结构体参数。
|
||||
2. 将 `taos_stmt_prepare()` 改为 `taos_stmt2_prepare()` 。
|
||||
3. 将 `taos_stmt_set_tbname_tags` ， `taos_stmt_bind_param()` 和 `taos_stmt_add_batch` 合并改为 `taos_stmt2_bind_param()`，将 `TAOS_MULTI_BIND` 改为 `TAOS_STMT2_BINDV` 结构体作为参数。
|
||||
4. 将 `taos_stmt_execute()` 改为 `taos_stmt2_exec()`，增加了 `affected_rows` 参数。
|
||||
5. 将 `taos_stmt_close()` 改为 `taos_stmt2_close()` 。
|
||||
|
||||
接口相关的具体函数如下（也可以参考 [stmt2_insert_demo.c](https://github.com/taosdata/TDengine/tree/main/docs/examples/c/stmt2_insert_demo.c) 文件中使用对应函数的方式）：
|
||||
|
||||
- `TAOS_STMT2 *taos_stmt2_init(TAOS *taos, TAOS_STMT2_OPTION *option)`
|
||||
- **接口说明**：初始化一个预编译的 SQL 语句对象。
|
||||
- **参数说明**：
|
||||
- `taos`：[入参] 指向数据库连接的指针，数据库连接是通过 `taos_connect()` 函数建立。
|
||||
- `option`：[入参] 创建配置，当选择高效写入模式需要将 `singleStbInsert` 和 `singleTableBindOnce` 设置为 `true`；当选择异步执行是，需要设置回调函数 `asyncExecFn` 和参数 `userdata`。
|
||||
- **返回值**：非 `NULL`：成功，返回 TAOS_STMT2 的一个结构体的指针，该结构体表示预编译的 SQL 语句对象。`NULL`：失败，详情请调用 taos_stmt_errstr() 函数来获取错误提示。
|
||||
|
||||
- `int taos_stmt2_prepare(TAOS_STMT2 *stmt, const char *sql, unsigned long length)`
|
||||
- **接口说明**：解析一条预编译的 SQL 语句，将解析结果和参数信息绑定到 stmt 上。
|
||||
- **参数说明**：
|
||||
- stmt：[入参] 指向一个有效的预编译的 SQL 语句对象指针。
|
||||
- sql：[入参] 需要解析的 SQL 语句。
|
||||
- length：[入参] 参数 sql 的长度。如果参数 length 大于 0，将使用此参数作为 SQL 语句的长度，如等于 0，将自动判断 SQL 语句的长度。
|
||||
- **返回值**：`0`：成功。非 `0`：失败，详情请参考错误码页面。
|
||||
|
||||
- `int taos_stmt2_bind_param(TAOS_STMT2 *stmt, TAOS_STMT2_BINDV *bindv, int32_t col_idx)`
|
||||
- **接口说明**：绑定一批参数到一个预编译的 SQL 语句。
|
||||
- **参数说明**：
|
||||
- stmt：[入参] 指向一个有效的预编译的 SQL 语句对象指针。
|
||||
- bindv：[入参] 指向一个有效的 TAOS_STMT2_BINDV 结构体指针，该结构体中包含要绑定的表名、标签、数据，`count`表示要绑定的表数量，一个`tbnames`可以对应一组`tags`以及多组`bind_cols`；如果是 `SELECT` 的语句，则只需要绑定 `bind_cols`。
|
||||
- col_idx：[入参] 表示要绑定的指定列的位置， `-1`表示全列绑定。
|
||||
- **返回值**：`0`：成功。非 `0`：失败，详情请参考错误码页面。
|
||||
|
||||
- `int taos_stmt2_exec(TAOS_STMT2 *stmt, int *affected_rows)`
|
||||
- **接口说明**：执行绑定完成数据的 SQL，可同步或异步，由 option 决定。
|
||||
- **参数说明**：
|
||||
- stmt：[入参] 指向一个有效的预编译的 SQL 语句对象指针。
|
||||
- affected_rows：[出参] 若为同步执行，表示本次执行影响到的的行数。
|
||||
- **返回值**：`0`：成功。非 `0`：失败，详情请参考错误码页面。
|
||||
|
||||
- `int taos_stmt2_close(TAOS_STMT2 *stmt)`
|
||||
- **接口说明**：执行完毕，释放所有资源。
|
||||
- **参数说明**：
|
||||
- stmt：[入参] 指向一个有效的预编译的 SQL 语句对象指针。
|
||||
- **返回值**：`0`：成功。非 `0`：失败，详情请参考错误码页面。
|
||||
|
||||
- `int taos_stmt2_get_fields(TAOS_STMT2 *stmt, int *count, TAOS_FIELD_ALL **fields)`
|
||||
- **接口说明**：获取 `?` 顺序对应的列数据的属性（列的名称、列的数据类型、列的长度、列的 schema 类型）的数组。
|
||||
- **参数说明**：
|
||||
- stmt：[入参] 指向一个有效的预编译的 SQL 语句对象指针。
|
||||
- count：[出参] 返回绑定 SQL 中 `?` 的数量。
|
||||
- fields：[出参] 返回 `?` 顺序对应的列数据的属性，如果是 `SELECT` 的语句，该结构体返回 `NULL`。
|
||||
- **返回值**：`0`：成功。非 `0`：失败，详情请参考错误码页面。
|
||||
|
||||
- `void taos_stmt2_free_fields(TAOS_STMT2 *stmt, TAOS_FIELD_ALL *fields)`
|
||||
- **接口说明**：释放 TAOS_FIELD_ALL 返回值的内存，一般用于 taos_stmt2_get_fields 之后。
|
||||
- **参数说明**：
|
||||
- stmt：[入参] 指向一个有效的预编译的 SQL 语句对象指针。
|
||||
- fields：[入参] 要释放资源的指针。
|
||||
- **返回值**：无。
|
||||
|
||||
- `TAOS_RES *taos_stmt2_result(TAOS_STMT2 *stmt)`
|
||||
- **接口说明**：获取执行 SQL 后返回的结果。
|
||||
- **参数说明**：
|
||||
- stmt：[入参] 指向一个有效的预编译的 SQL 语句对象指针。
|
||||
- **返回值**：返回一个指向 TAOS_RES 结构体的指针，该结构体包含了插入操作的结果。返回的 TAOS_RES 需要调用方来负责释放，否则会出现内存泄漏。
|
||||
|
||||
- `char *taos_stmt2_error(TAOS_STMT2 *stmt)`
|
||||
- **接口说明**：用于在其他 STMT2 API 返回错误（返回错误码或空指针）时获取错误信息。
|
||||
- **参数说明**：
|
||||
- stmt：[入参] 指向一个有效的预编译的 SQL 语句对象指针。
|
||||
- **返回值**：返回一个指向包含错误信息的字符串的指针。
|
||||
|
||||
<details>
|
||||
<summary>参数绑定（旧）</summary>
|
||||
|
||||
从 2.1.1.0 和 2.1.2.0 版本开始，TDengine 大幅改进了参数绑定接口对数据写入（INSERT）场景的支持。这样在通过参数绑定接口写入数据时，就避免了 SQL 语法解析的资源消耗，从而在绝大多数情况下显著提升写入性能。此时的典型操作步骤如下：
|
||||
|
||||
1. 调用 `taos_stmt_init()` 创建参数绑定对象；
|
||||
|
|
@ -981,6 +1077,7 @@ TDengine 的异步 API 均采用非阻塞调用模式。应用程序可以用多
|
|||
- **接口说明**：（2.1.3.0 版本新增）用于在其他 STMT API 返回错误（返回错误码或空指针）时获取错误信息。
|
||||
- stmt：[入参] 指向一个有效的预编译的 SQL 语句对象指针。
|
||||
- **返回值**：返回一个指向包含错误信息的字符串的指针。
|
||||
</details>
|
||||
|
||||
#### 无模式写入
|
||||
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ TDengine 的 JDBC 驱动实现尽可能与关系型数据库驱动保持一致
|
|||
|
||||
| taos-jdbcdriver 版本 | 主要变化 | TDengine 版本 |
|
||||
| ------------------| ---------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------- |
|
||||
| 3.7.0 | 1. 使用 Netty 替换 Java-WebSocket 库，提升小查询性能<br/> 2. 兼容 IPv6 网络协议<br/> 3. 支持 BLOB 二进制数据类型<br/> 4. 实现 TDengine 版本兼容性检查<br/> 5. 支持 `varcharAsString` 配置属性<br/> 6. 优化 WebSocket 查询内存使用效率<br/> 7. 修复 WebSocket 连接的时区问题<br/> | - |
|
||||
| 3.6.3 | 解决了订阅数据库和超级表时的数据类型转换 bug | - |
|
||||
| 3.6.2 | 1. 支持订阅数据库和超级表（不支持订阅元数据） <br/> 2. 解决了云服务订阅 bug <br/> 3. 优化了 setQueryTimeout 参数为 0 的实现 | - |
|
||||
| 3.6.1 | 解决 WebSocket 连接在小查询上的性能 bug | - |
|
||||
|
|
@ -108,6 +109,12 @@ JDBC 连接器可能报错的错误码包括 4 种：
|
|||
| 0x231c | httpEntity is null, sql | REST 连接中执行出现异常 |
|
||||
| 0x231d | can't create connection with server within | 通过增加参数 httpConnectTimeout 增加连接耗时，或是请检查与 taosAdapter 之间的连接情况。 |
|
||||
| 0x231e | failed to complete the task within the specified time | 通过增加参数 messageWaitTimeout 增加执行耗时，或是请检查与 taosAdapter 之间的连接情况。 |
|
||||
| 0x231f | restful client query exception | HTTP 请求错误，请查看详细信息。 |
|
||||
| 0x2320 | type convert exception | 检查是否没有使用正确的类型 |
|
||||
| 0x2321 | TDengine version incompatible | TDengine 版本不匹配，请升级至对应版本。 |
|
||||
| 0x2322 | resource has been freed | 资源已经释放，请确认操作正确。 |
|
||||
| 0x2323 | BLOB is unsupported on the server | 服务端不支持 BLOB 类型，需要升级。 |
|
||||
| 0x2324 | line bind mode is unsupported on the server | 服务端不支持行绑定模式，需要升级。 |
|
||||
| 0x2350 | unknown error | 未知异常，请在 github 反馈给开发人员。 |
|
||||
| 0x2352 | Unsupported encoding | 本地连接下指定了不支持的字符编码集 |
|
||||
| 0x2353 | internal error of database, please see taoslog for more details | 本地连接执行 prepareStatement 时出现错误，请检查 taos log 进行问题定位。 |
|
||||
|
|
@ -220,8 +227,8 @@ taos-jdbcdriver 实现了 JDBC 标准的 Driver 接口，提供了 3 个实现
|
|||
TDengine 的 JDBC URL 规范格式为：
|
||||
`jdbc:[TAOS|TAOS-WS|TAOS-RS]://[host_name]:[port]/[database_name]?[user={user}|&password={password}|&charset={charset}|&cfgdir={config_dir}|&locale={locale}|&timezone={timezone}|&batchfetch={batchfetch}]`
|
||||
|
||||
`host_name` 参数支持合法的域名或 IP 地址。taos-jdbcdriver 同时支持 IPv4 和 IPv6 格式。对于 IPv6 地址，必须使用中括号括起来（例如 `[::1]` 或 `[2001:db8:1234:5678::1]`），以避免端口号解析冲突。
|
||||
|
||||
`host_name` 参数支持合法的域名或 IP 地址。taos-jdbcdriver 同时支持 IPv4 和 IPv6 格式。对于 IPv6 地址，必须使用中括号括起来（例如 `[::1]` 或 `[2001:db8:1234:5678::1]`），以避免端口号解析冲突。
|
||||
JDBC URL 中支持设置 Properties 中所有属性，具体请参考下文 **Properties** 章节。
|
||||
|
||||
**原生连接**
|
||||
`jdbc:TAOS://taosdemo.com:6030/power?user=root&password=taosdata`，使用了 JDBC 原生连接的 TSDBDriver，建立了到 hostname 为 taosdemo.com，端口为 6030（TDengine 的默认端口），数据库名为 power 的连接。这个 URL
|
||||
|
|
@ -301,45 +308,48 @@ TDengine 中，只要保证 firstEp 和 secondEp 中一个节点有效，就可
|
|||
|
||||
#### Properties
|
||||
|
||||
除了通过指定的 URL 获取连接，还可以使用 Properties 指定建立连接时的参数。
|
||||
除了通过指定的 URL 获取连接，还可以使用 Properties 指定建立连接时的参数。
|
||||
所有 Properties 配置参数同样可以在 JDBC URL 中指定，方括号中的参数名可以用于 JDBC URL(如 TSDBDriver.PROPERTY_KEY_USER[`user`]，可以在 JDBC URL 中使用 `user=root` 来设置用户名)。
|
||||
|
||||
> **注意**：应用中设置的 client parameter 为进程级别的，即如果要更新 client 的参数，需要重启应用。这是因为 client parameter 是全局参数，仅在应用程序的第一次设置生效。
|
||||
|
||||
properties 中的配置参数如下：
|
||||
- TSDBDriver.PROPERTY_KEY_USER：登录 TDengine 用户名，默认值 'root'。
|
||||
- TSDBDriver.PROPERTY_KEY_PASSWORD：用户登录密码，默认值 'taosdata'。
|
||||
- TSDBDriver.PROPERTY_KEY_BATCH_LOAD：true：在执行查询时批量拉取结果集；false：逐行拉取结果集。默认值为：false。因历史原因使用 REST 连接时，若设置此参数为 true 会变成 WebSocket 连接。
|
||||
- TSDBDriver.PROPERTY_KEY_BATCH_ERROR_IGNORE：true：在执行 Statement 的 executeBatch 时，如果中间有一条 SQL 执行失败，继续执行下面的 sq 了。false：不再执行失败 SQL 后的任何语句。默认值为：false。
|
||||
- TSDBDriver.PROPERTY_KEY_CONFIG_DIR：仅在使用 JDBC 原生连接时生效。客户端配置文件目录路径，Linux OS 上默认值 `/etc/taos`，Windows OS 上默认值 `C:/TDengine/cfg`。
|
||||
- TSDBDriver.PROPERTY_KEY_CHARSET：客户端使用的字符集，默认值为系统字符集。
|
||||
- TSDBDriver.PROPERTY_KEY_LOCALE：仅在使用 JDBC 原生连接时生效。客户端语言环境，默认值系统当前 locale。
|
||||
- TSDBDriver.PROPERTY_KEY_TIME_ZONE：
|
||||
properties 中的配置参数如下：
|
||||
- TSDBDriver.PROPERTY_KEY_USER [`user`]：登录 TDengine 用户名，默认值 'root'。
|
||||
- TSDBDriver.PROPERTY_KEY_PASSWORD [`password`]：用户登录密码，默认值 'taosdata'。
|
||||
- TSDBDriver.PROPERTY_KEY_BATCH_LOAD [`batchfetch`]：true：在执行查询时批量拉取结果集；false：逐行拉取结果集。默认值为：false。因历史原因使用 REST 连接时，若设置此参数为 true 会变成 WebSocket 连接。
|
||||
- TSDBDriver.PROPERTY_KEY_BATCH_ERROR_IGNORE [`batchErrorIgnore`]：true：在执行 Statement 的 executeBatch 时，如果中间有一条 SQL 执行失败，继续执行下面的 sq 了。false：不再执行失败 SQL 后的任何语句。默认值为：false。
|
||||
- TSDBDriver.PROPERTY_KEY_CONFIG_DIR [`cfgdir`]：仅在使用 JDBC 原生连接时生效。客户端配置文件目录路径，Linux OS 上默认值 `/etc/taos`，Windows OS 上默认值 `C:/TDengine/cfg`。
|
||||
- TSDBDriver.PROPERTY_KEY_CHARSET [`charset`]：客户端使用的字符集，默认值为系统字符集。
|
||||
- TSDBDriver.PROPERTY_KEY_LOCALE [`locale`]：仅在使用 JDBC 原生连接时生效。客户端语言环境，默认值系统当前 locale。
|
||||
- TSDBDriver.PROPERTY_KEY_TIME_ZONE [`timezone`]：
|
||||
- 原生连接：客户端使用的时区，默认值为系统当前时区，全局生效。因为历史的原因，我们只支持 POSIX 标准的部分规范，如 UTC-8(代表中国上上海), GMT-8，Asia/Shanghai 这几种形式。
|
||||
- WebSocket 连接：客户端使用的时区，连接上生效，默认值为系统时区。仅支持 IANA 时区，即 Asia/Shanghai 这种形式。推荐不设置，使用系统时区性能更好。
|
||||
- TSDBDriver.HTTP_CONNECT_TIMEOUT：连接超时时间，单位 ms，默认值为 60000。仅在 REST 连接时生效。
|
||||
- TSDBDriver.HTTP_SOCKET_TIMEOUT：socket 超时时间，单位 ms，默认值为 60000。仅在 REST 连接且 batchfetch 设置为 false 时生效。
|
||||
- TSDBDriver.PROPERTY_KEY_MESSAGE_WAIT_TIMEOUT：消息超时时间，单位 ms，默认值为 60000。仅 WebSocket 连接下有效。
|
||||
- TSDBDriver.PROPERTY_KEY_USE_SSL：连接中是否使用 SSL。仅在 WebSocket/REST 连接时生效。
|
||||
- TSDBDriver.HTTP_POOL_SIZE：REST 并发请求大小，默认 20。
|
||||
- TSDBDriver.PROPERTY_KEY_ENABLE_COMPRESSION：传输过程是否启用压缩。仅在使用 REST/WebSocket 连接时生效。true：启用，false：不启用。默认为 false。
|
||||
- TSDBDriver.PROPERTY_KEY_ENABLE_AUTO_RECONNECT：是否启用自动重连。仅在使用 WebSocket 连接时生效。true：启用，false：不启用。默认为 false。
|
||||
- TSDBDriver.HTTP_CONNECT_TIMEOUT [`httpConnectTimeout`]：连接超时时间，单位 ms，默认值为 60000。仅在 REST 连接时生效。
|
||||
- TSDBDriver.HTTP_SOCKET_TIMEOUT [`httpSocketTimeout`]：socket 超时时间，单位 ms，默认值为 60000。仅在 REST 连接且 batchfetch 设置为 false 时生效。
|
||||
- TSDBDriver.PROPERTY_KEY_MESSAGE_WAIT_TIMEOUT [`messageWaitTimeout`]：消息超时时间，单位 ms，默认值为 60000。仅 WebSocket 连接下有效。
|
||||
- TSDBDriver.PROPERTY_KEY_USE_SSL [`useSSL`]：连接中是否使用 SSL。仅在 WebSocket/REST 连接时生效。
|
||||
- TSDBDriver.HTTP_POOL_SIZE [`httpPoolSize`]：REST 并发请求大小，默认 20。
|
||||
- TSDBDriver.PROPERTY_KEY_ENABLE_COMPRESSION [`enableCompression`]：传输过程是否启用压缩。仅在使用 REST/WebSocket 连接时生效。true：启用，false：不启用。默认为 false。
|
||||
- TSDBDriver.PROPERTY_KEY_ENABLE_AUTO_RECONNECT [`enableAutoReconnect`]：是否启用自动重连。仅在使用 WebSocket 连接时生效。true：启用，false：不启用。默认为 false。
|
||||
> **注意**：启用自动重连仅对简单执行 SQL 语句以及 无模式写入、数据订阅有效。对于参数绑定无效。自动重连仅对连接建立时通过参数指定数据库有效，对后面的 `use db` 语句切换数据库无效。
|
||||
|
||||
- TSDBDriver.PROPERTY_KEY_RECONNECT_INTERVAL_MS：自动重连重试间隔，单位毫秒，默认值 2000。仅在 PROPERTY_KEY_ENABLE_AUTO_RECONNECT 为 true 时生效。
|
||||
- TSDBDriver.PROPERTY_KEY_RECONNECT_RETRY_COUNT：自动重连重试次数，默认值 3，仅在 PROPERTY_KEY_ENABLE_AUTO_RECONNECT 为 true 时生效。
|
||||
- TSDBDriver.PROPERTY_KEY_DISABLE_SSL_CERT_VALIDATION：关闭 SSL 证书验证。仅在使用 WebSocket 连接时生效。true：启用，false：不启用。默认为 false。
|
||||
- TSDBDriver.PROPERTY_KEY_RECONNECT_INTERVAL_MS [`reconnectIntervalMs`]：自动重连重试间隔，单位毫秒，默认值 2000。仅在 PROPERTY_KEY_ENABLE_AUTO_RECONNECT 为 true 时生效。
|
||||
- TSDBDriver.PROPERTY_KEY_RECONNECT_RETRY_COUNT [`reconnectRetryCount`]：自动重连重试次数，默认值 3，仅在 PROPERTY_KEY_ENABLE_AUTO_RECONNECT 为 true 时生效。
|
||||
- TSDBDriver.PROPERTY_KEY_DISABLE_SSL_CERT_VALIDATION [`disableSSLCertValidation`]：关闭 SSL 证书验证。仅在使用 WebSocket 连接时生效。true：启用，false：不启用。默认为 false。
|
||||
|
||||
- TSDBDriver.PROPERTY_KEY_VARCHAR_AS_STRING：将 VARCHAR/BINARY 类型映射为 String，仅在使用 WebSocket 连接时生效。默认值为 false。
|
||||
- TSDBDriver.PROPERTY_KEY_APP_NAME：App 名称，可用于 `show connections` 查询结果显示。仅在使用 WebSocket 连接时生效。默认值为 java。
|
||||
- TSDBDriver.PROPERTY_KEY_APP_IP：App IP，可用于 `show connections` 查询结果显示。仅在使用 WebSocket 连接时生效。默认值为空。
|
||||
- TSDBDriver.PROPERTY_KEY_VARCHAR_AS_STRING [`varcharAsString`]：将 VARCHAR/BINARY 类型映射为 String，仅在使用 WebSocket 连接时生效。默认值为 false。
|
||||
- TSDBDriver.PROPERTY_KEY_APP_NAME [`app_name`]：App 名称，可用于 `show connections` 查询结果显示。仅在使用 WebSocket 连接时生效。默认值为 java。
|
||||
- TSDBDriver.PROPERTY_KEY_APP_IP [`app_ip`]：App IP，可用于 `show connections` 查询结果显示。仅在使用 WebSocket 连接时生效。默认值为空。
|
||||
|
||||
- TSDBDriver.PROPERTY_KEY_ASYNC_WRITE：高效写入模式，目前仅支持 `stmt` 方式。仅在使用 WebSocket 连接时生效。默认值为空，即不启用高效写入模式。
|
||||
- TSDBDriver.PROPERTY_KEY_BACKEND_WRITE_THREAD_NUM：高效写入模式下，后台写入线程数。仅在使用 WebSocket 连接时生效。默认值为 10。
|
||||
- TSDBDriver.PROPERTY_KEY_BATCH_SIZE_BY_ROW：高效写入模式下，写入数据的批大小，单位是行。仅在使用 WebSocket 连接时生效。默认值为 1000。
|
||||
- TSDBDriver.PROPERTY_KEY_CACHE_SIZE_BY_ROW：高效写入模式下，缓存的大小，单位是行。仅在使用 WebSocket 连接时生效。默认值为 10000。
|
||||
- TSDBDriver.PROPERTY_KEY_COPY_DATA：高效写入模式下，是否拷贝应用通过 addBatch 传入的二进制类型数据。仅在使用 WebSocket 连接时生效。默认值为 false。
|
||||
- TSDBDriver.PROPERTY_KEY_STRICT_CHECK：高效写入模式下，是否校验表名长度和变长数据类型长度。仅在使用 WebSocket 连接时生效。默认值为 false。
|
||||
- TSDBDriver.PROPERTY_KEY_RETRY_TIMES：高效写入模式下，写入失败重试次数。仅在使用 WebSocket 连接时生效。默认值为 3。
|
||||
- TSDBDriver.PROPERTY_KEY_ASYNC_WRITE [`asyncWrite`]：高效写入模式，目前仅支持 `stmt` 方式。仅在使用 WebSocket 连接时生效。默认值为空，即不启用高效写入模式。
|
||||
- TSDBDriver.PROPERTY_KEY_BACKEND_WRITE_THREAD_NUM [`backendWriteThreadNum`]：高效写入模式下，后台写入线程数。仅在使用 WebSocket 连接时生效。默认值为 10。
|
||||
- TSDBDriver.PROPERTY_KEY_BATCH_SIZE_BY_ROW [`batchSizeByRow`]：高效写入模式下，写入数据的批大小，单位是行。仅在使用 WebSocket 连接时生效。默认值为 1000。
|
||||
- TSDBDriver.PROPERTY_KEY_CACHE_SIZE_BY_ROW [`cacheSizeByRow`]：高效写入模式下，缓存的大小，单位是行。仅在使用 WebSocket 连接时生效。默认值为 10000。
|
||||
- TSDBDriver.PROPERTY_KEY_COPY_DATA [`copyData`]：高效写入模式下，是否拷贝应用通过 addBatch 传入的二进制类型数据。仅在使用 WebSocket 连接时生效。默认值为 false。
|
||||
- TSDBDriver.PROPERTY_KEY_STRICT_CHECK [`strictCheck`]：高效写入模式下，是否校验表名长度和变长数据类型长度。仅在使用 WebSocket 连接时生效。默认值为 false。
|
||||
- TSDBDriver.PROPERTY_KEY_RETRY_TIMES [`retryTimes`]：高效写入模式下，写入失败重试次数。仅在使用 WebSocket 连接时生效。默认值为 3。
|
||||
|
||||
- TSDBDriver.PROPERTY_KEY_PBS_MODE [`pbsMode`]：参数绑定序列化模式，目前是实验特性，仅支持 `line` 模式，在参数绑定一批绑定的数据中每个子表仅一条数据时可以提升性能。仅在使用 WebSocket 连接时生效，不支持高效写入模式。默认值为空。
|
||||
|
||||
**配置参数的优先级**
|
||||
|
||||
|
|
|
|||
|
|
@ -25,23 +25,26 @@ import RequestId from "./_request_id.mdx";
|
|||
|
||||
| driver-go 版本 | 主要变化 | TDengine 版本 |
|
||||
|--------------|--------------------------------------------|---------------|
|
||||
| v3.7.3 | 修复 WebSocket 连接 stmt 查询结果包含 decimal 数据崩溃 | - |
|
||||
| v3.7.2 | 支持 BLOB 类型 | - |
|
||||
| v3.7.1 | 支持 ipv6 连接 | - |
|
||||
| v3.7.0 | 支持 decimal 类型 | 3.3.6.0 及更高版本 |
|
||||
| v3.6.0 | stmt2 原生接口，DSN 支持密码包含特殊字符（url.QueryEscape） | 3.3.5.0 及更高版本 |
|
||||
| v3.5.8 | 修复空指针异常 | - |
|
||||
| v3.5.7 | taosWS 和 taosRestful 支持传入 request id | - |
|
||||
| v3.5.6 | 提升 websocket 查询和写入性能 | 3.3.2.0 及更高版本 |
|
||||
| v3.5.6 | 提升 WebSocket 查询和写入性能 | 3.3.2.0 及更高版本 |
|
||||
| v3.5.5 | restful 支持跳过 ssl 证书检查 | - |
|
||||
| v3.5.4 | 兼容 TDengine 3.3.0.0 tmq raw data | - |
|
||||
| v3.5.3 | 重构 taosWS | - |
|
||||
| v3.5.2 | websocket 压缩和优化消息订阅性能 | 3.2.3.0 及更高版本 |
|
||||
| v3.5.2 | WebSocket 压缩和优化消息订阅性能 | 3.2.3.0 及更高版本 |
|
||||
| v3.5.1 | 原生 stmt 查询和 geometry 类型支持 | 3.2.1.0 及更高版本 |
|
||||
| v3.5.0 | 获取消费进度及按照指定进度开始消费 | 3.0.5.0 及更高版本 |
|
||||
| v3.3.1 | 基于 websocket 的 schemaless 协议写入 | 3.0.4.1 及更高版本 |
|
||||
| v3.3.1 | 基于 WebSocket 的 schemaless 协议写入 | 3.0.4.1 及更高版本 |
|
||||
| v3.1.0 | 提供贴近 kafka 的订阅 api | - |
|
||||
| v3.0.4 | 新增 request id 相关接口 | 3.0.2.2 及更高版本 |
|
||||
| v3.0.3 | 基于 websocket 的 statement 写入 | - |
|
||||
| v3.0.2 | 基于 websocket 的数据查询和写入 | 3.0.1.5 及更高版本 |
|
||||
| v3.0.1 | 基于 websocket 的消息订阅 | - |
|
||||
| v3.0.3 | 基于 WebSocket 的 statement 写入 | - |
|
||||
| v3.0.2 | 基于 WebSocket 的数据查询和写入 | 3.0.1.5 及更高版本 |
|
||||
| v3.0.1 | 基于 WebSocket 的消息订阅 | - |
|
||||
| v3.0.0 | 适配 TDengine 3.0 查询和写入 | 3.0.0.0 及更高版本 |
|
||||
|
||||
## 异常和错误码
|
||||
|
|
@ -85,6 +88,7 @@ TDengine 其他功能模块的报错，请参考 [错误码](../../../reference/
|
|||
| GEOMETRY | []byte |
|
||||
| VARBINARY | []byte |
|
||||
| DECIMAL | string |
|
||||
| BLOB | []byte |
|
||||
|
||||
**注意**：JSON 类型仅在 tag 中支持。
|
||||
GEOMETRY 类型是 little endian 字节序的二进制数据，符合 WKB 规范。详细信息请参考 [数据类型](../../taos-sql/data-type/#数据类型)
|
||||
|
|
@ -146,6 +150,12 @@ username:password@protocol(address)/dbname?param=value
|
|||
|
||||
当密码中包含特殊字符时，需要使用 `url.QueryEscape` 进行转义。
|
||||
|
||||
当使用 IPv6 地址时（v3.7.1 及以上版本支持），地址需要用方括号括起来，例如：
|
||||
|
||||
```text
|
||||
root:taosdata@ws([::1]:6041)/testdb
|
||||
```
|
||||
|
||||
##### 原生连接
|
||||
|
||||
导入驱动：
|
||||
|
|
|
|||
|
|
@ -51,6 +51,7 @@ Python Connector 历史版本（建议使用最新版本的 `taospy`）：
|
|||
|
||||
| Python Connector 版本 | 主要变化 | TDengine 版本 |
|
||||
| -------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------- |
|
||||
| 2.8.3 | 支持 BLOB 数据类型 | - |
|
||||
| 2.8.2 | 连接参数设置支持跨平台 | - |
|
||||
| 2.8.1 | 增加 connect 属性设置函数 | - |
|
||||
| 2.8.0 | 移除 Apache Superset 连接驱动 | - |
|
||||
|
|
@ -136,6 +137,7 @@ TDengine 目前支持时间戳、数字、字符、布尔类型，与 Python 对
|
|||
| GEOMETRY | bytearray |
|
||||
| VARBINARY | bytearray |
|
||||
| DECIMAL | Decimal |
|
||||
| BLOB | bytearray |
|
||||
## 示例程序汇总
|
||||
|
||||
| 示例程序链接 | 示例程序内容 |
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ import RequestId from "./_request_id.mdx";
|
|||
|
||||
| Connector 版本 | 主要变化 | TDengine 版本 |
|
||||
|:-------------|:----------------------------|:--------------|
|
||||
| 3.1.7 | 支持 IPv6 连接，支持 DECIMAL 类型。 | 3.3.6.0 及更高版本 |
|
||||
| 3.1.6 | 优化 WebSocket 连接接收消息处理。 | - |
|
||||
| 3.1.5 | 修复 WebSocket 协议编码中文时长度错误。 | - |
|
||||
| 3.1.4 | 提升 WebSocket 查询和写入性能。 | 3.3.2.0 及更高版本 |
|
||||
|
|
@ -58,10 +59,17 @@ TDengine 其他功能模块的报错，请参考 [错误码](../../../reference/
|
|||
| JSON | byte[] |
|
||||
| VARBINARY | byte[] |
|
||||
| GEOMETRY | byte[] |
|
||||
| DECIMAL | decimal |
|
||||
|
||||
**注意**：JSON 类型仅在 tag 中支持。
|
||||
GEOMETRY 类型是 little endian 字节序的二进制数据，符合 WKB 规范。详细信息请参考 [数据类型](../../taos-sql/data-type/#数据类型)。
|
||||
**注意**：
|
||||
|
||||
- JSON 类型仅在 tag 中支持。
|
||||
- GEOMETRY 类型是 little endian 字节序的二进制数据，符合 WKB 规范。详细信息请参考 [数据类型](../../taos-sql/data-type/#数据类型)。
|
||||
WKB 规范请参考 [Well-Known Binary (WKB)](https://libgeos.org/specifications/wkb/)。
|
||||
- DECIMAL 类型在 C# 中使用 `decimal` 类型表示，支持高精度的十进制数值。因为 C# 的 `decimal` 类型与 TDengine 的 DECIMAL 类型在精度和范围上有所不同，
|
||||
C# `decimal` 精度最大 29 位，TDengine DECIMAL 类型精度最大 38 位，所以在使用时需要注意：
|
||||
- 未超出 C# decimal 类型范围时可以使用 `GetDecimal` 或 `GetValue` 获取。
|
||||
- 当超出 C# decimal 类型范围时 `GetDecimal` 和 `GetValue` 方法会抛出 `OverflowException`，此时可以使用 `GetString` 方法获取字符串表示形式。
|
||||
|
||||
## 示例程序汇总
|
||||
|
||||
|
|
@ -331,6 +339,15 @@ C# 驱动提供了符合 ADO.NET 标准的 `DbDataReader` 接口，提供了用
|
|||
- **返回值**：日期时间值。
|
||||
- **异常**：类型不对应抛出 `InvalidCastException` 异常。
|
||||
|
||||
- `public decimal GetDecimal(int ordinal)`
|
||||
- **接口说明**：获取指定列的十进制数值。
|
||||
- **参数说明**：
|
||||
- `ordinal`：列索引。
|
||||
- **返回值**：十进制数值。
|
||||
- **异常**：
|
||||
- 类型不对应抛出 `InvalidCastException` 异常。
|
||||
- 超出范围抛出 `OverflowException` 异常。
|
||||
|
||||
- `public double GetDouble(int ordinal)`
|
||||
- **接口说明**：获取指定列的双精度浮点数值。
|
||||
- **参数说明**：
|
||||
|
|
|
|||
|
|
@ -275,16 +275,19 @@ C 接口网络不可用相关错误码：
|
|||
- "JSON"
|
||||
- "VARBINARY"
|
||||
- "GEOMETRY"
|
||||
- "DECIMAL(precision, scale)"
|
||||
- "BLOB"
|
||||
|
||||
`VARBINARY` 和 `GEOMETRY` 类型返回数据为 Hex 字符串，样例：
|
||||
DECIMAL 类型的 precision 是指最大支持的有效数字个数，scale 是指最大支持的小数位数，例如 `DECIMAL(8, 4)` 可表示范围即 [-9999.9999, 9999.9999]。
|
||||
|
||||
`VARBINARY` 、 `GEOMETRY` 和 `BLOB` 类型返回数据为 Hex 字符串，样例：
|
||||
|
||||
准备数据
|
||||
|
||||
```bash
|
||||
create database demo
|
||||
use demo
|
||||
create table t(ts timestamp,c1 varbinary(20),c2 geometry(100))
|
||||
insert into t values(now,'\x7f8290','point(100 100)')
|
||||
create database demo;
|
||||
create table demo.t(ts timestamp,c1 varbinary(20),c2 geometry(100),c3 blob);
|
||||
insert into demo.t values(now,'\x7f8290','point(100 100)','\x010203ddff');
|
||||
```
|
||||
|
||||
执行查询
|
||||
|
|
@ -316,13 +319,19 @@ curl --location 'http://<fqdn>:<port>/rest/sql' \
|
|||
"c2",
|
||||
"GEOMETRY",
|
||||
100
|
||||
],
|
||||
[
|
||||
"c3",
|
||||
"BLOB",
|
||||
4194304
|
||||
]
|
||||
],
|
||||
"data": [
|
||||
[
|
||||
"2023-11-01T06:28:15.210Z",
|
||||
"2025-07-22T05:58:41.798Z",
|
||||
"7f8290",
|
||||
"010100000000000000000059400000000000005940"
|
||||
"010100000000000000000059400000000000005940",
|
||||
"010203ddff"
|
||||
]
|
||||
],
|
||||
"rows": 1
|
||||
|
|
|
|||
|
|
@ -85,28 +85,28 @@ description: TDengine 服务端的错误码列表和详细说明
|
|||
|
||||
## tsc
|
||||
|
||||
| 错误码 | 错误描述 | 可能的出错场景或者可能的原因 | 建议用户采取的措施 |
|
||||
| ---------- | --------------------------- | ---------------------------- | -------------------------------------------------------------------------- |
|
||||
| 0x80000207 | Invalid user name | 数据库用户名不合法 | 检查数据库用户名是否正确 |
|
||||
| 0x80000208 | Invalid password | 数据库密码不合法 | 检查数据库密码是否正确 |
|
||||
| 0x80000209 | Database name too long | 数据库名称不合法 | 检查数据库名称是否正确 |
|
||||
| 0x8000020A | Table name too long | 表名不合法 | 检查表名是否正确 |
|
||||
| 0x8000020F | Query terminated | 查询被中止 | 检查是否有用户中止了查询 |
|
||||
| 0x80000213 | Disconnected from server | 连接已中断 | 检查连接是否被人为中断或客户端正在退出 |
|
||||
| 0x80000216 | Syntax error in SQL | SQL 语法错误 | 检查 SQL 语句并修正错误 |
|
||||
| 0x80000219 | SQL statement too long | SQL 长度超出限制 | 检查 SQL 语句并修正错误 |
|
||||
| 0x8000021A | File is empty | 文件内容为空 | 检查输入文件内容 |
|
||||
| 0x8000021F | Invalid column length | 列长度错误 | 保留现场和日志，github 上报 issue |
|
||||
| 0x80000222 | Invalid JSON data type | JSON 数据类型错误 | 检查输入 JSON 内容 |
|
||||
| 0x80000224 | Value out of range | 数据大小超过类型范围 | 检查输入的数据值 |
|
||||
| 0x80000229 | Invalid tsc input | API 输入错误 | 检查应用调用 API 时传递的参数 |
|
||||
| 0x8000022A | Stmt API usage error | STMT API 使用错误 | 检查 STMT API 调用的顺序、适用场景、错误处理 |
|
||||
| 0x8000022B | Stmt table name not set | STMT 未正确设置 table name | 检查是否调用了设置 table name 接口 |
|
||||
| 0x8000022D | Query killed | 查询被中止 | 检查是否有用户中止了查询 |
|
||||
| 0x8000022E | No available execution node | 没有可用的查询执行节点 | 检查当前 query policy 配置，如果需要有 Qnode 参与确保系统中存在可用的 Qnode 节点 |
|
||||
| 0x8000022F | Table is not a super table | 当前语句中的表名不是超级表 | 检查当前语句中所用表名是否是超级表 |
|
||||
| 0x80000230 | Stmt cache error | STMT 内部缓存出错 | 保留现场和日志，github 上报 issue |
|
||||
| 0x80000231 | Tsc internal error | TSC 内部错误 | 保留现场和日志，github 上报 issue |
|
||||
| 错误码 | 错误描述 | 可能的出错场景或者可能的原因 | 建议用户采取的措施 |
|
||||
| ---------- | --------------------------------- | ---------------------------- | -------------------------------------------------------------------------------- |
|
||||
| 0x80000207 | Invalid user name | 数据库用户名不合法 | 检查数据库用户名是否正确 |
|
||||
| 0x80000208 | Invalid password | 数据库密码不合法 | 检查数据库密码是否正确 |
|
||||
| 0x80000209 | Database name too long | 数据库名称不合法 | 检查数据库名称是否正确 |
|
||||
| 0x8000020A | Table name too long | 表名不合法 | 检查表名是否正确 |
|
||||
| 0x8000020F | Query terminated | 查询被中止 | 检查是否有用户中止了查询 |
|
||||
| 0x80000213 | Disconnected from server | 连接已中断 | 检查连接是否被人为中断或客户端正在退出 |
|
||||
| 0x80000216 | Syntax error in SQL | SQL 语法错误 | 检查 SQL 语句并修正错误 |
|
||||
| 0x80000219 | SQL statement too long | SQL 长度超出限制 | 检查 SQL 语句并修正错误 |
|
||||
| 0x8000021A | File is empty | 文件内容为空 | 检查输入文件内容 |
|
||||
| 0x8000021F | Invalid column length | 列长度错误 | 保留现场和日志，github 上报 issue |
|
||||
| 0x80000222 | Invalid JSON data type | JSON 数据类型错误 | 检查输入 JSON 内容 |
|
||||
| 0x80000224 | Value out of range | 数据大小超过类型范围 | 检查输入的数据值 |
|
||||
| 0x80000229 | Invalid tsc input | API 输入错误 | 检查应用调用 API 时传递的参数 |
|
||||
| 0x8000022A | Stmt API usage error | STMT/STMT2 API 使用错误 | 检查 STMT/STMT2 API 调用的顺序、适用场景、错误处理 |
|
||||
| 0x8000022B | Stmt table name not set correctly | STMT/STMT2 未正确设置表名 | 检查 STMT/STMT2 绑定的表名是否合法 |
|
||||
| 0x8000022D | Query killed | 查询被中止 | 检查是否有用户中止了查询 |
|
||||
| 0x8000022E | No available execution node | 没有可用的查询执行节点 | 检查当前 query policy 配置，如果需要有 Qnode 参与确保系统中存在可用的 Qnode 节点 |
|
||||
| 0x8000022F | Table is not a super table | 当前语句中的表名不是超级表 | 检查当前语句中所用表名是否是超级表 |
|
||||
| 0x80000230 | Stmt cache error | STMT/STMT2 内部缓存出错 | 保留现场和日志，github 上报 issue |
|
||||
| 0x80000231 | Tsc internal error | TSC 内部错误 | 保留现场和日志，github 上报 issue |
|
||||
|
||||
|
||||
|
||||
|
|
@ -236,6 +236,19 @@ description: TDengine 服务端的错误码列表和详细说明
|
|||
| 0x80000484 | index not exist | 不存在 | 确认操作是否正确 |
|
||||
|
||||
|
||||
## Bnode
|
||||
|
||||
| 错误码 | 错误描述 | 可能的出错场景或者可能的原因 | 建议用户采取的措施 |
|
||||
| ---------- | -------------------------- | ---------------------------- | ---------------------- |
|
||||
| 0x80000450 | Bnode already exists | 已创建 | 检查节点状态 |
|
||||
| 0x80000451 | Bnode already deployed | 已部署 | 确认操作是否正确 |
|
||||
| 0x80000452 | Bnode not deployed | 内部错误 | 上报 issue |
|
||||
| 0x80000453 | Bnode not there | 不在线 | 确认操作是否正确 |
|
||||
| 0x80000454 | Bnode not found | 内部错误 | 上报 issue |
|
||||
| 0x80000455 | Bnode exec launch failed | 内部错误 | 上报 issue |
|
||||
| 0x8000261C | Invalid Bnode option | Bnode 选项值非法 | 检查并修正数据库选项值 |
|
||||
|
||||
|
||||
## dnode
|
||||
|
||||
| 错误码 | 错误描述 | 可能的出错场景或者可能的原因 | 建议用户采取的措施 |
|
||||
|
|
|
|||
|
|
@ -79,12 +79,13 @@ void Test(TAOS *taos, char *qstr, int index) {
|
|||
TAOS_RES *result;
|
||||
queryDB(taos, "use demo");
|
||||
|
||||
queryDB(taos, "create table m1 (ts timestamp, ti tinyint, si smallint, i int, bi bigint, f float, d double, b binary(10))");
|
||||
queryDB(taos, "create table m1 (ts timestamp, ti tinyint, si smallint, i int, bi bigint, f float, d double, b blob)");
|
||||
printf("success to create table\n");
|
||||
|
||||
int i = 0;
|
||||
for (i = 0; i < 10; ++i) {
|
||||
sprintf(qstr, "insert into m1 values (%" PRId64 ", %d, %d, %d, %d, %f, %lf, '%s')", (uint64_t)(1546300800000 + i * 1000), i, i, i, i*10000000, i*1.0, i*2.0, "hello");
|
||||
sprintf(qstr, "insert into m1 values (%" PRId64 ", %d, %d, %d, %d, %f, %lf, '%s')",
|
||||
(uint64_t)(1546300800000 + i * 1000), i, i, i, i * 10000000, i * 1.0, i * 2.0, "abc");
|
||||
printf("qstr: %s\n", qstr);
|
||||
|
||||
// note: how do you wanna do if taos_query returns non-NULL
|
||||
|
|
|
|||
|
|
@ -24,13 +24,6 @@ extern "C" {
|
|||
|
||||
#define S3_BLOCK_CACHE
|
||||
|
||||
extern int8_t tsS3StreamEnabled;
|
||||
extern int8_t tsS3Enabled;
|
||||
extern int32_t tsS3BlockSize;
|
||||
extern int32_t tsS3BlockCacheSize;
|
||||
extern int32_t tsS3PageCacheSize;
|
||||
extern int32_t tsS3UploadDelaySec;
|
||||
|
||||
int32_t s3Init();
|
||||
int32_t s3Begin();
|
||||
void s3End();
|
||||
|
|
|
|||
|
|
@ -1,24 +0,0 @@
|
|||
//
|
||||
// Created by mingming wanng on 2023/11/2.
|
||||
//
|
||||
|
||||
#ifndef TDENGINE_RSYNC_H
|
||||
#define TDENGINE_RSYNC_H
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#include "tarray.h"
|
||||
|
||||
void stopRsync();
|
||||
int32_t startRsync();
|
||||
int32_t uploadByRsync(const char* id, const char* path, int64_t checkpointId);
|
||||
int32_t downloadByRsync(const char* id, const char* path, int64_t checkpointId);
|
||||
int32_t deleteRsync(const char* pTaskId, int64_t checkpointId);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif // TDENGINE_RSYNC_H
|
||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue