Merge branch 'main' into misc/improve-e2e-tests

This commit is contained in:
undercover-cactus 2025-10-27 16:33:54 +01:00
commit f748d7dbbb
59 changed files with 1868 additions and 629 deletions

View file

@ -69,4 +69,4 @@ jobs:
uses: ./.github/workflows/task-e2e.yml
secrets: inherit
with:
image-tag: ${{ needs.docker-build-ci.outputs.image-tag }}
image-tag: ${{ needs.docker-build-ci.outputs.image-tag }}

View file

@ -46,6 +46,7 @@ jobs:
- uses: actions/checkout@v4
with:
submodules: recursive
fetch-depth: 0
- uses: oven-sh/setup-bun@v2
- name: Install Foundry
uses: foundry-rs/foundry-toolchain@v1
@ -129,26 +130,16 @@ jobs:
chmod +x tmp/bin/snowbridge-relay
docker rm -fv temp
tmp/bin/snowbridge-relay --help
- name: Pull DataHaven node image
- name: Pull DataHaven node image
run: |
docker pull ghcr.io/datahaven-xyz/datahaven/datahaven:${{ inputs.image-tag }}
docker tag ghcr.io/datahaven-xyz/datahaven/datahaven:${{ inputs.image-tag }} datahavenxyz/datahaven:local
- run: bun install
- name: Run E2E tests
run: bun test:e2e
# Try to collect all docker logs and upload it
- name: Collect docker logs
- name: Delete volumes not used
if: always()
run: |
mkdir ./logs
for name in `docker ps -a --format '{{.Names}}'`; do docker logs $name > ./logs/$name.log 2>&1; done
- name: Upload logs to GitHub
if: always()
uses: actions/upload-artifact@v4
with:
name: logs
path: logs/
retention-days: 1
run: podman system prune --volumes -f

View file

@ -31,6 +31,8 @@ jobs:
- name: Install Foundry
uses: foundry-rs/foundry-toolchain@v1
with:
version: v1.4.3
- name: Cache Foundry libraries
uses: actions/cache@v4

View file

@ -135,8 +135,11 @@ jobs:
- name: Cargo build
uses: ./.github/workflow-templates/publish-docker
with:
dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }}
dockerhub_password: ${{ secrets.DOCKERHUB_TOKEN }}
dockerfile: ./operator/Dockerfile
context: .
registry: docker.io
registry_username: ${{ secrets.DOCKERHUB_USERNAME }}
registry_password: ${{ secrets.DOCKERHUB_TOKEN }}
image_tags: ${{ steps.prep.outputs.tags }}
image_title: ${{ github.event.repository.name }}
image_description: ${{ github.event.repository.description }}

View file

@ -99,9 +99,7 @@ contract DataHavenServiceManager is ServiceManagerBase, IDataHavenServiceManager
) external payable onlyOwner {
// Send the new validator set message to the Snowbridge Gateway
bytes memory message = buildNewValidatorSetMessage();
_snowbridgeGateway.v2_sendMessage{
value: msg.value
}(
_snowbridgeGateway.v2_sendMessage{value: msg.value}(
message,
new bytes[](0), // No assets to send
bytes(""), // No claimer

View file

@ -0,0 +1,9 @@
apiVersion: v2
name: sh-mspbackend
description: A Helm chart for StorageHub MSP Backend API
type: application
version: 0.1.0
appVersion: "1.0.0"
maintainers:
- name: StorageHub Team
email: team@storagehub.io

View file

@ -0,0 +1,270 @@
# StorageHub MSP Backend Helm Chart
This Helm chart deploys the StorageHub MSP Backend API service that provides REST API access to the StorageHub network data.
## Overview
The StorageHub MSP Backend API:
- Connects to a StorageHub Indexer's database for indexed blockchain data
- Connects to a StorageHub MSP node for real-time blockchain queries
- Provides REST API endpoints for StorageHub operations
## Prerequisites
- Kubernetes 1.19+
- Helm 3.2.0+
- Running StorageHub Indexer database (PostgreSQL)
- Running StorageHub MSP node
## Installation
### Using base configuration
```bash
helm install sh-mspbackend ./charts/backend \
-f ./charts/backend/storagehub/sh-mspbackend.yaml
```
### For Local environment
```bash
helm install sh-mspbackend ./charts/backend \
-f ./charts/backend/storagehub/sh-mspbackend.yaml \
-f ./environments/local/sh-mspbackend.yaml \
-n kt-datahaven-local
```
### For Stagenet environment
```bash
helm install sh-mspbackend ./charts/backend \
-f ./charts/backend/storagehub/sh-mspbackend.yaml \
-f ./environments/stagenet/sh-mspbackend.yaml \
-n datahaven-stagenet
```
## Configuration
### Key Parameters
| Parameter | Description | Default |
|-----------|-------------|---------|
| `image.repository` | Container image repository | `moonsonglabs/storage-hub-msp-backend` |
| `image.tag` | Container image tag | `latest` |
| `replicaCount` | Number of replicas | `1` |
| `service.type` | Kubernetes service type | `ClusterIP` |
| `service.port` | Service port | `8080` |
| `service.targetPort` | Service target port | `80` |
| `backend.port` | Backend application port | `8080` |
| `backend.database.url` | PostgreSQL connection URL | `postgresql://storagehub:storagehub@sh-indexer-db:5432/storagehub` |
| `backend.rpc.endpoint` | WebSocket RPC endpoint | `ws://sh-idxnode:9944` |
| `backend.api.defaultPageSize` | Default page size for API results | `20` |
| `backend.api.maxPageSize` | Maximum page size for API results | `100` |
| `ingress.enabled` | Enable ingress | `false` |
### Configuration File
The backend uses a TOML configuration file passed via the `--config` CLI argument. This file is automatically generated from the Helm values and mounted as a ConfigMap at `/configs/config.toml`.
#### Basic Configuration:
```yaml
backend:
port: 8080
database:
url: postgresql://indexer:indexer@sh-idxnode-db-postgresql:5432/datahaven
rpc:
endpoint: ws://sh-mspnode-0:9955
api:
defaultPageSize: 20
maxPageSize: 100
auth:
jwtSecret: "your-secret-here"
args:
- "--config"
- "/configs/config.toml"
configMap:
enabled: true
```
#### Alternative: Building Database URL from Components
The chart can also construct the database URL from separate components:
```yaml
backend:
database:
host: sh-idxnode-db-postgresql
port: 5432
name: datahaven
user: indexer
password: production_password
```
**Note:** For production deployments, consider using Kubernetes Secrets or external secret management solutions for sensitive values like database passwords and JWT secrets.
### Environment Variables
Additional environment variables can be configured:
```yaml
backend:
env:
NODE_ENV: production
LOG_LEVEL: info
```
### Additional ConfigMap Data
You can add extra files to the ConfigMap:
```yaml
configMap:
enabled: true
data:
custom-config.yaml: |
# Your custom configuration here
key: value
```
### CLI Arguments
Additional CLI arguments can be specified to pass to the backend application:
```yaml
backend:
args:
- "--config"
- "/configs/config.toml"
- "--log-level"
- "debug"
```
### Using Environment Variables from ConfigMaps or Secrets
You can inject environment variables from existing ConfigMaps or Secrets:
```yaml
backend:
envFrom:
- configMapRef:
name: my-config
- secretRef:
name: my-secret
```
## Accessing the Service
### Local Environment
When deployed with `NodePort` service type:
```bash
# Access via NodePort (configured as 30300 in local environment)
curl http://localhost:30300/
# Or via ingress if enabled
curl http://sh-mspbackend.datahaven.local/
```
### Stagenet Environment
```bash
# Access via ingress
curl https://sh-mspbackend.datahaven-kt.xyz/
```
## Generated Configuration
The chart automatically generates a `config.toml` file with the following structure:
```toml
host = "0.0.0.0"
port = 8080
[api]
default_page_size = 20
max_page_size = 100
[storage_hub]
rpc_url = "ws://sh-mspnode-0:9955"
msp_callback_url = "http://sh-mspbackend:8080"
timeout_secs = 30
max_concurrent_requests = 100
verify_tls = true
mock_mode = false
[auth]
jwt_secret = "your-secret-here"
[database]
url = "postgresql://indexer:indexer@sh-idxnode-db-postgresql:5432/datahaven"
mock_mode = false
```
## Troubleshooting
### Check pod status
```bash
kubectl get pods -l app.kubernetes.io/name=sh-mspbackend -n <namespace>
```
### View logs
```bash
kubectl logs -l app.kubernetes.io/name=sh-mspbackend -n <namespace>
```
### Verify database connection
For local environment:
```bash
kubectl exec -it deployment/sh-mspbackend -n kt-datahaven-local -- nc -zv sh-idxnode-db-postgresql 5432
```
For stagenet environment:
```bash
kubectl exec -it deployment/sh-mspbackend -n datahaven-stagenet -- nc -zv sh-idxnode-db-postgresql 5432
```
### Verify RPC connection
For local environment:
```bash
kubectl exec -it deployment/sh-mspbackend -n kt-datahaven-local -- nc -zv sh-mspnode-0 9955
```
For stagenet environment:
```bash
kubectl exec -it deployment/sh-mspbackend -n datahaven-stagenet -- nc -zv sh-mspnode-0 9955
```
### View generated configuration
```bash
kubectl get configmap sh-mspbackend-config -n <namespace> -o yaml
```
## Uninstallation
```bash
# For local environment
helm uninstall sh-mspbackend -n kt-datahaven-local
# For stagenet environment
helm uninstall sh-mspbackend -n datahaven-stagenet
```
## Environment-Specific Examples
### Local Environment Values
See `environments/local/sh-mspbackend.yaml` for the complete local configuration, which includes:
- NodePort service on port 30300
- Debug logging
- Traefik ingress at `sh-mspbackend.datahaven.local`
- Minimal resource requests for development
### Stagenet Environment Values
See `environments/stagenet/sh-mspbackend.yaml` for the complete stagenet configuration, which includes:
- ClusterIP service with AWS NLB annotations
- Production logging levels
- ALB ingress with SSL at `sh-mspbackend.datahaven-kt.xyz`
- Production-level resource requests and limits

View file

@ -0,0 +1,64 @@
# StorageHub MSP Backend API base configuration
# This file contains the base configuration for the StorageHub MSP Backend API
# Chart metadata
fullnameOverride: sh-mspbackend
# Container image
image:
repository: moonsonglabs/storage-hub-msp-backend
tag: latest
pullPolicy: Always
# Service configuration
service:
type: ClusterIP
port: 8080
targetPort: 80
# Backend API configuration
backend:
port: 8080
# Database connection to StorageHub Indexer PostgreSQL
database:
url: postgresql://indexer:indexer@sh-idxnode-db-postgresql:5432/datahaven
# RPC connection to StorageHub Indexer node
rpc:
endpoint: ws://sh-mspnode-0:9955
# Authentication (set in environment-specific values)
# auth:
# jwtSecret: "set-in-environment-values"
# CLI arguments for the backend application
args:
- "--config"
- "/configs/config.toml"
# Resource limits
resources:
requests:
memory: "256Mi"
cpu: "100m"
limits:
memory: "512Mi"
cpu: "500m"
# Service account
serviceAccount:
create: true
name: sh-mspbackend
# Security context
securityContext:
runAsNonRoot: true
runAsUser: 1000
capabilities:
drop:
- ALL
# Ingress configuration (disabled by default, enabled per environment)
ingress:
enabled: false

View file

@ -0,0 +1,60 @@
{{/*
Expand the name of the chart.
*/}}
{{- define "backend.name" -}}
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
{{- end }}
{{/*
Create a default fully qualified app name.
*/}}
{{- define "backend.fullname" -}}
{{- if .Values.fullnameOverride }}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- $name := default .Chart.Name .Values.nameOverride }}
{{- if contains $name .Release.Name }}
{{- .Release.Name | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
{{- end }}
{{- end }}
{{- end }}
{{/*
Create chart name and version as used by the chart label.
*/}}
{{- define "backend.chart" -}}
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
{{- end }}
{{/*
Common labels
*/}}
{{- define "backend.labels" -}}
helm.sh/chart: {{ include "backend.chart" . }}
{{ include "backend.selectorLabels" . }}
{{- if .Chart.AppVersion }}
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
{{- end }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- end }}
{{/*
Selector labels
*/}}
{{- define "backend.selectorLabels" -}}
app.kubernetes.io/name: {{ include "backend.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
{{- end }}
{{/*
Create the name of the service account to use
*/}}
{{- define "backend.serviceAccountName" -}}
{{- if .Values.serviceAccount.create }}
{{- default (include "backend.fullname" .) .Values.serviceAccount.name }}
{{- else }}
{{- default "default" .Values.serviceAccount.name }}
{{- end }}
{{- end }}

View file

@ -0,0 +1,48 @@
{{- if .Values.configMap.enabled -}}
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "backend.fullname" . }}-config
labels:
{{- include "backend.labels" . | nindent 4 }}
data:
config.toml: |
# StorageHub Backend Configuration
host = "0.0.0.0"
port = {{ .Values.backend.port }}
[api]
default_page_size = {{ .Values.backend.api.defaultPageSize | default 20 }}
max_page_size = {{ .Values.backend.api.maxPageSize | default 100 }}
[storage_hub]
{{- if .Values.backend.rpc.endpoint }}
rpc_url = {{ .Values.backend.rpc.endpoint | quote }}
{{- end }}
msp_callback_url = "http://{{ include "backend.fullname" . }}:8080"
timeout_secs = 30
max_concurrent_requests = 100
verify_tls = true
mock_mode = false
[auth]
{{- if .Values.backend.auth.jwtSecret }}
jwt_secret = {{ .Values.backend.auth.jwtSecret | quote }}
{{- end }}
[database]
{{- if .Values.backend.database.url }}
url = {{ .Values.backend.database.url | quote }}
{{- else if .Values.backend.database.host }}
url = "postgresql://{{ .Values.backend.database.user | default "indexer" }}:{{ .Values.backend.database.password | default "password" }}@{{ .Values.backend.database.host }}:{{ .Values.backend.database.port | default 5432 }}/{{ .Values.backend.database.name | default "storagehub" }}"
{{- else }}
url = "postgresql://indexer:password@sh-indexer-db:5432/storagehub"
{{- end }}
mock_mode = false
{{- range $key, $value := .Values.configMap.data }}
{{ $key }}: {{ $value | quote }}
{{- end }}
{{- end }}

View file

@ -0,0 +1,80 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "backend.fullname" . }}
labels:
{{- include "backend.labels" . | nindent 4 }}
spec:
replicas: {{ .Values.replicaCount }}
selector:
matchLabels:
{{- include "backend.selectorLabels" . | nindent 6 }}
template:
metadata:
annotations:
checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
{{- with .Values.podAnnotations }}
{{- toYaml . | nindent 8 }}
{{- end }}
labels:
{{- include "backend.selectorLabels" . | nindent 8 }}
spec:
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
serviceAccountName: {{ include "backend.serviceAccountName" . }}
securityContext:
{{- toYaml .Values.podSecurityContext | nindent 8 }}
containers:
- name: {{ .Chart.Name }}
securityContext:
{{- toYaml .Values.securityContext | nindent 12 }}
image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
imagePullPolicy: {{ .Values.image.pullPolicy }}
ports:
- name: http
containerPort: {{ .Values.backend.port }}
protocol: TCP
env:
- name: PORT
value: {{ .Values.backend.port | quote }}
{{- range $key, $value := .Values.backend.env }}
- name: {{ $key }}
value: {{ $value | quote }}
{{- end }}
{{- with .Values.backend.envFrom }}
envFrom:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- if .Values.backend.args }}
args:
{{- toYaml .Values.backend.args | nindent 12 }}
{{- end }}
resources:
{{- toYaml .Values.resources | nindent 12 }}
{{- if .Values.configMap.enabled }}
volumeMounts:
- name: config
mountPath: "/configs/config.toml"
subPath: "config.toml"
readOnly: true
{{- end }}
{{- if .Values.configMap.enabled }}
volumes:
- name: config
configMap:
name: {{ include "backend.fullname" . }}-config
{{- end }}
{{- with .Values.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.tolerations }}
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}

View file

@ -0,0 +1,61 @@
{{- if .Values.ingress.enabled -}}
{{- $fullName := include "backend.fullname" . -}}
{{- $svcPort := .Values.service.port -}}
{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }}
{{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }}
{{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}}
{{- end }}
{{- end }}
{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}}
apiVersion: networking.k8s.io/v1
{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}}
apiVersion: networking.k8s.io/v1beta1
{{- else -}}
apiVersion: extensions/v1beta1
{{- end }}
kind: Ingress
metadata:
name: {{ $fullName }}
labels:
{{- include "backend.labels" . | nindent 4 }}
{{- with .Values.ingress.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
{{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }}
ingressClassName: {{ .Values.ingress.className }}
{{- end }}
{{- if .Values.ingress.tls }}
tls:
{{- range .Values.ingress.tls }}
- hosts:
{{- range .hosts }}
- {{ . | quote }}
{{- end }}
secretName: {{ .secretName }}
{{- end }}
{{- end }}
rules:
{{- range .Values.ingress.hosts }}
- host: {{ .host | quote }}
http:
paths:
{{- range .paths }}
- path: {{ .path }}
{{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }}
pathType: {{ .pathType }}
{{- end }}
backend:
{{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}
service:
name: {{ $fullName }}
port:
number: {{ $svcPort }}
{{- else }}
serviceName: {{ $fullName }}
servicePort: {{ $svcPort }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}

View file

@ -0,0 +1,13 @@
{{- if .Values.secrets.enabled -}}
apiVersion: v1
kind: Secret
metadata:
name: {{ include "backend.fullname" . }}-custom
labels:
{{- include "backend.labels" . | nindent 4 }}
type: Opaque
data:
{{- range $key, $value := .Values.secrets.data }}
{{ $key }}: {{ $value | b64enc | quote }}
{{- end }}
{{- end }}

View file

@ -0,0 +1,22 @@
apiVersion: v1
kind: Service
metadata:
name: {{ include "backend.fullname" . }}
labels:
{{- include "backend.labels" . | nindent 4 }}
{{- with .Values.service.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
type: {{ .Values.service.type }}
ports:
- port: {{ .Values.service.port }}
targetPort: http
protocol: TCP
name: http
{{- if and (eq .Values.service.type "NodePort") .Values.service.nodePort }}
nodePort: {{ .Values.service.nodePort }}
{{- end }}
selector:
{{- include "backend.selectorLabels" . | nindent 4 }}

View file

@ -0,0 +1,12 @@
{{- if .Values.serviceAccount.create -}}
apiVersion: v1
kind: ServiceAccount
metadata:
name: {{ include "backend.serviceAccountName" . }}
labels:
{{- include "backend.labels" . | nindent 4 }}
{{- with .Values.serviceAccount.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
{{- end }}

View file

@ -0,0 +1,122 @@
# Default values for backend.
# This is a YAML-formatted file.
# Declare variables to be passed into your templates.
replicaCount: 1
image:
repository: storagehub/backend
pullPolicy: IfNotPresent
tag: "latest"
imagePullSecrets: []
nameOverride: ""
fullnameOverride: ""
serviceAccount:
create: true
annotations: {}
name: ""
podAnnotations: {}
podSecurityContext: {}
# fsGroup: 2000
securityContext: {}
# capabilities:
# drop:
# - ALL
# readOnlyRootFilesystem: true
# runAsNonRoot: true
# runAsUser: 1000
service:
type: ClusterIP
port: 3000
targetPort: 3000
annotations: {}
ingress:
enabled: false
className: ""
annotations: {}
hosts:
- host: api.storagehub.local
paths:
- path: /
pathType: Prefix
tls: []
resources:
limits:
cpu: 500m
memory: 512Mi
requests:
cpu: 250m
memory: 256Mi
nodeSelector: {}
tolerations: []
affinity: {}
# Backend API Configuration
backend:
# Port the backend listens on
port: 3000
# Database configuration
database:
# Full database connection URL
url: "postgresql://storagehub:storagehub@sh-indexer-db:5432/storagehub"
# Use existing secret for database URL
existingSecret: ""
# Key in the secret containing the DATABASE_URL
existingSecretUrlKey: "database-url"
# RPC Node configuration
rpc:
endpoint: "ws://sh-idxnode:9944"
# Alternative HTTP endpoint if needed
httpEndpoint: "http://sh-idxnode:9933"
# API configuration
api:
defaultPageSize: 20
maxPageSize: 100
# Authentication configuration
auth:
jwtSecret: ""
# Environment variables
env:
NODE_ENV: "production"
LOG_LEVEL: "info"
# Additional environment variables from ConfigMap or Secret
envFrom: []
# CLI arguments to pass to the backend application
args: []
# Example:
# args:
# - "--config"
# - "/app/config.toml"
# - "--log-level"
# - "debug"
# ConfigMap for configuration files
configMap:
enabled: true
# Additional configuration to merge into config.json
extraConfig: {}
# Additional files to add to ConfigMap
data: {}
# Secrets for sensitive data
secrets:
enabled: false
data: {}

View file

@ -26,6 +26,7 @@ node:
- "--allow-private-ipv4"
- "--discover-local"
- "--network-backend libp2p"
- "--pool-type fork-aware"
ingress:
enabled: false

View file

@ -43,6 +43,7 @@ node:
persistGeneratedNodeKey: true
flags:
- "--network-backend libp2p"
- "--pool-type fork-aware"
# Note: Bootnode discovery will happen automatically via the chainspec downloaded from customChainspecUrl
enableOffchainIndexing: true

View file

@ -0,0 +1,47 @@
name: sh-bspnode
description: Datahaven BSP node
fullnameOverride: sh-bspnode
image:
repository: datahavenxyz/datahaven
tag: main
pullPolicy: Always
imagePullSecrets:
- name: datahaven-dockerhub
node:
command: datahaven-node
customChainspecUrl: http://dh-bootnode:8080/chainspec.json
forceDownloadChainspec: true
role: full
replicas: 2
chainData:
pruning: 1000
storageClass: "gp2"
chainKeystore:
storageClass: "gp2"
keys:
# This is Alice seed. To generate new seed run: docker run --rm datahavenxyz/datahaven:latest key generate
- seed: "bottom drive obey lake curtain smoke basket hold race lonely fit walk"
type: bcsv
scheme: ecdsa
# ${HOSTNAME##*-} will be evaluated as the pod index, pod-0: //Eve, pod-1: //Ferdie
extraDerivation: '$([ "${HOSTNAME##*-}" = "0" ] && echo "//Eve" || echo "//Ferdie")'
persistGeneratedNodeKey: true
flags:
- "--allow-private-ipv4"
- "--discover-local"
- "--network-backend libp2p"
- "--provider"
- "--provider-type bsp"
- "--max-storage-capacity 10737418240" # 10 GiB
- "--jump-capacity=1073741824" # 1 GiB
ingress:
enabled: false
perReplica: true
wildcardDomain: datahaven.local
# If enabled, this would generate:
# - sh-bspnode-0.datahaven.local
# - sh-bspnode-1.datahaven.local

View file

@ -0,0 +1,44 @@
name: sh-fisherman
description: Datahaven Fisherman node
fullnameOverride: sh-fisherman
image:
repository: datahavenxyz/datahaven
tag: main
pullPolicy: Always
imagePullSecrets:
- name: datahaven-dockerhub
node:
command: datahaven-node
customChainspecUrl: http://dh-bootnode:8080/chainspec.json
forceDownloadChainspec: true
role: full
replicas: 1
chainData:
pruning: 1000
storageClass: "gp2"
chainKeystore:
storageClass: "gp2"
keys:
# This is Alice seed. To generate new seed run: docker run --rm datahavenxyz/datahaven:latest key generate
- seed: "bottom drive obey lake curtain smoke basket hold race lonely fit walk"
type: bcsv
scheme: ecdsa
# ${HOSTNAME##*-} will be evaluated as the pod index, pod-0: //Gustavo, pod-1: //Hermano
extraDerivation: '$([ "${HOSTNAME##*-}" = "0" ] && echo "//Gustavo" || echo "//Hermano")'
persistGeneratedNodeKey: true
flags:
- "--allow-private-ipv4"
- "--discover-local"
- "--network-backend libp2p"
- "--fisherman"
- "--fisherman-database-url postgresql://indexer:indexer@sh-idxnode-db-postgresql:5432/datahaven"
ingress:
enabled: false
perReplica: false
wildcardDomain: datahaven.local
# If enabled, this would generate:
# - sh-fisherman-0.datahaven.local

View file

@ -0,0 +1,38 @@
name: sh-idxnode
description: Datahaven Indexer node
fullnameOverride: sh-idxnode
image:
repository: datahavenxyz/datahaven
tag: main
pullPolicy: Always
imagePullSecrets:
- name: datahaven-dockerhub
node:
command: datahaven-node
customChainspecUrl: http://dh-bootnode:8080/chainspec.json
forceDownloadChainspec: true
role: full
replicas: 1
chainData:
pruning: 1000
storageClass: "gp2"
chainKeystore:
storageClass: "gp2"
persistGeneratedNodeKey: true
flags:
- "--allow-private-ipv4"
- "--discover-local"
- "--network-backend libp2p"
- "--indexer"
- "--indexer-mode full"
- "--indexer-database-url postgresql://indexer:indexer@sh-idxnode-db-postgresql:5432/datahaven"
ingress:
enabled: false
perReplica: false
wildcardDomain: datahaven.local
# If enabled, this would generate:
# - sh-idxnode-0.datahaven.local

View file

@ -0,0 +1,49 @@
name: sh-mspnode
description: Datahaven MSP node
fullnameOverride: sh-mspnode
image:
repository: datahavenxyz/datahaven
tag: main
pullPolicy: Always
imagePullSecrets:
- name: datahaven-dockerhub
node:
command: datahaven-node
customChainspecUrl: http://dh-bootnode:8080/chainspec.json
forceDownloadChainspec: true
role: full
replicas: 2
chainData:
pruning: 1000
storageClass: "gp2"
chainKeystore:
storageClass: "gp2"
keys:
# This is Alice seed. To generate new seed run: docker run --rm datahavenxyz/datahaven:latest key generate
- seed: "bottom drive obey lake curtain smoke basket hold race lonely fit walk"
type: bcsv
scheme: ecdsa
# ${HOSTNAME##*-} will be evaluated as the pod index, pod-0: //Charlie, pod-1: //Dave
extraDerivation: '$([ "${HOSTNAME##*-}" = "0" ] && echo "//Charlie" || echo "//Dave")'
persistGeneratedNodeKey: true
allowUnsafeRpcMethods: true
flags:
- "--allow-private-ipv4"
- "--discover-local"
- "--network-backend libp2p"
- "--provider"
- "--provider-type msp"
- "--msp-charging-period 100" # in blocks, 100 blocks = 10 minutes
- "--max-storage-capacity 10737418240" # 10 GiB
- "--jump-capacity=1073741824" # 1 GiB
ingress:
enabled: false
perReplica: true
wildcardDomain: datahaven.local
# If enabled, this would generate:
# - sh-mspnode-0.datahaven.local
# - sh-mspnode-1.datahaven.local

View file

@ -15,7 +15,7 @@ imagePullSecrets:
ingress:
enabled: false
ingressClassName: traefik
host: dh-bootnode-0.datahaven-kt.local
host: dh-bootnode-0.datahaven.local
node:
chain: local

View file

@ -0,0 +1,44 @@
# BSP-specific settings for Local environment
global:
environment: local
namespace: kt-datahaven-local
image:
tag: local
pullPolicy: IfNotPresent
imagePullSecrets:
- name: datahaven-dockerhub
# Ingress disabled for local (can be enabled for testing)
ingress:
enabled: false
ingressClassName: traefik
perReplica: true
wildcardDomain: datahaven.local
# If enabled, this would generate:
# - sh-bspnode-0.datahaven.local
# - sh-bspnode-1.datahaven.local
node:
chain: local
customChainspecUrl:
forceDownloadChainspec: false
chainData:
storageClass: "hostpath"
persistence:
size: 10Gi
chainKeystore:
storageClass: "hostpath"
resources:
requests:
memory: "256Mi"
cpu: "100m"
limits:
memory: "512Mi"
cpu: "200m"
perNodeServices:
apiService:
enabled: true
type: NodePort

View file

@ -0,0 +1,44 @@
# Fisherman-specific settings for Local environment
global:
environment: local
namespace: kt-datahaven-local
image:
tag: local
pullPolicy: IfNotPresent
imagePullSecrets:
- name: datahaven-dockerhub
# Ingress disabled for local (can be enabled for testing)
ingress:
enabled: false
ingressClassName: traefik
perReplica: true
wildcardDomain: datahaven.local
# If enabled, this would generate:
# - sh-fisherman-0.datahaven.local
# - sh-fisherman-1.datahaven.local
node:
chain: local
customChainspecUrl:
forceDownloadChainspec: false
chainData:
storageClass: "hostpath"
persistence:
size: 10Gi
chainKeystore:
storageClass: "hostpath"
resources:
requests:
memory: "256Mi"
cpu: "100m"
limits:
memory: "512Mi"
cpu: "200m"
perNodeServices:
apiService:
enabled: true
type: NodePort

View file

@ -0,0 +1,21 @@
# Install PostgreSQL into local k8S cluster
# helm repo add bitnami https://charts.bitnami.com/bitnami
# helm repo update
# helm install sh-idxnode-db bitnami/postgresql -f ./deploy/environments/local/sh-idxnode-db.yaml -n datahaven-local
auth:
username: indexer
password: indexer
database: datahaven
postgresPassword: postgres
primary:
persistence:
enabled: true
size: 10Gi # requires a default StorageClass
resources:
requests: { cpu: "100m", memory: "256Mi" }
limits: { cpu: "500m", memory: "512Mi" }
volumePermissions:
enabled: true # helps with FS perms on some storage classes
metrics:
enabled: true # optional: Prometheus exporter

View file

@ -0,0 +1,33 @@
# Indexer-specific settings for Local environment
global:
environment: local
namespace: kt-datahaven-local
image:
tag: local
pullPolicy: IfNotPresent
imagePullSecrets:
- name: datahaven-dockerhub
ingress:
enabled: false
node:
chain: local
customChainspecUrl:
forceDownloadChainspec: false
chainData:
storageClass: "hostpath"
persistence:
size: 10Gi
chainKeystore:
storageClass: "hostpath"
resources:
requests:
memory: "256Mi"
cpu: "100m"
limits:
memory: "512Mi"
cpu: "200m"

View file

@ -0,0 +1,73 @@
# StorageHub MSP Backend API configuration for Local environment
global:
environment: local
namespace: kt-datahaven-local
# Use local image
image:
repository: moonsonglabs/storage-hub-msp-backend
tag: latest
pullPolicy: Always
imagePullSecrets:
- name: datahaven-dockerhub
# Single replica for local development
replicaCount: 1
# Service configuration for local access
service:
type: NodePort
port: 8080
nodePort: 30300 # Fixed NodePort for local access
# Backend configuration for local environment
backend:
database:
url: postgresql://indexer:indexer@sh-idxnode-db-postgresql:5432/datahaven
rpc:
url: ws://sh-idxnode:9944
auth:
jwtSecret: "local-development-secret"
env:
LOG_LEVEL: debug
# CLI arguments for local development
args:
- "--config"
- "/app/config/config.toml"
# Minimal resources for local development
resources:
requests:
memory: "128Mi"
cpu: "50m"
limits:
memory: "256Mi"
cpu: "200m"
# Enable ingress for local development with Traefik
ingress:
enabled: true
className: traefik
annotations:
traefik.ingress.kubernetes.io/router.entrypoints: web
hosts:
- host: sh-mspbackend.datahaven.local
paths:
- path: /
pathType: Prefix
# ConfigMap for local environment
configMap:
enabled: true
extraConfig:
environment: "local"
features:
debug: true
swagger: true
metrics: true

View file

@ -0,0 +1,44 @@
# MSP-specific settings for Local environment
global:
environment: local
namespace: kt-datahaven-local
image:
tag: local
pullPolicy: IfNotPresent
imagePullSecrets:
- name: datahaven-dockerhub
# Ingress disabled for local (can be enabled for testing)
ingress:
enabled: false
ingressClassName: traefik
perReplica: true
wildcardDomain: datahaven.local
# If enabled, this would generate:
# - sh-mspnode-0.datahaven.local
# - sh-mspnode-1.datahaven.local
node:
chain: local
customChainspecUrl:
forceDownloadChainspec: false
chainData:
storageClass: "hostpath"
persistence:
size: 10Gi
chainKeystore:
storageClass: "hostpath"
resources:
requests:
memory: "256Mi"
cpu: "100m"
limits:
memory: "512Mi"
cpu: "200m"
perNodeServices:
apiService:
enabled: true
type: NodePort

View file

@ -5,7 +5,7 @@ global:
namespace: kt-datahaven-stagenet
image:
tag: main
tag: latest
pullPolicy: Always
imagePullSecrets:

View file

@ -5,7 +5,7 @@ global:
namespace: kt-datahaven-stagenet
image:
tag: main
tag: latest
pullPolicy: Always
imagePullSecrets:

View file

@ -0,0 +1,51 @@
# BSP-specific settings for stagenet
global:
environment: stagenet
namespace: kt-datahaven-stagenet
image:
tag: latest
pullPolicy: Always
imagePullSecrets:
- name: datahaven-dockerhub
ingress:
enabled: true
perReplica: true
wildcardDomain: datahaven-kt.xyz
# This will generate:
# - sh-bspnode-0.datahaven-kt.xyz
# - sh-bspnode-1.datahaven-kt.xyz
# (based on replica count)
annotations:
kubernetes.io/ingress.class: alb
alb.ingress.kubernetes.io/scheme: internet-facing
alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80}, {"HTTPS":443}]'
alb.ingress.kubernetes.io/load-balancer-name: sh-bspnode
alb.ingress.kubernetes.io/target-type: ip
alb.ingress.kubernetes.io/ssl-policy: ELBSecurityPolicy-TLS-1-2-2017-01
alb.ingress.kubernetes.io/load-balancer-attributes: idle_timeout.timeout_seconds=120
alb.ingress.kubernetes.io/target-group-attributes: deregistration_delay.timeout_seconds=30
alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-east-1:601766312530:certificate/61c2fc57-4ce4-4a80-90b8-2810c1221f09
external-dns.alpha.kubernetes.io/hostname: sh-bspnode.datahaven-kt.xyz
meta.helm.sh/release-name: sh-bspnode
meta.helm.sh/release-namespace: kt-datahaven-stagenet
node:
chain: stagenet-local
chainData:
persistence:
size: 20Gi
resources:
requests:
memory: "512Mi"
cpu: "200m"
limits:
memory: "1Gi"
cpu: "500m"
perNodeServices:
apiService:
enabled: true
type: NodePort

View file

@ -0,0 +1,51 @@
# Fisherman-specific settings for stagenet
global:
environment: stagenet
namespace: kt-datahaven-stagenet
image:
tag: latest
pullPolicy: Always
imagePullSecrets:
- name: datahaven-dockerhub
ingress:
enabled: true
perReplica: true
wildcardDomain: datahaven-kt.xyz
# This will generate:
# - sh-fisherman-0.datahaven-kt.xyz
# - sh-fisherman-1.datahaven-kt.xyz
# (based on replica count)
annotations:
kubernetes.io/ingress.class: alb
alb.ingress.kubernetes.io/scheme: internet-facing
alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80}, {"HTTPS":443}]'
alb.ingress.kubernetes.io/load-balancer-name: sh-fisherman
alb.ingress.kubernetes.io/target-type: ip
alb.ingress.kubernetes.io/ssl-policy: ELBSecurityPolicy-TLS-1-2-2017-01
alb.ingress.kubernetes.io/load-balancer-attributes: idle_timeout.timeout_seconds=120
alb.ingress.kubernetes.io/target-group-attributes: deregistration_delay.timeout_seconds=30
alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-east-1:601766312530:certificate/61c2fc57-4ce4-4a80-90b8-2810c1221f09
external-dns.alpha.kubernetes.io/hostname: sh-fisherman.datahaven-kt.xyz
meta.helm.sh/release-name: sh-fisherman
meta.helm.sh/release-namespace: kt-datahaven-stagenet
node:
chain: stagenet-local
chainData:
persistence:
size: 20Gi
resources:
requests:
memory: "512Mi"
cpu: "200m"
limits:
memory: "1Gi"
cpu: "500m"
perNodeServices:
apiService:
enabled: true
type: NodePort

View file

@ -0,0 +1,22 @@
# Install PostgreSQL into local k8S cluster
# helm repo add bitnami https://charts.bitnami.com/bitnami
# helm repo update
# helm install sh-idxnode-db bitnami/postgresql -f ./deploy/environments/local/sh-idxnode-db.yaml -n kt-datahaven-stagenet
auth:
username: indexer
password: indexer
database: datahaven
postgresPassword: postgres
primary:
persistence:
enabled: true
size: 10Gi # requires a default StorageClass
storageClass: "gp2"
resources:
requests: { cpu: "100m", memory: "256Mi" }
limits: { cpu: "500m", memory: "512Mi" }
volumePermissions:
enabled: true # helps with FS perms on some storage classes
metrics:
enabled: true # optional: Prometheus exporter

View file

@ -0,0 +1,28 @@
# Indexer-specific settings for stagenet
global:
environment: stagenet
namespace: kt-datahaven-stagenet
image:
tag: latest
pullPolicy: Always
imagePullSecrets:
- name: datahaven-dockerhub
ingress:
enabled: false
node:
chain: stagenet-local
chainData:
persistence:
size: 20Gi
resources:
requests:
memory: "512Mi"
cpu: "200m"
limits:
memory: "1Gi"
cpu: "500m"

View file

@ -0,0 +1,71 @@
# StorageHub MSP Backend API configuration for Stagenet environment
global:
environment: stagenet
namespace: datahaven-stagenet
# Stagenet image configuration
image:
repository: moonsonglabs/storage-hub-msp-backend
tag: latest
pullPolicy: Always
imagePullSecrets:
- name: datahaven-dockerhub
replicaCount: 1
# Service configuration
service:
type: ClusterIP
port: 8080
annotations:
service.beta.kubernetes.io/aws-load-balancer-type: "nlb"
# Backend configuration for stagenet
backend:
database:
url: postgresql://indexer:indexer@sh-idxnode-db-postgresql:5432/datahaven
rpc:
endpoint: ws://sh-mspnode-0:9955
auth:
jwtSecret: "0xdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef"
# CLI arguments for production
args:
- "--config"
- "/configs/config.toml"
# Production-ready resources
resources:
requests:
memory: "512Mi"
cpu: "250m"
limits:
memory: "1Gi"
cpu: "1000m"
# Enable ingress with SSL
ingress:
enabled: true
host: sh-mspbackend.datahaven-kt.xyz
annotations:
kubernetes.io/ingress.class: alb
alb.ingress.kubernetes.io/scheme: internet-facing
alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80}, {"HTTPS":443}]'
alb.ingress.kubernetes.io/load-balancer-name: sh-mspbackend
alb.ingress.kubernetes.io/target-type: ip
alb.ingress.kubernetes.io/ssl-policy: ELBSecurityPolicy-TLS-1-2-2017-01
alb.ingress.kubernetes.io/load-balancer-attributes: idle_timeout.timeout_seconds=120
alb.ingress.kubernetes.io/target-group-attributes: deregistration_delay.timeout_seconds=30
alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-east-1:601766312530:certificate/61c2fc57-4ce4-4a80-90b8-2810c1221f09
external-dns.alpha.kubernetes.io/hostname: sh-mspbackend.datahaven-kt.xyz
meta.helm.sh/release-name: sh-mspbackend
meta.helm.sh/release-namespace: kt-datahaven-stagenet
hosts:
- host: sh-mspbackend.datahaven-kt.xyz
paths:
- path: /
pathType: Prefix

View file

@ -0,0 +1,51 @@
# MSP-specific settings for stagenet
global:
environment: stagenet
namespace: kt-datahaven-stagenet
image:
tag: latest
pullPolicy: Always
imagePullSecrets:
- name: datahaven-dockerhub
ingress:
enabled: true
perReplica: true
wildcardDomain: datahaven-kt.xyz
# This will generate:
# - sh-mspnode-0.datahaven-kt.xyz
# - sh-mspnode-1.datahaven-kt.xyz
# (based on replica count)
annotations:
kubernetes.io/ingress.class: alb
alb.ingress.kubernetes.io/scheme: internet-facing
alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80}, {"HTTPS":443}]'
alb.ingress.kubernetes.io/load-balancer-name: sh-mspnode
alb.ingress.kubernetes.io/target-type: ip
alb.ingress.kubernetes.io/ssl-policy: ELBSecurityPolicy-TLS-1-2-2017-01
alb.ingress.kubernetes.io/load-balancer-attributes: idle_timeout.timeout_seconds=120
alb.ingress.kubernetes.io/target-group-attributes: deregistration_delay.timeout_seconds=30
alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-east-1:601766312530:certificate/61c2fc57-4ce4-4a80-90b8-2810c1221f09
external-dns.alpha.kubernetes.io/hostname: sh-mspnode.datahaven-kt.xyz
meta.helm.sh/release-name: sh-mspnode
meta.helm.sh/release-namespace: kt-datahaven-stagenet
node:
chain: stagenet-local
chainData:
persistence:
size: 20Gi
resources:
requests:
memory: "512Mi"
cpu: "200m"
limits:
memory: "1Gi"
cpu: "500m"
perNodeServices:
apiService:
enabled: true
type: NodePort

View file

@ -1,5 +1,5 @@
# --- Setup Build Environment ---
FROM docker.io/paritytech/ci-unified:bullseye-1.88.0 AS base
FROM rust:latest AS base
ARG MOLD_VERSION=2.40.4
ARG PROTOC_VER=21.12
@ -10,42 +10,18 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
xz-utils \
clang \
libpq-dev \
&& echo "Installing mold v${MOLD_VERSION}..." \
&& curl -Lo mold.tar.gz "https://github.com/rui314/mold/releases/download/v${MOLD_VERSION}/mold-${MOLD_VERSION}-x86_64-linux.tar.gz" \
&& tar -xf mold.tar.gz --strip-components=1 -C /usr/local \
&& rm mold.tar.gz \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/* \
&& echo "Installing protoc v${PROTOC_VER}..." \
&& curl -Lo protoc.zip "https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VER}/protoc-${PROTOC_VER}-linux-x86_64.zip" \
&& unzip -q protoc.zip -d /usr/local/ \
&& rm protoc.zip \
&& echo "Installing sccache v${SCCACHE_VERSION}..." \
&& curl -Lo sccache.tar.gz "https://github.com/mozilla/sccache/releases/download/v${SCCACHE_VERSION}/sccache-v${SCCACHE_VERSION}-x86_64-unknown-linux-musl.tar.gz" \
&& tar -xf sccache.tar.gz --strip-components=1 -C /usr/local/bin sccache-v${SCCACHE_VERSION}-x86_64-unknown-linux-musl/sccache \
&& rm sccache.tar.gz
RUN cargo install cargo-chef --version 0.1.72 --locked
ENV RUSTC_WRAPPER=sccache \
SCCACHE_DIR=/usr/local/sccache \
SCCACHE_CACHE_SIZE=25G \
RUSTFLAGS="-Clinker=clang -Clink-arg=-fuse-ld=/usr/local/bin/mold"
# --- Prepare build plan with cargo-chef ---
FROM base AS planner
WORKDIR /datahaven
COPY . .
RUN cargo chef prepare --recipe-path recipe.json
&& rm protoc.zip
# --- Build dependencies using cargo-chef ---
FROM base AS builder
WORKDIR /datahaven
COPY --from=planner /datahaven/recipe.json recipe.json
RUN --mount=type=cache,target=/usr/local/cargo/registry \
--mount=type=cache,target=/usr/local/cargo/git \
cargo chef cook --recipe-path recipe.json --release
COPY . .
COPY . /datahaven
RUN --mount=type=cache,target=/usr/local/cargo/registry \
--mount=type=cache,target=/usr/local/cargo/git \
if [ "$FAST_RUNTIME" = "TRUE" ]; then \
@ -55,30 +31,11 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \
fi
# --- Create final lightweight runtime image ---
FROM docker.io/parity/base-bin:latest
# Copy CA certificates and shared libraries from builder
COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
COPY --from=builder \
/lib/x86_64-linux-gnu/libpq.so.5 \
/lib/x86_64-linux-gnu/libssl.so.3 \
/lib/x86_64-linux-gnu/libcrypto.so.3 \
/lib/x86_64-linux-gnu/libgssapi_krb5.so.2 \
/lib/x86_64-linux-gnu/libldap.so.2 \
/lib/x86_64-linux-gnu/libz.so.1 \
/lib/x86_64-linux-gnu/libzstd.so.1 \
/lib/x86_64-linux-gnu/libkrb5.so.3 \
/lib/x86_64-linux-gnu/libk5crypto.so.3 \
/lib/x86_64-linux-gnu/libcom_err.so.2 \
/lib/x86_64-linux-gnu/libkrb5support.so.0 \
/lib/x86_64-linux-gnu/liblber.so.2 \
/lib/x86_64-linux-gnu/libsasl2.so.2 \
/lib/x86_64-linux-gnu/libkeyutils.so.1 \
/lib/x86_64-linux-gnu/
FROM debian:trixie-slim
COPY --from=builder /datahaven/target/release/datahaven-node /usr/local/bin
USER root
RUN apt-get update && apt-get install -y gcc libc6-dev libpq-dev && rm -rf /var/lib/apt/lists/*
RUN useradd -m -u 1001 -U -s /bin/sh -d /datahaven datahaven && \
mkdir -p /data /datahaven/.local/share && \
chown -R datahaven:datahaven /data && \

View file

@ -15,7 +15,7 @@ WORKDIR /
RUN echo "*** Installing Basic dependencies ***"
RUN apt-get update && apt-get install -y ca-certificates && update-ca-certificates
RUN apt install --assume-yes git clang curl libpq-dev libssl-dev llvm libudev-dev make pkg-config unzip
RUN apt install --assume-yes git clang curl libldap2-dev libpq-dev libssl-dev llvm libudev-dev make pkg-config unzip
RUN echo "*** Installing protoc v${PROTOC_VER} ***"
RUN curl -Lo /tmp/protoc.zip "https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VER}/protoc-${PROTOC_VER}-linux-x86_64.zip" \
@ -50,14 +50,14 @@ COPY --from=builder \
/lib/x86_64-linux-gnu/libssl.so.3 \
/lib/x86_64-linux-gnu/libcrypto.so.3 \
/lib/x86_64-linux-gnu/libgssapi_krb5.so.2 \
/lib/x86_64-linux-gnu/libldap.so.2 \
/lib/x86_64-linux-gnu/libldap-2.5.so.0 \
/lib/x86_64-linux-gnu/libz.so.1 \
/lib/x86_64-linux-gnu/libzstd.so.1 \
/lib/x86_64-linux-gnu/libkrb5.so.3 \
/lib/x86_64-linux-gnu/libk5crypto.so.3 \
/lib/x86_64-linux-gnu/libcom_err.so.2 \
/lib/x86_64-linux-gnu/libkrb5support.so.0 \
/lib/x86_64-linux-gnu/liblber.so.2 \
/lib/x86_64-linux-gnu/liblber-2.5.so.0 \
/lib/x86_64-linux-gnu/libsasl2.so.2 \
/lib/x86_64-linux-gnu/libkeyutils.so.1 \
/lib/x86_64-linux-gnu/

112
operator/Cargo.lock generated
View file

@ -1517,7 +1517,7 @@ dependencies = [
"pallet-message-queue",
"parity-scale-codec",
"scale-info",
"snowbridge-core 0.3.0",
"snowbridge-core 0.3.1",
"sp-core",
"sp-runtime",
"sp-std",
@ -2902,7 +2902,7 @@ dependencies = [
[[package]]
name = "datahaven-mainnet-runtime"
version = "0.3.0"
version = "0.3.1"
dependencies = [
"bridge-hub-common 0.13.1",
"datahaven-runtime-common",
@ -3006,8 +3006,8 @@ dependencies = [
"shp-traits",
"shp-treasury-funding",
"smallvec",
"snowbridge-beacon-primitives 0.3.0",
"snowbridge-core 0.3.0",
"snowbridge-beacon-primitives 0.3.1",
"snowbridge-core 0.3.1",
"snowbridge-inbound-queue-primitives",
"snowbridge-merkle-tree",
"snowbridge-outbound-queue-primitives",
@ -3049,7 +3049,7 @@ dependencies = [
[[package]]
name = "datahaven-node"
version = "0.3.0"
version = "0.3.1"
dependencies = [
"async-channel 1.9.0",
"clap",
@ -3159,7 +3159,7 @@ dependencies = [
[[package]]
name = "datahaven-runtime-common"
version = "0.3.0"
version = "0.3.1"
dependencies = [
"fp-account",
"frame-support",
@ -3188,7 +3188,7 @@ dependencies = [
[[package]]
name = "datahaven-stagenet-runtime"
version = "0.3.0"
version = "0.3.1"
dependencies = [
"bridge-hub-common 0.13.1",
"datahaven-runtime-common",
@ -3292,8 +3292,8 @@ dependencies = [
"shp-traits",
"shp-treasury-funding",
"smallvec",
"snowbridge-beacon-primitives 0.3.0",
"snowbridge-core 0.3.0",
"snowbridge-beacon-primitives 0.3.1",
"snowbridge-core 0.3.1",
"snowbridge-inbound-queue-primitives",
"snowbridge-merkle-tree",
"snowbridge-outbound-queue-primitives",
@ -3335,7 +3335,7 @@ dependencies = [
[[package]]
name = "datahaven-testnet-runtime"
version = "0.3.0"
version = "0.3.1"
dependencies = [
"bridge-hub-common 0.13.1",
"datahaven-runtime-common",
@ -3439,8 +3439,8 @@ dependencies = [
"shp-traits",
"shp-treasury-funding",
"smallvec",
"snowbridge-beacon-primitives 0.3.0",
"snowbridge-core 0.3.0",
"snowbridge-beacon-primitives 0.3.1",
"snowbridge-core 0.3.1",
"snowbridge-inbound-queue-primitives",
"snowbridge-merkle-tree",
"snowbridge-outbound-queue-primitives",
@ -3631,7 +3631,7 @@ dependencies = [
[[package]]
name = "dhp-bridge"
version = "0.3.0"
version = "0.3.1"
dependencies = [
"frame-support",
"frame-system",
@ -3639,7 +3639,7 @@ dependencies = [
"pallet-datahaven-native-transfer",
"pallet-external-validators",
"parity-scale-codec",
"snowbridge-core 0.3.0",
"snowbridge-core 0.3.1",
"snowbridge-inbound-queue-primitives",
"sp-core",
"sp-std",
@ -9058,7 +9058,7 @@ dependencies = [
[[package]]
name = "pallet-datahaven-native-transfer"
version = "0.3.0"
version = "0.3.1"
dependencies = [
"frame-benchmarking",
"frame-support",
@ -9066,7 +9066,7 @@ dependencies = [
"pallet-balances",
"parity-scale-codec",
"scale-info",
"snowbridge-core 0.3.0",
"snowbridge-core 0.3.1",
"snowbridge-outbound-queue-primitives",
"sp-core",
"sp-io",
@ -9170,7 +9170,7 @@ dependencies = [
[[package]]
name = "pallet-evm-precompile-balances-erc20"
version = "0.3.0"
version = "0.3.1"
dependencies = [
"fp-evm",
"frame-support",
@ -9193,7 +9193,7 @@ dependencies = [
[[package]]
name = "pallet-evm-precompile-batch"
version = "0.3.0"
version = "0.3.1"
dependencies = [
"evm",
"fp-evm",
@ -9232,7 +9232,7 @@ dependencies = [
[[package]]
name = "pallet-evm-precompile-call-permit"
version = "0.3.0"
version = "0.3.1"
dependencies = [
"evm",
"fp-evm",
@ -9366,7 +9366,7 @@ dependencies = [
[[package]]
name = "pallet-evm-precompile-proxy"
version = "0.3.0"
version = "0.3.1"
dependencies = [
"evm",
"fp-evm",
@ -9410,7 +9410,7 @@ dependencies = [
[[package]]
name = "pallet-evm-precompile-registry"
version = "0.3.0"
version = "0.3.1"
dependencies = [
"fp-evm",
"frame-support",
@ -9447,7 +9447,7 @@ dependencies = [
[[package]]
name = "pallet-external-validators"
version = "0.3.0"
version = "0.3.1"
dependencies = [
"frame-benchmarking",
"frame-support",
@ -9469,7 +9469,7 @@ dependencies = [
[[package]]
name = "pallet-external-validators-rewards"
version = "0.3.0"
version = "0.3.1"
dependencies = [
"cumulus-primitives-core",
"frame-benchmarking",
@ -9484,7 +9484,7 @@ dependencies = [
"polkadot-primitives",
"polkadot-runtime-parachains",
"scale-info",
"snowbridge-core 0.3.0",
"snowbridge-core 0.3.1",
"snowbridge-merkle-tree",
"snowbridge-outbound-queue-primitives",
"sp-core",
@ -9496,7 +9496,7 @@ dependencies = [
[[package]]
name = "pallet-external-validators-rewards-runtime-api"
version = "0.3.0"
version = "0.3.1"
dependencies = [
"parity-scale-codec",
"snowbridge-merkle-tree",
@ -9719,7 +9719,7 @@ dependencies = [
[[package]]
name = "pallet-outbound-commitment-store"
version = "0.3.0"
version = "0.3.1"
dependencies = [
"frame-support",
"frame-system",
@ -16042,7 +16042,7 @@ dependencies = [
[[package]]
name = "snowbridge-beacon-primitives"
version = "0.3.0"
version = "0.3.1"
dependencies = [
"byte-slice-cast",
"frame-support",
@ -16086,7 +16086,7 @@ dependencies = [
[[package]]
name = "snowbridge-core"
version = "0.3.0"
version = "0.3.1"
dependencies = [
"bp-relayers",
"ethabi-decode",
@ -16186,8 +16186,8 @@ dependencies = [
"log",
"parity-scale-codec",
"scale-info",
"snowbridge-beacon-primitives 0.3.0",
"snowbridge-core 0.3.0",
"snowbridge-beacon-primitives 0.3.1",
"snowbridge-core 0.3.1",
"snowbridge-verification-primitives",
"sp-core",
"sp-io",
@ -16200,7 +16200,7 @@ dependencies = [
[[package]]
name = "snowbridge-merkle-tree"
version = "0.3.0"
version = "0.3.1"
dependencies = [
"array-bytes",
"hex",
@ -16241,7 +16241,7 @@ dependencies = [
[[package]]
name = "snowbridge-outbound-queue-primitives"
version = "0.3.0"
version = "0.3.1"
dependencies = [
"alloy-core",
"ethabi-decode",
@ -16253,7 +16253,7 @@ dependencies = [
"parity-scale-codec",
"polkadot-parachain-primitives",
"scale-info",
"snowbridge-core 0.3.0",
"snowbridge-core 0.3.1",
"snowbridge-verification-primitives",
"sp-arithmetic",
"sp-core",
@ -16267,12 +16267,12 @@ dependencies = [
[[package]]
name = "snowbridge-outbound-queue-v2-runtime-api"
version = "0.3.0"
version = "0.3.1"
dependencies = [
"frame-support",
"parity-scale-codec",
"scale-info",
"snowbridge-core 0.3.0",
"snowbridge-core 0.3.1",
"snowbridge-merkle-tree",
"snowbridge-outbound-queue-primitives",
"sp-api",
@ -16282,7 +16282,7 @@ dependencies = [
[[package]]
name = "snowbridge-pallet-ethereum-client"
version = "0.3.0"
version = "0.3.1"
dependencies = [
"frame-benchmarking",
"frame-support",
@ -16295,8 +16295,8 @@ dependencies = [
"scale-info",
"serde",
"serde_json",
"snowbridge-beacon-primitives 0.3.0",
"snowbridge-core 0.3.0",
"snowbridge-beacon-primitives 0.3.1",
"snowbridge-core 0.3.1",
"snowbridge-ethereum 0.3.0",
"snowbridge-inbound-queue-primitives",
"snowbridge-pallet-ethereum-client-fixtures",
@ -16312,8 +16312,8 @@ name = "snowbridge-pallet-ethereum-client-fixtures"
version = "0.9.0"
dependencies = [
"hex-literal 0.3.4",
"snowbridge-beacon-primitives 0.3.0",
"snowbridge-core 0.3.0",
"snowbridge-beacon-primitives 0.3.1",
"snowbridge-core 0.3.1",
"snowbridge-inbound-queue-primitives",
"sp-core",
"sp-std",
@ -16321,7 +16321,7 @@ dependencies = [
[[package]]
name = "snowbridge-pallet-inbound-queue-v2"
version = "0.3.0"
version = "0.3.1"
dependencies = [
"alloy-core",
"bp-relayers",
@ -16335,8 +16335,8 @@ dependencies = [
"parity-scale-codec",
"scale-info",
"serde",
"snowbridge-beacon-primitives 0.3.0",
"snowbridge-core 0.3.0",
"snowbridge-beacon-primitives 0.3.1",
"snowbridge-core 0.3.1",
"snowbridge-inbound-queue-primitives",
"snowbridge-pallet-ethereum-client",
"snowbridge-pallet-inbound-queue-v2-fixtures",
@ -16357,8 +16357,8 @@ name = "snowbridge-pallet-inbound-queue-v2-fixtures"
version = "0.10.0"
dependencies = [
"hex-literal 0.3.4",
"snowbridge-beacon-primitives 0.3.0",
"snowbridge-core 0.3.0",
"snowbridge-beacon-primitives 0.3.1",
"snowbridge-core 0.3.1",
"snowbridge-inbound-queue-primitives",
"sp-core",
"sp-std",
@ -16388,7 +16388,7 @@ dependencies = [
[[package]]
name = "snowbridge-pallet-outbound-queue-v2"
version = "0.3.0"
version = "0.3.1"
dependencies = [
"alloy-core",
"bp-relayers",
@ -16402,8 +16402,8 @@ dependencies = [
"parity-scale-codec",
"scale-info",
"serde",
"snowbridge-beacon-primitives 0.3.0",
"snowbridge-core 0.3.0",
"snowbridge-beacon-primitives 0.3.1",
"snowbridge-core 0.3.1",
"snowbridge-inbound-queue-primitives",
"snowbridge-merkle-tree",
"snowbridge-outbound-queue-primitives",
@ -16434,7 +16434,7 @@ dependencies = [
"parity-scale-codec",
"polkadot-primitives",
"scale-info",
"snowbridge-core 0.3.0",
"snowbridge-core 0.3.1",
"snowbridge-outbound-queue-primitives",
"snowbridge-pallet-outbound-queue",
"sp-core",
@ -16447,7 +16447,7 @@ dependencies = [
[[package]]
name = "snowbridge-pallet-system-v2"
version = "0.3.0"
version = "0.3.1"
dependencies = [
"frame-benchmarking",
"frame-support",
@ -16459,7 +16459,7 @@ dependencies = [
"parity-scale-codec",
"polkadot-primitives",
"scale-info",
"snowbridge-core 0.3.0",
"snowbridge-core 0.3.1",
"snowbridge-outbound-queue-primitives",
"snowbridge-pallet-outbound-queue-v2",
"snowbridge-pallet-system",
@ -16475,10 +16475,10 @@ dependencies = [
[[package]]
name = "snowbridge-system-v2-runtime-api"
version = "0.3.0"
version = "0.3.1"
dependencies = [
"parity-scale-codec",
"snowbridge-core 0.3.0",
"snowbridge-core 0.3.1",
"sp-api",
"sp-std",
"staging-xcm",
@ -16486,7 +16486,7 @@ dependencies = [
[[package]]
name = "snowbridge-test-utils"
version = "0.3.0"
version = "0.3.1"
dependencies = [
"frame-benchmarking",
"frame-support",
@ -16506,12 +16506,12 @@ dependencies = [
[[package]]
name = "snowbridge-verification-primitives"
version = "0.3.0"
version = "0.3.1"
dependencies = [
"frame-support",
"parity-scale-codec",
"scale-info",
"snowbridge-beacon-primitives 0.3.0",
"snowbridge-beacon-primitives 0.3.1",
"sp-core",
"sp-std",
]

View file

@ -5,7 +5,7 @@ edition = "2021"
homepage = "https://datahaven.xyz/"
license = "GPL-3"
repository = "https://github.com/datahavenxyz/datahaven"
version = "0.3.0"
version = "0.3.1"
[workspace]
members = [

25
operator/Dockerfile.local Normal file
View file

@ -0,0 +1,25 @@
# Dockerfile.local - Fast local image build using pre-built binary
# Usage: docker build -f Dockerfile.local -t datahaven:local .
# Prerequisite: cargo build --release (or --release --features fast-runtime)
FROM docker.io/parity/base-bin:latest
# Copy the pre-built binary from local target directory
COPY ./target/release/datahaven-node /usr/local/bin/datahaven-node
# Make sure binary is executable
RUN chmod +x /usr/local/bin/datahaven-node
USER root
RUN useradd -m -u 1001 -U -s /bin/sh -d /datahaven datahaven && \
mkdir -p /data /datahaven/.local/share && \
chown -R datahaven:datahaven /data && \
ln -s /data /datahaven/.local/share/datahaven && \
/usr/local/bin/datahaven-node --version
USER datahaven
EXPOSE 30333 9933 9944 9615
VOLUME ["/data"]
ENTRYPOINT ["/usr/local/bin/datahaven-node"]

View file

@ -184,7 +184,7 @@ impl pallet_storage_providers::Config for Runtime {
type SpMinCapacity = ConstU64<2>;
type DepositPerData = ConstU128<2>;
type MaxFileSize = ConstU64<{ u64::MAX }>;
type MaxMultiAddressSize = ConstU32<100>;
type MaxMultiAddressSize = ConstU32<200>;
type MaxMultiAddressAmount = ConstU32<5>;
type MaxProtocols = ConstU32<100>;
type BucketDeposit = BucketDeposit;

View file

@ -123,7 +123,7 @@ pub const VERSION: RuntimeVersion = RuntimeVersion {
// `spec_version`, and `authoring_version` are the same between Wasm and native.
// This value is set to 200 to notify Polkadot-JS App (https://polkadot.js.org/apps) to use
// the compatible custom types.
spec_version: 300,
spec_version: 310,
impl_version: 1,
apis: RUNTIME_API_VERSIONS,
transaction_version: 1,

View file

@ -184,7 +184,7 @@ impl pallet_storage_providers::Config for Runtime {
type SpMinCapacity = ConstU64<2>;
type DepositPerData = ConstU128<2>;
type MaxFileSize = ConstU64<{ u64::MAX }>;
type MaxMultiAddressSize = ConstU32<100>;
type MaxMultiAddressSize = ConstU32<200>;
type MaxMultiAddressAmount = ConstU32<5>;
type MaxProtocols = ConstU32<100>;
type BucketDeposit = BucketDeposit;

View file

@ -126,7 +126,7 @@ pub const VERSION: RuntimeVersion = RuntimeVersion {
// `spec_version`, and `authoring_version` are the same between Wasm and native.
// This value is set to 200 to notify Polkadot-JS App (https://polkadot.js.org/apps) to use
// the compatible custom types.
spec_version: 300,
spec_version: 310,
impl_version: 1,
apis: RUNTIME_API_VERSIONS,
transaction_version: 1,

View file

@ -205,7 +205,7 @@ pub mod dynamic_params {
/// volunteered BSPs is ~1%.
#[codec(index = 23)]
#[allow(non_upper_case_globals)]
pub static BasicReplicationTarget: ReplicationTargetType = 7;
pub static BasicReplicationTarget: ReplicationTargetType = 1;
/// The amount of BSPs that a standard security storage request should use as the replication target.
///
@ -214,7 +214,7 @@ pub mod dynamic_params {
/// volunteered BSPs is ~0.1%.
#[codec(index = 24)]
#[allow(non_upper_case_globals)]
pub static StandardReplicationTarget: ReplicationTargetType = 12;
pub static StandardReplicationTarget: ReplicationTargetType = 2;
/// The amount of BSPs that a high security storage request should use as the replication target.
///
@ -223,7 +223,7 @@ pub mod dynamic_params {
/// volunteered BSPs is ~0.01%.
#[codec(index = 25)]
#[allow(non_upper_case_globals)]
pub static HighSecurityReplicationTarget: ReplicationTargetType = 17;
pub static HighSecurityReplicationTarget: ReplicationTargetType = 3;
/// The amount of BSPs that a super high security storage request should use as the replication target.
///
@ -232,7 +232,7 @@ pub mod dynamic_params {
/// volunteered BSPs is ~0.001%.
#[codec(index = 26)]
#[allow(non_upper_case_globals)]
pub static SuperHighSecurityReplicationTarget: ReplicationTargetType = 22;
pub static SuperHighSecurityReplicationTarget: ReplicationTargetType = 4;
/// The amount of BSPs that an ultra high security storage request should use as the replication target.
///
@ -241,7 +241,7 @@ pub mod dynamic_params {
/// volunteered BSPs is ~0.0001%.
#[codec(index = 27)]
#[allow(non_upper_case_globals)]
pub static UltraHighSecurityReplicationTarget: ReplicationTargetType = 26;
pub static UltraHighSecurityReplicationTarget: ReplicationTargetType = 5;
/// The maximum amount of BSPs that a user can require a storage request to use as the replication target.
///

View file

@ -184,7 +184,7 @@ impl pallet_storage_providers::Config for Runtime {
type SpMinCapacity = ConstU64<2>;
type DepositPerData = ConstU128<2>;
type MaxFileSize = ConstU64<{ u64::MAX }>;
type MaxMultiAddressSize = ConstU32<100>;
type MaxMultiAddressSize = ConstU32<200>;
type MaxMultiAddressAmount = ConstU32<5>;
type MaxProtocols = ConstU32<100>;
type BucketDeposit = BucketDeposit;

View file

@ -122,7 +122,7 @@ pub const VERSION: RuntimeVersion = RuntimeVersion {
// `spec_version`, and `authoring_version` are the same between Wasm and native.
// This value is set to 200 to notify Polkadot-JS App (https://polkadot.js.org/apps) to use
// the compatible custom types.
spec_version: 300,
spec_version: 310,
impl_version: 1,
apis: RUNTIME_API_VERSIONS,
transaction_version: 1,

Binary file not shown.

View file

@ -9,6 +9,7 @@ import { deployDataHavenSolochain } from "./datahaven";
import { deployKurtosis } from "./kurtosis";
import { setParametersFromCollection } from "./parameters";
import { deployRelayers } from "./relayer";
import { deployStorageHubComponents } from "./storagehub";
import { performValidatorOperations } from "./validator";
// Non-optional properties determined by having default values
@ -39,6 +40,7 @@ export interface DeployOptions {
skipValidatorOperations: boolean;
skipSetParameters: boolean;
skipRelayers: boolean;
skipStorageHub: boolean;
}
const deployFunction = async (options: DeployOptions, launchedNetwork: LaunchedNetwork) => {
@ -82,6 +84,8 @@ const deployFunction = async (options: DeployOptions, launchedNetwork: LaunchedN
await deployRelayers(options, launchedNetwork);
await deployStorageHubComponents(options, launchedNetwork);
// Cleaning up the port forwarding for the validator.
await validatorPortForwardCleanup();

View file

@ -0,0 +1,216 @@
import path from "node:path";
import { $ } from "bun";
import invariant from "tiny-invariant";
import { logger, printDivider, printHeader } from "utils";
import { waitFor } from "utils/waits";
import { isNetworkReady } from "../../../launcher/datahaven";
import type { LaunchedNetwork } from "../../../launcher/types/launchedNetwork";
import { forwardPort } from "../common/kubernetes";
import type { DeployOptions } from ".";
/**
* Deploys StorageHub components (MSP, BSP, Indexer, Fisherman nodes and databases) in a Kubernetes namespace.
*
* @param options - Configuration options for launching the network.
* @param launchedNetwork - An instance of LaunchedNetwork to track the network's state.
* @returns A promise that resolves when all StorageHub components are deployed.
*/
export const deployStorageHubComponents = async (
options: DeployOptions,
launchedNetwork: LaunchedNetwork
): Promise<void> => {
if (options.skipStorageHub) {
logger.info("🏳️ Skipping StorageHub components deployment");
printDivider();
return;
}
printHeader("Deploying StorageHub Components");
invariant(options.datahavenImageTag, "❌ DataHaven image tag not defined");
if (!options.dockerUsername) {
await checkTagExists(options.datahavenImageTag);
}
// Deploy StorageHub Indexer database first (Indexer PostgreSQL database)
await deployStorageHubDatabase(options, launchedNetwork);
// Deploy StorageHub nodes (MSP, BSP, Indexer, Fisherman)
await deployStorageHubNodes(options, launchedNetwork);
// Deploy StorageHub MSP Backend API
await deployStorageHubBackend(options, launchedNetwork);
await registerStorageHubNodes(launchedNetwork);
printDivider();
};
/**
* Deploys StorageHub PostgreSQL databases for Indexer and Fisherman nodes.
*/
const deployStorageHubDatabase = async (
options: DeployOptions,
launchedNetwork: LaunchedNetwork
): Promise<void> => {
logger.info("🗄️ Deploying StorageHub PostgreSQL database...");
const deployDatabase = async (name: string, component: string) => {
const timeout = "3m";
const args = [
"upgrade",
"--install",
name,
"oci://registry-1.docker.io/bitnamicharts/postgresql",
"-f",
`environments/${options.environment}/${component}-db.yaml`,
"-n",
launchedNetwork.kubeNamespace,
"--wait",
"--timeout",
timeout
];
logger.info(`📦 Deploying ${name} database...`);
logger.debug(await $`helm ${args}`.cwd(path.join(process.cwd(), "../deploy")).text());
logger.success(`${name} database deployed successfully`);
};
// Deploy Indexer database
await deployDatabase("sh-indexer-db", "sh-idxnode");
};
/**
* Deploys StorageHub nodes (MSP, BSP, Indexer, Fisherman).
*/
const deployStorageHubNodes = async (
options: DeployOptions,
launchedNetwork: LaunchedNetwork
): Promise<void> => {
logger.info("🚀 Deploying StorageHub nodes...");
const deployNode = async (name: string, component: string) => {
const timeout = "5m";
const args = [
"upgrade",
"--install",
name,
"charts/node",
"-f",
`charts/node/storagehub/${component}.yaml`,
"-f",
`environments/${options.environment}/${component}.yaml`,
"-n",
launchedNetwork.kubeNamespace,
"--wait",
"--timeout",
timeout
];
logger.info(`🏗️ Deploying ${name}...`);
logger.debug(await $`helm ${args}`.cwd(path.join(process.cwd(), "../deploy")).text());
logger.success(`${name} deployed successfully`);
};
// Deploy StorageHub nodes in dependency order
await deployNode("sh-mspnode", "sh-mspnode");
await deployNode("sh-bspnode", "sh-bspnode");
await deployNode("sh-idxnode", "sh-idxnode");
await deployNode("sh-fisherman", "sh-fisherman");
};
/**
* Deploys StorageHub MSP Backend API.
*/
const deployStorageHubBackend = async (
options: DeployOptions,
launchedNetwork: LaunchedNetwork
): Promise<void> => {
logger.info("🚀 Deploying StorageHub MSP Backend API...");
const timeout = "3m";
const args = [
"upgrade",
"--install",
"sh-mspbackend",
"charts/backend",
"-f",
"charts/backend/storagehub/sh-mspbackend.yaml",
"-f",
`environments/${options.environment}/sh-mspbackend.yaml`,
"-n",
launchedNetwork.kubeNamespace,
"--wait",
"--timeout",
timeout
];
logger.debug(await $`helm ${args}`.cwd(path.join(process.cwd(), "../deploy")).text());
logger.success("StorageHub MSP Backend API deployed successfully");
};
/**
* Waits for StorageHub Indexer node to be ready and registers nodes in LaunchedNetwork.
*/
const registerStorageHubNodes = async (launchedNetwork: LaunchedNetwork): Promise<void> => {
// Forward port from indexer node to localhost for health checks
const indexerPort = 9944;
const { cleanup: indexerPortForwardCleanup } = await forwardPort(
"sh-idxnode-0",
indexerPort,
indexerPort + 100, // Use different local port to avoid conflicts
launchedNetwork
);
// Wait for the StorageHub Indexer to start
logger.info("⌛️ Waiting for StorageHub Indexer to start...");
const timeoutMs = 5000; // 5 second timeout
const delayMs = 5000; // 5 second delay between iterations
await waitFor({
lambda: async () => {
logger.info(`📡 Checking if StorageHub Indexer is ready (timeout: ${timeoutMs / 1000}s)...`);
const isReady = await isNetworkReady(indexerPort + 100, timeoutMs);
if (!isReady) {
logger.info(
`⌛️ StorageHub Indexer not ready, waiting ${delayMs / 1000}s to check again...`
);
}
return isReady;
},
iterations: 12, // 12 iterations of 5 + 5 = 2 minutes
delay: delayMs,
errorMessage: "StorageHub Indexer not ready"
});
logger.success("StorageHub Indexer is ready");
// Clean up the port forwarding
await indexerPortForwardCleanup();
// Register StorageHub nodes in LaunchedNetwork
launchedNetwork.addContainer("sh-mspnode-0", { ws: 9944 });
launchedNetwork.addContainer("sh-bspnode-0", { ws: 9944 });
launchedNetwork.addContainer("sh-idxnode-0", { ws: 9944 });
launchedNetwork.addContainer("sh-fisherman-0", { ws: 9944 });
logger.info("📝 StorageHub nodes successfully registered in launchedNetwork.");
};
/**
* Checks if an image exists in Docker Hub.
*
* @param tag - The tag of the image to check.
* @returns A promise that resolves when the image is found.
*/
const checkTagExists = async (tag: string) => {
const cleanTag = tag.trim();
logger.debug(`Checking if image ${cleanTag} is available on Docker Hub`);
const result = await $`docker manifest inspect ${cleanTag}`.nothrow().quiet();
invariant(
result.exitCode === 0,
`❌ Image ${tag} not found.\n Does this image exist?\n Are you logged and have access to the repository?`
);
logger.success(`Image ${cleanTag} found on Docker Hub`);
};

View file

@ -10,6 +10,7 @@ import { launchDataHavenSolochain } from "./datahaven";
import { launchKurtosis } from "./kurtosis";
import { setParametersFromCollection } from "./parameters";
import { launchRelayers } from "./relayer";
import { launchStorageHubComponents } from "./storagehub";
import { performSummaryOperations } from "./summary";
import { performValidatorOperations } from "./validator";
@ -44,6 +45,7 @@ export interface LaunchOptions {
setParameters?: boolean;
relayer?: boolean;
relayerImageTag: string;
storagehub?: boolean;
cleanNetwork?: boolean;
injectContracts?: boolean;
}
@ -155,6 +157,8 @@ const launchFunction = async (options: LaunchOptions, launchedNetwork: LaunchedN
await launchRelayers(options, launchedNetwork);
await launchStorageHubComponents(options, launchedNetwork);
await performSummaryOperations(options, launchedNetwork);
const fullEnd = performance.now();
const fullMinutes = ((fullEnd - timeStart) / (1000 * 60)).toFixed(1);
@ -181,7 +185,8 @@ export const launchPreActionHook = (
launchKurtosis,
relayer,
setParameters,
injectContracts
injectContracts,
storagehub
} = thisCmd.opts();
// Check for conflicts with --all flag
@ -194,10 +199,11 @@ export const launchPreActionHook = (
fundValidators === false ||
setupValidators === false ||
setParameters === false ||
relayer === false)
relayer === false ||
storagehub === false)
) {
thisCmd.error(
"--all cannot be used with --no-datahaven, --no-build-datahaven, --no-launch-kurtosis, --no-deploy-contracts, --no-fund-validators, --no-setup-validators, --no-update-validator-set, --no-set-parameters, or --no-relayer"
"--all cannot be used with --no-datahaven, --no-build-datahaven, --no-launch-kurtosis, --no-deploy-contracts, --no-fund-validators, --no-setup-validators, --no-update-validator-set, --no-set-parameters, --no-relayer, or --no-storagehub"
);
}
@ -211,6 +217,7 @@ export const launchPreActionHook = (
thisCmd.setOptionValue("setupValidators", true);
thisCmd.setOptionValue("setParameters", true);
thisCmd.setOptionValue("relayer", true);
thisCmd.setOptionValue("storagehub", true);
thisCmd.setOptionValue("cleanNetwork", true);
}

View file

@ -0,0 +1,36 @@
import { logger, printHeader } from "utils";
import type { LaunchedNetwork } from "../../../launcher/types/launchedNetwork";
import { deployStorageHubComponents } from "../deploy/storagehub";
import type { LaunchOptions } from ".";
/**
* Launches StorageHub components by delegating to the deploy function.
*
* @param options - Launch options.
* @param launchedNetwork - The launched network instance.
* @returns A promise that resolves when StorageHub components are launched.
*/
export const launchStorageHubComponents = async (
options: LaunchOptions,
launchedNetwork: LaunchedNetwork
): Promise<void> => {
// Convert launch options to deploy options format
const deployOptions = {
environment: "local" as const, // Launch is typically used for local development
skipStorageHub: !options.storagehub,
datahavenImageTag: options.datahavenImageTag,
dockerUsername: undefined,
dockerPassword: undefined,
dockerEmail: undefined
};
printHeader("Launching StorageHub Components");
logger.info(
"🚀 Launching StorageHub components (MSP, BSP, Indexer, Fisherman nodes and databases)..."
);
// Reuse the deploy StorageHub function
await deployStorageHubComponents(deployOptions as any, launchedNetwork);
logger.success("StorageHub components launched successfully");
};

View file

@ -49,7 +49,7 @@ program
`🫎 DataHaven: Network Deployer CLI for deploying a full DataHaven network stack to a Kubernetes cluster
It will deploy:
- DataHaven solochain validators (all envs),
- Storage providers (all envs) (TODO),
- StorageHub components: MSP, BSP, Indexer, Fisherman nodes and databases (local & stagenet envs),
- Kurtosis Ethereum private network (stagenet env),
- Snowbridge Relayers (all envs)
`
@ -98,6 +98,11 @@ program
.option("--skip-validator-operations", "Skip performing validator operations", false)
.option("--skip-set-parameters", "Skip setting DataHaven runtime parameters", false)
.option("--skip-relayers", "Skip deploying Snowbridge Relayers", false)
.option(
"--skip-storage-hub",
"Skip deploying StorageHub components (MSP, BSP, Indexer, Fisherman, databases)",
false
)
.hook("preAction", deployPreActionHook)
.action(deploy);
@ -109,7 +114,7 @@ program
`🫎 DataHaven: Network Launcher CLI for launching a full DataHaven network.
Complete with:
- Solo-chain validators,
- Storage providers (TODO),
- StorageHub components: MSP, BSP, Indexer, Fisherman nodes and databases,
- Ethereum Private network,
- Snowbridge Relayers
`

View file

@ -1,105 +0,0 @@
import { describe, expect, it } from "bun:test";
import { logger, parseDeploymentsFile } from "utils";
import { BaseTestSuite } from "../framework";
class ContractsTestSuite extends BaseTestSuite {
constructor() {
super({
suiteName: "contracts"
});
this.setupHooks();
}
}
// Create the test suite instance
const suite = new ContractsTestSuite();
describe("Smart Contract Interactions", () => {
it("should query contract deployment addresses", async () => {
const _connectors = suite.getTestConnectors();
const deployments = await parseDeploymentsFile();
// Check that we have basic contract addresses
expect(deployments.BeefyClient).toBeDefined();
expect(deployments.Gateway).toBeDefined();
expect(deployments.ServiceManager).toBeDefined();
logger.info(`BeefyClient deployed at: ${deployments.BeefyClient}`);
logger.info(`Gateway deployed at: ${deployments.Gateway}`);
logger.info(`ServiceManager deployed at: ${deployments.ServiceManager}`);
});
it("should check contract code exists", async () => {
const connectors = suite.getTestConnectors();
const deployments = await parseDeploymentsFile();
// Get deployment transaction receipt for BeefyClient
const code = await connectors.publicClient.getCode({
address: deployments.BeefyClient as `0x${string}`
});
expect(code).toBeDefined();
expect(code?.length).toBeGreaterThan(2); // More than just "0x"
logger.info(`BeefyClient contract code size: ${code?.length} bytes`);
});
it("should check contract balances", async () => {
const connectors = suite.getTestConnectors();
const deployments = await parseDeploymentsFile();
// Check ETH balance of contracts
const beefyBalance = await connectors.publicClient.getBalance({
address: deployments.BeefyClient as `0x${string}`
});
const serviceManagerBalance = await connectors.publicClient.getBalance({
address: deployments.ServiceManager as `0x${string}`
});
logger.info(`BeefyClient ETH balance: ${beefyBalance}`);
logger.info(`ServiceManager ETH balance: ${serviceManagerBalance}`);
// Contracts typically start with 0 balance
expect(beefyBalance).toBeGreaterThanOrEqual(0n);
expect(serviceManagerBalance).toBeGreaterThanOrEqual(0n);
});
it("should verify contract addresses are valid", async () => {
const connectors = suite.getTestConnectors();
const deployments = await parseDeploymentsFile();
// List of expected contracts
const expectedContracts = [
"BeefyClient",
"ServiceManager",
"RewardsRegistry",
"AVSDirectory",
"DelegationManager",
"StrategyManager"
];
for (const contractName of expectedContracts) {
const address = deployments[contractName as keyof typeof deployments];
if (address && typeof address === "string") {
// Verify it's a valid address format
expect(address.startsWith("0x")).toBeTrue();
expect(address.length).toBe(42);
// Verify contract exists (has code)
const code = await connectors.publicClient.getCode({
address: address as `0x${string}`
});
expect(code).toBeDefined();
expect(code?.length).toBeGreaterThan(2);
logger.info(`${contractName} deployed at ${address}`);
} else {
logger.warn(`⚠️ ${contractName} not found in deployments`);
}
}
});
});

View file

@ -1,108 +0,0 @@
import { beforeAll, describe, expect, it } from "bun:test";
import type { PolkadotSigner } from "polkadot-api";
import { getPapiSigner, logger, SUBSTRATE_FUNDED_ACCOUNTS } from "utils";
import { BaseTestSuite } from "../framework";
class CrossChainTestSuite extends BaseTestSuite {
constructor() {
super({
suiteName: "cross-chain"
});
this.setupHooks();
}
override async onSetup(): Promise<void> {
// Relayers initialization is handled by the network setup
logger.info("Cross-chain test setup complete");
}
}
// Create the test suite instance
const suite = new CrossChainTestSuite();
describe("Cross-Chain Communication", () => {
let _signer: PolkadotSigner;
beforeAll(() => {
_signer = getPapiSigner();
});
it("should query Ethereum client state on DataHaven", async () => {
const connectors = suite.getTestConnectors();
// Check basic chain connectivity
const blockNumber = await connectors.papiClient.getBlockHeader();
logger.info(`Connected to DataHaven at block: ${blockNumber.number}`);
expect(blockNumber.number).toBeGreaterThan(0);
});
it("should check beacon relayer status", async () => {
const connectors = suite.getTestConnectors();
// Check if we can access chain state
try {
const blockHash = await connectors.papiClient.getFinalizedBlock();
logger.info(`Finalized block hash: ${blockHash}`);
expect(blockHash).toBeDefined();
} catch (_error) {
logger.warn("Unable to get finalized block - relayers may still be syncing");
}
});
it("should verify validator registry connection", async () => {
const connectors = suite.getTestConnectors();
// For now, just check that we can connect
// The specific storage items depend on the runtime configuration
const blockNumber = await connectors.papiClient.getBlockHeader();
logger.info(`Current block number: ${blockNumber.number}`);
expect(blockNumber.number).toBeGreaterThan(0);
});
it("should check system information", async () => {
const connectors = suite.getTestConnectors();
// Query basic system information
const blockNumber = await connectors.dhApi.query.System.Number.getValue();
const parentHash = await connectors.dhApi.query.System.ParentHash.getValue();
logger.info(`Current block: ${blockNumber}`);
logger.info(`Parent hash: ${parentHash}`);
expect(blockNumber).toBeGreaterThan(0);
expect(parentHash).toBeDefined();
});
it("should query ethereum client pallet", async () => {
const connectors = suite.getTestConnectors();
// Check if we can access account info
const accountInfo = await connectors.dhApi.query.System.Account.getValue(
SUBSTRATE_FUNDED_ACCOUNTS.ALITH.publicKey
);
logger.info(`Account nonce: ${accountInfo.nonce}`);
logger.info(`Account providers: ${accountInfo.providers}`);
expect(accountInfo.providers).toBeGreaterThan(0);
});
it("should check BEEFY consensus status", async () => {
const connectors = suite.getTestConnectors();
// Query BEEFY validator set
const validatorSet = await connectors.papiClient.getUnsafeApi().apis.BeefyApi.validator_set();
if (validatorSet) {
logger.info(`BEEFY validator set ID: ${validatorSet.id}`);
logger.info(`BEEFY validator count: ${validatorSet.validators.length}`);
expect(validatorSet.validators.length).toBeGreaterThan(0);
} else {
logger.warn("BEEFY validator set not yet available");
}
});
});

View file

@ -1,93 +0,0 @@
import { beforeAll, describe, expect, it } from "bun:test";
import type { PolkadotSigner } from "polkadot-api";
import {
getPapiSigner,
isValidatorNodeRunning,
launchDatahavenValidator,
logger,
SUBSTRATE_FUNDED_ACCOUNTS,
TestAccounts
} from "utils";
import { isAddress } from "viem";
import { BaseTestSuite } from "../framework";
class DataHavenSubstrateTestSuite extends BaseTestSuite {
constructor() {
super({
suiteName: "datahaven-substrate"
});
this.setupHooks();
}
override async onSetup(): Promise<void> {
await launchDatahavenValidator(TestAccounts.Charlie, {
launchedNetwork: this.getConnectors().launchedNetwork
});
}
public getNetworkId(): string {
return this.getConnectors().launchedNetwork.networkId;
}
}
// Create the test suite instance
const suite = new DataHavenSubstrateTestSuite();
describe("DataHaven Substrate Operations", () => {
let _signer: PolkadotSigner;
beforeAll(() => {
_signer = getPapiSigner();
});
it("should query runtime API", async () => {
const connectors = suite.getTestConnectors();
const address = await connectors.dhApi.apis.EthereumRuntimeRPCApi.author();
logger.info(`Author address: ${address.asHex()}`);
expect(isAddress(address.asHex())).toBeTrue();
});
it("should lookup account balance", async () => {
const connectors = suite.getTestConnectors();
const {
data: { free: freeBalance }
} = await connectors.dhApi.query.System.Account.getValue(
SUBSTRATE_FUNDED_ACCOUNTS.ALITH.publicKey
);
logger.info(`Balance of ALITH: ${freeBalance}`);
expect(freeBalance).toBeGreaterThan(0n);
});
it("should listen to events", async () => {
const connectors = suite.getTestConnectors();
// Pull next ExtrinsicSuccess event
const event = await connectors.dhApi.event.System.ExtrinsicSuccess.pull();
expect(event).not.toBeEmpty();
expect(event[0].payload.dispatch_info.weight.ref_time).toBeGreaterThan(0n);
logger.info(
`Caught ExtrinsicSuccess event with weight: ${event[0].payload.dispatch_info.weight.ref_time}`
);
});
it("should query block information", async () => {
const connectors = suite.getTestConnectors();
// Get current block
const blockHeader = await connectors.papiClient.getBlockHeader();
expect(blockHeader.number).toBeGreaterThan(0);
logger.info(`Current block #${blockHeader.number}`);
});
it("should see Charlie running", async () => {
const isRunning = await isValidatorNodeRunning(TestAccounts.Charlie, suite.getNetworkId());
expect(isRunning).toBe(true);
});
});

View file

@ -1,173 +0,0 @@
import { describe, expect, it } from "bun:test";
import { ANVIL_FUNDED_ACCOUNTS, generateRandomAccount, logger } from "utils";
import { parseEther } from "viem";
import { BaseTestSuite } from "../framework";
class EthereumBasicTestSuite extends BaseTestSuite {
constructor() {
super({
suiteName: "ethereum-basic"
});
// Set up hooks in constructor
this.setupHooks();
}
}
// Create the test suite instance
const suite = new EthereumBasicTestSuite();
describe("Ethereum Basic Operations", () => {
it("should query block number", async () => {
const connectors = suite.getTestConnectors();
const blockNumber = await connectors.publicClient.getBlockNumber();
expect(blockNumber).toBeGreaterThan(0n);
logger.info(`Current block number: ${blockNumber}`);
});
it("should check funded account balance", async () => {
const connectors = suite.getTestConnectors();
const balance = await connectors.publicClient.getBalance({
address: ANVIL_FUNDED_ACCOUNTS[0].publicKey
});
expect(balance).toBeGreaterThan(parseEther("1"));
logger.info(`Account balance: ${balance} wei`);
});
it("should send ETH transaction", async () => {
const connectors = suite.getTestConnectors();
const amount = parseEther("1");
const randomAccount = generateRandomAccount();
// Check initial balance
const balanceBefore = await connectors.publicClient.getBalance({
address: randomAccount.address
});
expect(balanceBefore).toBe(0n);
// Check balance of the sender
const balance = await connectors.publicClient.getBalance({
address: connectors.walletClient.account.address
});
expect(balance).toBeGreaterThan(amount);
// Send transaction
if (!connectors.walletClient.account) {
throw new Error("Wallet client account not available");
}
const hash = await connectors.walletClient.sendTransaction({
account: connectors.walletClient.account,
chain: null,
to: randomAccount.address as `0x${string}`,
value: amount
});
// Wait for receipt
const receipt = await connectors.publicClient.waitForTransactionReceipt({ hash });
expect(receipt.status).toBe("success");
// Check final balance
const balanceAfter = await connectors.publicClient.getBalance({
address: randomAccount.address
});
expect(balanceAfter).toBe(amount);
logger.info(`Successfully sent ${amount} wei to ${randomAccount.address}`);
});
it("should interact with multiple accounts", async () => {
const connectors = suite.getTestConnectors();
const factory = suite.getConnectorFactory();
// Create wallet clients for multiple accounts
const wallet1 = factory.createWalletClient(ANVIL_FUNDED_ACCOUNTS[1].privateKey);
const wallet2 = factory.createWalletClient(ANVIL_FUNDED_ACCOUNTS[2].privateKey);
const recipient = generateRandomAccount();
const amount = parseEther("0.5");
// Fund wallet1 and wallet2 with 1ETH to successfully send transaction
const initialAmount = parseEther("1");
// Give 1ETH to wallet1
const hashInit1 = await connectors.walletClient.sendTransaction({
account: connectors.walletClient.account,
chain: null,
to: wallet1.account.address as `0x${string}`,
value: initialAmount
});
// Wait for receipt
const receiptInit1 = await connectors.publicClient.waitForTransactionReceipt({
hash: hashInit1
});
expect(receiptInit1.status).toBe("success");
const balance1 = await connectors.publicClient.getBalance({
address: wallet1.account.address
});
expect(balance1).toBeGreaterThan(parseEther("1"));
// Give 1ETH to wallet2
const hashInit2 = await connectors.walletClient.sendTransaction({
account: connectors.walletClient.account,
chain: null,
to: wallet2.account.address as `0x${string}`,
value: initialAmount
});
// Wait for receipt
const receiptInit2 = await connectors.publicClient.waitForTransactionReceipt({
hash: hashInit2
});
expect(receiptInit2.status).toBe("success");
const balance2 = await connectors.publicClient.getBalance({
address: wallet2.account.address
});
expect(balance2).toBeGreaterThan(parseEther("1"));
// Send from account 1
if (!wallet1.account) {
throw new Error("Wallet1 account not available");
}
const hash1 = await wallet1.sendTransaction({
account: wallet1.account,
chain: null,
to: recipient.address as `0x${string}`,
value: amount
});
// Send from account 2
if (!wallet2.account) {
throw new Error("Wallet2 account not available");
}
const hash2 = await wallet2.sendTransaction({
account: wallet2.account,
chain: null,
to: recipient.address as `0x${string}`,
value: amount
});
// Wait for both transactions
const [receipt1, receipt2] = await Promise.all([
connectors.publicClient.waitForTransactionReceipt({ hash: hash1 }),
connectors.publicClient.waitForTransactionReceipt({ hash: hash2 })
]);
expect(receipt1.status).toBe("success");
expect(receipt2.status).toBe("success");
// Check final balance
const finalBalance = await connectors.publicClient.getBalance({
address: recipient.address
});
expect(finalBalance).toBe(amount * 2n);
logger.info(`Received total of ${finalBalance} wei from multiple accounts`);
}, 20_000);
});