datahaven/test/utils/docker.ts
Tim B 82145b882b
test: 🐳 Add docker support for datahaven nodes (#71)
> [!NOTE]  
> This is  `Part 3` of the ongoing _Docker Series._


## New Additions:
- Launching Datahaven network will spin up containers, as opposed to
native binaries
- `stop:docker` script to kill all dh containers
- `e2e` test suite for datahaven solochain network
- Contains reference test file that uses papi for storage queries,
submitting exts, runtime calls (good job on that facu and tobi)
- Added new utils:
  - `waitForLog()` to wait for log lines in docker container logs
- `createPapiConnectors()` helper for test cases to build and connect to
dh network
- `getPapiSigner()` helper to return a papi compatible signer using our
prefunded accounts (alith by default)
- `sendTxn()` helper to submit txn and wait for block inclusion, instead
of finalization, which std library provides

## Changes:

> [!CAUTION] 
> Launching native binaries for datahaven no longer supported.


- Datahaven binary location cli option changed to `-i,
--datahaven-image-tag`
- To locally run this you'll need a datahaven docker image handy, you'll
need to either:
- Point to remote dockerhub e.g. `moonsonglabs/datahaven:main` (must be
logged in and have permission)
  - Build this locally with `bun build:docker:operator`

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

- **New Features**
- Added end-to-end tests for the Datahaven solochain, including runtime
API queries, storage lookups, extrinsic submissions, and event
listening.
- Introduced CLI option to specify the Datahaven Docker image tag, with
a default value.
  - Added CLI option to disable the Relayer.
- Provided new scripts to stop Docker containers associated with
Datahaven.
- Added utility functions for Docker log monitoring and container
startup checks.
- Introduced utilities for interacting with the Datahaven Polkadot API.

- **Improvements**
- Switched Datahaven network launch from local binaries to Docker
containers.
- Enhanced cache accuracy in build workflows by including Rust source
files in cache keys.
- Improved build performance with TypeScript incremental build options.
  - Increased timeout for end-to-end tests for better reliability.
  - Updated CLI version to 0.2.0.
  - Modified Dockerfile build to enable the `fast-runtime` feature.
- Extended network launch summary to include relayer and container
details.

- **Bug Fixes**
- Fixed cleanup logic by tracking and preparing for forced removal of
Docker containers after tests.

- **Chores**
  - Updated workflow steps for Docker image handling and network checks.
- Adjusted scripts and workflow logic for improved Docker and test
management.
- Removed top-level disk usage summaries from cleanup workflow for
streamlined reporting.
- Enhanced shell command utility to support asynchronous wait during
execution.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

---------

Co-authored-by: Facundo Farall <37149322+ffarall@users.noreply.github.com>
2025-05-16 15:17:05 +01:00

174 lines
5.1 KiB
TypeScript

import { type Duplex, PassThrough, Transform } from "node:stream";
import Docker from "dockerode";
import invariant from "tiny-invariant";
import { type ServiceInfo, StandardServiceMappings, logger } from "utils";
const docker = new Docker({});
export const getServicesFromDocker = async (): Promise<ServiceInfo[]> => {
const containers = await docker.listContainers();
const services: ServiceInfo[] = [];
for (const mapping of StandardServiceMappings) {
try {
const container = containers.find((container) =>
container.Names.some((name) => name.includes(mapping.containerPattern))
);
if (!container) {
logger.warn(`Container with pattern "${mapping.containerPattern}" not found.`);
services.push({
service: mapping.service,
port: "Not found",
url: "N/A"
});
continue;
}
const portMappings = container.Ports.filter(
(port) => port.PrivatePort === mapping.internalPort && port.Type === mapping.protocol
);
let selectedMapping = portMappings.find((port) => port.IP === "0.0.0.0" || port.IP === ":::");
if (!selectedMapping && portMappings.length > 0) {
selectedMapping = portMappings[0];
}
if (!selectedMapping || !selectedMapping.PublicPort) {
logger.warn(
`Port mapping not found for ${mapping.service} (${mapping.internalPort}/${mapping.protocol}).`
);
services.push({
service: mapping.service,
port: "Not found",
url: "N/A"
});
continue;
}
services.push({
service: mapping.service,
port: selectedMapping.PublicPort.toString(),
url: `http://127.0.0.1:${selectedMapping.PublicPort}`
});
} catch (error) {
logger.error(`Error getting info for ${mapping.service}:`, error);
services.push({
service: mapping.service,
port: "Error",
url: "N/A"
});
}
}
return services;
};
export const getPublicPort = async (
containerName: string,
internalPort: number
): Promise<number> => {
const docker = new Docker();
const containers = await docker.listContainers();
const container = containers.find((container) =>
container.Names.some((name) => name.includes(containerName))
);
invariant(container, `❌ container ${container} cannot be found in running container list`);
const portMappings = container.Ports.find(
(port) => port.PrivatePort === internalPort && port.Type === "tcp"
);
logger.debug(`Port mappings for ${containerName}:${internalPort}`, portMappings);
invariant(portMappings, `❌ port mapping not found for ${containerName}:${internalPort}`);
return portMappings.PublicPort;
};
export async function waitForLog(opts: {
search: string | RegExp;
containerName: string;
timeoutSeconds?: number;
}): Promise<string> {
const container = docker.getContainer(opts.containerName);
await container.inspect();
const timeoutMs = (opts.timeoutSeconds ?? 10) * 1_000;
const rawStream = (await container.logs({
stdout: true,
stderr: true,
follow: true,
since: 0
})) as Duplex;
const pass = new PassThrough();
container.modem.demuxStream(rawStream, pass, pass);
const { readable } = Transform.toWeb(pass);
const decoder = new TextDecoder();
const timer = setTimeout(
() =>
pass.destroy(
new Error(
`Timed out after ${timeoutMs} ms waiting for “${opts.search}” in ${opts.containerName}`
)
),
timeoutMs
);
try {
for await (const chunk of readable) {
const text = decoder.decode(chunk as Uint8Array, { stream: false });
const hit =
typeof opts.search === "string" ? text.includes(opts.search) : opts.search.test(text);
if (hit) return text.trim();
}
throw new Error(
`Log stream ended before “${opts.search}” appeared for container ${opts.containerName}`
);
} finally {
if (timer) {
clearTimeout(timer);
}
if (pass && typeof pass.destroy === "function" && !pass.destroyed) {
pass.destroy();
}
if (rawStream) {
if (typeof rawStream.destroy === "function" && !rawStream.destroyed) {
rawStream.destroy();
}
const socket = (rawStream as any).socket;
if (socket && typeof socket.destroy === "function" && !socket.destroyed) {
socket.destroy();
}
}
}
}
export const waitForContainerToStart = async (
containerName: string,
options?: { timeoutSeconds?: number }
) => {
logger.debug(`Waiting for container ${containerName} to start...`);
const docker = new Docker();
const seconds = options?.timeoutSeconds ?? 30;
for (let i = 0; i < seconds; i++) {
const containers = await docker.listContainers();
const container = containers.find((container) =>
container.Names.some((name) => name.includes(containerName))
);
if (container) {
logger.debug(`Container ${containerName} started after ${i} seconds`);
return;
}
await Bun.sleep(1000);
}
invariant(
false,
`❌ container ${containerName} cannot be found in running container list after ${seconds} seconds`
);
};