Elgato_dark/hyperdx
1
0
Fork 0
mirror of https://github.com/hyperdxio/hyperdx synced 2026-04-21 13:37:15 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

fix: cache the result conditionally (SimpleCache) (#286)

Browse source 
In most of cases, we don't want to cache the result if its empty (because the data hasnt been ingested yet)
This commit is contained in:
Warren 2024-01-30 19:20:10 -08:00 committed by GitHub
parent 9a026e3304
commit a49726ee58
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 107 additions and 57 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
.changeset/thin-toes-smile.md Normal file
View file

@ -0,0 +1,6 @@
---
'@hyperdx/api': patch
'@hyperdx/app': patch
---
fix: cache the result conditionally (SimpleCache)

89
packages/api/src/routers/api/chart.ts
View file

@ -49,45 +49,56 @@ router.get('/services', async (req, res, next) => {
const simpleCache = new SimpleCache<
Awaited<ReturnType<typeof clickhouse.getMultiSeriesChart>>[]
>(`chart-services-${teamId}`, ms('10m'), () =>
Promise.all([
clickhouse.getMultiSeriesChart({
series: [
{
aggFn: clickhouse.AggFn.Count,
groupBy: targetGroupByFields,
table: 'logs',
type: 'table',
where: '',
},
],
endTime,
granularity: undefined,
maxNumGroups: MAX_NUM_GROUPS,
startTime,
tableVersion: team.logStreamTableVersion,
teamId: teamId.toString(),
seriesReturnType: clickhouse.SeriesReturnType.Column,
}),
clickhouse.getMultiSeriesChart({
series: [
{
aggFn: clickhouse.AggFn.Count,
groupBy: ['service'],
table: 'logs',
type: 'table',
where: '',
},
],
endTime,
granularity: undefined,
maxNumGroups: MAX_NUM_GROUPS,
startTime,
tableVersion: team.logStreamTableVersion,
teamId: teamId.toString(),
seriesReturnType: clickhouse.SeriesReturnType.Column,
}),
]),
>(
`chart-services-${teamId}`,
ms('10m'),
() =>
Promise.all([
clickhouse.getMultiSeriesChart({
series: [
{
aggFn: clickhouse.AggFn.Count,
groupBy: targetGroupByFields,
table: 'logs',
type: 'table',
where: '',
},
],
endTime,
granularity: undefined,
maxNumGroups: MAX_NUM_GROUPS,
startTime,
tableVersion: team.logStreamTableVersion,
teamId: teamId.toString(),
seriesReturnType: clickhouse.SeriesReturnType.Column,
}),
clickhouse.getMultiSeriesChart({
series: [
{
aggFn: clickhouse.AggFn.Count,
groupBy: ['service'],
table: 'logs',
type: 'table',
where: '',
},
],
endTime,
granularity: undefined,
maxNumGroups: MAX_NUM_GROUPS,
startTime,
tableVersion: team.logStreamTableVersion,
teamId: teamId.toString(),
seriesReturnType: clickhouse.SeriesReturnType.Column,
}),
]),
results => {
for (const result of results) {
if (result.rows != null && result.rows > 0) {
return true;
}
}
return false;
},
);
const [customAttrsResults, servicesResults] = await simpleCache.get();

23
packages/api/src/routers/api/metrics.ts
View file

@ -19,13 +19,22 @@ router.get('/tags', async (req, res, next) => {
const nowInMs = Date.now();
const simpleCache = new SimpleCache<
Awaited<ReturnType<typeof clickhouse.getMetricsTags>>
>(`metrics-tags-${teamId}`, ms('10m'), () =>
clickhouse.getMetricsTags({
// FIXME: fix it 5 days ago for now
startTime: nowInMs - ms('5d'),
endTime: nowInMs,
teamId: teamId.toString(),
}),
>(
`metrics-tags-${teamId}`,
ms('10m'),
() =>
clickhouse.getMetricsTags({
// FIXME: fix it 5 days ago for now
startTime: nowInMs - ms('5d'),
endTime: nowInMs,
teamId: teamId.toString(),
}),
result => {
if (result.rows != null) {
return result.rows > 0;
}
return false;
},
);
res.json(await simpleCache.get());
} catch (e) {

23
packages/api/src/routers/external-api/v1/index.ts
View file

@ -195,13 +195,22 @@ router.get(
const nowInMs = Date.now();
const simpleCache = new SimpleCache<
Awaited<ReturnType<typeof clickhouse.getMetricsTags>>
>(`metrics-tags-${teamId}`, ms('10m'), () =>
clickhouse.getMetricsTags({
// FIXME: fix it 5 days ago for now
startTime: nowInMs - ms('5d'),
endTime: nowInMs,
teamId: teamId.toString(),
}),
>(
`metrics-tags-${teamId}`,
ms('10m'),
() =>
clickhouse.getMetricsTags({
// FIXME: fix it 5 days ago for now
startTime: nowInMs - ms('5d'),
endTime: nowInMs,
teamId: teamId.toString(),
}),
result => {
if (result.rows != null) {
return result.rows > 0;
}
return false;
},
);
const tags = await simpleCache.get();
res.json({

23
packages/api/src/utils/redis.ts
View file

@ -12,13 +12,31 @@ client.on('error', (err: any) => {
logger.error('Redis error: ', serializeError(err));
});
// TODO: add tests
class SimpleCache<T> {
constructor(
private readonly key: string,
private readonly ttlInMs: number,
private readonly fetcher: () => Promise<T>,
private readonly shouldRefreshOnResult: (result: T) => boolean = () => true,
) {}
async refresh() {
const dt = Date.now();
const result = await this.fetcher();
if (this.shouldRefreshOnResult(result)) {
logger.info({
message: 'SimpleCache: refresh',
key: this.key,
duration: Date.now() - dt,
});
await client.set(this.key, JSON.stringify(result), {
PX: this.ttlInMs,
});
}
return result;
}
async get(): Promise<T> {
const cached = await client.get(this.key);
if (cached != null) {
@ -32,10 +50,7 @@ class SimpleCache<T> {
message: 'SimpleCache: cache miss',
key: this.key,
});
const result = await this.fetcher();
await client.set(this.key, JSON.stringify(result), {
PX: this.ttlInMs,
});
const result = await this.refresh();
return result;
}
}