fix(cdn-worker): improve logging and ignore analytics engine errors (#6477)

2026-05-24 01:28:32 +00:00 · 2025-02-06 11:56:36 +01:00 · 2025-02-06 11:56:36 +01:00 · 1b3086facc
commit 1b3086facc
parent 4c2550551e
3 changed files with 114 additions and 68 deletions
--- a/packages/services/cdn-worker/src/analytics.ts
+++ b/packages/services/cdn-worker/src/analytics.ts
@ -89,54 +89,59 @@ export function createAnalytics(
        return;
      }

-      switch (event.type) {
-        case 'artifact':
-          return engines.usage.writeDataPoint({
-            blobs: [event.version, event.value, targetId],
-            indexes: [targetId.substring(0, 32)],
-          });
-        case 'error':
-          return engines.error.writeDataPoint({
-            blobs: event.value,
-          });
-        case 'r2':
-        case 's3':
-          return engines[event.type].writeDataPoint({
-            blobs: [event.action, event.statusCodeOrErrCode.toString(), targetId],
-            doubles: [event.duration],
-            indexes: [targetId.substring(0, 32)],
-          });
-        case 'response':
-          return engines.response.writeDataPoint({
-            blobs: [event.statusCode.toString(), event.requestPath, targetId],
-            indexes: [targetId.substring(0, 32)],
-          });
-        case 'key-validation':
-          switch (event.value.type) {
-            case 'cache-hit':
-              return engines.keyValidation.writeDataPoint({
-                blobs: [
-                  'cache-hit',
-                  event.value.version,
-                  event.value.isValid ? 'valid' : 'invalid',
-                ],
-                indexes: [targetId.substring(0, 32)],
-              });
-            case 'cache-write':
-              return engines.keyValidation.writeDataPoint({
-                blobs: [
-                  'cache-write',
-                  event.value.version,
-                  event.value.isValid ? 'valid' : 'invalid',
-                ],
-                indexes: [targetId.substring(0, 32)],
-              });
-            case 's3-key-validation':
-              return engines.keyValidation.writeDataPoint({
-                blobs: ['s3-key-validation', event.value.version, event.value.status],
-                indexes: [targetId.substring(0, 32)],
-              });
-          }
+      try {
+        switch (event.type) {
+          case 'artifact':
+            return engines.usage.writeDataPoint({
+              blobs: [event.version, event.value, targetId],
+              indexes: [targetId.substring(0, 32)],
+            });
+          case 'error':
+            return engines.error.writeDataPoint({
+              blobs: event.value,
+            });
+          case 'r2':
+          case 's3':
+            return engines[event.type].writeDataPoint({
+              blobs: [event.action, event.statusCodeOrErrCode.toString(), targetId],
+              doubles: [event.duration],
+              indexes: [targetId.substring(0, 32)],
+            });
+          case 'response':
+            return engines.response.writeDataPoint({
+              blobs: [event.statusCode.toString(), event.requestPath, targetId],
+              indexes: [targetId.substring(0, 32)],
+            });
+          case 'key-validation':
+            switch (event.value.type) {
+              case 'cache-hit':
+                return engines.keyValidation.writeDataPoint({
+                  blobs: [
+                    'cache-hit',
+                    event.value.version,
+                    event.value.isValid ? 'valid' : 'invalid',
+                  ],
+                  indexes: [targetId.substring(0, 32)],
+                });
+              case 'cache-write':
+                return engines.keyValidation.writeDataPoint({
+                  blobs: [
+                    'cache-write',
+                    event.value.version,
+                    event.value.isValid ? 'valid' : 'invalid',
+                  ],
+                  indexes: [targetId.substring(0, 32)],
+                });
+              case 's3-key-validation':
+                return engines.keyValidation.writeDataPoint({
+                  blobs: ['s3-key-validation', event.value.version, event.value.status],
+                  indexes: [targetId.substring(0, 32)],
+                });
+            }
+        }
+      } catch (err) {
+        console.log(`Analytics.track: Error while trying to write databapoints`);
+        console.log(err);
      }
    },
  };
--- a/packages/services/cdn-worker/src/artifact-storage-reader.ts
+++ b/packages/services/cdn-worker/src/artifact-storage-reader.ts
@ -238,9 +238,15 @@ export class ArtifactStorageReader {
      method: 'GET',
      headers,
      onAttempt: args => {
-        this.breadcrumb(
-          `Fetch attempt (source=${args.isMirror ? 'mirror' : 'primary'}, attempt=${args.attempt} duration=${args.duration}, result=${args.result.type}, key=${key})`,
-        );
+        if (args.result.type === 'error') {
+          this.breadcrumb(
+            `Fetch attempt failed (source=${args.isMirror ? 'mirror' : 'primary'}, attempt=${args.attempt} duration=${args.duration}, result=${args.result.type}, key=${key}, message=${args.result.error.message})`,
+          );
+        } else {
+          this.breadcrumb(
+            `Fetch attempt succeeded (source=${args.isMirror ? 'mirror' : 'primary'}, attempt=${args.attempt} duration=${args.duration}, result=${args.result.type}, key=${key})`,
+          );
+        }

        this.analytics?.track(
          {
@ -296,9 +302,15 @@ export class ArtifactStorageReader {
      key,
      method: 'HEAD',
      onAttempt: args => {
-        this.breadcrumb(
-          `Fetch attempt (source=${args.isMirror ? 'mirror' : 'primary'}, attempt=${args.attempt} duration=${args.duration}, result=${args.result.type}, key=${key})`,
-        );
+        if (args.result.type === 'error') {
+          this.breadcrumb(
+            `Fetch attempt failed (source=${args.isMirror ? 'mirror' : 'primary'}, attempt=${args.attempt} duration=${args.duration}, result=${args.result.type}, key=${key}, message=${args.result.error.message})`,
+          );
+        } else {
+          this.breadcrumb(
+            `Fetch attempt succeeded (source=${args.isMirror ? 'mirror' : 'primary'}, attempt=${args.attempt} duration=${args.duration}, result=${args.result.type}, key=${key})`,
+          );
+        }
        this.analytics?.track(
          {
            type: args.isMirror ? 's3' : 'r2',
@ -336,9 +348,15 @@ export class ArtifactStorageReader {
      method: 'GET',
      headers,
      onAttempt: args => {
-        this.breadcrumb(
-          `Fetch attempt (source=${args.isMirror ? 'mirror' : 'primary'}, attempt=${args.attempt} duration=${args.duration}, result=${args.result.type}, key=${key})`,
-        );
+        if (args.result.type === 'error') {
+          this.breadcrumb(
+            `Fetch attempt failed (source=${args.isMirror ? 'mirror' : 'primary'}, attempt=${args.attempt} duration=${args.duration}, result=${args.result.type}, key=${key}, message=${args.result.error.message})`,
+          );
+        } else {
+          this.breadcrumb(
+            `Fetch attempt succeeded (source=${args.isMirror ? 'mirror' : 'primary'}, attempt=${args.attempt} duration=${args.duration}, result=${args.result.type}, key=${key})`,
+          );
+        }
        this.analytics?.track(
          {
            type: args.isMirror ? 's3' : 'r2',
@ -381,9 +399,15 @@ export class ArtifactStorageReader {
      key,
      method: 'GET',
      onAttempt: args => {
-        this.breadcrumb(
-          `Fetch attempt (source=${args.isMirror ? 'mirror' : 'primary'}, attempt=${args.attempt} duration=${args.duration}, result=${args.result.type}, key=${key})`,
-        );
+        if (args.result.type === 'error') {
+          this.breadcrumb(
+            `Fetch attempt failed (source=${args.isMirror ? 'mirror' : 'primary'}, attempt=${args.attempt} duration=${args.duration}, result=${args.result.type}, key=${key}, message=${args.result.error.message})`,
+          );
+        } else {
+          this.breadcrumb(
+            `Fetch attempt succeeded (source=${args.isMirror ? 'mirror' : 'primary'}, attempt=${args.attempt} duration=${args.duration}, result=${args.result.type}, key=${key})`,
+          );
+        }
        this.analytics?.track(
          {
            type: args.isMirror ? 's3' : 'r2',
@ -410,9 +434,16 @@ export class ArtifactStorageReader {
      key,
      method: 'GET',
      onAttempt: args => {
-        this.breadcrumb(
-          `Fetch attempt (source=${args.isMirror ? 'mirror' : 'primary'}, attempt=${args.attempt} duration=${args.duration}, result=${args.result.type}, key=${key})`,
-        );
+        if (args.result.type === 'error') {
+          this.breadcrumb(
+            `Fetch attempt failed (source=${args.isMirror ? 'mirror' : 'primary'}, attempt=${args.attempt} duration=${args.duration}, result=${args.result.type}, key=${key}, message=${args.result.error.message})`,
+          );
+        } else {
+          this.breadcrumb(
+            `Fetch attempt succeeded (source=${args.isMirror ? 'mirror' : 'primary'}, attempt=${args.attempt} duration=${args.duration}, result=${args.result.type}, key=${key})`,
+          );
+        }
+
        this.analytics?.track(
          {
            type: args.isMirror ? 'r2' : 's3',
--- a/packages/services/cdn-worker/src/aws.ts
+++ b/packages/services/cdn-worker/src/aws.ts
@ -176,22 +176,32 @@ export class AwsClient {
      try {
        const response = await this._fetch(...(await this.sign(input, init)));
        const duration = performance.now() - attemptStart;
-        init.onAttempt?.({
-          attempt: retryCounter,
-          duration,
-          result: { type: 'success', response },
-        });
+
+        console.log(`AwsClient.fetch (url=${input.toString()}, retry=${retryCounter})`);

        if (
          (response.status < 500 && response.status !== 429 && response.status !== 499) ||
          retryCounter === maximumRetryCount
        ) {
          if (init.isResponseOk && !init.isResponseOk(response)) {
+            console.log(
+              `AwsClient.fetch: Response is not okay (url=${input.toString()}, status=${response.status}, retry=${retryCounter})`,
+            );
            throw new ResponseNotOkayError(response);
          }

+          init.onAttempt?.({
+            attempt: retryCounter,
+            duration,
+            result: { type: 'success', response },
+          });
+
          return response;
        }
+
+        console.log(
+          `AwsClient.fetch: Schedule another retry (url=${input.toString()}, status=${response.status}, retry=${retryCounter})`,
+        );
      } catch (error) {
        const duration = performance.now() - attemptStart;
        // Retry also when there's an exception