chore: add some basic instrumentation to Node WebSocket server (#14354)

* chore: add some basic instrumentation * Switch to statsd using hot-shots * Cleanup a few leftover bits
2026-04-07 10:31:50 +00:00 · 2021-04-26 14:05:10 -07:00
parent 2a1235c0c2
commit 8dd0620fd3
7 changed files with 160 additions and 1 deletions
--- a/superset-websocket/README.md
+++ b/superset-websocket/README.md
@@ -93,6 +93,22 @@ GLOBAL_ASYNC_QUERIES_JWT_SECRET

 More info on Superset configuration values for async queries: https://github.com/apache/superset/blob/master/CONTRIBUTING.md#async-chart-queries

+## StatsD monitoring
+
+The application is tracking a couple of metrics with `statsd` using the [hot-shots](https://www.npmjs.com/package/hot-shots) library, such as the number of connected clients and the number of failed attempts to send a message to a client.
+
+`statsd` can be configured with the `statsd` object in the configuration file. See the [hot-shots](https://www.npmjs.com/package/hot-shots) readme for more info. The default configuration is:
+
+```json
+{
+  "statsd": {
+    "host": "127.0.0.1",
+    "port": 8125,
+    "globalTags": []
+  }
+}
+```
+
 ## Running

 Running locally via dev server:
--- a/superset-websocket/config.example.json
+++ b/superset-websocket/config.example.json
@@ -3,6 +3,11 @@
  "logLevel": "info",
  "logToFile": false,
  "logFilename": "app.log",
+  "statsd": {
+    "host": "127.0.0.1",
+    "port": 8125,
+    "globalTags": []
+  },
  "redis": {
    "port": 6379,
    "host": "127.0.0.1",
--- a/superset-websocket/config.test.json
+++ b/superset-websocket/config.test.json
@@ -6,6 +6,11 @@
    "db": 10,
    "ssl": false
  },
+  "statsd": {
+    "host": "127.0.0.1",
+    "port": 8125,
+    "globalTags": []
+  },
  "redisStreamPrefix": "test-async-events-",
  "jwtSecret": "test123-test123-test123-test123-test123-test123-test123",
  "jwtCookieName": "test-async-token"
--- a/superset-websocket/package-lock.json
+++ b/superset-websocket/package-lock.json
@@ -9,6 +9,7 @@
      "license": "Apache-2.0",
      "dependencies": {
        "cookie": "^0.4.1",
+        "hot-shots": "^8.3.1",
        "ioredis": "^4.16.1",
        "jsonwebtoken": "^8.5.1",
        "uuid": "^8.3.2",
@@ -1580,6 +1581,15 @@
        "tweetnacl": "^0.14.3"
      }
    },
+    "node_modules/bindings": {
+      "version": "1.5.0",
+      "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz",
+      "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==",
+      "optional": true,
+      "dependencies": {
+        "file-uri-to-path": "1.0.0"
+      }
+    },
    "node_modules/brace-expansion": {
      "version": "1.1.11",
      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
@@ -2988,6 +2998,12 @@
        "node": "^10.12.0 || >=12.0.0"
      }
    },
+    "node_modules/file-uri-to-path": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz",
+      "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==",
+      "optional": true
+    },
    "node_modules/fill-range": {
      "version": "7.0.1",
      "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
@@ -3358,6 +3374,17 @@
      "integrity": "sha512-f/wzC2QaWBs7t9IYqB4T3sR1xviIViXJRJTWBlx2Gf3g0Xi5vI7Yy4koXQ1c9OYDGHN9sBy1DQ2AB8fqZBWhUg==",
      "dev": true
    },
+    "node_modules/hot-shots": {
+      "version": "8.3.1",
+      "resolved": "https://registry.npmjs.org/hot-shots/-/hot-shots-8.3.1.tgz",
+      "integrity": "sha512-eB20m2neM+uV7IM3JDCZg8/NU1+WGM7Qh7kpvC/sGIWNg4Ng+9O8JT/zrLuA+2JStiQLEy9z/9VEDFH0BzYf0A==",
+      "engines": {
+        "node": ">=6.0.0"
+      },
+      "optionalDependencies": {
+        "unix-dgram": "2.0.x"
+      }
+    },
    "node_modules/html-encoding-sniffer": {
      "version": "2.0.1",
      "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-2.0.1.tgz",
@@ -5007,6 +5034,12 @@
      "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
      "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
    },
+    "node_modules/nan": {
+      "version": "2.14.2",
+      "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.2.tgz",
+      "integrity": "sha512-M2ufzIiINKCuDfBSAUr1vWQ+vuVcA9kqx8JJUsbQi6yf1uGRyb7HfpdfUr5qLXf3B/t8dPvcjhKMmlfnP47EzQ==",
+      "optional": true
+    },
    "node_modules/nanomatch": {
      "version": "1.2.13",
      "resolved": "https://registry.npmjs.org/nanomatch/-/nanomatch-1.2.13.tgz",
@@ -7003,6 +7036,20 @@
        "node": ">=0.10.0"
      }
    },
+    "node_modules/unix-dgram": {
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/unix-dgram/-/unix-dgram-2.0.4.tgz",
+      "integrity": "sha512-7tpK6x7ls7J7pDrrAU63h93R0dVhRbPwiRRCawR10cl+2e1VOvF3bHlVJc6WI1dl/8qk5He673QU+Ogv7bPNaw==",
+      "hasInstallScript": true,
+      "optional": true,
+      "dependencies": {
+        "bindings": "^1.3.0",
+        "nan": "^2.13.2"
+      },
+      "engines": {
+        "node": ">=0.10.48"
+      }
+    },
    "node_modules/unset-value": {
      "version": "1.0.0",
      "resolved": "https://registry.npmjs.org/unset-value/-/unset-value-1.0.0.tgz",
@@ -8765,6 +8812,15 @@
        "tweetnacl": "^0.14.3"
      }
    },
+    "bindings": {
+      "version": "1.5.0",
+      "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz",
+      "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==",
+      "optional": true,
+      "requires": {
+        "file-uri-to-path": "1.0.0"
+      }
+    },
    "brace-expansion": {
      "version": "1.1.11",
      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
@@ -9911,6 +9967,12 @@
        "flat-cache": "^3.0.4"
      }
    },
+    "file-uri-to-path": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz",
+      "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==",
+      "optional": true
+    },
    "fill-range": {
      "version": "7.0.1",
      "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
@@ -10203,6 +10265,14 @@
      "integrity": "sha512-f/wzC2QaWBs7t9IYqB4T3sR1xviIViXJRJTWBlx2Gf3g0Xi5vI7Yy4koXQ1c9OYDGHN9sBy1DQ2AB8fqZBWhUg==",
      "dev": true
    },
+    "hot-shots": {
+      "version": "8.3.1",
+      "resolved": "https://registry.npmjs.org/hot-shots/-/hot-shots-8.3.1.tgz",
+      "integrity": "sha512-eB20m2neM+uV7IM3JDCZg8/NU1+WGM7Qh7kpvC/sGIWNg4Ng+9O8JT/zrLuA+2JStiQLEy9z/9VEDFH0BzYf0A==",
+      "requires": {
+        "unix-dgram": "2.0.x"
+      }
+    },
    "html-encoding-sniffer": {
      "version": "2.0.1",
      "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-2.0.1.tgz",
@@ -11544,6 +11614,12 @@
      "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
      "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
    },
+    "nan": {
+      "version": "2.14.2",
+      "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.2.tgz",
+      "integrity": "sha512-M2ufzIiINKCuDfBSAUr1vWQ+vuVcA9kqx8JJUsbQi6yf1uGRyb7HfpdfUr5qLXf3B/t8dPvcjhKMmlfnP47EzQ==",
+      "optional": true
+    },
    "nanomatch": {
      "version": "1.2.13",
      "resolved": "https://registry.npmjs.org/nanomatch/-/nanomatch-1.2.13.tgz",
@@ -13148,6 +13224,16 @@
        "set-value": "^2.0.1"
      }
    },
+    "unix-dgram": {
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/unix-dgram/-/unix-dgram-2.0.4.tgz",
+      "integrity": "sha512-7tpK6x7ls7J7pDrrAU63h93R0dVhRbPwiRRCawR10cl+2e1VOvF3bHlVJc6WI1dl/8qk5He673QU+Ogv7bPNaw==",
+      "optional": true,
+      "requires": {
+        "bindings": "^1.3.0",
+        "nan": "^2.13.2"
+      }
+    },
    "unset-value": {
      "version": "1.0.0",
      "resolved": "https://registry.npmjs.org/unset-value/-/unset-value-1.0.0.tgz",
--- a/superset-websocket/package.json
+++ b/superset-websocket/package.json
@@ -16,6 +16,7 @@
  "license": "Apache-2.0",
  "dependencies": {
    "cookie": "^0.4.1",
+    "hot-shots": "^8.3.1",
    "ioredis": "^4.16.1",
    "jsonwebtoken": "^8.5.1",
    "uuid": "^8.3.2",
--- a/superset-websocket/spec/index.test.ts
+++ b/superset-websocket/spec/index.test.ts
@@ -56,11 +56,19 @@ const streamReturnValue: server.StreamResult[] = [
 ];

 import * as server from '../src/index';
+import { statsd } from '../src/index';

 describe('server', () => {
+  let statsdIncrementMock: jest.SpyInstance;
+
  beforeEach(() => {
    mockRedisXrange.mockClear();
    server.resetState();
+    statsdIncrementMock = jest.spyOn(statsd, 'increment').mockReturnValue();
+  });
+
+  afterEach(() => {
+    statsdIncrementMock.mockRestore();
  });

  describe('HTTP requests', () => {
@@ -90,7 +98,7 @@ describe('server', () => {
      expect(endMock).toHaveBeenLastCalledWith('OK');
    });

-    test('reponds with a 404 otherwise', () => {
+    test('reponds with a 404 when not found', () => {
      const endMock = jest.fn();
      const writeHeadMock = jest.fn();

@@ -168,9 +176,17 @@ describe('server', () => {
      const ws = new wsMock('localhost');
      const sendMock = jest.spyOn(ws, 'send');
      const socketInstance = { ws: ws, channel: channelId, pongTs: Date.now() };
+
+      expect(statsdIncrementMock).toBeCalledTimes(0);
      server.trackClient(channelId, socketInstance);
+      expect(statsdIncrementMock).toBeCalledTimes(1);
+      expect(statsdIncrementMock).toHaveBeenNthCalledWith(
+        1,
+        'ws_connected_client',
+      );

      server.processStreamResults(streamReturnValue);
+      expect(statsdIncrementMock).toBeCalledTimes(1);

      const message1 = `{"id":"1615426152415-0","channel_id":"${channelId}","job_id":"c9b99965-8f1e-4ce5-aa43-d6fc94d6a510","user_id":"1","status":"done","errors":[],"result_url":"/superset/explore_json/data/ejr-37281682b1282cdb8f25e0de0339b386"}`;
      const message2 = `{"id":"1615426152516-0","channel_id":"${channelId}","job_id":"f1e5bb1f-f2f1-4f21-9b2f-c9b91dcc9b59","user_id":"1","status":"done","errors":[],"result_url":"/api/v1/chart/data/qc-64e8452dc9907dd77746cb75a19202de"}`;
@@ -182,7 +198,9 @@ describe('server', () => {
      const ws = new wsMock('localhost');
      const sendMock = jest.spyOn(ws, 'send');

+      expect(statsdIncrementMock).toBeCalledTimes(0);
      server.processStreamResults(streamReturnValue);
+      expect(statsdIncrementMock).toBeCalledTimes(0);

      expect(sendMock).not.toHaveBeenCalled();
    });
@@ -194,9 +212,21 @@ describe('server', () => {
      });
      const cleanChannelMock = jest.spyOn(server, 'cleanChannel');
      const socketInstance = { ws: ws, channel: channelId, pongTs: Date.now() };
+
+      expect(statsdIncrementMock).toBeCalledTimes(0);
      server.trackClient(channelId, socketInstance);
+      expect(statsdIncrementMock).toBeCalledTimes(1);
+      expect(statsdIncrementMock).toHaveBeenNthCalledWith(
+        1,
+        'ws_connected_client',
+      );

      server.processStreamResults(streamReturnValue);
+      expect(statsdIncrementMock).toBeCalledTimes(2);
+      expect(statsdIncrementMock).toHaveBeenNthCalledWith(
+        2,
+        'ws_client_send_error',
+      );

      expect(sendMock).toHaveBeenCalled();
      expect(cleanChannelMock).toHaveBeenCalledWith(channelId);
--- a/superset-websocket/src/index.ts
+++ b/superset-websocket/src/index.ts
@@ -23,6 +23,7 @@ import { v4 as uuidv4 } from 'uuid';
 import jwt from 'jsonwebtoken';
 import cookie from 'cookie';
 import Redis from 'ioredis';
+import StatsD from 'hot-shots';

 import { createLogger } from './logger';

@@ -84,6 +85,11 @@ export const opts = {
  logLevel: 'info',
  logToFile: false,
  logFilename: 'app.log',
+  statsd: {
+    host: '127.0.0.1',
+    port: 8125,
+    globalTags: [],
+  },
  redis: {
    port: 6379,
    host: '127.0.0.1',
@@ -121,6 +127,13 @@ const logger = createLogger({
  logFilename: opts.logFilename,
 });

+export const statsd = new StatsD({
+  ...opts.statsd,
+  errorHandler: (e: Error) => {
+    logger.error(e);
+  },
+});
+
 // enforce JWT secret length
 if (startServer && opts.jwtSecret.length < 32)
  throw new Error('Please provide a JWT secret at least 32 bytes long');
@@ -160,6 +173,8 @@ export const trackClient = (
  channel: string,
  socketInstance: SocketInstance,
 ): string => {
+  statsd.increment('ws_connected_client');
+
  const socketId = uuidv4();
  sockets[socketId] = socketInstance;

@@ -189,6 +204,7 @@ export const sendToChannel = (channel: string, value: EventValue): void => {
    try {
      socketInstance.ws.send(strData);
    } catch (err) {
+      statsd.increment('ws_client_send_error');
      logger.debug(`Error sending to socket: ${err}`);
      // check that the connection is still active
      cleanChannel(channel);