feat(SIP-39): Websocket sidecar app (#11498)

* WIP node.js websocket app * Load testing * Multi-stream publish with blocking reads * Use JWT for auth and channel ID * Update ws jwt cookie name * Typescript * Frontend WebSocket transport support * ws server ping/pong and GC logic * ws server unit tests * GC interval config, debug logging * Cleanup JWT cookie logic * Refactor asyncEvents.ts to support non-Redux use cases * Update tests for refactored asyncEvents * Add eslint, write READMEs, reorg files * CI workflow * Moar Apache license headers * pylint found something * adjust GH actions workflow * Improve documentation & comments * Prettier * Add configurable logging via Winston * Add SSL support for Redis connections * Fix incompatible logger statements * Apply suggestions from code review Co-authored-by: David Aaron Suddjian <1858430+suddjian@users.noreply.github.com> * rename streamPrefix config Co-authored-by: David Aaron Suddjian <1858430+suddjian@users.noreply.github.com>
2026-04-18 23:55:00 +00:00 · 2021-04-08 11:12:03 -07:00
parent 6a81a7961c
commit 806fb73d25
33 changed files with 17198 additions and 12 deletions
--- a/superset-websocket/utils/loadtest.js
+++ b/superset-websocket/utils/loadtest.js
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+const { v4: uuidv4 } = require('uuid');
+const Redis = require('ioredis');
+const config = require('../config.json');
+const redis = new Redis(config.redis);
+
+const numClients = 256;
+const globalEventStreamName = `${config.redisStreamPrefix}full`;
+
+function pushData() {
+  for (let i = 0; i < numClients; i++) {
+    const channelId = String(i);
+    const streamId = `${config.redisStreamPrefix}${channelId}`;
+    const data = {
+      channel_id: channelId,
+      job_id: uuidv4(),
+      status: 'pending',
+    };
+
+    // push to channel stream
+    redis
+      .xadd(streamId, 'MAXLEN', 1000, '*', 'data', JSON.stringify(data))
+      .then(resp => {
+        console.log('stream response', resp);
+      });
+
+    // push to firehose (all events) stream
+    redis
+      .xadd(
+        globalEventStreamName,
+        'MAXLEN',
+        100000,
+        '*',
+        'data',
+        JSON.stringify(data),
+      )
+      .then(resp => {
+        console.log('stream response', resp);
+      });
+  }
+}
+
+pushData();
+setInterval(pushData, 1000);