feat(SIP-39): Websocket sidecar app (#11498)

* WIP node.js websocket app

* Load testing

* Multi-stream publish with blocking reads

* Use JWT for auth and channel ID

* Update ws jwt cookie name

* Typescript

* Frontend WebSocket transport support

* ws server ping/pong and GC logic

* ws server unit tests

* GC interval config, debug logging

* Cleanup JWT cookie logic

* Refactor asyncEvents.ts to support non-Redux use cases

* Update tests for refactored asyncEvents

* Add eslint, write READMEs, reorg files

* CI workflow

* Moar Apache license headers

* pylint found something

* adjust GH actions workflow

* Improve documentation & comments

* Prettier

* Add configurable logging via Winston

* Add SSL support for Redis connections

* Fix incompatible logger statements

* Apply suggestions from code review

Co-authored-by: David Aaron Suddjian <1858430+suddjian@users.noreply.github.com>

* rename streamPrefix config

Co-authored-by: David Aaron Suddjian <1858430+suddjian@users.noreply.github.com>

superset-websocket/utils/client-ws-app/.nvmrc

@@ -0,0 +1 @@
+v14.15.5

superset-websocket/utils/client-ws-app/README.md

@@ -0,0 +1,42 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+# Test client application
+This Express web application is provided for testing the WebSocket server. It is not required for running the server application, and is provided here for testing and development purposes only.
+
+## Running
+First, start the WebSocket server:
+```
+cd ..
+npm run dev-server
+```
+
+Then run the client application:
+```
+cd client-ws-app
+npm install
+npm start
+```
+
+Open http://127.0.0.1:3000 in your web browser.
+
+You can customize the number of WebSocket connections by passing the count in the `sockets` query param, e.g. `http://127.0.0.1:3000?sockets=180`, though beware that browsers limit the number of open WebSocket connections to around 200.
+
+Run in conjunction with the `loadtest.js` script to populate the Redis streams with event data.
+
+**Note:** this test application is configured to use the server's local `config.json` values, so care should be taken to not overwrite any sensitive data.

superset-websocket/utils/client-ws-app/app.js

@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+var createError = require('http-errors');
+var express = require('express');
+var path = require('path');
+var cookieParser = require('cookie-parser');
+var logger = require('morgan');
+
+var indexRouter = require('./routes/index');
+
+var app = express();
+
+// view engine setup
+app.set('views', path.join(__dirname, 'views'));
+app.set('view engine', 'jade');
+
+app.use(logger('dev'));
+app.use(express.json());
+app.use(express.urlencoded({ extended: false }));
+app.use(cookieParser());
+app.use(express.static(path.join(__dirname, 'public')));
+
+app.use('/', indexRouter);
+
+// catch 404 and forward to error handler
+app.use(function (req, res, next) {
+  next(createError(404));
+});
+
+// error handler
+app.use(function (err, req, res) {
+  // set locals, only providing error in development
+  res.locals.message = err.message;
+  res.locals.error = req.app.get('env') === 'development' ? err : {};
+
+  // render the error page
+  res.status(err.status || 500);
+  res.render('error');
+});
+
+module.exports = app;

superset-websocket/utils/client-ws-app/bin/www

@@ -0,0 +1,108 @@
+#!/usr/bin/env node
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * Module dependencies.
+ */
+
+var app = require('../app');
+var debug = require('debug')('client-ws-app:server');
+var http = require('http');
+
+/**
+ * Get port from environment and store in Express.
+ */
+
+var port = normalizePort(process.env.PORT || '3000');
+app.set('port', port);
+
+/**
+ * Create HTTP server.
+ */
+
+var server = http.createServer(app);
+
+/**
+ * Listen on provided port, on all network interfaces.
+ */
+
+server.listen(port);
+server.on('error', onError);
+server.on('listening', onListening);
+
+/**
+ * Normalize a port into a number, string, or false.
+ */
+
+function normalizePort(val) {
+  var port = parseInt(val, 10);
+
+  if (isNaN(port)) {
+    // named pipe
+    return val;
+  }
+
+  if (port >= 0) {
+    // port number
+    return port;
+  }
+
+  return false;
+}
+
+/**
+ * Event listener for HTTP server "error" event.
+ */
+
+function onError(error) {
+  if (error.syscall !== 'listen') {
+    throw error;
+  }
+
+  var bind = typeof port === 'string'
+    ? 'Pipe ' + port
+    : 'Port ' + port;
+
+  // handle specific listen errors with friendly messages
+  switch (error.code) {
+    case 'EACCES':
+      console.error(bind + ' requires elevated privileges');
+      process.exit(1);
+      break;
+    case 'EADDRINUSE':
+      console.error(bind + ' is already in use');
+      process.exit(1);
+      break;
+    default:
+      throw error;
+  }
+}
+
+/**
+ * Event listener for HTTP server "listening" event.
+ */
+
+function onListening() {
+  var addr = server.address();
+  var bind = typeof addr === 'string'
+    ? 'pipe ' + addr
+    : 'port ' + addr.port;
+  debug('Listening on ' + bind);
+}

superset-websocket/utils/client-ws-app/package.json

@@ -0,0 +1,17 @@
+{
+  "name": "client-ws-app",
+  "version": "0.0.0",
+  "private": true,
+  "scripts": {
+    "start": "node ./bin/www"
+  },
+  "dependencies": {
+    "cookie-parser": "~1.4.4",
+    "debug": "~2.6.9",
+    "express": "~4.16.1",
+    "http-errors": "~1.6.3",
+    "jade": "~1.11.0",
+    "jsonwebtoken": "^8.5.1",
+    "morgan": "~1.9.1"
+  }
+}

superset-websocket/utils/client-ws-app/public/javascripts/app.js

@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+let socketCount = 0;
+let messageCount = 0;
+let lastMessage;
+
+function ts() {
+  return new Date().getTime();
+}
+
+const cookieName = document.getElementById('cookie').innerHTML;
+const tokenData = document.getElementById('tokens').innerHTML;
+const tokens = JSON.parse(tokenData);
+
+function connect() {
+  if (socketCount >= tokens.length) return;
+
+  // using https://github.com/js-cookie/js-cookie
+  // eslint-disable-next-line no-undef
+  Cookies.set(cookieName, tokens[socketCount], { path: '' });
+
+  // Create WebSocket connection.
+  let url = `ws://127.0.0.1:8080?last_id=${ts()}`;
+  const socket = new WebSocket(url);
+
+  // Connection opened
+  socket.addEventListener('open', function () {
+    socketCount++;
+    document.getElementById('socket-count').innerHTML = socketCount;
+    connect();
+
+    socket.send('Hello Server!');
+  });
+
+  // Listen for messages
+  socket.addEventListener('message', function (event) {
+    messageCount++;
+    lastMessage = event.data;
+  });
+}
+
+connect();
+
+setInterval(() => {
+  document.getElementById('message-count').innerHTML = messageCount;
+  document.getElementById('message-debug').innerHTML = lastMessage;
+}, 250);

superset-websocket/utils/client-ws-app/public/stylesheets/style.css

@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+body {
+  padding: 50px;
+  font: 14px 'Lucida Grande', Helvetica, Arial, sans-serif;
+}
+
+a {
+  color: #00b7ff;
+}
+
+span {
+  font-weight: bold;
+}
+
+code {
+  display: inline-block;
+  margin-top: 5px;
+  border: 1px #999 solid;
+  padding: 5px;
+}

superset-websocket/utils/client-ws-app/routes/index.js

@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+const express = require('express');
+const router = express.Router();
+const jwt = require('jsonwebtoken');
+const config = require('../../../config.json');
+
+router.get('/', function (req, res) {
+  let numTokens = req.query.sockets ? Number(req.query.sockets) : 100;
+  let tokens = [];
+  for (let i = 0; i < numTokens; i++) {
+    const token = jwt.sign({ channel: String(i) }, config.jwtSecret);
+    tokens.push(token);
+  }
+
+  res.render('index', {
+    tokens: JSON.stringify(tokens),
+    c: config.jwtCookieName,
+  });
+});
+
+module.exports = router;

superset-websocket/utils/client-ws-app/views/error.jade

@@ -0,0 +1,23 @@
+//-
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+extends layout
+
+block content
+  h1= message
+  h2= error.status
+  pre #{error.stack}

superset-websocket/utils/client-ws-app/views/index.jade

@@ -0,0 +1,32 @@
+//-
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+extends layout
+
+block content
+  h1 Websocket test app
+  #tokens(style="display:none")= tokens
+  #cookie(style="display:none")= c
+
+  div Sockets connected:
+    span#socket-count 0
+  div Messages recevied:
+    span#message-count 0
+  div Last message received:
+  code#message-debug
+
+  script(src="/javascripts/app.js")

superset-websocket/utils/client-ws-app/views/layout.jade

@@ -0,0 +1,26 @@
+//-
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+doctype html
+html
+  head
+    title= title
+    link(rel='stylesheet', href='/stylesheets/style.css')
+    script(src="https://cdn.jsdelivr.net/npm/js-cookie@rc/dist/js.cookie.min.js")
+
+  body
+    block content