Compare commits

..

1 Commits

Author SHA1 Message Date
Maxime Beauchemin
dd6f107902 0.24.0rc1 2018-03-19 13:42:40 -07:00
5993 changed files with 322835 additions and 1258866 deletions

View File

@@ -1,81 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# https://cwiki.apache.org/confluence/display/INFRA/.asf.yaml+features+for+git+repositories
---
github:
description: "Apache Superset is a Data Visualization and Data Exploration Platform"
homepage: https://superset.apache.org/
labels:
- superset
- apache
- apache-superset
- data-visualization
- data-viz
- analytics
- business-intelligence
- data-science
- data-engineering
- asf
- bi
- business-analytics
- data-analytics
- data-analysis
- data-science
- python
- react
- sql-editor
- flask
features:
# Enable issues management
issues: true
# Enable projects for project management boards
projects: true
# Enable wiki for documentation
wiki: true
enabled_merge_buttons:
squash: true
merge: false
rebase: false
protected_branches:
master:
required_status_checks:
# strict means "Require branches to be up to date before merging".
strict: false
# contexts are the names of checks that must pass
contexts:
- check
- cypress-matrix (1, chrome)
- cypress-matrix (2, chrome)
- cypress-matrix (3, chrome)
- docker-build
- frontend-build
- pre-commit (3.9)
- python-lint (3.9)
- test-mysql (3.9)
- test-postgres (3.9)
- test-postgres (3.10)
- test-sqlite (3.9)
required_pull_request_reviews:
dismiss_stale_reviews: false
require_code_owner_reviews: true
required_approving_review_count: 1
required_signatures: false

View File

@@ -1,35 +0,0 @@
codecov:
notify:
after_n_builds: 4
ignore:
- "superset/migrations/versions/*.py"
- "superset-frontend/packages/superset-ui-demo/**/*"
- "**/*.stories.tsx"
- "**/*.stories.jsx"
coverage:
status:
project:
default:
informational: true
# Commits pushed to master should not make the overall
# project coverage decrease:
target: auto
threshold: 0%
core-packages-ts:
target: 100%
paths:
- 'superset-frontend/packages'
- '!superset-frontend/packages/**/*.jsx'
- '!superset-frontend/packages/**/*.tsx'
core-packages-tsx:
target: 50%
paths:
- 'superset-frontend/packages/**/*.jsx'
- 'superset-frontend/packages/**/*.tsx'
patch:
default:
informational: true
threshold: 0%
flag_management:
default_rules:
carryforward: true

View File

@@ -1,47 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
**/__pycache__/
**/.git
**/.apache_superset.egg-info
**/.github
**/.mypy_cache
**/.pytest_cache
**/.tox
**/.vscode
**/.idea
**/.coverage
**/.DS_Store
**/.eggs
**/.python-version
**/*.egg-info
**/*.bak
**/*.db
**/*.pyc
**/*.sqllite
**/*.swp
**/.terser-plugin-cache/
**/.storybook/
**/node_modules/
tests/
docs/
install/
superset-frontend/cypress-base/
superset-frontend/coverage/
superset/static/assets/
superset-websocket/dist/
venv

View File

@@ -1,47 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# EditorConfig is awesome: https://EditorConfig.org
# top-most EditorConfig file
root = true
# Unix-style newlines with a newline ending every file
[*]
end_of_line = lf
insert_final_newline = true
charset = utf-8
# 4 space indentation for Python files
[*.py]
indent_style = space
indent_size = 4
max_line_length=88
# 2 space indentation for Frontend files
[*.{js,jsx,ts,tsx,html,less,css}]
indent_style = space
indent_size = 2
# 2 space indentation for json and yaml files
[*.{json,yml}]
indent_style = space
indent_size = 2
# Tab indentation
[Makefile]
indent_style = tab

View File

@@ -1,18 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
FLASK_APP="superset.app:create_app()"
FLASK_DEBUG=true

View File

@@ -1,36 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Generated by FOSSA CLI (https://github.com/fossas/fossa-cli)
# Visit https://fossa.com to learn more
version: 2
cli:
server: https://app.fossa.com
fetcher: custom
analyze:
modules:
- name: assets
type: npm
target: superset-frontend
path: superset-frontend
- name: base
type: pip
target: .
path: .
options:
requirements: ./requirements/base.txt

1
.gitattributes vendored
View File

@@ -1 +0,0 @@
docker/**/*.sh text eol=lf

29
.github/CODEOWNERS vendored
View File

@@ -1,29 +0,0 @@
# Notify all committers of DB migration changes, per SIP-59
# https://github.com/apache/superset/issues/13351
/superset/migrations/ @apache/superset-committers
# Notify Preset team when ephemeral env settings are changed
.github/workflows/ecs-task-definition.json @robdiciuccio @craig-rueda @rusackas @eschutho @dpgaspar @nytai @mistercrunch
.github/workflows/docker-ephemeral-env.yml @robdiciuccio @craig-rueda @rusackas @eschutho @dpgaspar @nytai @mistercrunch
.github/workflows/ephemeral*.yml @robdiciuccio @craig-rueda @rusackas @eschutho @dpgaspar @nytai @mistercrunch
# Notify some committers of changes in the components
/superset-frontend/src/components/Select/ @michael-s-molina @geido @kgabryje
/superset-frontend/src/components/MetadataBar/ @michael-s-molina @geido @kgabryje
/superset-frontend/src/components/DropdownContainer/ @michael-s-molina @geido @kgabryje
# Notify Helm Chart maintainers about changes in it
/helm/superset/ @craig-rueda @dpgaspar @villebro
# Notify E2E test maintainers of changes
/superset-frontend/cypress-base/ @jinghua-qa @geido @eschutho @rusackas @betodealmeida
# Notify PMC members of changes to Github Actions
/.github/ @villebro @geido @eschutho @rusackas @betodealmeida @nytai @mistercrunch @craig-rueda @john-bodley @kgabryje

View File

@@ -1,49 +0,0 @@
---
name: Bug report
about: "Create a report to help us improve Superset's stability! For feature requests please open a discussion [here](https://github.com/apache/superset/discussions/categories/ideas)."
labels: bug
---
A clear and concise description of what the bug is.
#### How to reproduce the bug
1. Go to '...'
2. Click on '....'
3. Scroll down to '....'
4. See error
### Expected results
what you expected to happen.
### Actual results
what actually happens.
#### Screenshots
If applicable, add screenshots to help explain your problem.
### Environment
(please complete the following information):
- browser type and version:
- superset version: `superset version`
- python version: `python --version`
- node.js version: `node -v`
- any feature flags active:
### Checklist
Make sure to follow these steps before submitting your issue - thank you!
- [ ] I have checked the superset logs for python stacktraces and included it here as text if there are any.
- [ ] I have reproduced the issue with at least the latest released version of superset.
- [ ] I have checked the issue tracker for the same issue and I haven't found one similar.
### Additional context
Add any other context about the problem here.

View File

@@ -1,19 +0,0 @@
---
name: Cosmetic Issue
about: Describe a cosmetic issue with CSS, positioning, layout, labeling, or similar
labels: "cosmetic-issue"
---
## Screenshot
[drag & drop image(s) here!]
## Description
[describe the issue here!]
## Design input
[describe any input/collaboration you'd like from designers, and
tag accordingly. For design review, add the
label `design:review`. If this includes a design proposal,
include the label `design:suggest`]

View File

@@ -1,36 +0,0 @@
---
name: SIP
about: "Superset Improvement Proposal. See [here](https://github.com/apache/superset/issues/5602) for details."
labels: sip
title: "[SIP] Your Title Here (do not add SIP number)"
asignees: "apache/superset-committers"
---
*Please make sure you are familiar with the SIP process documented*
(here)[https://github.com/apache/superset/issues/5602]. The SIP will be numbered by a committer upon acceptance.
## [SIP] Proposal for ...<title>
### Motivation
Description of the problem to be solved.
### Proposed Change
Describe how the feature will be implemented, or the problem will be solved. If possible, include mocks, screenshots, or screencasts (even if from different tools).
### New or Changed Public Interfaces
Describe any new additions to the model, views or `REST` endpoints. Describe any changes to existing visualizations, dashboards and React components. Describe changes that affect the Superset CLI and how Superset is deployed.
### New dependencies
Describe any `npm`/`PyPI` packages that are required. Are they actively maintained? What are their licenses?
### Migration Plan and Compatibility
Describe any database migrations that are necessary, or updates to stored URLs.
### Rejected Alternatives
Describe alternative approaches that were considered and rejected.

View File

@@ -1,27 +0,0 @@
<!---
Please write the PR title following the conventions at https://www.conventionalcommits.org/en/v1.0.0/
Example:
fix(dashboard): load charts correctly
-->
### SUMMARY
<!--- Describe the change below, including rationale and design decisions -->
### BEFORE/AFTER SCREENSHOTS OR ANIMATED GIF
<!--- Skip this if not applicable -->
### TESTING INSTRUCTIONS
<!--- Required! What steps can be taken to manually verify the changes? -->
### ADDITIONAL INFORMATION
<!--- Check any relevant boxes with "x" -->
<!--- HINT: Include "Fixes #nnn" if you are fixing an existing issue -->
- [ ] Has associated issue:
- [ ] Required feature flags:
- [ ] Changes UI
- [ ] Includes DB Migration (follow approval process in [SIP-59](https://github.com/apache/superset/issues/13351))
- [ ] Migration is atomic, supports rollback & is backwards-compatible
- [ ] Confirm DB migration upgrade and downgrade tested
- [ ] Runtime estimates and downtime expectations provided
- [ ] Introduces new feature or API
- [ ] Removes existing feature or API

38
.github/SECURITY.md vendored
View File

@@ -1,38 +0,0 @@
# Security Policy
This is a project of the [Apache Software Foundation](https://apache.org) and follows the
ASF [vulnerability handling process](https://apache.org/security/#vulnerability-handling).
## Reporting Vulnerabilities
**⚠️ Please do not file GitHub issues for security vulnerabilities as they are public! ⚠️**
Apache Software Foundation takes a rigorous standpoint in annihilating the security issues
in its software projects. Apache Superset is highly sensitive and forthcoming to issues
pertaining to its features and functionality.
If you have any concern or believe you have found a vulnerability in Apache Superset,
please get in touch with the Apache Security Team privately at
e-mail address [security@apache.org](mailto:security@apache.org).
More details can be found on the ASF website at
[ASF vulnerability reporting process](https://apache.org/security/#reporting-a-vulnerability)
We kindly ask you to include the following information in your report:
- Apache Superset version that you are using
- A sanitized copy of your `superset_config.py` file or any config overrides
- Detailed steps to reproduce the vulnerability
Note that Apache Superset is not responsible for any third-party dependencies that may
have security issues. Any vulnerabilities found in third-party dependencies should be
reported to the maintainers of those projects. Results from security scans of Apache
Superset dependencies found on its official Docker image can be remediated at release time
by extending the image itself.
**Your responsible disclosure and collaboration are invaluable.**
## Extra Information
- [Apache Superset documentation](https://superset.apache.org/docs/security)
- [Common Vulnerabilities and Exposures by release](https://superset.apache.org/docs/security/cves)
- [How Security Vulnerabilities are Reported & Handled in Apache Superset (Blog)](https://preset.io/blog/how-security-vulnerabilities-are-reported-and-handled-in-apache-superset/)

15
.github/config.yml vendored
View File

@@ -1,15 +0,0 @@
# Configuration for request-info - https://github.com/behaviorbot/request-info
# *Required* Comment to reply with
requestInfoReplyComment: >
We would appreciate it if you could provide us with more info about this issue/pr!
Please do not leave the `title` or `description` empty.
# *OPTIONAL* default titles to check against for lack of descriptiveness
# MUST BE ALL LOWERCASE
requestInfoDefaultTitles:
- update readme.md
- updates
# *OPTIONAL* Label to be added to Issues and Pull Requests with insufficient information given
requestInfoLabelToAdd: "need:more-info"

View File

@@ -1,50 +0,0 @@
version: 2
updates:
- package-ecosystem: "npm"
directory: "/superset-frontend/"
schedule:
interval: "daily"
labels:
- npm
- dependabot
versioning-strategy: increase
- package-ecosystem: "pip"
directory: "/requirements/"
schedule:
interval: "daily"
labels:
- pip
- dependabot
- package-ecosystem: "npm"
directory: ".github/actions"
schedule:
interval: "daily"
open-pull-requests-limit: 0
versioning-strategy: increase
- package-ecosystem: "npm"
directory: "/docs/"
schedule:
interval: "daily"
open-pull-requests-limit: 0
versioning-strategy: increase
- package-ecosystem: "npm"
directory: "/superset-websocket/"
schedule:
interval: "daily"
labels:
- npm
- dependabot
versioning-strategy: increase
- package-ecosystem: "npm"
directory: "/superset-websocket/utils/client-ws-app/"
schedule:
interval: "daily"
labels:
- npm
- dependabot
versioning-strategy: increase

View File

@@ -1,5 +0,0 @@
# for Issue Label Bot https://github.com/marketplace/issue-label-bot
label-alias:
bug: '#bug'
feature_request: '#enhancement'
question: '#question'

27
.github/move.yml vendored
View File

@@ -1,27 +0,0 @@
# Configuration for Move Issues - https://github.com/dessant/move-issues
# Delete the command comment when it contains no other content
deleteCommand: true
# Close the source issue after moving
closeSourceIssue: true
# Lock the source issue after moving
lockSourceIssue: false
# Mention issue and comment authors
mentionAuthors: true
# Preserve mentions in the issue content
keepContentMentions: false
# Move labels that also exist on the target repository
moveLabels: true
# Set custom aliases for targets
# aliases:
# r: repo
# or: owner/repo
# Repository to extend settings from
# _extends: repo

19
.github/stale.yml vendored
View File

@@ -1,19 +0,0 @@
# Number of days of inactivity before an issue becomes stale
daysUntilStale: 60
# Number of days of inactivity before a stale issue is closed
daysUntilClose: 7
# Issues with these labels will never be considered stale
exemptLabels:
- "#SIP"
- ".pinned"
- ".security"
# Label to use when marking an issue as stale
staleLabel: inactive
# Comment to post when marking an issue as stale. Set to `false` to disable
markComment: >
This issue has been automatically marked as stale because it has not had
recent activity. It will be closed if no further activity occurs.
Thank you for your contributions. For admin, please label this issue `.pinned`
to prevent stale bot from closing the issue.
# Comment to post when closing a stale issue. Set to `false` to disable
closeComment: false

View File

@@ -1,246 +0,0 @@
#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
set -e
GITHUB_WORKSPACE=${GITHUB_WORKSPACE:-.}
ASSETS_MANIFEST="$GITHUB_WORKSPACE/superset/static/assets/manifest.json"
# Rounded job start time, used to create a unique Cypress build id for
# parallelization so we can manually rerun a job after 20 minutes
NONCE=$(echo "$(date "+%Y%m%d%H%M") - ($(date +%M)%20)" | bc)
# Echo only when not in parallel mode
say() {
if [[ $(echo "$INPUT_PARALLEL" | tr '[:lower:]' '[:upper:]') != 'TRUE' ]]; then
echo "$1"
fi
}
# default command to run when the `run` input is empty
default-setup-command() {
apt-get-install
pip-upgrade
}
apt-get-install() {
say "::group::apt-get install dependencies"
sudo apt-get update && sudo apt-get install --yes \
libsasl2-dev \
libldap2-dev
say "::endgroup::"
}
pip-upgrade() {
say "::group::Upgrade pip"
pip install --upgrade pip
say "::endgroup::"
}
# prepare (lint and build) frontend code
npm-install() {
cd "$GITHUB_WORKSPACE/superset-frontend"
# cache-restore npm
say "::group::Install npm packages"
echo "npm: $(npm --version)"
echo "node: $(node --version)"
npm ci
say "::endgroup::"
# cache-save npm
}
build-assets() {
cd "$GITHUB_WORKSPACE/superset-frontend"
say "::group::Build static assets"
npm run build
say "::endgroup::"
}
build-instrumented-assets() {
cd "$GITHUB_WORKSPACE/superset-frontend"
say "::group::Build static assets with JS instrumented for test coverage"
cache-restore instrumented-assets
if [[ -f "$ASSETS_MANIFEST" ]]; then
echo 'Skip frontend build because instrumented static assets already exist.'
else
npm run build-instrumented
cache-save instrumented-assets
fi
say "::endgroup::"
}
setup-postgres() {
say "::group::Install dependency for unit tests"
sudo apt-get update && sudo apt-get install --yes libecpg-dev
say "::group::Initialize database"
psql "postgresql://superset:superset@127.0.0.1:15432/superset" <<-EOF
DROP SCHEMA IF EXISTS sqllab_test_db CASCADE;
DROP SCHEMA IF EXISTS admin_database CASCADE;
CREATE SCHEMA sqllab_test_db;
CREATE SCHEMA admin_database;
EOF
say "::endgroup::"
}
setup-mysql() {
say "::group::Initialize database"
mysql -h 127.0.0.1 -P 13306 -u root --password=root <<-EOF
DROP DATABASE IF EXISTS superset;
CREATE DATABASE superset DEFAULT CHARACTER SET utf8 COLLATE utf8_unicode_ci;
DROP DATABASE IF EXISTS sqllab_test_db;
CREATE DATABASE sqllab_test_db DEFAULT CHARACTER SET utf8 COLLATE utf8_unicode_ci;
DROP DATABASE IF EXISTS admin_database;
CREATE DATABASE admin_database DEFAULT CHARACTER SET utf8 COLLATE utf8_unicode_ci;
CREATE USER 'superset'@'%' IDENTIFIED BY 'superset';
GRANT ALL ON *.* TO 'superset'@'%';
FLUSH PRIVILEGES;
EOF
say "::endgroup::"
}
testdata() {
cd "$GITHUB_WORKSPACE"
say "::group::Load test data"
# must specify PYTHONPATH to make `tests.superset_test_config` importable
export PYTHONPATH="$GITHUB_WORKSPACE"
pip install -e .
superset db upgrade
superset load_test_users
superset load_examples --load-test-data
superset init
say "::endgroup::"
}
codecov() {
say "::group::Upload code coverage"
bash ".github/workflows/codecov.sh" "$@"
say "::endgroup::"
}
cypress-install() {
cd "$GITHUB_WORKSPACE/superset-frontend/cypress-base"
cache-restore cypress
say "::group::Install Cypress"
npm ci
say "::endgroup::"
cache-save cypress
}
# Run Cypress and upload coverage reports
cypress-run() {
cd "$GITHUB_WORKSPACE/superset-frontend/cypress-base"
local page=$1
local group=${2:-Default}
local cypress="./node_modules/.bin/cypress run"
local browser=${CYPRESS_BROWSER:-chrome}
export TERM="xterm"
say "::group::Run Cypress for [$page]"
if [[ -z $CYPRESS_KEY ]]; then
$cypress --spec "cypress/e2e/$page" --browser "$browser"
else
export CYPRESS_RECORD_KEY=$(echo $CYPRESS_KEY | base64 --decode)
# additional flags for Cypress dashboard recording
$cypress --spec "cypress/e2e/$page" --browser "$browser" \
--record --group "$group" --tag "${GITHUB_REPOSITORY},${GITHUB_EVENT_NAME}" \
--parallel --ci-build-id "${GITHUB_SHA:0:8}-${NONCE}"
fi
# don't add quotes to $record because we do want word splitting
say "::endgroup::"
}
cypress-run-all() {
# Start Flask and run it in background
# --no-debugger means disable the interactive debugger on the 500 page
# so errors can print to stderr.
local flasklog="${HOME}/flask.log"
local port=8081
export CYPRESS_BASE_URL="http://localhost:${port}"
nohup flask run --no-debugger -p $port >"$flasklog" 2>&1 </dev/null &
local flaskProcessId=$!
cypress-run "*/**/*"
# After job is done, print out Flask log for debugging
say "::group::Flask log for default run"
cat "$flasklog"
say "::endgroup::"
# Rerun SQL Lab tests with backend persist disabled
export SUPERSET_CONFIG=tests.integration_tests.superset_test_config_sqllab_backend_persist_off
# Restart Flask with new configs
kill $flaskProcessId
nohup flask run --no-debugger -p $port >"$flasklog" 2>&1 </dev/null &
local flaskProcessId=$!
cypress-run "sqllab/*" "Backend persist"
# Upload code coverage separately so each page can have separate flags
# -c will clean existing coverage reports, -F means add flags
# || true to prevent CI failure on codecov upload
codecov -c -F "cypress" || true
say "::group::Flask log for backend persist"
cat "$flasklog"
say "::endgroup::"
# make sure the program exits
kill $flaskProcessId
}
eyes-storybook-dependencies() {
say "::group::install eyes-storyook dependencies"
sudo apt-get update -y && sudo apt-get -y install gconf-service ca-certificates libxshmfence-dev fonts-liberation libappindicator3-1 libasound2 libatk-bridge2.0-0 libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 libgbm1 libgcc1 libgconf-2-4 libglib2.0-0 libgdk-pixbuf2.0-0 libgtk-3-0 libnspr4 libnss3 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 lsb-release xdg-utils libappindicator1
say "::endgroup::"
}
cypress-run-applitools() {
cd "$GITHUB_WORKSPACE/superset-frontend/cypress-base"
local flasklog="${HOME}/flask.log"
local port=8081
local cypress="./node_modules/.bin/cypress run"
local browser=${CYPRESS_BROWSER:-chrome}
export CYPRESS_BASE_URL="http://localhost:${port}"
nohup flask run --no-debugger -p $port >"$flasklog" 2>&1 </dev/null &
local flaskProcessId=$!
$cypress --spec "cypress/e2e/*/**/*.applitools.test.ts" --browser "$browser" --headless --config ignoreTestFiles="[]"
codecov -c -F "cypress" || true
say "::group::Flask log for default run"
cat "$flasklog"
say "::endgroup::"
# make sure the program exits
kill $flaskProcessId
}

View File

@@ -1,58 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
// always use absolute directory
const workspaceDirectory = process.env.GITHUB_WORKSPACE;
const homeDirectory = process.env.HOME;
const assetsConfig = {
path: [`${workspaceDirectory}/superset/static/assets`],
hashFiles: [
`${workspaceDirectory}/superset-frontend/src/**/*`,
`${workspaceDirectory}/superset-frontend/packages/**/*`,
`${workspaceDirectory}/superset-frontend/plugins/**/*`,
`${workspaceDirectory}/superset-frontend/*.js`,
`${workspaceDirectory}/superset-frontend/*.json`,
],
// dont use restore keys as it may give an invalid older build
restoreKeys: '',
};
// Multi-layer cache definition
module.exports = {
pip: {
path: [`${homeDirectory}/.cache/pip`],
hashFiles: [`${workspaceDirectory}/requirements/*.txt`],
},
npm: {
path: [`${homeDirectory}/.npm`],
hashFiles: [`${workspaceDirectory}/superset-frontend/package-lock.json`],
},
assets: assetsConfig,
// use separate cache for instrumented JS files and regular assets
// one is built with `npm run build`,
// another is built with `npm run build-instrumented`
'instrumented-assets': assetsConfig,
cypress: {
path: [`${homeDirectory}/.cache/Cypress`],
hashFiles: [
`${workspaceDirectory}/superset-frontend/cypress-base/package-lock.json`,
],
},
};

View File

@@ -1,43 +0,0 @@
name: Cancel Duplicates
on:
workflow_run:
workflows:
- "Miscellaneous"
types:
- requested
jobs:
cancel-duplicate-runs:
name: Cancel duplicate workflow runs
runs-on: ubuntu-20.04
permissions:
actions: write
contents: read
steps:
- name: Check number of queued tasks
id: check_queued
env:
GITHUB_TOKEN: ${{ github.token }}
GITHUB_REPO: ${{ github.repository }}
run: |
get_count() {
echo $(curl -s -H "Authorization: token $GITHUB_TOKEN" \
"https://api.github.com/repos/$GITHUB_REPO/actions/runs?status=$1" | \
jq ".total_count")
}
count=$(( `get_count queued` + `get_count in_progress` ))
echo "Found $count unfinished jobs."
echo "::set-output name=count::$count"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
if: steps.check_queued.outputs.count >= 20
uses: actions/checkout@v3
- name: Cancel duplicate workflow runs
if: steps.check_queued.outputs.count >= 20
env:
GITHUB_TOKEN: ${{ github.token }}
GITHUB_REPOSITORY: ${{ github.repository }}
run: |
pip install click requests typing_extensions python-dateutil
python ./scripts/cancel_github_workflows.py

View File

@@ -1,63 +0,0 @@
name: Check DB migration conflict
on:
push:
paths:
- "superset/migrations/**"
jobs:
check_db_migration_conflict:
name: Check DB migration conflict
runs-on: ubuntu-20.04
permissions:
contents: read
pull-requests: write
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v3
- name: Check and notify
uses: actions/github-script@v3
with:
github-token: ${{ github.token }}
script: |
// API reference: https://octokit.github.io/rest.js
const currentBranch = context.ref.replace('refs/heads/', '');
// Find all pull requests to current branch
const opts = github.pulls.list.endpoint.merge({
owner: context.repo.owner,
repo: context.repo.repo,
base: context.ref,
state: 'open',
sort: 'updated',
per_page: 100,
});
const pulls = await github.paginate(opts);
if (pulls.length > 0) {
console.log(`Found ${pulls.length} open PRs for base branch "${currentBranch}"`)
}
for (const pull of pulls) {
const listFilesOpts = await github.pulls.listFiles.endpoint.merge({
owner: context.repo.owner,
repo: context.repo.repo,
pull_number: pull.number,
});
const files = await github.paginate(listFilesOpts);
if (
files.some(x => x.contents_url.includes('/contents/superset/migrations'))
) {
console.log(`PR #${pull.number} "${pull.title}" also added db migration`)
await github.issues.createComment({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: pull.number,
body:
`# 🙅‍♂️ 🙅‍♂️ 🙅‍♂️ 🙅‍♂️ 🙅‍♂️ 🙅‍♂️ 🙅‍♂️ 🙅‍♂️ 🙅‍♂️ 🙅‍♂️ 🙅‍♂️ 🙅‍♂️ 🙅‍♂️ 🙅‍♂️ 🙅‍♂️ 🙅‍♂️` +
`❗ @${pull.user.login} Your base branch \`${currentBranch}\` has ` +
'also updated `superset/migrations`.\n' +
'\n' +
'**Please consider rebasing your branch and [resolving potential db migration conflicts](https://github.com/apache/superset/blob/master/CONTRIBUTING.md#merging-db-migrations).**',
});
}
}

View File

@@ -1,72 +0,0 @@
# .github/workflows/chromatic.yml
# seee https://www.chromatic.com/docs/github-actions
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Workflow name
name: 'Chromatic Storybook Master'
# Event for the workflow
# Only run if changes were made in superset-frontend folder of repo on merge to Master
on:
# This will trigger when a branch merges to master when the PR has changes in the frontend folder updating the chromatic baseline
push:
branches:
- master
paths:
- "superset-frontend/**"
# List of jobs
jobs:
config:
runs-on: "ubuntu-latest"
outputs:
has-secrets: ${{ steps.check.outputs.has-secrets }}
steps:
- name: "Check for secrets"
id: check
shell: bash
run: |
if [ -n "${{ (secrets.CHROMATIC_PROJECT_TOKEN != '') || '' }}" ]; then
echo "has-secrets=1" >> "$GITHUB_OUTPUT"
fi
chromatic-deployment:
needs: config
if: needs.config.outputs.has-secrets
# Operating System
runs-on: ubuntu-latest
# Job steps
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0 # 👈 Required to retrieve git history
- name: Install dependencies
run: npm ci
working-directory: superset-frontend
# 👇 Build and publish Storybook to Chromatic
- name: Build and publish Storybook to Chromatic
id: chromatic-master
uses: chromaui/action@v1
# Required options for the Chromatic GitHub Action
with:
# 👇 Location of package.json from root of mono-repo
workingDir: superset-frontend
# 👇 Chromatic projectToken, refer to the manage page to obtain it.
projectToken: ${{ secrets.CHROMATIC_PROJECT_TOKEN }}
exitZeroOnChanges: true # 👈 Option to prevent the workflow from failing
autoAcceptChanges: true # 👈 Option to accept all changes when merging to master

File diff suppressed because it is too large Load Diff

View File

@@ -1,50 +0,0 @@
name: "CodeQL"
on:
push:
branches: [ "master" ]
paths:
- 'superset/**'
pull_request:
# The branches below must be a subset of the branches above
branches: [ "master" ]
paths:
- 'superset/**'
schedule:
- cron: '0 4 * * *'
jobs:
analyze:
name: Analyze
runs-on: ubuntu-22.04
permissions:
actions: read
contents: read
security-events: write
strategy:
fail-fast: false
matrix:
language: [ 'python', 'javascript' ]
# Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
steps:
- name: Checkout repository
uses: actions/checkout@v3
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@v2
with:
languages: ${{ matrix.language }}
# If you wish to specify custom queries, you can do so here or in a config file.
# By default, queries listed here will override any specified in a config file.
# Prefix the list here with "+" to use these queries and those in the config file.
# For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
# queries: security-extended,security-and-quality
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v2
with:
category: "/language:${{matrix.language}}"

View File

@@ -1,25 +0,0 @@
# Dependency Review Action
#
# This Action will scan dependency manifest files that change as part of a Pull Request, surfacing known-vulnerable versions of the packages declared or updated in the PR. Once installed, if the workflow run is marked as required, PRs introducing known-vulnerable packages will be blocked from merging.
#
# Source repository: https://github.com/actions/dependency-review-action
# Public documentation: https://docs.github.com/en/code-security/supply-chain-security/understanding-your-software-supply-chain/about-dependency-review#dependency-review-enforcement
name: 'Dependency Review'
on: [pull_request]
permissions:
contents: read
jobs:
dependency-review:
runs-on: ubuntu-latest
steps:
- name: 'Checkout Repository'
uses: actions/checkout@v3
- name: 'Dependency Review'
uses: actions/dependency-review-action@v2
with:
fail-on-severity: high
# compatible/incompatible licenses addressed here: https://www.apache.org/legal/resolved.html
# find SPDX identifiers here: https://spdx.org/licenses/
deny-licenses: MS-LPL, BUSL-1.1, QPL-1.0, Sleepycat, SSPL-1.0, CPOL-1.02, AGPL-3.0, GPL-1.0+, BSD-4-Clause-UC, NPL-1.0, NPL-1.1, JSON

View File

@@ -1,100 +0,0 @@
name: Push ephemeral env image
on:
workflow_run:
workflows: ["Docker"]
types:
- completed
jobs:
config:
runs-on: "ubuntu-latest"
if: github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.conclusion == 'success'
outputs:
has-secrets: ${{ steps.check.outputs.has-secrets }}
steps:
- name: "Check for secrets"
id: check
shell: bash
run: |
if [ -n "${{ (secrets.AWS_ACCESS_KEY_ID != '' &&
secrets.AWS_ACCESS_KEY_ID != '' &&
secrets.AWS_SECRET_ACCESS_KEY != '' &&
secrets.AWS_SECRET_ACCESS_KEY != '') || '' }}" ]; then
echo "has-secrets=1" >> "$GITHUB_OUTPUT"
echo "has secrets!"
else
echo "has-secrets=0" >> "$GITHUB_OUTPUT"
echo "no secrets!"
fi
docker_ephemeral_env:
needs: config
if: needs.config.outputs.has-secrets
name: Push ephemeral env Docker image to ECR
runs-on: ubuntu-latest
steps:
- name: "Download artifact"
uses: actions/github-script@v3.1.0
with:
script: |
const artifacts = await github.actions.listWorkflowRunArtifacts({
owner: context.repo.owner,
repo: context.repo.repo,
run_id: ${{ github.event.workflow_run.id }},
});
core.info('*** artifacts')
core.info(JSON.stringify(artifacts))
const matchArtifact = artifacts.data.artifacts.filter((artifact) => {
return artifact.name == "build"
})[0];
if(!matchArtifact) return core.setFailed("Build artifacts not found")
const download = await github.actions.downloadArtifact({
owner: context.repo.owner,
repo: context.repo.repo,
artifact_id: matchArtifact.id,
archive_format: 'zip',
});
var fs = require('fs');
fs.writeFileSync('${{github.workspace}}/build.zip', Buffer.from(download.data));
- run: unzip build.zip
- name: Display downloaded files (debug)
run: ls -la
- name: Get SHA
id: get-sha
run: echo "::set-output name=sha::$(cat ./SHA)"
- name: Get PR
id: get-pr
run: echo "::set-output name=num::$(cat ./PR-NUM)"
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: us-west-2
- name: Login to Amazon ECR
id: login-ecr
uses: aws-actions/amazon-ecr-login@v1
- name: Load, tag and push image to ECR
id: push-image
env:
ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
ECR_REPOSITORY: superset-ci
SHA: ${{ steps.get-sha.outputs.sha }}
IMAGE_TAG: pr-${{ steps.get-pr.outputs.num }}
run: |
docker load < $SHA.tar.gz
docker tag $SHA $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
docker tag $SHA $ECR_REGISTRY/$ECR_REPOSITORY:$SHA
docker push -a $ECR_REGISTRY/$ECR_REPOSITORY

View File

@@ -1,37 +0,0 @@
name: Docker
on:
release:
types: [published]
jobs:
config:
runs-on: "ubuntu-latest"
outputs:
has-secrets: ${{ steps.check.outputs.has-secrets }}
steps:
- name: "Check for secrets"
id: check
shell: bash
run: |
if [ -n "${{ (secrets.DOCKERHUB_USER != '' && secrets.DOCKERHUB_TOKEN != '') || '' }}" ]; then
echo "has-secrets=1" >> "$GITHUB_OUTPUT"
fi
docker-release:
needs: config
if: needs.config.outputs.has-secrets
name: docker-release
runs-on: ubuntu-latest
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v3
with:
persist-credentials: false
submodules: recursive
ref: ${{ github.ref }}
- shell: bash
env:
DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
run: |
.github/workflows/docker_build_push.sh

View File

@@ -1,60 +0,0 @@
name: Docker
on:
push:
branches:
- 'master'
pull_request:
types: [synchronize, opened, reopened, ready_for_review]
jobs:
config:
runs-on: "ubuntu-latest"
outputs:
has-secrets: ${{ steps.check.outputs.has-secrets }}
steps:
- name: "Check for secrets"
id: check
shell: bash
run: |
if [ -n "${{ (secrets.DOCKERHUB_USER != '' && secrets.DOCKERHUB_TOKEN != '') || '' }}" ]; then
echo "has-secrets=1" >> "$GITHUB_OUTPUT"
echo "has secrets!"
else
echo "has-secrets=0" >> "$GITHUB_OUTPUT"
echo "no secrets!"
fi
docker-build:
needs: config
if: needs.config.outputs.has-secrets
name: docker-build
runs-on: ubuntu-latest
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v3
with:
persist-credentials: false
- shell: bash
env:
DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
run: |
.github/workflows/docker_build_push.sh
- name: Build ephemeral env image
if: github.event_name == 'pull_request'
run: |
mkdir -p ./build
echo ${{ github.sha }} > ./build/SHA
echo ${{ github.event.pull_request.number }} > ./build/PR-NUM
DOCKER_BUILDKIT=1 docker build --target ci -t ${{ github.sha }} -t "pr-${{ github.event.pull_request.number }}" .
docker save ${{ github.sha }} | gzip > ./build/${{ github.sha }}.tar.gz
- name: Upload build artifacts
if: github.event_name == 'pull_request'
uses: actions/upload-artifact@v3
with:
name: build
path: build/

View File

@@ -1,118 +0,0 @@
#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
set -eo pipefail
SHA=$(git rev-parse HEAD)
REPO_NAME="apache/superset"
if [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then
REFSPEC=$(echo "${GITHUB_HEAD_REF}" | sed 's/[^a-zA-Z0-9]/-/g' | head -c 40)
PR_NUM=$(echo "${GITHUB_REF}" | sed 's:refs/pull/::' | sed 's:/merge::')
LATEST_TAG="pr-${PR_NUM}"
elif [[ "${GITHUB_EVENT_NAME}" == "release" ]]; then
REFSPEC=$(echo "${GITHUB_REF}" | sed 's:refs/tags/::' | head -c 40)
LATEST_TAG="${REFSPEC}"
else
REFSPEC=$(echo "${GITHUB_REF}" | sed 's:refs/heads/::' | sed 's/[^a-zA-Z0-9]/-/g' | head -c 40)
LATEST_TAG="${REFSPEC}"
fi
if [[ "${REFSPEC}" == "master" ]]; then
LATEST_TAG="latest"
fi
cat<<EOF
Rolling with tags:
- ${REPO_NAME}:${SHA}
- ${REPO_NAME}:${REFSPEC}
- ${REPO_NAME}:${LATEST_TAG}
EOF
#
# Build the "lean" image
#
DOCKER_BUILDKIT=1 docker build --target lean \
-t "${REPO_NAME}:${SHA}" \
-t "${REPO_NAME}:${REFSPEC}" \
-t "${REPO_NAME}:${LATEST_TAG}" \
--label "sha=${SHA}" \
--label "built_at=$(date)" \
--label "target=lean" \
--label "build_actor=${GITHUB_ACTOR}" \
.
#
# Build the "lean310" image
#
DOCKER_BUILDKIT=1 docker build --target lean \
-t "${REPO_NAME}:${SHA}-py310" \
-t "${REPO_NAME}:${REFSPEC}-py310" \
-t "${REPO_NAME}:${LATEST_TAG}-py310" \
--build-arg PY_VER="3.10-slim-bookworm"\
--label "sha=${SHA}" \
--label "built_at=$(date)" \
--label "target=lean310" \
--label "build_actor=${GITHUB_ACTOR}" \
.
#
# Build the "websocket" image
#
DOCKER_BUILDKIT=1 docker build \
-t "${REPO_NAME}:${SHA}-websocket" \
-t "${REPO_NAME}:${REFSPEC}-websocket" \
-t "${REPO_NAME}:${LATEST_TAG}-websocket" \
--label "sha=${SHA}" \
--label "built_at=$(date)" \
--label "target=websocket" \
--label "build_actor=${GITHUB_ACTOR}" \
superset-websocket
#
# Build the dev image
#
DOCKER_BUILDKIT=1 docker build --target dev \
-t "${REPO_NAME}:${SHA}-dev" \
-t "${REPO_NAME}:${REFSPEC}-dev" \
-t "${REPO_NAME}:${LATEST_TAG}-dev" \
--label "sha=${SHA}" \
--label "built_at=$(date)" \
--label "target=dev" \
--label "build_actor=${GITHUB_ACTOR}" \
.
#
# Build the dockerize image
#
DOCKER_BUILDKIT=1 docker build \
-t "${REPO_NAME}:dockerize" \
--label "sha=${SHA}" \
--label "built_at=$(date)" \
--label "build_actor=${GITHUB_ACTOR}" \
-f dockerize.Dockerfile \
.
if [ -z "${DOCKERHUB_TOKEN}" ]; then
# Skip if secrets aren't populated -- they're only visible for actions running in the repo (not on forks)
echo "Skipping Docker push"
else
# Login and push
docker logout
docker login --username "${DOCKERHUB_USER}" --password "${DOCKERHUB_TOKEN}"
docker push --all-tags "${REPO_NAME}"
fi

View File

@@ -1,59 +0,0 @@
{
"containerDefinitions": [
{
"name": "superset-ci",
"image": "apache/superset:latest",
"cpu": 0,
"links": [],
"portMappings": [
{
"containerPort": 8080,
"hostPort": 8080,
"protocol": "tcp"
}
],
"essential": true,
"entryPoint": [],
"command": [],
"environment": [
{
"name": "SUPERSET_LOAD_EXAMPLES",
"value": "yes"
},
{
"name": "SUPERSET_PORT",
"value": "8080"
},
{
"name": "SUPERSET_SECRET_KEY",
"value": "super-secret-for-ephemerals"
},
{
"name": "TALISMAN_ENABLED",
"value": "False"
}
],
"mountPoints": [],
"volumesFrom": [],
"logConfiguration": {
"logDriver": "awslogs",
"options": {
"awslogs-group": "/ecs/superset-ci",
"awslogs-region": "us-west-2",
"awslogs-stream-prefix": "ecs"
}
}
}
],
"family": "superset-ci",
"taskRoleArn": "ecsTaskExecutionRole",
"executionRoleArn": "ecsTaskExecutionRole",
"networkMode": "awsvpc",
"volumes": [],
"placementConstraints": [],
"requiresCompatibilities": [
"FARGATE"
],
"cpu": "512",
"memory": "1024"
}

View File

@@ -1,38 +0,0 @@
name: Embedded SDK Release
on:
push:
branches:
- 'master'
jobs:
config:
runs-on: "ubuntu-latest"
outputs:
has-secrets: ${{ steps.check.outputs.has-secrets }}
steps:
- name: "Check for secrets"
id: check
shell: bash
run: |
if [ -n "${{ (secrets.NPM_TOKEN != '') || '' }}" ]; then
echo "has-secrets=1" >> "$GITHUB_OUTPUT"
fi
build:
needs: config
if: needs.config.outputs.has-secrets
runs-on: ubuntu-20.04
defaults:
run:
working-directory: superset-embedded-sdk
steps:
- uses: actions/checkout@v3
- uses: actions/setup-node@v3
with:
node-version: "16"
registry-url: 'https://registry.npmjs.org'
- run: npm ci
- run: npm run ci:release
env:
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}

View File

@@ -1,23 +0,0 @@
name: Embedded SDK PR Checks
on:
pull_request:
paths:
- "superset-embedded-sdk/**"
types: [synchronize, opened, reopened, ready_for_review]
jobs:
embedded-sdk-test:
runs-on: ubuntu-20.04
defaults:
run:
working-directory: superset-embedded-sdk
steps:
- uses: actions/checkout@v3
- uses: actions/setup-node@v3
with:
node-version: "16"
registry-url: 'https://registry.npmjs.org'
- run: npm ci
- run: npm test
- run: npm run build

View File

@@ -1,75 +0,0 @@
name: Cleanup ephemeral envs (PR close)
on:
pull_request_target:
types: [closed]
jobs:
config:
runs-on: "ubuntu-latest"
outputs:
has-secrets: ${{ steps.check.outputs.has-secrets }}
steps:
- name: "Check for secrets"
id: check
shell: bash
run: |
if [ -n "${{ (secrets.AWS_ACCESS_KEY_ID != '' && secrets.AWS_SECRET_ACCESS_KEY != '') || '' }}" ]; then
echo "has-secrets=1" >> "$GITHUB_OUTPUT"
fi
ephemeral-env-cleanup:
needs: config
if: needs.config.outputs.has-secrets
name: Cleanup ephemeral envs
runs-on: ubuntu-latest
permissions:
pull-requests: write
steps:
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: us-west-2
- name: Describe ECS service
id: describe-services
run: |
echo "::set-output name=active::$(aws ecs describe-services --cluster superset-ci --services pr-${{ github.event.number }}-service | jq '.services[] | select(.status == "ACTIVE") | any')"
- name: Delete ECS service
if: steps.describe-services.outputs.active == 'true'
id: delete-service
run: |
aws ecs delete-service \
--cluster superset-ci \
--service pr-${{ github.event.number }}-service \
--force
- name: Login to Amazon ECR
if: steps.describe-services.outputs.active == 'true'
id: login-ecr
uses: aws-actions/amazon-ecr-login@v1
- name: Delete ECR image tag
if: steps.describe-services.outputs.active == 'true'
id: delete-image-tag
run: |
aws ecr batch-delete-image \
--registry-id $(echo "${{ steps.login-ecr.outputs.registry }}" | grep -Eo "^[0-9]+") \
--repository-name superset-ci \
--image-ids imageTag=pr-${{ github.event.number }}
- name: Comment (success)
if: steps.describe-services.outputs.active == 'true'
uses: actions/github-script@v3
with:
github-token: ${{github.token}}
script: |
github.issues.createComment({
issue_number: ${{ github.event.number }},
owner: context.repo.owner,
repo: context.repo.repo,
body: 'Ephemeral environment shutdown and build artifacts deleted.'
})

View File

@@ -1,214 +0,0 @@
name: Ephemeral env workflow
on:
issue_comment:
types: [created]
jobs:
config:
runs-on: "ubuntu-latest"
if: github.event.issue.pull_request
outputs:
has-secrets: ${{ steps.check.outputs.has-secrets }}
steps:
- name: "Check for secrets"
id: check
shell: bash
run: |
if [ -n "${{ (secrets.AWS_ACCESS_KEY_ID != '' && secrets.AWS_SECRET_ACCESS_KEY != '') || '' }}" ]; then
echo "has-secrets=1" >> "$GITHUB_OUTPUT"
fi
ephemeral_env_comment:
needs: config
if: needs.config.outputs.has-secrets
name: Evaluate ephemeral env comment trigger (/testenv)
runs-on: ubuntu-latest
permissions:
pull-requests: write
outputs:
slash-command: ${{ steps.eval-body.outputs.result }}
feature-flags: ${{ steps.eval-feature-flags.outputs.result }}
steps:
- name: Debug
run: |
echo "Comment on PR #${{ github.event.issue.number }} by ${{ github.event.issue.user.login }}, ${{ github.event.comment.author_association }}"
- name: Eval comment body for /testenv slash command
uses: actions/github-script@v3
id: eval-body
with:
result-encoding: string
script: |
const pattern = /^\/testenv (up|down)/
const result = pattern.exec(context.payload.comment.body)
return result === null ? 'noop' : result[1]
- name: Eval comment body for feature flags
uses: actions/github-script@v3
id: eval-feature-flags
with:
script: |
const pattern = /FEATURE_(\w+)=(\w+)/g;
let results = [];
[...context.payload.comment.body.matchAll(pattern)].forEach(match => {
const config = {
name: `SUPERSET_FEATURE_${match[1]}`,
value: match[2],
};
results.push(config);
});
return results;
- name: Limit to committers
if: >
steps.eval-body.outputs.result != 'noop' &&
github.event.comment.author_association != 'MEMBER' &&
github.event.comment.author_association != 'OWNER'
uses: actions/github-script@v3
with:
github-token: ${{github.token}}
script: |
const errMsg = '@${{ github.event.comment.user.login }} Ephemeral environment creation is currently limited to committers.'
github.issues.createComment({
issue_number: ${{ github.event.issue.number }},
owner: context.repo.owner,
repo: context.repo.repo,
body: errMsg
})
core.setFailed(errMsg)
ephemeral_env_up:
needs: ephemeral_env_comment
if: needs.ephemeral_env_comment.outputs.slash-command == 'up'
name: Spin up an ephemeral environment
runs-on: ubuntu-latest
permissions:
contents: read
pull-requests: write
steps:
- uses: actions/checkout@v3
with:
persist-credentials: false
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: us-west-2
- name: Login to Amazon ECR
id: login-ecr
uses: aws-actions/amazon-ecr-login@v1
- name: Check target image exists in ECR
id: check-image
continue-on-error: true
run: |
aws ecr describe-images \
--registry-id $(echo "${{ steps.login-ecr.outputs.registry }}" | grep -Eo "^[0-9]+") \
--repository-name superset-ci \
--image-ids imageTag=pr-${{ github.event.issue.number }}
- name: Fail on missing container image
if: steps.check-image.outcome == 'failure'
uses: actions/github-script@v3
with:
github-token: ${{github.token}}
script: |
const errMsg = '@${{ github.event.comment.user.login }} Container image not yet published for this PR. Please try again when build is complete.'
github.issues.createComment({
issue_number: ${{ github.event.issue.number }},
owner: context.repo.owner,
repo: context.repo.repo,
body: errMsg
})
core.setFailed(errMsg)
- name: Fill in the new image ID in the Amazon ECS task definition
id: task-def
uses: aws-actions/amazon-ecs-render-task-definition@v1
with:
task-definition: .github/workflows/ecs-task-definition.json
container-name: superset-ci
image: ${{ steps.login-ecr.outputs.registry }}/superset-ci:pr-${{ github.event.issue.number }}
- name: Update env vars in the Amazon ECS task definition
run: |
cat <<< "$(jq '.containerDefinitions[0].environment += ${{ needs.ephemeral_env_comment.outputs.feature-flags }}' < ${{ steps.task-def.outputs.task-definition }})" > ${{ steps.task-def.outputs.task-definition }}
- name: Describe ECS service
id: describe-services
run: |
echo "::set-output name=active::$(aws ecs describe-services --cluster superset-ci --services pr-${{ github.event.issue.number }}-service | jq '.services[] | select(.status == "ACTIVE") | any')"
- name: Create ECS service
if: steps.describe-services.outputs.active != 'true'
id: create-service
env:
ECR_SUBNETS: subnet-0e15a5034b4121710,subnet-0e8efef4a72224974
ECR_SECURITY_GROUP: sg-092ff3a6ae0574d91
run: |
aws ecs create-service \
--cluster superset-ci \
--service-name pr-${{ github.event.issue.number }}-service \
--task-definition superset-ci \
--launch-type FARGATE \
--desired-count 1 \
--platform-version LATEST \
--network-configuration "awsvpcConfiguration={subnets=[$ECR_SUBNETS],securityGroups=[$ECR_SECURITY_GROUP],assignPublicIp=ENABLED}" \
--tags key=pr,value=${{ github.event.issue.number }} key=github_user,value=${{ github.actor }}
- name: Deploy Amazon ECS task definition
id: deploy-task
uses: aws-actions/amazon-ecs-deploy-task-definition@v1
with:
task-definition: ${{ steps.task-def.outputs.task-definition }}
service: pr-${{ github.event.issue.number }}-service
cluster: superset-ci
wait-for-service-stability: true
wait-for-minutes: 10
- name: List tasks
id: list-tasks
run: |
echo "::set-output name=task::$(aws ecs list-tasks --cluster superset-ci --service-name pr-${{ github.event.issue.number }}-service | jq '.taskArns | first')"
- name: Get network interface
id: get-eni
run: |
echo "::set-output name=eni::$(aws ecs describe-tasks --cluster superset-ci --tasks ${{ steps.list-tasks.outputs.task }} | jq '.tasks | .[0] | .attachments | .[0] | .details | map(select(.name=="networkInterfaceId")) | .[0] | .value')"
- name: Get public IP
id: get-ip
run: |
echo "::set-output name=ip::$(aws ec2 describe-network-interfaces --network-interface-ids ${{ steps.get-eni.outputs.eni }} | jq -r '.NetworkInterfaces | first | .Association.PublicIp')"
- name: Comment (success)
if: ${{ success() }}
uses: actions/github-script@v3
with:
github-token: ${{github.token}}
script: |
github.issues.createComment({
issue_number: ${{ github.event.issue.number }},
owner: context.repo.owner,
repo: context.repo.repo,
body: '@${{ github.event.comment.user.login }} Ephemeral environment spinning up at http://${{ steps.get-ip.outputs.ip }}:8080. Credentials are `admin`/`admin`. Please allow several minutes for bootstrapping and startup.'
})
- name: Comment (failure)
if: ${{ failure() }}
uses: actions/github-script@v3
with:
github-token: ${{github.token}}
script: |
github.issues.createComment({
issue_number: ${{ github.event.issue.number }},
owner: context.repo.owner,
repo: context.repo.repo,
body: '@${{ github.event.comment.user.login }} Ephemeral environment creation failed. Please check the Actions logs for details.'
})

View File

@@ -1,32 +0,0 @@
name: Tags
on:
release:
types: [published] # This makes it run only when a new released is published
jobs:
latest-release:
name: Add/update tag to new release
runs-on: ubuntu-latest
permissions:
contents: write
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v3
with:
persist-credentials: false
submodules: recursive
- name: Check for latest tag
id: latest-tag
run: |
source ./scripts/tag_latest_release.sh $(echo ${{ github.event.release.tag_name }}) --dry-run
- name: Run latest-tag
uses: ./.github/actions/latest-tag
if: (! ${{ steps.latest-tag.outputs.SKIP_TAG }} )
with:
description: Superset latest release
tag-name: latest
env:
GITHUB_TOKEN: ${{ github.token }}

View File

@@ -1,66 +0,0 @@
name: License Check
on:
push:
branches-ignore:
- "dependabot/**"
pull_request:
jobs:
config:
runs-on: "ubuntu-latest"
outputs:
has-secrets: ${{ steps.check.outputs.has-secrets }}
steps:
- name: "Check for secrets"
id: check
shell: bash
run: |
if [ -n "${{ (secrets.FOSSA_API_KEY != '' ) || '' }}" ]; then
echo "has-secrets=1" >> "$GITHUB_OUTPUT"
fi
license_check:
needs: config
if: needs.config.outputs.has-secrets
name: License Check
runs-on: ubuntu-20.04
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v3
with:
persist-credentials: false
submodules: recursive
- name: Setup Java
uses: actions/setup-java@v1
with:
java-version: 8
- name: Generate fossa report
env:
FOSSA_API_KEY: ${{ secrets.FOSSA_API_KEY }}
run: |
set -eo pipefail
if [[ "${{github.event_name}}" != "pull_request" ]]; then
./scripts/fossa.sh
exit 0
fi
URL="https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/files"
FILES=$(curl -s -X GET -G $URL | jq -r '.[] | .filename')
cat<<EOF
CHANGED FILES:
$FILES
EOF
if [[ "${FILES}" =~ (.*package*\.json|requirements\/[a-z_-]+\.txt|setup\.py) ]]; then
echo "Detected dependency changes... running fossa check"
./scripts/fossa.sh
else
echo "No dependency changes... skiping fossa check"
fi
shell: bash
- name: Run license check
run: ./scripts/check_license.sh

View File

@@ -1,31 +0,0 @@
name: PR Lint
on:
pull_request:
# By default, a workflow only runs when a pull_request's activity type is opened, synchronize, or reopened. We
# explicity override here so that PR titles are re-linted when the PR text content is edited.
#
# Possible values: https://help.github.com/en/actions/reference/events-that-trigger-workflows#pull-request-event-pull_request
types: [opened, edited, reopened, synchronize]
jobs:
check:
runs-on: ubuntu-latest
permissions:
contents: read
pull-requests: write
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v3
with:
persist-credentials: false
submodules: recursive
- uses: ./.github/actions/pr-lint-action
with:
title-regex: "^(build|chore|ci|docs|feat|fix|perf|refactor|style|test|other)(\\(.+\\))?(\\!)?:\\s.+"
on-failed-regex-fail-action: true
on-failed-regex-request-changes: false
on-failed-regex-create-review: false
on-failed-regex-comment:
"Please format your PR title to match: `%regex%`!"
repo-token: "${{ github.token }}"

View File

@@ -1,59 +0,0 @@
name: Prefer Typescript
on:
push:
branches-ignore:
- "dependabot/**"
pull_request:
jobs:
prefer_typescript:
if: github.ref == 'ref/heads/master' && github.event_name == 'pull_request'
name: Prefer Typescript
runs-on: ubuntu-latest
permissions:
contents: read
pull-requests: write
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v3
with:
persist-credentials: false
submodules: recursive
- name: Get changed files
id: changed
uses: ./.github/actions/file-changes-action
with:
githubToken: ${{ github.token }}
- name: Determine if a .js or .jsx file was added
id: check
run: |
js_files_added() {
jq -r '
map(
select(
endswith(".js") or endswith(".jsx")
)
) | join("\n")
' ${HOME}/files_added.json
}
echo ::set-output name=js_files_added::$(js_files_added)
- if: steps.check.outputs.js_files_added
name: Add Comment to PR
uses: ./.github/actions/comment-on-pr
continue-on-error: true
env:
GITHUB_TOKEN: ${{ github.token }}
with:
msg: |
### WARNING: Prefer TypeScript
Looks like your PR contains new `.js` or `.jsx` files:
```
${{steps.check.outputs.js_files_added}}
```
As decided in [SIP-36](https://github.com/apache/superset/issues/9101), all new frontend code should be written in TypeScript. Please convert above files to TypeScript then re-request review.

View File

@@ -1,113 +0,0 @@
name: release-workflow
on:
push:
branches:
- 'master'
jobs:
config:
runs-on: "ubuntu-latest"
outputs:
has-secrets: ${{ steps.check.outputs.has-secrets }}
steps:
- name: "Check for secrets"
id: check
shell: bash
run: |
if [ -n "${{ (secrets.NPM_TOKEN != '' && secrets.GH_PERSONAL_ACCESS_TOKEN != '') || '' }}" ]; then
echo "has-secrets=1" >> "$GITHUB_OUTPUT"
fi
build:
needs: config
if: needs.config.outputs.has-secrets
name: Bump version and publish package(s)
runs-on: ubuntu-20.04
strategy:
matrix:
node-version: [16]
steps:
- uses: actions/checkout@v3
with:
# pulls all commits (needed for lerna / semantic release to correctly version)
fetch-depth: 0
- name: Get tags and filter trigger tags
run: |
if ! git fetch --depth=1 origin "+refs/tags/*:refs/tags/*"; then
echo "::notice title=Workflow skipped::No tags present in repository"
exit
fi
echo "HAS_TAGS=1" >> $GITHUB_ENV"
git fetch --prune --unshallow
git tag -d `git tag | grep -E '^trigger-'`
- name: Use Node.js ${{ matrix.node-version }}
if: env.HAS_TAGS
uses: actions/setup-node@v3
with:
node-version: ${{ matrix.node-version }}
- name: Cache npm
if: env.HAS_TAGS
uses: actions/cache@v1
with:
path: ~/.npm # npm cache files are stored in `~/.npm` on Linux/macOS
key: ${{ runner.OS }}-node-${{ hashFiles('**/package-lock.json') }}
restore-keys: |
${{ runner.OS }}-node-
${{ runner.OS }}-
- name: Get npm cache directory path
if: env.HAS_TAGS
id: npm-cache-dir-path
run: echo "::set-output name=dir::$(npm config get cache)"
- name: Cache npm
if: env.HAS_TAGS
uses: actions/cache@v1
id: npm-cache # use this to check for `cache-hit` (`steps.npm-cache.outputs.cache-hit != 'true'`)
with:
path: ${{ steps.npm-cache-dir-path.outputs.dir }}
key: ${{ runner.os }}-npm-${{ hashFiles('**/package-lock.json') }}
restore-keys: |
${{ runner.os }}-npm-
- name: Install dependencies
if: env.HAS_TAGS
working-directory: ./superset-frontend
run: npm ci
- name: Run unit tests
if: env.HAS_TAGS
working-directory: ./superset-frontend
run: npm run test -- plugins packages
- name: Build packages
if: env.HAS_TAGS
working-directory: ./superset-frontend
run: npm run plugins:build
- name: Configure npm and git
if: env.HAS_TAGS
run: |
echo "@superset-ui:registry=https://registry.npmjs.org/" > .npmrc
echo "registry=https://registry.npmjs.org/" >> .npmrc
echo "//registry.npmjs.org/:_authToken=\${NPM_TOKEN}" >> $HOME/.npmrc 2> /dev/null
npm whoami
git config --local user.email "action@github.com"
git config --local user.name "GitHub Action"
env:
NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
GITHUB_TOKEN: ${{ github.token }}
- name: Bump version and publish package(s)
if: env.HAS_TAGS
working-directory: ./superset-frontend
run: |
git tag -d `git tag | grep -E '^trigger-'`
npm run plugins:release-from-tag
env:
NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
GITHUB_TOKEN: ${{ github.token }}
GH_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}

View File

@@ -1,103 +0,0 @@
name: Applitools Cypress
on:
schedule:
- cron: "0 1 * * *"
jobs:
config:
runs-on: "ubuntu-latest"
outputs:
has-secrets: ${{ steps.check.outputs.has-secrets }}
steps:
- name: "Check for secrets"
id: check
shell: bash
run: |
if [ -n "${{ (secrets.APPLITOOLS_API_KEY != '' && secrets.APPLITOOLS_API_KEY != '') || '' }}" ]; then
echo "has-secrets=1" >> "$GITHUB_OUTPUT"
fi
cypress-applitools:
needs: config
if: needs.config.outputs.has-secrets
runs-on: ubuntu-20.04
strategy:
fail-fast: false
matrix:
browser: ["chrome"]
node: [16]
env:
SUPERSET_ENV: development
SUPERSET_CONFIG: tests.integration_tests.superset_test_config
SUPERSET__SQLALCHEMY_DATABASE_URI: postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset
PYTHONPATH: ${{ github.workspace }}
REDIS_PORT: 16379
GITHUB_TOKEN: ${{ github.token }}
APPLITOOLS_APP_NAME: Superset
APPLITOOLS_API_KEY: ${{ secrets.APPLITOOLS_API_KEY }}
APPLITOOLS_BATCH_ID: ${{ github.sha }}
APPLITOOLS_BATCH_NAME: Superset Cypress
services:
postgres:
image: postgres:15-alpine
env:
POSTGRES_USER: superset
POSTGRES_PASSWORD: superset
ports:
- 15432:5432
redis:
image: redis:7-alpine
ports:
- 16379:6379
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v3
with:
persist-credentials: false
submodules: recursive
ref: master
- name: Setup Python
uses: actions/setup-python@v4
with:
python-version: "3.9"
- name: OS dependencies
uses: ./.github/actions/cached-dependencies
with:
run: apt-get-install
- name: Install python dependencies
uses: ./.github/actions/cached-dependencies
with:
run: |
pip-upgrade
pip install -r requirements/testing.txt
- name: Setup postgres
uses: ./.github/actions/cached-dependencies
with:
run: setup-postgres
- name: Import test data
uses: ./.github/actions/cached-dependencies
with:
run: testdata
- name: Setup Node.js
uses: actions/setup-node@v3
with:
node-version: ${{ matrix.node }}
- name: Install npm dependencies
uses: ./.github/actions/cached-dependencies
with:
run: npm-install
- name: Build javascript packages
uses: ./.github/actions/cached-dependencies
with:
run: build-instrumented-assets
- name: Install cypress
uses: ./.github/actions/cached-dependencies
with:
run: cypress-install
- name: Run Cypress
uses: ./.github/actions/cached-dependencies
env:
CYPRESS_BROWSER: ${{ matrix.browser }}
with:
run: cypress-run-applitools

View File

@@ -1,55 +0,0 @@
name: Applitools Storybook
on:
schedule:
- cron: "0 0 * * *"
env:
APPLITOOLS_APP_NAME: Superset
APPLITOOLS_API_KEY: ${{ secrets.APPLITOOLS_API_KEY }}
APPLITOOLS_BATCH_ID: ${{ github.sha }}
APPLITOOLS_BATCH_NAME: Superset Storybook
jobs:
config:
runs-on: "ubuntu-latest"
outputs:
has-secrets: ${{ steps.check.outputs.has-secrets }}
steps:
- name: "Check for secrets"
id: check
shell: bash
run: |
if [ -n "${{ (secrets.APPLITOOLS_API_KEY != '' && secrets.APPLITOOLS_API_KEY != '') || '' }}" ]; then
echo "has-secrets=1" >> "$GITHUB_OUTPUT"
fi
cron:
needs: config
if: needs.config.outputs.has-secrets
runs-on: ubuntu-20.04
strategy:
matrix:
node: [16]
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v3
with:
persist-credentials: false
submodules: recursive
ref: master
- name: Set up Node.js
uses: actions/setup-node@v3.1.1
with:
node-version: ${{ matrix.node }}
- name: Install eyes-storybook dependencies
uses: ./.github/actions/cached-dependencies
with:
run: eyes-storybook-dependencies
- name: Install NPM dependencies
uses: ./.github/actions/cached-dependencies
with:
run: npm-install
- name: Run Applitools Eyes-Storybook
working-directory: ./superset-frontend
run: npx eyes-storybook -u https://superset-storybook.netlify.app/

View File

@@ -1,75 +0,0 @@
name: Superset CLI tests
on:
push:
branches-ignore:
- "dependabot/npm_and_yarn/**"
pull_request:
types: [synchronize, opened, reopened, ready_for_review]
jobs:
test-load-examples:
runs-on: ubuntu-20.04
strategy:
matrix:
python-version: ["3.9"]
env:
PYTHONPATH: ${{ github.workspace }}
SUPERSET_CONFIG: tests.integration_tests.superset_test_config
REDIS_PORT: 16379
SUPERSET__SQLALCHEMY_DATABASE_URI: postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset
services:
postgres:
image: postgres:15-alpine
env:
POSTGRES_USER: superset
POSTGRES_PASSWORD: superset
ports:
# Use custom ports for services to avoid accidentally connecting to
# GitHub action runner's default installations
- 15432:5432
redis:
image: redis:7-alpine
ports:
- 16379:6379
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v3
with:
persist-credentials: false
submodules: recursive
- name: Check if python changes are present
id: check
env:
GITHUB_REPO: ${{ github.repository }}
PR_NUMBER: ${{ github.event.pull_request.number }}
continue-on-error: true
run: ./scripts/ci_check_no_file_changes.sh python
- name: Setup Python
if: steps.check.outcome == 'failure'
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
cache: 'pip'
cache-dependency-path: 'requirements/testing.txt'
- name: Install dependencies
if: steps.check.outcome == 'failure'
uses: ./.github/actions/cached-dependencies
with:
run: |
apt-get-install
pip-upgrade
pip install wheel
pip install -r requirements/testing.txt
setup-postgres
- name: superset init
if: steps.check.outcome == 'failure'
run: |
pip install -e .
superset db upgrade
superset load_test_users
- name: superset load_examples
if: steps.check.outcome == 'failure'
run: |
# load examples without test data
superset load_examples --load-big-data

View File

@@ -1,56 +0,0 @@
name: Docs
on:
push:
paths:
- "docs/**"
pull_request:
paths:
- "docs/**"
jobs:
config:
runs-on: "ubuntu-latest"
outputs:
has-secrets: ${{ steps.check.outputs.has-secrets }}
steps:
- name: "Check for secrets"
id: check
shell: bash
run: |
if [ -n "${{ (secrets.SUPERSET_SITE_BUILD != '' && secrets.SUPERSET_SITE_BUILD != '') || '' }}" ]; then
echo "has-secrets=1" >> "$GITHUB_OUTPUT"
fi
build-deploy:
needs: config
if: needs.config.outputs.has-secrets
name: Build & Deploy
runs-on: ubuntu-20.04
defaults:
run:
working-directory: docs
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v3
with:
persist-credentials: false
submodules: recursive
- name: yarn install
run: |
yarn install --check-cache
- name: yarn build
run: |
yarn build
- name: deploy docs
if: github.ref == 'refs/heads/master'
uses: ./.github/actions/github-action-push-to-another-repository
env:
API_TOKEN_GITHUB: ${{ secrets.SUPERSET_SITE_BUILD }}
with:
source-directory: './docs/build'
destination-github-username: 'apache'
destination-repository-name: 'superset-site'
target-branch: 'asf-site'
commit-message: "deploying docs: ${{ github.event.head_commit.message }} (apache/superset@${{ github.sha }})"
user-email: dev@superset.apache.org

View File

@@ -1,126 +0,0 @@
name: E2E
on:
push:
branches-ignore:
- "dependabot/**/docs/**"
paths-ignore:
- "docs/**"
pull_request:
types: [synchronize, opened, reopened, ready_for_review]
jobs:
cypress-matrix:
runs-on: ubuntu-20.04
permissions:
contents: read
pull-requests: read
strategy:
# when one test fails, DO NOT cancel the other
# containers, because this will kill Cypress processes
# leaving the Dashboard hanging ...
# https://github.com/cypress-io/github-action/issues/48
fail-fast: false
matrix:
containers: [1, 2, 3]
browser: ["chrome"]
env:
SUPERSET_ENV: development
SUPERSET_CONFIG: tests.integration_tests.superset_test_config
SUPERSET__SQLALCHEMY_DATABASE_URI: postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset
PYTHONPATH: ${{ github.workspace }}
REDIS_PORT: 16379
GITHUB_TOKEN: ${{ github.token }}
services:
postgres:
image: postgres:15-alpine
env:
POSTGRES_USER: superset
POSTGRES_PASSWORD: superset
ports:
- 15432:5432
redis:
image: redis:7-alpine
ports:
- 16379:6379
steps:
- name: "Checkout (pull) ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v3
if: github.event_name == 'push'
with:
persist-credentials: false
submodules: recursive
- name: "Checkout (pull_request) ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v3
if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target'
with:
ref: "refs/pull/${{ github.event.number }}/merge"
persist-credentials: false
submodules: recursive
- name: Check if python or frontend changes are present
id: check
env:
GITHUB_REPO: ${{ github.repository }}
PR_NUMBER: ${{ github.event.pull_request.number }}
continue-on-error: true
run: ./scripts/ci_check_no_file_changes.sh python frontend
- name: Setup Python
if: steps.check.outcome == 'failure'
uses: actions/setup-python@v4
with:
python-version: "3.9"
- name: OS dependencies
if: steps.check.outcome == 'failure'
uses: ./.github/actions/cached-dependencies
with:
run: apt-get-install
- name: Install python dependencies
if: steps.check.outcome == 'failure'
uses: ./.github/actions/cached-dependencies
with:
run: |
pip-upgrade
pip install -r requirements/testing.txt
- name: Setup postgres
if: steps.check.outcome == 'failure'
uses: ./.github/actions/cached-dependencies
with:
run: setup-postgres
- name: Import test data
if: steps.check.outcome == 'failure'
uses: ./.github/actions/cached-dependencies
with:
run: testdata
- name: Setup Node.js
uses: actions/setup-node@v3
with:
node-version: "16"
- name: Install npm dependencies
if: steps.check.outcome == 'failure'
uses: ./.github/actions/cached-dependencies
with:
run: npm-install
- name: Build javascript packages
if: steps.check.outcome == 'failure'
uses: ./.github/actions/cached-dependencies
with:
run: build-instrumented-assets
- name: Install cypress
if: steps.check.outcome == 'failure'
uses: ./.github/actions/cached-dependencies
with:
run: cypress-install
- name: Run Cypress
if: steps.check.outcome == 'failure'
uses: ./.github/actions/cached-dependencies
env:
CYPRESS_BROWSER: ${{ matrix.browser }}
CYPRESS_KEY: YjljODE2MzAtODcwOC00NTA3LWE4NmMtMTU3YmFmMjIzOTRhCg==
with:
run: cypress-run-all
- name: Upload Artifacts
uses: actions/upload-artifact@v3
if: failure()
with:
name: screenshots
path: ${{ github.workspace }}/superset-frontend/cypress-base/cypress/screenshots

View File

@@ -1,71 +0,0 @@
name: Frontend
on:
push:
branches-ignore:
- "dependabot/**/docs/**"
- "dependabot/**/cypress-base/**"
pull_request:
types: [synchronize, opened, reopened, ready_for_review]
jobs:
frontend-build:
runs-on: ubuntu-20.04
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v3
with:
persist-credentials: false
submodules: recursive
- name: Check npm lock file version
run: ./scripts/ci_check_npm_lock_version.sh ./superset-frontend/package-lock.json
- name: Check if frontend changes are present
id: check
env:
GITHUB_REPO: ${{ github.repository }}
PR_NUMBER: ${{ github.event.pull_request.number }}
continue-on-error: true
run: ./scripts/ci_check_no_file_changes.sh frontend
- name: Setup Node.js
if: steps.check.outcome == 'failure'
uses: actions/setup-node@v3
with:
node-version: "16"
- name: Install dependencies
if: steps.check.outcome == 'failure'
uses: ./.github/actions/cached-dependencies
with:
run: npm-install
- name: lint
if: steps.check.outcome == 'failure'
working-directory: ./superset-frontend
run: |
npm run lint -- --quiet
npm run prettier-check
- name: Build plugins packages
if: steps.check.outcome == 'failure'
working-directory: ./superset-frontend
run: npm run plugins:build
- name: Build plugins Storybook
if: steps.check.outcome == 'failure'
working-directory: ./superset-frontend
run: npm run plugins:build-storybook
- name: superset-ui/core coverage
if: steps.check.outcome == 'failure'
working-directory: ./superset-frontend
run: |
npm run core:cover
- name: unit tests
if: steps.check.outcome == 'failure'
working-directory: ./superset-frontend
run: |
npm run test -- --coverage --silent
# todo: remove this step when fix generator as a project in root jest.config.js
- name: generator-superset unit tests
if: steps.check.outcome == 'failure'
working-directory: ./superset-frontend/packages/generator-superset
run: npx jest
- name: Upload code coverage
if: steps.check.outcome == 'failure'
working-directory: ./superset-frontend
run: ../.github/workflows/codecov.sh -c -F javascript

View File

@@ -1,47 +0,0 @@
name: Lint and Test Charts
on:
pull_request:
types: [opened, edited, reopened, synchronize]
jobs:
lint-test:
runs-on: ubuntu-latest
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v3
with:
persist-credentials: false
submodules: recursive
fetch-depth: 0
- name: Set up Helm
uses: azure/setup-helm@v1
with:
version: v3.5.4
- uses: actions/setup-python@v4
with:
python-version: "3.9"
- name: Set up chart-testing
uses: ./.github/actions/chart-testing-action
- name: Run chart-testing (list-changed)
id: list-changed
run: |
changed=$(ct list-changed --print-config)
if [[ -n "$changed" ]]; then
echo "::set-output name=changed::true"
fi
env:
CT_CHART_DIRS: helm
CT_SINCE: HEAD
- name: Run chart-testing (lint)
run: ct lint --print-config
env:
CT_CHART_DIRS: helm
CT_LINT_CONF: lintconf.yaml
CT_SINCE: HEAD
CT_CHART_REPOS: bitnami=https://charts.bitnami.com/bitnami

View File

@@ -1,43 +0,0 @@
name: Release Charts
on:
push:
branches:
- "master"
paths:
- "helm/**"
jobs:
release:
runs-on: ubuntu-latest
permissions:
contents: write
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v3
with:
persist-credentials: false
submodules: recursive
fetch-depth: 0
- name: Configure Git
run: |
git config user.name "$GITHUB_ACTOR"
git config user.email "$GITHUB_ACTOR@users.noreply.github.com"
- name: Install Helm
uses: azure/setup-helm@v1
with:
version: v3.5.4
- name: Add bitnami repo dependency
run: helm repo add bitnami https://charts.bitnami.com/bitnami
- name: Run chart-releaser
uses: ./.github/actions/chart-releaser-action
with:
charts_dir: helm
env:
CR_TOKEN: "${{ github.token }}"
CR_RELEASE_NAME_TEMPLATE: "superset-helm-chart-{{ .Version }}"

View File

@@ -1,199 +0,0 @@
# Python integration tests
name: Python-Integration
on:
push:
branches-ignore:
- "dependabot/npm_and_yarn/**"
pull_request:
types: [synchronize, opened, reopened, ready_for_review]
jobs:
test-mysql:
runs-on: ubuntu-20.04
strategy:
matrix:
python-version: ["3.9"]
env:
PYTHONPATH: ${{ github.workspace }}
SUPERSET_CONFIG: tests.integration_tests.superset_test_config
REDIS_PORT: 16379
SUPERSET__SQLALCHEMY_DATABASE_URI: |
mysql+mysqldb://superset:superset@127.0.0.1:13306/superset?charset=utf8mb4&binary_prefix=true
services:
mysql:
image: mysql:5.7
env:
MYSQL_ROOT_PASSWORD: root
ports:
- 13306:3306
redis:
image: redis:7-alpine
options: --entrypoint redis-server
ports:
- 16379:6379
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v3
with:
persist-credentials: false
submodules: recursive
- name: Check if python changes are present
id: check
env:
GITHUB_REPO: ${{ github.repository }}
PR_NUMBER: ${{ github.event.pull_request.number }}
continue-on-error: true
run: ./scripts/ci_check_no_file_changes.sh python
- name: Setup Python
if: steps.check.outcome == 'failure'
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
cache: 'pip'
cache-dependency-path: 'requirements/testing.txt'
- name: Install dependencies
if: steps.check.outcome == 'failure'
uses: ./.github/actions/cached-dependencies
with:
run: |
apt-get-install
pip-upgrade
pip install wheel
pip install -r requirements/testing.txt
setup-mysql
- name: Run celery
if: steps.check.outcome == 'failure'
run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 &
- name: Python integration tests (MySQL)
if: steps.check.outcome == 'failure'
run: |
./scripts/python_tests.sh
- name: Upload code coverage
if: steps.check.outcome == 'failure'
run: |
bash .github/workflows/codecov.sh -c -F python -F mysql
test-postgres:
runs-on: ubuntu-20.04
strategy:
matrix:
python-version: ["3.9", "3.10"]
env:
PYTHONPATH: ${{ github.workspace }}
SUPERSET_CONFIG: tests.integration_tests.superset_test_config
REDIS_PORT: 16379
SUPERSET__SQLALCHEMY_DATABASE_URI: postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset
services:
postgres:
image: postgres:15-alpine
env:
POSTGRES_USER: superset
POSTGRES_PASSWORD: superset
ports:
# Use custom ports for services to avoid accidentally connecting to
# GitHub action runner's default installations
- 15432:5432
redis:
image: redis:7-alpine
ports:
- 16379:6379
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v3
with:
persist-credentials: false
submodules: recursive
- name: Check if python changes are present
id: check
env:
GITHUB_REPO: ${{ github.repository }}
PR_NUMBER: ${{ github.event.pull_request.number }}
continue-on-error: true
run: ./scripts/ci_check_no_file_changes.sh python
- name: Setup Python
if: steps.check.outcome == 'failure'
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
cache: 'pip'
cache-dependency-path: 'requirements/testing.txt'
- name: Install dependencies
if: steps.check.outcome == 'failure'
uses: ./.github/actions/cached-dependencies
with:
run: |
apt-get-install
pip-upgrade
pip install wheel
pip install -r requirements/testing.txt
setup-postgres
- name: Run celery
if: steps.check.outcome == 'failure'
run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 &
- name: Python integration tests (PostgreSQL)
if: steps.check.outcome == 'failure'
run: |
./scripts/python_tests.sh
- name: Upload code coverage
if: steps.check.outcome == 'failure'
run: |
bash .github/workflows/codecov.sh -c -F python -F postgres
test-sqlite:
runs-on: ubuntu-20.04
strategy:
matrix:
python-version: ["3.9"]
env:
PYTHONPATH: ${{ github.workspace }}
SUPERSET_CONFIG: tests.integration_tests.superset_test_config
REDIS_PORT: 16379
SUPERSET__SQLALCHEMY_DATABASE_URI: |
sqlite:///${{ github.workspace }}/.temp/unittest.db
services:
redis:
image: redis:7-alpine
ports:
- 16379:6379
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v3
with:
persist-credentials: false
submodules: recursive
- name: Check if python changes are present
id: check
env:
GITHUB_REPO: ${{ github.repository }}
PR_NUMBER: ${{ github.event.pull_request.number }}
continue-on-error: true
run: ./scripts/ci_check_no_file_changes.sh python
- name: Setup Python
if: steps.check.outcome == 'failure'
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
cache: 'pip'
cache-dependency-path: 'requirements/testing.txt'
- name: Install dependencies
if: steps.check.outcome == 'failure'
uses: ./.github/actions/cached-dependencies
with:
run: |
apt-get-install
pip-upgrade
pip install wheel
pip install -r requirements/testing.txt
mkdir ${{ github.workspace }}/.temp
- name: Run celery
if: steps.check.outcome == 'failure'
run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 &
- name: Python integration tests (SQLite)
if: steps.check.outcome == 'failure'
run: |
./scripts/python_tests.sh
- name: Upload code coverage
if: steps.check.outcome == 'failure'
run: |
bash .github/workflows/codecov.sh -c -F python -F sqlite

View File

@@ -1,117 +0,0 @@
# Python Misc unit tests
name: Python Misc
on:
push:
branches-ignore:
- "dependabot/npm_and_yarn/**"
pull_request:
types: [synchronize, opened, reopened, ready_for_review]
jobs:
python-lint:
runs-on: ubuntu-20.04
strategy:
matrix:
python-version: ["3.9"]
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v3
with:
persist-credentials: false
submodules: recursive
- name: Check if python changes are present
id: check
env:
GITHUB_REPO: ${{ github.repository }}
PR_NUMBER: ${{ github.event.pull_request.number }}
continue-on-error: true
run: ./scripts/ci_check_no_file_changes.sh python
- name: Setup Python
if: steps.check.outcome == 'failure'
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
cache: "pip"
cache-dependency-path: "requirements/testing.txt"
- name: Install dependencies
if: steps.check.outcome == 'failure'
uses: ./.github/actions/cached-dependencies
with:
run: |
apt-get-install
pip-upgrade
pip install wheel
pip install -r requirements/testing.txt
- name: pylint
if: steps.check.outcome == 'failure'
# `-j 0` run Pylint in parallel
run: pylint -j 0 superset
pre-commit:
runs-on: ubuntu-20.04
strategy:
matrix:
python-version: ["3.9"]
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v3
with:
persist-credentials: false
submodules: recursive
- name: Setup Python
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
cache: "pip"
cache-dependency-path: |
requirements/base.txt
requirements/integration.txt
- name: Install dependencies
uses: ./.github/actions/cached-dependencies
with:
run: |
apt-get-install
pip-upgrade
pip install wheel
pip install -r requirements/base.txt
pip install -r requirements/integration.txt
# Add brew to the path - see https://github.com/actions/runner-images/issues/6283
- name: Enable brew and helm-docs
run: |
echo "/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/sbin" >> $GITHUB_PATH
eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"
echo "HOMEBREW_PREFIX=$HOMEBREW_PREFIX" >>"${GITHUB_ENV}"
echo "HOMEBREW_CELLAR=$HOMEBREW_CELLAR" >>"${GITHUB_ENV}"
echo "HOMEBREW_REPOSITORY=$HOMEBREW_REPOSITORY" >>"${GITHUB_ENV}"
brew install norwoodj/tap/helm-docs
- name: pre-commit
run: pre-commit run --all-files
babel-extract:
runs-on: ubuntu-20.04
strategy:
matrix:
python-version: ["3.9"]
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v3
with:
persist-credentials: false
submodules: recursive
- name: Setup Python
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
cache: "pip"
cache-dependency-path: "requirements/base.txt"
- name: Install dependencies
uses: ./.github/actions/cached-dependencies
with:
run: |
apt-get-install
pip-upgrade
pip install wheel
pip install -r requirements/base.txt
- name: Test babel extraction
run: flask fab babel-extract --target superset/translations --output superset/translations/messages.pot --config superset/translations/babel.cfg -k _,__,t,tn,tct

View File

@@ -1,163 +0,0 @@
# Python Presto/Hive unit tests
name: Python Presto/Hive
on:
push:
branches-ignore:
- "dependabot/npm_and_yarn/**"
pull_request:
types: [synchronize, opened, reopened, ready_for_review]
jobs:
test-postgres-presto:
runs-on: ubuntu-20.04
strategy:
matrix:
python-version: ["3.9"]
env:
PYTHONPATH: ${{ github.workspace }}
SUPERSET_CONFIG: tests.integration_tests.superset_test_config
REDIS_PORT: 16379
SUPERSET__SQLALCHEMY_DATABASE_URI: postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset
SUPERSET__SQLALCHEMY_EXAMPLES_URI: presto://localhost:15433/memory/default
services:
postgres:
image: postgres:15-alpine
env:
POSTGRES_USER: superset
POSTGRES_PASSWORD: superset
ports:
# Use custom ports for services to avoid accidentally connecting to
# GitHub action runner's default installations
- 15432:5432
presto:
image: starburstdata/presto:350-e.6
env:
POSTGRES_USER: superset
POSTGRES_PASSWORD: superset
ports:
# Use custom ports for services to avoid accidentally connecting to
# GitHub action runner's default installations
- 15433:8080
redis:
image: redis:7-alpine
ports:
- 16379:6379
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v3
with:
persist-credentials: false
submodules: recursive
- name: Check if python changes are present
id: check
env:
GITHUB_REPO: ${{ github.repository }}
PR_NUMBER: ${{ github.event.pull_request.number }}
continue-on-error: true
run: ./scripts/ci_check_no_file_changes.sh python
- name: Setup Python
if: steps.check.outcome == 'failure'
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
cache: 'pip'
cache-dependency-path: 'requirements/testing.txt'
- name: Install dependencies
if: steps.check.outcome == 'failure'
uses: ./.github/actions/cached-dependencies
with:
run: |
apt-get-install
pip-upgrade
pip install wheel
pip install -r requirements/testing.txt
setup-postgres
- name: Run celery
if: steps.check.outcome == 'failure'
run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 &
- name: Python unit tests (PostgreSQL)
if: steps.check.outcome == 'failure'
run: |
./scripts/python_tests.sh -m 'chart_data_flow or sql_json_flow'
- name: Upload code coverage
if: steps.check.outcome == 'failure'
run: |
bash .github/workflows/codecov.sh -c -F python -F presto
test-postgres-hive:
runs-on: ubuntu-20.04
strategy:
matrix:
python-version: ["3.9"]
env:
PYTHONPATH: ${{ github.workspace }}
SUPERSET_CONFIG: tests.integration_tests.superset_test_config
REDIS_PORT: 16379
SUPERSET__SQLALCHEMY_DATABASE_URI: postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset
SUPERSET__SQLALCHEMY_EXAMPLES_URI: hive://localhost:10000/default
UPLOAD_FOLDER: /tmp/.superset/uploads/
services:
postgres:
image: postgres:15-alpine
env:
POSTGRES_USER: superset
POSTGRES_PASSWORD: superset
ports:
# Use custom ports for services to avoid accidentally connecting to
# GitHub action runner's default installations
- 15432:5432
redis:
image: redis:7-alpine
ports:
- 16379:6379
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v3
with:
persist-credentials: false
submodules: recursive
- name: Check if python changes are present
id: check
env:
GITHUB_REPO: ${{ github.repository }}
PR_NUMBER: ${{ github.event.pull_request.number }}
continue-on-error: true
run: ./scripts/ci_check_no_file_changes.sh python
- name: Create csv upload directory
if: steps.check.outcome == 'failure'
run: sudo mkdir -p /tmp/.superset/uploads
- name: Give write access to the csv upload directory
if: steps.check.outcome == 'failure'
run: sudo chown -R $USER:$USER /tmp/.superset
- name: Start hadoop and hive
if: steps.check.outcome == 'failure'
run: docker-compose -f scripts/databases/hive/docker-compose.yml up -d
- name: Setup Python
if: steps.check.outcome == 'failure'
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
cache: 'pip'
cache-dependency-path: 'requirements/testing.txt'
- name: Install dependencies
if: steps.check.outcome == 'failure'
uses: ./.github/actions/cached-dependencies
with:
run: |
apt-get-install
pip-upgrade
pip install wheel
pip install -r requirements/testing.txt
setup-postgres
- name: Run celery
if: steps.check.outcome == 'failure'
run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 &
- name: Python unit tests (PostgreSQL)
if: steps.check.outcome == 'failure'
run: |
./scripts/python_tests.sh -m 'chart_data_flow or sql_json_flow'
- name: Upload code coverage
if: steps.check.outcome == 'failure'
run: |
bash .github/workflows/codecov.sh -c -F python -F hive

View File

@@ -1,59 +0,0 @@
# Python unit tests
name: Python-Unit
on:
push:
branches-ignore:
- "dependabot/npm_and_yarn/**"
pull_request:
types: [synchronize, opened, reopened, ready_for_review]
jobs:
unit-tests:
runs-on: ubuntu-20.04
strategy:
matrix:
python-version: ["3.9", "3.10"]
env:
PYTHONPATH: ${{ github.workspace }}
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v3
with:
persist-credentials: false
submodules: recursive
- name: Check if python changes are present
id: check
env:
GITHUB_REPO: ${{ github.repository }}
PR_NUMBER: ${{ github.event.pull_request.number }}
continue-on-error: true
run: ./scripts/ci_check_no_file_changes.sh python
- name: Setup Python
if: steps.check.outcome == 'failure'
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
cache: 'pip'
cache-dependency-path: 'requirements/testing.txt'
# TODO: separated requirements.txt file just for unit tests
- name: Install dependencies
if: steps.check.outcome == 'failure'
uses: ./.github/actions/cached-dependencies
with:
run: |
apt-get-install
pip-upgrade
pip install wheel
pip install -r requirements/testing.txt
mkdir ${{ github.workspace }}/.temp
- name: Python unit tests
if: steps.check.outcome == 'failure'
env:
SUPERSET_TESTENV: true
run: |
pytest --durations-min=0.5 --cov-report= --cov=superset ./tests/common ./tests/unit_tests --cache-clear
- name: Upload code coverage
if: steps.check.outcome == 'failure'
run: |
bash .github/workflows/codecov.sh -c -F python -F unit

View File

@@ -1,55 +0,0 @@
name: Translations
on:
push:
branches-ignore:
- "dependabot/npm_and_yarn/**"
pull_request:
types: [synchronize, opened, reopened, ready_for_review]
jobs:
frontend-check:
runs-on: ubuntu-20.04
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v3
with:
persist-credentials: false
submodules: recursive
- name: Setup Node.js
uses: actions/setup-node@v3
with:
node-version: '16'
- name: Install dependencies
uses: ./.github/actions/cached-dependencies
with:
run: npm-install
- name: lint
working-directory: ./superset-frontend
run: |
npm run check-translation
babel-extract:
runs-on: ubuntu-20.04
strategy:
matrix:
python-version: ["3.9"]
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v3
with:
persist-credentials: false
submodules: recursive
- name: Setup Python
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
uses: ./.github/actions/cached-dependencies
with:
run: |
apt-get-install
pip-upgrade
pip install -r requirements/base.txt
- name: Test babel extraction
run: ./scripts/babel_update.sh

View File

@@ -1,32 +0,0 @@
name: WebSocket server
on:
push:
paths:
- "superset-websocket/**"
pull_request:
paths:
- "superset-websocket/**"
jobs:
app-checks:
runs-on: ubuntu-20.04
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v3
with:
persist-credentials: false
- name: Install dependencies
working-directory: ./superset-websocket
run: npm ci
- name: lint
working-directory: ./superset-websocket
run: npm run lint
- name: prettier
working-directory: ./superset-websocket
run: npm run prettier-check
- name: unit tests
working-directory: ./superset-websocket
run: npm run test
- name: build
working-directory: ./superset-websocket
run: npm run build

View File

@@ -1,22 +0,0 @@
name: Welcome New Contributor
on:
pull_request_target:
types: [opened]
jobs:
welcome:
runs-on: ubuntu-latest
permissions:
pull-requests: write
steps:
- name: Welcome Message
uses: actions/first-interaction@v1
continue-on-error: true
with:
repo-token: ${{ github.token }}
pr-message: |-
Congrats on making your first PR and thank you for contributing to Superset! :tada: :heart:
We hope to see you in our [Slack](https://apache-superset.slack.com/) community too! Not signed up? Use our [Slack App](http://bit.ly/join-superset-slack) to self-register.

121
.gitignore vendored
View File

@@ -1,112 +1,43 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
*.bak
*.db
*.pyc
*.sqllite
*.swp
__pycache__
.local
.cache
.bento*
.cache-loader
.coverage
cover
yarn-error.log
_modules
superset/assets/coverage/*
changelog.sh
.DS_Store
.eggs
.env
.envrc
.idea
.mypy_cache
.python-version
.tox
.vscode
.coverage
_build
_static
_images
_modules
_static
build
app.db
apache_superset.egg-info/
changelog.sh
dist
dump.rdb
env
venv*
env_py3
envpy3
env36
local_config.py
/superset_config.py
/superset_text.yml
superset.egg-info/
superset/bin/supersetc
env_py3
.eggs
build
*.db
tmp
rat-results.txt
superset/app/
superset_config.py
local_config.py
env
dist
superset.egg-info/
app.db
*.bak
.idea
*.sqllite
.vscode
.python-version
.tox
dump.rdb
# Node.js, webpack artifacts, storybook
# Node.js, webpack artifacts
*.entry.js
*.js.map
node_modules
npm-debug.log*
superset/static/assets
superset/static/version_info.json
superset-frontend/**/esm/*
superset-frontend/**/lib/*
superset-frontend/**/storybook-static/*
yarn-error.log
*.map
*.min.js
test-changelog.md
*.tsbuildinfo
# Ignore package-lock in packages
plugins/*/package-lock.json
packages/*/package-lock.json
# For country map geojson conversion script
.ipynb_checkpoints/
scripts/*.zip
yarn.lock
superset/assets/version_info.json
# IntelliJ
*.iml
venv
@eaDir/
# PyCharm
.run
# Test data
celery_results.sqlite
celerybeat-schedule
celerydb.sqlite
celerybeat.pid
geckodriver.log
ghostdriver.log
testCSV.csv
.terser-plugin-cache/
apache-superset-*.tar.gz*
release.json
# Translation binaries
messages.mo
docker/requirements-local.txt
cache/

41
.gitmodules vendored
View File

@@ -1,41 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
[submodule ".github/actions/latest-tag"]
path = .github/actions/latest-tag
url = https://github.com/EndBug/latest-tag
[submodule ".github/actions/pr-lint-action"]
path = .github/actions/pr-lint-action
url = https://github.com/morrisoncole/pr-lint-action
[submodule ".github/actions/file-changes-action"]
path = .github/actions/file-changes-action
url = https://github.com/trilom/file-changes-action
[submodule ".github/actions/cached-dependencies"]
path = .github/actions/cached-dependencies
url = https://github.com/apache-superset/cached-dependencies
[submodule ".github/actions/comment-on-pr"]
path = .github/actions/comment-on-pr
url = https://github.com/unsplash/comment-on-pr
[submodule ".github/actions/chart-testing-action"]
path = .github/actions/chart-testing-action
url = https://github.com/helm/chart-testing-action
[submodule ".github/actions/chart-releaser-action"]
path = .github/actions/chart-releaser-action
url = https://github.com/helm/chart-releaser-action
[submodule ".github/actions/github-action-push-to-another-repository"]
path = .github/actions/github-action-push-to-another-repository
url = https://github.com/cpina/github-action-push-to-another-repository

View File

@@ -1,4 +0,0 @@
{
"no-bare-urls": false,
"line-length": false
}

View File

@@ -1,96 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
repos:
- repo: https://github.com/MarcoGorelli/auto-walrus
rev: v0.2.2
hooks:
- id: auto-walrus
- repo: https://github.com/asottile/pyupgrade
rev: v3.4.0
hooks:
- id: pyupgrade
args:
- --py39-plus
- repo: https://github.com/hadialqattan/pycln
rev: v2.1.2
hooks:
- id: pycln
args:
- --disable-all-dunder-policy
- --exclude=superset/config.py
- --extend-exclude=tests/integration_tests/superset_test_config.*.py
- repo: https://github.com/PyCQA/isort
rev: 5.12.0
hooks:
- id: isort
- repo: https://github.com/pre-commit/mirrors-mypy
rev: v1.3.0
hooks:
- id: mypy
args: [--check-untyped-defs]
additional_dependencies: [
types-simplejson,
types-python-dateutil,
types-requests,
# types-redis 4.6.0.5 is failing mypy
# because of https://github.com/python/typeshed/pull/10531
types-redis==4.6.0.4,
types-pytz,
types-croniter,
types-PyYAML,
types-setuptools,
types-paramiko,
types-Markdown,
]
- repo: https://github.com/peterdemin/pip-compile-multi
rev: v2.6.2
hooks:
- id: pip-compile-multi-verify
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.4.0
hooks:
- id: check-docstring-first
- id: check-added-large-files
exclude: \.(geojson)$
- id: check-yaml
exclude: ^helm/superset/templates/
- id: debug-statements
- id: end-of-file-fixer
- id: trailing-whitespace
args: ["--markdown-linebreak-ext=md"]
- repo: https://github.com/psf/black
rev: 23.1.0
hooks:
- id: black
language_version: python3
- repo: https://github.com/pre-commit/mirrors-prettier
rev: v2.4.1 # Use the sha or tag you want to point at
hooks:
- id: prettier
args: ["--ignore-path=./superset-frontend/.prettierignore"]
files: "superset-frontend"
# blacklist unsafe functions like make_url (see #19526)
- repo: https://github.com/skorokithakis/blacklist-pre-commit-hook
rev: e2f070289d8eddcaec0b580d3bde29437e7c8221
hooks:
- id: blacklist
args: ["--blacklisted-names=make_url", "--ignore=tests/"]
- repo: https://github.com/norwoodj/helm-docs
rev: v1.11.0
hooks:
- id: helm-docs
files: helm

107
.pylintrc
View File

@@ -1,19 +1,3 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
[MASTER]
# Specify a configuration file.
@@ -25,7 +9,7 @@
# Add files or directories to the blacklist. They should be base names, not
# paths.
ignore=CVS,migrations
ignore=CVS
# Add files or directories matching the regex patterns to the blacklist. The
# regex matches against base names, not paths.
@@ -39,7 +23,7 @@ persistent=yes
load-plugins=
# Use multiple processes to speed up Pylint.
jobs=2
jobs=1
# Allow loading of arbitrary C extensions. Extensions are imported into the
# active Python interpreter and may run arbitrary code.
@@ -48,7 +32,16 @@ unsafe-load-any-extension=no
# A comma-separated list of package or module names from where C extensions may
# be loaded. Extensions are loading into the active Python interpreter and may
# run arbitrary code
extension-pkg-whitelist=pyarrow
extension-pkg-whitelist=
# Allow optimization of some AST trees. This will activate a peephole AST
# optimizer, which will apply various small optimizations. For instance, it can
# be used to obtain the result of joining multiple strings with the addition
# operator. Joining a lot of strings can lead to a maximum recursion error in
# Pylint and this flag can prevent that. It has one side effect, the resulting
# AST will be different than the one from reality. This option is deprecated
# and it will be removed in Pylint 2.0.
optimize-ast=no
[MESSAGES CONTROL]
@@ -61,8 +54,7 @@ confidence=
# either give multiple identifier separated by comma (,) or put this option
# multiple time (only on the command line, not in the configuration file where
# it should appear only once). See also the "--disable" option for examples.
enable=
useless-suppression,
#enable=
# Disable the message, report, category or checker with the given id(s). You
# can either give multiple identifiers separated by comma (,) or put this
@@ -73,13 +65,8 @@ enable=
# --enable=similarities". If you want to run only the classes checker, but have
# no Warning level messages displayed, use"--disable=all --enable=classes
# --disable=W"
disable=
cyclic-import, # re-enable once this no longer raises false positives
no-member, # re-enable once this no longer raises false positives. This will become redundant after the min required version is 3.11
missing-docstring,
duplicate-code,
unspecified-encoding,
too-many-instance-attributes # re-enable once this no longer raises false positives
disable=standarderror-builtin,long-builtin,dict-view-method,intern-builtin,suppressed-message,no-absolute-import,unpacking-in-except,apply-builtin,delslice-method,indexing-exception,old-raise-syntax,print-statement,cmp-builtin,reduce-builtin,useless-suppression,coerce-method,input-builtin,cmp-method,raw_input-builtin,nonzero-method,backtick,basestring-builtin,setslice-method,reload-builtin,oct-method,map-builtin-not-iterating,execfile-builtin,old-octal-literal,zip-builtin-not-iterating,buffer-builtin,getslice-method,metaclass-assignment,xrange-builtin,long-suffix,round-builtin,range-builtin-not-iterating,next-method-called,dict-iter-method,parameter-unpacking,unicode-builtin,unichr-builtin,import-star-module-level,raising-string,filter-builtin-not-iterating,old-ne-operator,using-cmp-argument,coerce-builtin,file-builtin,old-division,hex-method,invalid-unary-operand-type
[REPORTS]
@@ -88,6 +75,12 @@ disable=
# mypackage.mymodule.MyReporterClass.
output-format=text
# Put messages in a separate file for each module / package specified on the
# command line instead of printing them on stdout. Reports (if any) will be
# written in a file name "pylint_global.[txt|html]". This option is deprecated
# and it will be removed in Pylint 2.0.
files-output=no
# Tells whether to display a full report or only the messages
reports=yes
@@ -106,10 +99,10 @@ evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / stateme
[BASIC]
# Good variable names which should always be accepted, separated by a comma
good-names=_,df,ex,f,i,id,j,k,l,o,pk,Run,ts,v,x,y
good-names=i,j,k,ex,Run,_,d,e,v,o,l,x,ts
# Bad variable names which should always be refused, separated by a comma
bad-names=fd,foo,bar,baz,toto,tutu,tata
bad-names=foo,bar,baz,toto,tutu,tata,d,fd
# Colon-delimited sets of names that determine each other's naming style when
# the name regexes allow several styles.
@@ -120,40 +113,68 @@ include-naming-hint=no
# List of decorators that produce properties, such as abc.abstractproperty. Add
# to this list to register other decorators that produce valid properties.
property-classes=
abc.abstractproperty,
sqlalchemy.ext.hybrid.hybrid_property
property-classes=abc.abstractproperty
# Regular expression matching correct argument names
argument-rgx=[a-z_][a-z0-9_]{2,30}$
# Naming hint for argument names
argument-name-hint=[a-z_][a-z0-9_]{2,30}$
# Regular expression matching correct method names
method-rgx=[a-z_][a-z0-9_]{2,30}$
# Naming hint for method names
method-name-hint=[a-z_][a-z0-9_]{2,30}$
# Regular expression matching correct variable names
variable-rgx=[a-z_][a-z0-9_]{1,30}$
# Naming hint for variable names
variable-name-hint=[a-z_][a-z0-9_]{2,30}$
# Regular expression matching correct inline iteration names
inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$
# Naming hint for inline iteration names
inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$
# Regular expression matching correct constant names
const-rgx=(([A-Za-z_][A-Za-z0-9_]*)|(__.*__))$
# Naming hint for constant names
const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$
# Regular expression matching correct class names
class-rgx=[A-Z_][a-zA-Z0-9]+$
# Naming hint for class names
class-name-hint=[A-Z_][a-zA-Z0-9]+$
# Regular expression matching correct class attribute names
class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$
# Naming hint for class attribute names
class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$
# Regular expression matching correct module names
module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
# Naming hint for module names
module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
# Regular expression matching correct attribute names
attr-rgx=[a-z_][a-z0-9_]{2,30}$
# Naming hint for attribute names
attr-name-hint=[a-z_][a-z0-9_]{2,30}$
# Regular expression matching correct function names
function-rgx=[a-z_][a-z0-9_]{2,30}$
# Naming hint for function names
function-name-hint=[a-z_][a-z0-9_]{2,30}$
# Regular expression which should only match function or class names that do
# not require a docstring.
no-docstring-rgx=^_
@@ -181,6 +202,12 @@ ignore-long-lines=^\s*(# )?<?https?://\S+>?$
# else.
single-line-if-stmt=no
# List of optional constructs for which whitespace checking is disabled. `dict-
# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}.
# `trailing-comma` allows a space between comma and closing bracket: (a, ).
# `empty-line` allows space-only lines.
no-space-check=trailing-comma,dict-separator
# Maximum number of lines in a module
max-module-lines=1000
@@ -205,13 +232,13 @@ logging-modules=logging
[MISCELLANEOUS]
# List of note tags to take in consideration, separated by a comma.
notes=FIXME,XXX
notes=FIXME,XXX,TODO
[SIMILARITIES]
# Minimum lines number of a similarity.
min-similarity-lines=5
min-similarity-lines=4
# Ignore comments when computing similarities.
ignore-comments=yes
@@ -255,7 +282,7 @@ ignored-modules=numpy,pandas,alembic.op,sqlalchemy,alembic.context,flask_appbuil
# List of class names for which member attributes should not be checked (useful
# for classes with dynamically set attributes). This supports the use of
# qualified names.
ignored-classes=contextlib.closing,optparse.Values,thread._local,_thread._local
ignored-classes=optparse.Values,thread._local,_thread._local,sqlalchemy.orm.scoping.scoped_session
# List of members which are set dynamically and missed by pylint inference
# system, and so shouldn't trigger E1101 when accessed. Python regular
@@ -319,10 +346,10 @@ ignored-argument-names=_.*
max-locals=15
# Maximum number of return / yield for function / method body
max-returns=10
max-returns=6
# Maximum number of branch for function / method body
max-branches=15
max-branches=12
# Maximum number of statements in function / method body
max-statements=50
@@ -331,7 +358,7 @@ max-statements=50
max-parents=7
# Maximum number of attributes for a class (see R0902).
max-attributes=8
max-attributes=7
# Minimum number of public methods for a class (see R0903).
min-public-methods=2
@@ -377,4 +404,4 @@ analyse-fallback-blocks=no
# Exceptions that will emit a warning when being caught. Defaults to
# "Exception"
overgeneral-exceptions=builtins.Exception
overgeneral-exceptions=Exception

View File

@@ -1,64 +0,0 @@
# Note: these patterns are applied to single files or directories, not full paths
.gitignore
docs/README.md
.gitattributes
.gitkeep
.coverage
.coveragerc
.codecov.yml
.eslintrc
.eslintignore
.flake8
.nvmrc
.prettierrc
.rat-excludes
.*log
.*pyc
.*lock
.*geojson
DISCLAIMER
licenses/*
node_modules/*
rat-results.txt
babel-node
dist
superset/static/*
build
superset.egg-info
apache_superset.egg-info
.idea
.*sql
.*zip
.*lock
# json and csv in general cannot have comments
.*json
.*csv
# Generated doc files
env/*
docs/.htaccess*
docs-v2/.htaccess*
.nojekyll
_build/*
_static/*
.buildinfo
searchindex.js
# auto generated
requirements/*
# vendorized
vendor/*
# github configuration
.github/*
.*mdx
# skip license check in superset-ui
tmp/*
lib/*
esm/*
tsconfig.tsbuildinfo
.*ipynb
.*yml
.*iml
.esprintrc
.prettierignore
generator-superset/*
temporary_superset_ui/*

30
.travis.yml Normal file
View File

@@ -0,0 +1,30 @@
language: python
services:
- redis-server
cache:
directories:
- $HOME/.wheelhouse/
env:
global:
- TRAVIS_CACHE=$HOME/.travis_cache/
matrix:
- TOX_ENV=flake8
- TOX_ENV=javascript
- TOX_ENV=pylint
- TOX_ENV=py34-postgres
- TOX_ENV=py34-sqlite
- TOX_ENV=py27-mysql
- TOX_ENV=py27-sqlite
before_script:
- mysql -u root -e "DROP DATABASE IF EXISTS superset; CREATE DATABASE superset DEFAULT CHARACTER SET utf8 COLLATE utf8_unicode_ci"
- mysql -u root -e "CREATE USER 'mysqluser'@'localhost' IDENTIFIED BY 'mysqluserpassword';"
- mysql -u root -e "GRANT ALL ON superset.* TO 'mysqluser'@'localhost';"
- psql -U postgres -c "CREATE DATABASE superset;"
- psql -U postgres -c "CREATE USER postgresuser WITH PASSWORD 'pguserpassword';"
- export PATH=${PATH}:/tmp/hive/bin
install:
- pip install --upgrade pip
- pip install codecov tox tox-travis
script: tox -e $TOX_ENV
after_success:
- codecov

File diff suppressed because it is too large Load Diff

View File

@@ -1,138 +1,84 @@
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
# Code of Conduct
http://www.apache.org/licenses/LICENSE-2.0
## 1. Purpose
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
A primary goal of Apache Superset is to be inclusive to the largest number of contributors, with the most varied and diverse backgrounds possible. As such, we are committed to providing a friendly, safe and welcoming environment for all, regardless of gender, sexual orientation, ability, ethnicity, socioeconomic status, and religion (or lack thereof).
# CODE OF CONDUCT
This code of conduct outlines our expectations for all those who participate in our community, as well as the consequences for unacceptable behavior.
*The following is copied for your convenience from <https://www.apache.org/foundation/policies/conduct.html>. If there's a discrepancy between the two, let us know or submit a PR to fix it.*
We invite all those who participate in Apache Superset to help us create safe and positive experiences for everyone.
## INTRODUCTION
## 2. Open Source Citizenship
This code of conduct applies to all spaces managed by the Apache Software Foundation, including IRC, all public and private mailing lists, issue trackers, wikis, blogs, Twitter, and any other communication channel used by our communities. A code of conduct which is specific to in-person events (ie., conferences) is codified in the published ASF anti-harassment policy.
A supplemental goal of this Code of Conduct is to increase open source citizenship by encouraging participants to recognize and strengthen the relationships between our actions and their effects on our community.
We expect this code of conduct to be honored by everyone who participates in the Apache community formally or informally, or claims any affiliation with the Foundation, in any Foundation-related activities and especially when representing the ASF, in any role.
Communities mirror the societies in which they exist and positive action is essential to counteract the many forms of inequality and abuses of power that exist in society.
This code is __not exhaustive or complete__. It serves to distill our common understanding of a collaborative, shared environment and goals. We expect it to be followed in spirit as much as in the letter, so that it can enrich all of us and the technical communities in which we participate.
If you see someone who is making an extra effort to ensure our community is welcoming, friendly, and encourages all participants to contribute to the fullest extent, we want to know.
## SPECIFIC GUIDELINES
## 3. Expected Behavior
We strive to:
The following behaviors are expected and requested of all community members:
1. **Be open.** We invite anyone to participate in our community. We preferably use public methods of communication for project-related messages, unless discussing something sensitive. This applies to messages for help or project-related support, too; not only is a public support request much more likely to result in an answer to a question, it also makes sure that any inadvertent mistakes made by people answering will be more easily detected and corrected.
* Participate in an authentic and active way. In doing so, you contribute to the health and longevity of this community.
* Exercise consideration and respect in your speech and actions.
* Attempt collaboration before conflict.
* Refrain from demeaning, discriminatory, or harassing behavior and speech.
* Be mindful of your surroundings and of your fellow participants. Alert community leaders if you notice a dangerous situation, someone in distress, or violations of this Code of Conduct, even if they seem inconsequential.
* Remember that community event venues may be shared with members of the public; please be respectful to all patrons of these locations.
2. **Be empathetic, welcoming, friendly, and patient.** We work together to resolve conflict, assume good intentions, and do our best to act in an empathetic fashion. We may all experience some frustration from time to time, but we do not allow frustration to turn into a personal attack. A community where people feel uncomfortable or threatened is not a productive one. We should be respectful when dealing with other community members as well as with people outside our community.
## 4. Unacceptable Behavior
3. **Be collaborative.** Our work will be used by other people, and in turn we will depend on the work of others. When we make something for the benefit of the project, we are willing to explain to others how it works, so that they can build on the work to make it even better. Any decision we make will affect users and colleagues, and we take those consequences seriously when making decisions.
The following behaviors are considered harassment and are unacceptable within our community:
4. **Be inquisitive.** Nobody knows everything! Asking questions early avoids many problems later, so questions are encouraged, though they may be directed to the appropriate forum. Those who are asked should be responsive and helpful, within the context of our shared goal of improving Apache project code.
* Violence, threats of violence or violent language directed against another person.
* Sexist, racist, homophobic, transphobic, ableist or otherwise discriminatory jokes and language.
* Posting or displaying sexually explicit or violent material.
* Posting or threatening to post other peoples personally identifying information ("doxing").
* Personal insults, particularly those related to gender, sexual orientation, race, religion, or disability.
* Inappropriate photography or recording.
* Inappropriate physical contact. You should have someones consent before touching them.
* Unwelcome sexual attention. This includes, sexualized comments or jokes; inappropriate touching, groping, and unwelcomed sexual advances.
* Deliberate intimidation, stalking or following (online or in person).
* Advocating for, or encouraging, any of the above behavior.
* Sustained disruption of community events, including talks and presentations.
5. **Be careful in the words that we choose.** Whether we are participating as professionals or volunteers, we value professionalism in all interactions, and take responsibility for our own speech. Be kind to others. Do not insult or put down other participants. Harassment and other exclusionary behavior are not acceptable. This includes, but is not limited to:
* Violent threats or language directed against another person.
* Sexist, racist, or otherwise discriminatory jokes and language.
* Posting sexually explicit or violent material.
* Posting (or threatening to post) other people's personally identifying information ("doxing").
* Sharing private content, such as emails sent privately or non-publicly, or unlogged forums such as IRC channel history.
* Personal insults, especially those using racist or sexist terms.
* Unwelcome sexual attention.
* Excessive or unnecessary profanity.
* Repeated harassment of others. In general, if someone asks you to stop, then stop.
* Advocating for, or encouraging, any of the above behavior.
## 5. Consequences of Unacceptable Behavior
6. **Be concise.** Keep in mind that what you write once will be read by hundreds of persons. Writing a short email means people can understand the conversation as efficiently as possible. Short emails should always strive to be empathetic, welcoming, friendly and patient. When a long explanation is necessary, consider adding a summary.
Unacceptable behavior from any community member, including sponsors and those with decision-making authority, will not be tolerated.
Try to bring new ideas to a conversation so that each mail adds something unique to the thread, keeping in mind that the rest of the thread still contains the other messages with arguments that have already been made.
Anyone asked to stop unacceptable behavior is expected to comply immediately.
Try to stay on topic, especially in discussions that are already fairly large.
If a community member engages in unacceptable behavior, the community organizers may take any action they deem appropriate, up to and including a temporary ban or permanent expulsion from the community without warning (and without refund in the case of a paid event).
7. **Step down considerately.** Members of every project come and go. When somebody leaves or disengages from the project they should tell people they are leaving and take the proper steps to ensure that others can pick up where they left off. In doing so, they should remain respectful of those who continue to participate in the project and should not misrepresent the project's goals or achievements. Likewise, community members should respect any individual's choice to leave the project.
## 6. Reporting Guidelines
## DIVERSITY STATEMENT
Apache welcomes and encourages participation by everyone. We are committed to being a community that everyone feels good about joining. Although we may not be able to satisfy everyone, we will always work to treat everyone well.
No matter how you identify yourself or how others perceive you: we welcome you. Though no list can hope to be comprehensive, we explicitly honour diversity in: age, culture, ethnicity, genotype, gender identity or expression, language, national origin, neurotype, phenotype, political beliefs, profession, race, religion, sexual orientation, socioeconomic status, subculture and technical ability.
Though we welcome people fluent in all languages, Apache development is conducted in English.
Standards for behaviour in the Apache community are detailed in the Code of Conduct above. We expect participants in our community to meet these standards in all their interactions and to help others to do so as well.
## REPORTING GUIDELINES
While this code of conduct should be adhered to by participants, we recognize that sometimes people may have a bad day, or be unaware of some of the guidelines in this code of conduct. When that happens, you may reply to them and point out this code of conduct. Such messages may be in public or in private, whatever is most appropriate. However, regardless of whether the message is public or not, it should still adhere to the relevant parts of this code of conduct; in particular, it should not be abusive or disrespectful.
If you believe someone is violating this code of conduct, you may reply to them and point out this code of conduct. Such messages may be in public or in private, whatever is most appropriate. Assume good faith; it is more likely that participants are unaware of their bad behaviour than that they intentionally try to degrade the quality of the discussion. Should there be difficulties in dealing with the situation, you may report your compliance issues in confidence to either:
* President of the Apache Software Foundation: Sam Ruby (rubys at intertwingly dot net)
Or one of our volunteers:
* [Mark Thomas](http://home.apache.org/~markt/coc.html)
* [Joan Touzet](http://home.apache.org/~wohali/)
* [Sharan Foga](http://home.apache.org/~sharan/coc.html)
If the violation is in documentation or code, for example inappropriate pronoun usage or word choice within official documentation, we ask that people report these privately to the project in question at <private@project.apache.org>, and, if they have sufficient ability within the project, to resolve or remove the concerning material, being mindful of the perspective of the person originally reporting the issue.
## ENDNOTES
This Code defines **empathy** as "a vicarious participation in the emotions, ideas, or opinions of others; the ability to imagine oneself in the condition or predicament of another." **Empathetic** is the adjectival form of empathy.
This statement thanks the following, on which it draws for content and inspiration:
* [CouchDB Project Code of conduct](http://couchdb.apache.org/conduct.html)
* [Fedora Project Code of Conduct](http://fedoraproject.org/code-of-conduct)
* [Speak Up! Code of Conduct](http://speakup.io/coc.html)
* [Django Code of Conduct](https://www.djangoproject.com/conduct/)
* [Debian Code of Conduct](http://www.debian.org/vote/2014/vote_002)
* [Twitter Open Source Code of Conduct](https://github.com/twitter/code-of-conduct/blob/master/code-of-conduct.md)
* [Mozilla Code of Conduct/Draft](https://wiki.mozilla.org/Code_of_Conduct/Draft#Conflicts_of_Interest)
* [Python Diversity Appendix](https://www.python.org/community/diversity/)
* [Python Mentors Home Page](http://pythonmentors.com)
If you are subject to or witness unacceptable behavior, or have any other concerns, please notify a community organizer as soon as possible. dev@superset.incubator.apache.org .
# Slack Community Guidelines
If you decide to join the [Community Slack](http://bit.ly/join-superset-slack), please adhere to the following rules:
Additionally, community organizers are available to help community members engage with local law enforcement or to otherwise help those experiencing unacceptable behavior feel safe. In the context of in-person events, organizers will also provide escorts as desired by the person experiencing distress.
**1. Treat everyone in the community with respect.**
## 7. Addressing Grievances
- We strive to make this community a warm place for people from all industries, use cases, geographies, and backgrounds. Harassment of any kind is not acceptable and wont be tolerated.
- Please follow the guidelines as outlined in the Superset Community [code of conduct here](https://github.com/apache/superset/blob/master/CODE_OF_CONDUCT.md).
If you feel you have been falsely or unfairly accused of violating this Code of Conduct, you should notify Apache with a concise description of your grievance. Your grievance will be handled in accordance with our existing governing policies.
**2. Use the right channel.**
- Channels are an effective way to organize and focus discussions while also empowering members to opt-in to the types of content theyre interested in. When questions are posted or discussions are started in the wrong channel, it dilutes the trust of the members in the channel and, more practically, makes it harder for your questions to be answered.
**3. Ask thoughtful questions.**
## 8. Scope
- Were all here to help each other out. The best way to get help is by investing effort into your questions. First check and see if your question is answered in [the Superset documentation](https://superset.apache.org/faq.html) or on [Stack Overflow](https://stackoverflow.com/search?q=apache+superset). You can also check [GitHub issues](https://github.com/apache/superset/issues) to see if your question or feature request has been submitted before. Then, use Slack search to see if your question has already been asked and answered in the past. If you still feel the need to ask a question, make sure you include:
We expect all community participants (contributors, paid or otherwise; sponsors; and other guests) to abide by this Code of Conduct in all community venuesonline and in-personas well as in all one-on-one communications pertaining to community business.
- The steps youve already taken
- Relevant details presented cleanly (text stacktraces, formatted markdown, or screenshots. Please dont paste large blocks of code unformatted or post photos of your screen from your phone)
- The specific question you have or the specific type of help you're seeking
This code of conduct and its related procedures also applies to unacceptable behavior occurring outside the scope of community activities when such behavior has the potential to adversely affect the safety and well-being of community members.
**4. Avoid double posting**
## 9. Contact info
- This Slack community is not a customer support channel and all members are here voluntarily. If you arent getting a response to a question you have, make sure you look at rules 1, 2, and 3. Its also worth remembering that there may not be someone in the community who has the context to help you out.
dev@superset.incubator.apache.org
**5. Communicate openly**
## 10. License and attribution
- Unless you have explicit permission from the person, please avoid sending direct messages to individuals. Communicating in public channels ensures that were all respecting each others attentions and we can scalably moderate our communication to mitigate harassment or discrimination. Do not use direct messages to pitch products and services. If you are receiving unwelcome direct messages, please notify an admin.
This Code of Conduct is distributed under a [Creative Commons Attribution-ShareAlike license](http://creativecommons.org/licenses/by-sa/3.0/).
**6. Practice good Slack hygiene by using threads for discussions and emojis for light reactions.**
Portions of text derived from the [Django Code of Conduct](https://www.djangoproject.com/conduct/) and the [Geek Feminism Anti-Harassment Policy](http://geekfeminism.wikia.com/wiki/Conference_anti-harassment/Policy).
- The medium is the message. Slack can foster a warm, collaborative, and organized community when used effectively. We want to respect peoples attentions (thread notifications > channel notifications > DM notifications) and we want to improve information density (a member should be able to browse and explore many convo threads, not just see one thread discussed in a top level channel).
Retrieved on November 22, 2016 from [http://citizencodeofconduct.org/](http://citizencodeofconduct.org/)

File diff suppressed because it is too large Load Diff

View File

@@ -1,139 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
######################################################################
# Node stage to deal with static asset construction
######################################################################
ARG PY_VER=3.9-slim-bookworm
# if BUILDPLATFORM is null, set it to 'amd64' (or leave as is otherwise).
ARG BUILDPLATFORM=${BUILDPLATFORM:-amd64}
FROM --platform=${BUILDPLATFORM} node:16-slim AS superset-node
ARG NPM_BUILD_CMD="build"
RUN apt-get update -q \
&& apt-get install -yq --no-install-recommends \
python3 \
make \
gcc \
g++
ENV BUILD_CMD=${NPM_BUILD_CMD} \
PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true
# NPM ci first, as to NOT invalidate previous steps except for when package.json changes
WORKDIR /app/superset-frontend
RUN --mount=type=bind,target=/frontend-mem-nag.sh,src=./docker/frontend-mem-nag.sh \
/frontend-mem-nag.sh
COPY superset-frontend/package*.json ./
RUN npm ci
COPY ./superset-frontend ./
# This seems to be the most expensive step
RUN npm run ${BUILD_CMD}
######################################################################
# Final lean image...
######################################################################
FROM python:${PY_VER} AS lean
WORKDIR /app
ENV LANG=C.UTF-8 \
LC_ALL=C.UTF-8 \
SUPERSET_ENV=production \
FLASK_APP="superset.app:create_app()" \
PYTHONPATH="/app/pythonpath" \
SUPERSET_HOME="/app/superset_home" \
SUPERSET_PORT=8088
RUN mkdir -p ${PYTHONPATH} superset/static superset-frontend apache_superset.egg-info requirements \
&& useradd --user-group -d ${SUPERSET_HOME} -m --no-log-init --shell /bin/bash superset \
&& apt-get update -q \
&& apt-get install -yq --no-install-recommends \
build-essential \
curl \
default-libmysqlclient-dev \
libsasl2-dev \
libsasl2-modules-gssapi-mit \
libpq-dev \
libecpg-dev \
libldap2-dev \
&& apt-get autoremove -yqq --purge && rm -rf /var/lib/apt/lists/* /var/[log,tmp]/* /tmp/* && apt-get clean \
&& touch superset/static/version_info.json \
&& chown -R superset:superset ./*
COPY --chown=superset:superset ./requirements/*.txt requirements/
COPY --chown=superset:superset setup.py MANIFEST.in README.md ./
# setup.py uses the version information in package.json
COPY --chown=superset:superset superset-frontend/package.json superset-frontend/
RUN pip install --no-cache-dir -r requirements/local.txt
COPY --chown=superset:superset --from=superset-node /app/superset/static/assets superset/static/assets
## Lastly, let's install superset itself
COPY --chown=superset:superset superset superset
RUN pip install --no-cache-dir -e . \
&& flask fab babel-compile --target superset/translations \
&& chown -R superset:superset superset/translations
COPY --chmod=755 ./docker/run-server.sh /usr/bin/
USER superset
HEALTHCHECK CMD curl -f "http://localhost:$SUPERSET_PORT/health"
EXPOSE ${SUPERSET_PORT}
CMD ["/usr/bin/run-server.sh"]
######################################################################
# Dev image...
######################################################################
FROM lean AS dev
ARG GECKODRIVER_VERSION=v0.33.0 \
FIREFOX_VERSION=117.0.1
USER root
RUN apt-get update -q \
&& apt-get install -yq --no-install-recommends \
libnss3 \
libdbus-glib-1-2 \
libgtk-3-0 \
libx11-xcb1 \
libasound2 \
libxtst6 \
wget \
# Install GeckoDriver WebDriver
&& wget https://github.com/mozilla/geckodriver/releases/download/${GECKODRIVER_VERSION}/geckodriver-${GECKODRIVER_VERSION}-linux64.tar.gz -O - | tar xfz - -C /usr/local/bin \
# Install Firefox
&& wget https://download-installer.cdn.mozilla.net/pub/firefox/releases/${FIREFOX_VERSION}/linux-x86_64/en-US/firefox-${FIREFOX_VERSION}.tar.bz2 -O - | tar xfj - -C /opt \
&& ln -s /opt/firefox/firefox /usr/local/bin/firefox \
&& apt-get autoremove -yqq --purge wget && rm -rf /var/lib/apt/lists/* /var/[log,tmp]/* /tmp/* && apt-get clean
# Cache everything for dev purposes...
RUN pip install --no-cache-dir -r requirements/docker.txt
USER superset
######################################################################
# CI image...
######################################################################
FROM lean AS ci
COPY --chown=superset:superset --chmod=755 ./docker/*.sh /app/docker/
CMD ["/app/docker/docker-ci.sh"]

View File

@@ -1,23 +0,0 @@
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
# INSTALL / BUILD instructions for Apache Superset
At this time, the docker file at RELEASING/Dockerfile.from_local_tarball
constitutes the recipe on how to get to a working release from a source
release tarball.

19
ISSUE_TEMPLATE.md Normal file
View File

@@ -0,0 +1,19 @@
Make sure these boxes are checked before submitting your issue - thank you!
- [ ] I have checked the superset logs for python stacktraces and included it here as text if any
- [ ] I have reproduced the issue with at least the latest released version of superset
- [ ] I have checked the issue tracker for the same issue and I haven't found one similar
### Superset version
### Expected results
### Actual results
### Steps to reproduce

View File

@@ -199,18 +199,3 @@ distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
============================================================================
APACHE SUPERSET SUBCOMPONENTS:
The Apache Superset project contains subcomponents with separate copyright
notices and license terms. Your use of the source code for the these
subcomponents is subject to the terms and conditions of the following
licenses.
========================================================================
Third party SIL Open Font License v1.1 (OFL-1.1)
========================================================================
(SIL OPEN FONT LICENSE Version 1.1) The Inter font family (https://github.com/rsms/inter)
(SIL OPEN FONT LICENSE Version 1.1) The Fira Code font family (https://github.com/tonsky/FiraCode)

View File

@@ -1,27 +1,12 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
include NOTICE
include LICENSE.txt
graft licenses/
include README.md
include superset-frontend/package.json
recursive-include superset/examples *
recursive-include superset/data *
recursive-include superset/migrations *
recursive-include superset/static *
recursive-exclude superset/static/assets/docs *
recursive-exclude superset/static/assets/images/viz_thumbnails_large *
recursive-exclude superset/static/docs *
recursive-exclude superset/static/spec *
recursive-exclude superset/static/images/viz_thumbnails_large *
recursive-exclude superset/static/assets/node_modules *
recursive-include superset/templates *
recursive-include superset/translations *
recursive-include superset/static *
recursive-exclude tests *

117
Makefile
View File

@@ -1,117 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Python version installed; we need 3.9-3.11
PYTHON=`command -v python3.11 || command -v python3.10 || command -v python3.9`
.PHONY: install superset venv pre-commit
install: superset pre-commit
superset:
# Install external dependencies
pip install -r requirements/local.txt
# Install Superset in editable (development) mode
pip install -e .
# Create an admin user in your metadata database
superset fab create-admin \
--username admin \
--firstname "Admin I."\
--lastname Strator \
--email admin@superset.io \
--password general
# Initialize the database
superset db upgrade
# Create default roles and permissions
superset init
# Load some data to play with
superset load-examples
# Install node packages
cd superset-frontend; npm ci
update: update-py update-js
update-py:
# Install external dependencies
pip install -r requirements/local.txt
# Install Superset in editable (development) mode
pip install -e .
# Initialize the database
superset db upgrade
# Create default roles and permissions
superset init
update-js:
# Install js packages
cd superset-frontend; npm ci
venv:
# Create a virtual environment and activate it (recommended)
if ! [ -x "${PYTHON}" ]; then echo "You need Python 3.9, 3.10 or 3.11 installed"; exit 1; fi
test -d venv || ${PYTHON} -m venv venv # setup a python3 virtualenv
. venv/bin/activate
activate:
. venv/bin/activate
pre-commit:
# setup pre commit dependencies
pip3 install -r requirements/integration.txt
pre-commit install
format: py-format js-format
py-format: pre-commit
pre-commit run black --all-files
py-lint: pre-commit
pylint -j 0 superset
js-format:
cd superset-frontend; npm run prettier
flask-app:
flask run -p 8088 --with-threads --reload --debugger
node-app:
cd superset-frontend; npm run dev-server
build-cypress:
cd superset-frontend; npm run build-instrumented
cd superset-frontend/cypress-base; npm ci
open-cypress:
if ! [ $(port) ]; then cd superset-frontend/cypress-base; CYPRESS_BASE_URL=http://localhost:9000 npm run cypress open; fi
cd superset-frontend/cypress-base; CYPRESS_BASE_URL=http://localhost:$(port) npm run cypress open
report-celery-worker:
celery --app=superset.tasks.celery_app:app worker
report-celery-beat:
celery --app=superset.tasks.celery_app:app beat --pidfile /tmp/celerybeat.pid --schedule /tmp/celerybeat-schedulecd
admin-user:
superset fab create-admin

5
NOTICE
View File

@@ -1,5 +0,0 @@
Apache Superset
Copyright 2016-2021 The Apache Software Foundation
This product includes software developed at The Apache Software
Foundation (http://www.apache.org/).

328
README.md
View File

@@ -1,189 +1,203 @@
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
Superset
=========
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
# Superset
[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
[![GitHub release (latest SemVer)](https://img.shields.io/github/v/release/apache/superset?sort=semver)](https://github.com/apache/superset/tree/latest)
[![Build Status](https://github.com/apache/superset/workflows/Python/badge.svg)](https://github.com/apache/superset/actions)
[![PyPI version](https://badge.fury.io/py/apache-superset.svg)](https://badge.fury.io/py/apache-superset)
[![Coverage Status](https://codecov.io/github/apache/superset/coverage.svg?branch=master)](https://codecov.io/github/apache/superset)
[![PyPI](https://img.shields.io/pypi/pyversions/apache-superset.svg?maxAge=2592000)](https://pypi.python.org/pypi/apache-superset)
[![Get on Slack](https://img.shields.io/badge/slack-join-orange.svg)](http://bit.ly/join-superset-slack)
[![Documentation](https://img.shields.io/badge/docs-apache.org-blue.svg)](https://superset.apache.org)
[![Build Status](https://travis-ci.org/apache/incubator-superset.svg?branch=master)](https://travis-ci.org/apache/incubator-superset)
[![PyPI version](https://badge.fury.io/py/superset.svg)](https://badge.fury.io/py/superset)
[![Coverage Status](https://codecov.io/github/apache/incubator-superset/coverage.svg?branch=master)](https://codecov.io/github/apache/incubator-superset)
[![PyPI](https://img.shields.io/pypi/pyversions/superset.svg?maxAge=2592000)](https://pypi.python.org/pypi/superset)
[![Join the chat at https://gitter.im/airbnb/superset](https://badges.gitter.im/apache/incubator-superset.svg)](https://gitter.im/airbnb/superset?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
[![Documentation](https://img.shields.io/badge/docs-apache.org-blue.svg)](https://superset.incubator.apache.org)
[![dependencies Status](https://david-dm.org/apache/incubator-superset/status.svg?path=superset/assets)](https://david-dm.org/apache/incubator-superset?path=superset/assets)
<img
src="https://github.com/apache/superset/raw/master/superset-frontend/src/assets/branding/superset-logo-horiz-apache.png"
src="https://cloud.githubusercontent.com/assets/130878/20946612/49a8a25c-bbc0-11e6-8314-10bef902af51.png"
alt="Superset"
width="500"
/>
A modern, enterprise-ready business intelligence web application.
**Apache Superset** (incubating) is a modern, enterprise-ready
business intelligence web application
[**Why Superset?**](#why-superset) |
[**Supported Databases**](#supported-databases) |
[**Installation and Configuration**](#installation-and-configuration) |
[**Release Notes**](RELEASING/README.md#release-notes-for-recent-releases) |
[**Get Involved**](#get-involved) |
[**Contributor Guide**](#contributor-guide) |
[**Resources**](#resources) |
[**Organizations Using Superset**](RESOURCES/INTHEWILD.md)
[this project used to be named **Caravel**, and **Panoramix** in the past]
## Why Superset?
Superset is a modern data exploration and data visualization platform. Superset can replace or augment proprietary business intelligence tools for many teams. Superset integrates well with a variety of data sources.
Screenshots & Gifs
------------------
Superset provides:
**View Dashboards**
- A **no-code interface** for building charts quickly
- A powerful, web-based **SQL Editor** for advanced querying
- A **lightweight semantic layer** for quickly defining custom dimensions and metrics
- Out of the box support for **nearly any SQL** database or data engine
- A wide array of **beautiful visualizations** to showcase your data, ranging from simple bar charts to geospatial visualizations
- Lightweight, configurable **caching layer** to help ease database load
- Highly extensible **security roles and authentication** options
- An **API** for programmatic customization
- A **cloud-native architecture** designed from the ground up for scale
## Screenshots & Gifs
**Video Overview**
https://user-images.githubusercontent.com/64562059/234390129-321d4f35-cb4b-45e8-89d9-20ae292f34fc.mp4
![superset-dashboard](https://cloud.githubusercontent.com/assets/130878/20371438/a703a2a0-ac19-11e6-80c4-00a47c2eb644.gif)
<br/>
**Large Gallery of Visualizations**
**View/Edit a Slice**
<kbd><img title="Gallery" src="superset-frontend/src/assets/images/screenshots/gallery.jpg"/></kbd><br/>
![superset-explore-slice](https://cloud.githubusercontent.com/assets/130878/20372732/410392f4-ac22-11e6-9c6d-3ef512e81212.gif)
**Craft Beautiful, Dynamic Dashboards**
<br/>
<kbd><img title="View Dashboards" src="superset-frontend/src/assets/images/screenshots/slack_dash.jpg"/></kbd><br/>
**Query and Visualize with SQL Lab**
**No-Code Chart Builder**
![superset-sql-lab-visualization](https://cloud.githubusercontent.com/assets/130878/20372911/7c3b3358-ac23-11e6-8f24-923ef1b35715.gif)
<kbd><img title="Slice & dice your data" src="superset-frontend/src/assets/images/screenshots/explore.jpg"/></kbd><br/>
<br/>
**Powerful SQL Editor**
![superset-dashboard-misc](https://cloud.githubusercontent.com/assets/130878/20234704/0f40778c-a835-11e6-9556-983a62ea061b.png)
<kbd><img title="SQL Lab" src="superset-frontend/src/assets/images/screenshots/sql_lab.jpg"/></kbd><br/>
![superset-edit-table](https://cloud.githubusercontent.com/assets/130878/20234705/0f415c88-a835-11e6-8b03-f7c35d56dd7d.png)
## Supported Databases
![superset-query-search](https://cloud.githubusercontent.com/assets/130878/20234706/0f430a10-a835-11e6-8a0d-8b26cc2e6bbd.png)
Superset can query data from any SQL-speaking datastore or data engine (Presto, Trino, Athena, [and more](https://superset.apache.org/docs/databases/installing-database-drivers/)) that has a Python DB-API driver and a SQLAlchemy dialect.
Apache Superset
---------------
Apache Superset is a data exploration and visualization web application.
Here are some of the major database solutions that are supported:
<p align="center">
<img src="superset-frontend/src/assets/images/redshift.png" alt="redshift" border="0" width="200" height="80"/>
<img src="superset-frontend/src/assets/images/google-biquery.png" alt="google-biquery" border="0" width="200" height="80"/>
<img src="superset-frontend/src/assets/images/snowflake.png" alt="snowflake" border="0" width="200" height="80"/>
<img src="superset-frontend/src/assets/images/trino.png" alt="trino" border="0" width="200" height="80"/>
<img src="superset-frontend/src/assets/images/presto.png" alt="presto" border="0" width="200" height="80"/>
<img src="superset-frontend/src/assets/images/databricks.png" alt="databricks" border="0" width="200" height="80" />
<img src="superset-frontend/src/assets/images/druid.png" alt="druid" border="0" width="200" height="80" />
<img src="superset-frontend/src/assets/images/firebolt.png" alt="firebolt" border="0" width="200" height="80" />
<img src="superset-frontend/src/assets/images/timescale.png" alt="timescale" border="0" width="200" height="80" />
<img src="superset-frontend/src/assets/images/rockset.png" alt="rockset" border="0" width="200" height="80" />
<img src="superset-frontend/src/assets/images/postgresql.png" alt="postgresql" border="0" width="200" height="80" />
<img src="superset-frontend/src/assets/images/mysql.png" alt="mysql" border="0" width="200" height="80" />
<img src="superset-frontend/src/assets/images/mssql-server.png" alt="mssql-server" border="0" width="200" height="80" />
<img src="superset-frontend/src/assets/images/db2.png" alt="db2" border="0" width="200" height="80" />
<img src="superset-frontend/src/assets/images/sqlite.png" alt="sqlite" border="0" width="200" height="80" />
<img src="superset-frontend/src/assets/images/sybase.png" alt="sybase" border="0" width="200" height="80" />
<img src="superset-frontend/src/assets/images/mariadb.png" alt="mariadb" border="0" width="200" height="80" />
<img src="superset-frontend/src/assets/images/vertica.png" alt="vertica" border="0" width="200" height="80" />
<img src="superset-frontend/src/assets/images/oracle.png" alt="oracle" border="0" width="200" height="80" />
<img src="superset-frontend/src/assets/images/firebird.png" alt="firebird" border="0" width="200" height="80" />
<img src="superset-frontend/src/assets/images/greenplum.png" alt="greenplum" border="0" width="200" height="80" />
<img src="superset-frontend/src/assets/images/clickhouse.png" alt="clickhouse" border="0" width="200" height="80" />
<img src="superset-frontend/src/assets/images/exasol.png" alt="exasol" border="0" width="200" height="80" />
<img src="superset-frontend/src/assets/images/monet-db.png" alt="monet-db" border="0" width="200" height="80" />
<img src="superset-frontend/src/assets/images/apache-kylin.png" alt="apache-kylin" border="0" width="200" height="80"/>
<img src="superset-frontend/src/assets/images/hologres.png" alt="hologres" border="0" width="200" height="80"/>
<img src="superset-frontend/src/assets/images/netezza.png" alt="netezza" border="0" width="200" height="80"/>
<img src="superset-frontend/src/assets/images/pinot.png" alt="pinot" border="0" width="200" height="80"/>
<img src="superset-frontend/src/assets/images/teradata.png" alt="teradata" border="0" width="200" height="80"/>
<img src="superset-frontend/src/assets/images/yugabyte.png" alt="yugabyte" border="0" width="200" height="80"/>
<img src="superset-frontend/src/assets/images/starrocks.png" alt="starrocks" border="0" width="200" height="80"/>
</p>
**A more comprehensive list of supported databases** along with the configuration instructions can be found [here](https://superset.apache.org/docs/databases/installing-database-drivers).
Want to add support for your datastore or data engine? Read more [here](https://superset.apache.org/docs/frequently-asked-questions#does-superset-work-with-insert-database-engine-here) about the technical requirements.
## Installation and Configuration
[Extended documentation for Superset](https://superset.apache.org/docs/installation/installing-superset-using-docker-compose)
## Get Involved
- Ask and answer questions on [StackOverflow](https://stackoverflow.com/questions/tagged/apache-superset) using the **apache-superset** tag
- [Join our community's Slack](http://bit.ly/join-superset-slack)
and please read our [Slack Community Guidelines](https://github.com/apache/superset/blob/master/CODE_OF_CONDUCT.md#slack-community-guidelines)
- [Join our dev@superset.apache.org Mailing list](https://lists.apache.org/list.html?dev@superset.apache.org)
## Contributor Guide
Interested in contributing? Check out our
[CONTRIBUTING.md](https://github.com/apache/superset/blob/master/CONTRIBUTING.md)
to find resources around contributing along with a detailed guide on
how to set up a development environment.
## Resources
Superset 2.0!
- [Superset 2.0 Meetup](https://preset.io/events/superset-2-0-meetup/)
- [Superset 2.0 Release Notes](https://github.com/apache/superset/tree/master/RELEASING/release-notes-2-0)
Understanding the Superset Points of View
- [The Case for Dataset-Centric Visualization](https://preset.io/blog/dataset-centric-visualization/)
- [Understanding the Superset Semantic Layer](https://preset.io/blog/understanding-superset-semantic-layer/)
Superset provides:
* An intuitive interface to explore and visualize datasets, and
create interactive dashboards.
* A wide array of beautiful visualizations to showcase your data.
* Easy, code-free, user flows to drill down and slice and dice the data
underlying exposed dashboards. The dashboards and charts acts as a starting
point for deeper analysis.
* A state of the art SQL editor/IDE exposing a rich metadata browser, and
an easy workflow to create visualizations out of any result set.
* An extensible, high granularity security model allowing intricate rules
on who can access which product features and datasets.
Integration with major
authentication backends (database, OpenID, LDAP, OAuth, REMOTE_USER, ...)
* A lightweight semantic layer, allowing to control how data sources are
exposed to the user by defining dimensions and metrics
* Out of the box support for most SQL-speaking databases
* Deep integration with Druid allows for Superset to stay blazing fast while
slicing and dicing large, realtime datasets
* Fast loading dashboards with configurable caching
- Getting Started with Superset
- [Superset in 2 Minutes using Docker Compose](https://superset.apache.org/docs/installation/installing-superset-using-docker-compose#installing-superset-locally-using-docker-compose)
- [Installing Database Drivers](https://superset.apache.org/docs/databases/docker-add-drivers/)
- [Building New Database Connectors](https://preset.io/blog/building-database-connector/)
- [Create Your First Dashboard](https://superset.apache.org/docs/creating-charts-dashboards/first-dashboard)
- [Comprehensive Tutorial for Contributing Code to Apache Superset
](https://preset.io/blog/tutorial-contributing-code-to-apache-superset/)
- [Resources to master Superset by Preset](https://preset.io/resources/)
Database Support
----------------
- Deploying Superset
- [Official Docker image](https://hub.docker.com/r/apache/superset)
- [Helm Chart](https://github.com/apache/superset/tree/master/helm/superset)
Superset speaks many SQL dialects through SQLAlchemy, a Python
ORM that is compatible with
[most common databases](http://docs.sqlalchemy.org/en/rel_1_0/core/engines.html).
- Recordings of Past [Superset Community Events](https://preset.io/events)
- [Mixed Time Series Charts](https://preset.io/events/mixed-time-series-visualization-in-superset-workshop/)
- [How the Bing Team Customized Superset for the Internal Self-Serve Data & Analytics Platform](https://preset.io/events/how-the-bing-team-heavily-customized-superset-for-their-internal-data/)
- [Live Demo: Visualizing MongoDB and Pinot Data using Trino](https://preset.io/events/2021-04-13-visualizing-mongodb-and-pinot-data-using-trino/)
- [Introduction to the Superset API](https://preset.io/events/introduction-to-the-superset-api/)
- [Building a Database Connector for Superset](https://preset.io/events/2021-02-16-building-a-database-connector-for-superset/)
Superset can be used to visualize data out of most databases:
* MySQL
* Postgres
* Vertica
* Oracle
* Microsoft SQL Server
* SQLite
* Greenplum
* Firebird
* MariaDB
* Sybase
* IBM DB2
* Exasol
* MonetDB
* Snowflake
* Redshift
* **more!** look for the availability of a SQLAlchemy dialect for your database
to find out whether it will work with Superset
- Visualizations
- [Creating Viz Plugins](https://superset.apache.org/docs/contributing/creating-viz-plugins/)
- [Managing and Deploying Custom Viz Plugins](https://medium.com/nmc-techblog/apache-superset-manage-custom-viz-plugins-in-production-9fde1a708e55)
- [Why Apache Superset is Betting on Apache ECharts](https://preset.io/blog/2021-4-1-why-echarts/)
- [Superset API](https://superset.apache.org/docs/rest-api)
Druid!
------
<!-- telemetry/analytics pixel: -->
<img referrerpolicy="no-referrer-when-downgrade" src="https://static.scarf.sh/a.png?x-pxid=bc1c90cd-bc04-4e11-8c7b-289fb2839492" />
On top of having the ability to query your relational databases,
Superset has ships with deep integration with Druid (a real time distributed
column-store). When querying Druid,
Superset can query humongous amounts of data on top of real time dataset.
Note that Superset does not require Druid in any way to function, it's simply
another database backend that it can query.
Here's a description of Druid from the http://druid.io website:
*Druid is an open-source analytics data store designed for
business intelligence (OLAP) queries on event data. Druid provides low
latency (real-time) data ingestion, flexible data exploration,
and fast data aggregation. Existing Druid deployments have scaled to
trillions of events and petabytes of data. Druid is best used to
power analytic dashboards and applications.*
Installation & Configuration
----------------------------
[See in the documentation](https://superset.incubator.apache.org/installation.html)
Resources
-------------
* [Mailing list](https://lists.apache.org/list.html?dev@superset.apache.org/)
* [Gitter (live chat) Channel](https://gitter.im/airbnb/superset)
* [Docker image](https://hub.docker.com/r/amancevice/superset/) (community contributed)
* [Slides from Strata (March 2016)](https://drive.google.com/open?id=0B5PVE0gzO81oOVJkdF9aNkJMSmM)
* [Stackoverflow tag](https://stackoverflow.com/questions/tagged/apache-superset)
* [DEPRECATED Google Group](https://groups.google.com/forum/#!forum/airbnb_superset)
Contributing
------------
Interested in contributing? Casual hacking? Check out
[Contributing.MD](https://github.com/airbnb/superset/blob/master/CONTRIBUTING.md)
Who uses Apache Superset (incubating)?
--------------------------------------
Here's a list of organizations who have taken the time to send a PR to let
the world know they are using Superset. Join our growing community!
- [AiHello](https://www.aihello.com)
- [Airbnb](https://github.com/airbnb)
- [Aktia Bank plc](https://www.aktia.com)
- [Amino](https://amino.com)
- [Brilliant.org](https://brilliant.org/)
- [Capital Service S.A.](http://capitalservice.pl)
- [Clark.de](http://clark.de/)
- [Digit Game Studios](https://www.digitgaming.com/)
- [Douban](https://www.douban.com/)
- [Endress+Hauser](http://www.endress.com/)
- [FBK - ICT center](http://ict.fbk.eu)
- [Faasos](http://faasos.com/)
- [GfK Data Lab](http://datalab.gfk.com)
- [Konfío](http://konfio.mx)
- [Lyft](https://www.lyft.com/)
- [Maieutical Labs](https://cloudschooling.it)
- [PeopleDoc](https://www.people-doc.com)
- [Ona](https://ona.io)
- [Pronto Tools](http://www.prontotools.io)
- [Qunar](https://www.qunar.com/)
- [ScopeAI](https://www.getscopeai.com)
- [Shopee](https://shopee.sg)
- [Shopkick](https://www.shopkick.com)
- [Tails.com](https://tails.com)
- [Tobii](http://www.tobii.com/)
- [Tooploox](https://www.tooploox.com/)
- [Twitter](https://twitter.com/)
- [Udemy](https://www.udemy.com/)
- [VIPKID](https://www.vipkid.com.cn/)
- [Yahoo!](https://yahoo.com/)
- [Zaihang](http://www.zaih.com/)
- [Zalando](https://www.zalando.com)
More screenshots
----------------
![superset-security-menu](https://cloud.githubusercontent.com/assets/130878/20234707/0f565886-a835-11e6-9277-b4f5f4aa2fcc.png)
![superset-slice-bubble](https://cloud.githubusercontent.com/assets/130878/20234708/0f57f3d0-a835-11e6-8268-fcefe8f868c8.png)
![superset-slice-map](https://cloud.githubusercontent.com/assets/130878/20234709/0f5a5a44-a835-11e6-987a-1b6f8ac9922b.png)
![superset-slice-multiline](https://cloud.githubusercontent.com/assets/130878/20234710/0f632d68-a835-11e6-98d1-542dcb618193.png)
![superset-slice-sankey](https://cloud.githubusercontent.com/assets/130878/20234711/0f639136-a835-11e6-8721-fe5e48dab8e7.png)
![superset-slice-view](https://cloud.githubusercontent.com/assets/130878/20234712/0f63c4c6-a835-11e6-8595-6091a6428fa9.png)
![superset-sql-lab-2](https://cloud.githubusercontent.com/assets/130878/20234713/0f67b856-a835-11e6-9d50-7a52168f66fd.png)
![superset-sql-lab](https://cloud.githubusercontent.com/assets/130878/20234714/0f68f45a-a835-11e6-9467-f47ad0af7e79.png)

View File

@@ -1,67 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
FROM python:3.9-buster
RUN useradd --user-group --create-home --no-log-init --shell /bin/bash superset
# Configure environment
ENV LANG=C.UTF-8 \
LC_ALL=C.UTF-8
RUN apt-get update -y
# Install dependencies to fix `curl https support error` and `delaying package configuration warning`
RUN apt-get install -y apt-transport-https apt-utils
# Install superset dependencies
# https://superset.apache.org/docs/installation/installing-superset-from-scratch
RUN apt-get install -y build-essential libssl-dev \
libffi-dev python3-dev libsasl2-dev libldap2-dev libxi-dev chromium
# Install nodejs for custom build
# https://nodejs.org/en/download/package-manager/
RUN curl -sL https://deb.nodesource.com/setup_16.x | bash - \
&& apt-get install -y nodejs
RUN mkdir -p /home/superset
RUN chown superset /home/superset
WORKDIR /home/superset
ARG VERSION
ARG SUPERSET_RELEASE_RC_TARBALL
# Can fetch source from svn or copy tarball from local mounted directory
COPY $SUPERSET_RELEASE_RC_TARBALL ./
RUN tar -xvf *.tar.gz
WORKDIR /home/superset/apache-superset-$VERSION/superset-frontend
RUN npm ci \
&& npm run build \
&& rm -rf node_modules
WORKDIR /home/superset/apache-superset-$VERSION
RUN pip install --upgrade setuptools pip \
&& pip install -r requirements/base.txt \
&& pip install --no-cache-dir .
RUN flask fab babel-compile --target superset/translations
ENV PATH=/home/superset/superset/bin:$PATH \
PYTHONPATH=/home/superset/superset/:$PYTHONPATH \
SUPERSET_TESTENV=true
COPY from_tarball_entrypoint.sh /entrypoint.sh
ENTRYPOINT ["/entrypoint.sh"]

View File

@@ -1,67 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
FROM python:3.9-buster
RUN useradd --user-group --create-home --no-log-init --shell /bin/bash superset
# Configure environment
ENV LANG=C.UTF-8 \
LC_ALL=C.UTF-8
RUN apt-get update -y
# Install dependencies to fix `curl https support error` and `delaying package configuration warning`
RUN apt-get install -y apt-transport-https apt-utils
# Install superset dependencies
# https://superset.apache.org/docs/installation/installing-superset-from-scratch
RUN apt-get install -y build-essential libssl-dev \
libffi-dev python3-dev libsasl2-dev libldap2-dev libxi-dev chromium
# Install nodejs for custom build
# https://nodejs.org/en/download/package-manager/
RUN curl -sL https://deb.nodesource.com/setup_16.x | bash - \
&& apt-get install -y nodejs
RUN mkdir -p /home/superset
RUN chown superset /home/superset
WORKDIR /home/superset
ARG VERSION
# Can fetch source from svn or copy tarball from local mounted directory
RUN svn co https://dist.apache.org/repos/dist/dev/superset/$VERSION ./
RUN tar -xvf *.tar.gz
WORKDIR apache-superset-$VERSION
RUN cd superset-frontend \
&& npm ci \
&& npm run build \
&& rm -rf node_modules
WORKDIR /home/superset/apache-superset-$VERSION
RUN pip install --upgrade setuptools pip \
&& pip install -r requirements/base.txt \
&& pip install --no-cache-dir .
RUN flask fab babel-compile --target superset/translations
ENV PATH=/home/superset/superset/bin:$PATH \
PYTHONPATH=/home/superset/superset/:$PYTHONPATH
COPY from_tarball_entrypoint.sh /entrypoint.sh
ENTRYPOINT ["/entrypoint.sh"]

View File

@@ -1,28 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
FROM python:3.9-buster
ARG VERSION
RUN git clone --depth 1 --branch ${VERSION} https://github.com/apache/superset.git /superset
WORKDIR /superset
# install doc dependencies
RUN pip install -r requirements/base.txt
# build the docs
RUN python setup.py build_sphinx
COPY make_docs_entrypoint.sh /entrypoint.sh
ENTRYPOINT ["/entrypoint.sh"]

View File

@@ -1,23 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
FROM python:3.9-buster
RUN apt-get update -y
RUN apt-get install -y jq
COPY make_tarball_entrypoint.sh /entrypoint.sh
ENTRYPOINT ["/entrypoint.sh"]

View File

@@ -1,488 +0,0 @@
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
# Apache Releases
Until things settle and we create scripts that streamline this,
you'll probably want to run these commands manually and understand what
they do prior to doing so.
For coordinating on releases, on operational topics that require more
synchronous communications, we recommend using the `#apache-releases` channel
on the Superset Slack. People crafting releases and those interested in
partaking in the process should join the channel.
## Release notes for recent releases
- [2.0](release-notes-2-0/README.md)
- [1.5](release-notes-1-5/README.md)
- [1.4](release-notes-1-4/README.md)
- [1.3](release-notes-1-3/README.md)
- [1.2](release-notes-1-2/README.md)
- [1.1](release-notes-1-1/README.md)
- [1.0](release-notes-1-0/README.md)
- [0.38](release-notes-0-38/README.md)
## Release setup (First Time Only)
First you need to setup a few things. This is a one-off and doesn't
need to be done at every release.
```bash
# Create PGP Key, and use your @apache.org email address
gpg --gen-key
# Checkout ASF dist repo
svn checkout https://dist.apache.org/repos/dist/dev/superset/ ~/svn/superset_dev
svn checkout https://dist.apache.org/repos/dist/release/superset/ ~/svn/superset
cd ~/svn/superset
# Add your GPG pub key to KEYS file. Replace "Maxime Beauchemin" with your name
export SUPERSET_PGP_FULLNAME="Maxime Beauchemin"
(gpg --list-sigs "${SUPERSET_PGP_FULLNAME}" && gpg --armor --export "${SUPERSET_PGP_FULLNAME}" ) >> KEYS
# Commit the changes
svn commit -m "Add PGP keys of new Superset committer"
# push the changes
svn update
```
To minimize the risk of mixing up your local development environment, it's recommended to work on the
release in a different directory than where the devenv is located. In this example, we'll clone
the repo directly from the main `apache/superset` repo to a new directory `superset-release`:
```bash
cd <MY PROJECTS PATH>
git clone https://github.com/apache/superset.git superset-release
cd superset-release
```
We recommend setting up a virtual environment to isolate the python dependencies from your main
setup:
```bash
virtualenv venv
source venv/bin/activate
```
In addition, we recommend using the [`cherrytree`](https://pypi.org/project/cherrytree/) tool for
automating cherry picking, as it will help speed up the release process. To install `cherrytree`
and other dependencies that are required for the release process, run the following commands:
```bash
pip install -r RELEASING/requirements.txt
```
## Setting up the release environment (do every time)
As the vote process takes a minimum of 72h, sometimes stretching over several weeks
of calendar time if votes don't pass, chances are
the same terminal session won't be used for crafting the release candidate and the
final release. Therefore, it's a good idea to do the following every time you
work on a new phase of the release process to make sure you aren't releasing
the wrong files/using wrong names. There's a script to help you set correctly all the
necessary environment variables. Change your current directory to `RELEASING`
and execute the `set_release_env.sh` script with the relevant parameters:
Usage (MacOS/ZSH):
```bash
cd RELEASING
source set_release_env.sh <SUPERSET_RC_VERSION> <PGP_KEY_FULLNAME>
```
Usage (BASH):
```bash
. set_release_env.sh <SUPERSET_RC_VERSION> <PGP_KEY_FULLNAME>
```
Example:
```bash
source set_release_env.sh 1.5.1rc1 myid@apache.org
```
The script will output the exported variables. Here's example for 1.5.1rc1:
```
-------------------------------
Set Release env variables
SUPERSET_VERSION=1.5.1
SUPERSET_RC=1
SUPERSET_GITHUB_BRANCH=1.5
SUPERSET_PGP_FULLNAME=villebro@apache.org
SUPERSET_VERSION_RC=1.5.1rc1
SUPERSET_RELEASE=apache-superset-1.5.1
SUPERSET_RELEASE_RC=apache-superset-1.5.1rc1
SUPERSET_RELEASE_TARBALL=apache-superset-1.5.1-source.tar.gz
SUPERSET_RELEASE_RC_TARBALL=apache-superset-1.5.1rc1-source.tar.gz
SUPERSET_TMP_ASF_SITE_PATH=/tmp/incubator-superset-site-1.5.1
-------------------------------
```
## Crafting a source release
When crafting a new minor or major release we create
a branch named with the release MAJOR.MINOR version (on this example 0.37).
This new branch will hold all PATCH and release candidates
that belong to the MAJOR.MINOR version.
### Creating an initial minor release (e.g. 1.5.0)
The MAJOR.MINOR branch is normally a "cut" from a specific point in time from the master branch.
When creating the initial minor release (e.g. 1.5.0), create a new branch:
```bash
git checkout master
git pull
git checkout -b ${SUPERSET_GITHUB_BRANCH}
git push origin $SUPERSET_GITHUB_BRANCH
```
Note that this initializes a new "release cut", and is NOT needed when creating a patch release
(e.g. 1.5.1).
### Creating a patch release (e.g. 1.5.1)
When getting ready to bake a patch release, simply checkout the relevant branch:
```bash
git checkout master
git pull
git checkout ${SUPERSET_GITHUB_BRANCH}
```
### Cherry picking
It is customary to label PRs that have been introduced after the cut with the label
`v<MAJOR>.<MINOR>`. For example, for any PRs that should be included in the 1.5 branch, the
label `v1.5` should be added.
To see how well the labelled PRs would apply to the current branch, run the following command:
```bash
cherrytree bake -r apache/superset -m master -l v${SUPERSET_GITHUB_BRANCH} ${SUPERSET_GITHUB_BRANCH}
```
This requires the presence of an environment variable `GITHUB_TOKEN`. Alternatively,
you can pass the token directly via the `--access-token` parameter (`-at` for short).
#### Happy path: no conflicts
This will show how many cherries will apply cleanly. If there are no conflicts, you can simply apply all cherries
by adding the `--no-dry-run` flag (`-nd` for short):
```bash
cherrytree bake -r apache/superset -m master -l v${SUPERSET_GITHUB_BRANCH} -nd ${SUPERSET_GITHUB_BRANCH}
```
#### Resolving conflicts
If there are conflicts, you can issue the following command to apply all cherries up until the conflict automatically, and then
break by adding the `-error-mode break` flag (`-e break` for short):
```bash
cherrytree bake -r apache/superset -m master -l v${SUPERSET_GITHUB_BRANCH} -nd -e break ${SUPERSET_GITHUB_BRANCH}
```
After applying the cleanly merged cherries, `cherrytree` will specify the SHA of the conflicted cherry. To resolve the conflict,
simply issue the following command:
```bash
git cherry-pick <SHA>
```
Then fix all conflicts, followed by
```bash
git add -u # add all changes
git cherry-pick --continue
```
After this, rerun all the above steps until all cherries have been picked, finally pushing all new commits to the release branch
on the main repo:
```bash
git push
```
### Updating changelog
Next, update the `CHANGELOG.md` with all the changes that are included in the release.
Make sure the branch has been pushed to `origin` to ensure the changelog generator
can pick up changes since the previous release.
Similar to `cherrytree`, the change log script requires a github token, either as an env var
(`GITHUB_TOKEN`) or as the parameter `--access_token`.
#### Initial release (e.g. 1.5.0)
When generating the changelog for an initial minor relese, you should compare with
the previous release (in the example, the previous release branch is `1.4`, so remember to
update it accordingly):
```bash
python changelog.py --previous_version 1.4 --current_version ${SUPERSET_GITHUB_BRANCH} changelog
```
You can get a list of pull requests with labels started with blocking, risk, hold, revert and security by using the parameter `--risk`.
Example:
```bash
python changelog.py --previous_version 0.37 --current_version 0.38 changelog --access_token {GITHUB_TOKEN} --risk
```
The script will checkout both branches, compare all the PRs, and output the lines that are needed to be added to the
`CHANGELOG.md` file in the root of the repo. Remember to also make sure to update the branch id (with the above command
`1.5` needs to be changed to `1.5.0`)
Then, in `UPDATING.md`, a file that contains a list of notifications around
deprecations and upgrading-related topics,
make sure to move the content now under the `Next Version` section under a new
section for the new release.
#### Patch release (e.g. 1.5.1)
To compare the forthcoming patch release with the latest release from the same branch, set
`--previous_version` as the tag of the previous release (in this example `1.5.0`; remember to update accordingly)
```bash
python changelog.py --previous_version 1.5.0 --current_version ${SUPERSET_GITHUB_BRANCH} changelog
```
### Set version number
Finally, bump the version number on `superset-frontend/package.json` (replace with whichever version is being released excluding the RC version):
```
"version": "0.38.0"
```
Commit the change with the version number, then git tag the version with the release candidate and push to the branch:
```
# add changed files and commit
git add ...
git commit ...
# push new tag
git tag ${SUPERSET_VERSION_RC}
git push origin ${SUPERSET_VERSION_RC}
```
## Preparing the release candidate
The first step of preparing an Apache Release is packaging a release candidate
to be voted on. Make sure you have correctly prepared and tagged the ready to ship
release on Superset's repo (MAJOR.MINOR branch), the following script will clone
the tag and create a signed source tarball from it:
```bash
# make_tarball will use the previously set environment variables
# you can override by passing arguments: make_tarball.sh <SUPERSET_VERSION> <SUPERSET_VERSION_RC> "<PGP_KEY_FULLNAME>"
./make_tarball.sh
```
Note that `make_tarball.sh`:
- By default, the script assumes you have already executed an SVN checkout to `$HOME/svn/superset_dev`.
This can be overridden by setting `SUPERSET_SVN_DEV_PATH` environment var to a different svn dev directory
- Will refuse to craft a new release candidate if a release already exists on your local svn dev directory
- Will check `package.json` version number and fails if it's not correctly set
### Build and test the created source tarball
To build and run the **local copy** of the recently created tarball:
```bash
# Build and run a release candidate tarball
./test_run_tarball.sh local
# you should be able to access localhost:5001 on your browser
# login using admin/admin
```
### Shipping to SVN
Now let's ship this RC into svn's dev folder
```bash
cd ~/svn/superset_dev/
svn add ${SUPERSET_VERSION_RC}
svn commit -m "Release ${SUPERSET_VERSION_RC}"
svn update
```
### Build and test from SVN source tarball
To build and run the recently created tarball **from SVN**:
```bash
# Build and run a release candidate tarball
./test_run_tarball.sh
# you should be able to access localhost:5001 on your browser
# login using admin/admin
```
## Create a release on Github
After submitting the tag and testing the release candidate, follow the steps [here](https://docs.github.com/en/repositories/releasing-projects-on-github/managing-releases-in-a-repository) to create the release on GitHub. Use the vote email text as the content for the release description. Make sure to check the "This is a pre-release" checkbox for release candidates. You can check previous releases if you need an example.
## Voting
Now you're ready to start the [VOTE] thread. Here's an example of a
previous release vote thread:
https://lists.apache.org/thread.html/e60f080ebdda26896214f7d3d5be1ccadfab95d48fbe813252762879@<dev.superset.apache.org>
To easily send a voting request to Superset community, still on the `superset/RELEASING` directory:
```bash
# Note: use Superset's virtualenv
(venv)$ python generate_email.py vote_pmc
```
The script will generate the email text that should be sent to dev@superset.apache.org using an email client. The release version and release candidate number are fetched from the previously set environment variables.
Once 3+ binding votes (by PMC members) have been cast and at
least 72 hours have past, you can post a [RESULT] thread:
https://lists.apache.org/thread.html/50a6b134d66b86b237d5d7bc89df1b567246d125a71394d78b45f9a8@%3Cdev.superset.apache.org%3E
To easily send the result email, still on the `superset/RELEASING` directory:
```bash
# Note: use Superset's virtualenv
python generate_email.py result_pmc
```
The script will interactively ask for extra information needed to fill out the email template. Based on the
voting description, it will generate a passing, non passing or non conclusive email.
Here's an example:
```
A List of people with +1 binding vote (ex: Max,Grace,Krist): Daniel,Alan,Max,Grace
A List of people with +1 non binding vote (ex: Ville): Ville
A List of people with -1 vote (ex: John):
```
The script will generate the email text that should be sent to dev@superset.apache.org using an email client. The release version and release candidate number are fetched from the previously set environment variables.
## Validating a release
https://www.apache.org/info/verification.html
## Publishing a successful release
Upon a successful vote, you'll have to copy the folder into the non-"dev/" folder.
```bash
cp -r ~/svn/superset_dev/${SUPERSET_VERSION_RC}/ ~/svn/superset/${SUPERSET_VERSION}/
cd ~/svn/superset/
# Rename the RC (0.34.1rc1) to the actual version being released (0.34.1)
for f in ${SUPERSET_VERSION}/*; do mv "$f" "${f/${SUPERSET_VERSION_RC}/${SUPERSET_VERSION}}"; done
svn add ${SUPERSET_VERSION}
svn commit -m "Release ${SUPERSET_VERSION}"
svn update
```
Then tag the final release:
```bash
# Go to the root directory of the repo, e.g. `~/src/superset`
cd ~/src/superset/
# make sure you're on the correct branch (e.g. 0.34)
git branch
# Create the release tag
git tag -f ${SUPERSET_VERSION}
# push the tag to the remote
git push origin ${SUPERSET_VERSION}
```
### Update CHANGELOG and UPDATING on superset
Now that we have a final Apache source release we need to open a pull request on Superset
with the changes on `CHANGELOG.md` and `UPDATING.md`.
### Publishing a Convenience Release to PyPI
Extract the release to the `/tmp` folder to build the PiPY release. Files in the `/tmp` folder will be automatically deleted by the OS.
```bash
mkdir -p /tmp/superset && cd /tmp/superset
tar xfvz ~/svn/superset/${SUPERSET_VERSION}/${SUPERSET_RELEASE_TARBALL}
```
Create a virtual environment and install the dependencies
```bash
cd ${SUPERSET_RELEASE_RC}
python3 -m venv venv
source venv/bin/activate
pip install -r requirements/base.txt
pip install twine
```
Create the distribution
```bash
cd superset-frontend/
npm ci && npm run build
cd ../
flask fab babel-compile --target superset/translations
python setup.py sdist
```
Publish to PyPI
You may need to ask a fellow committer to grant
you access to it if you don't have access already. Make sure to create
an account first if you don't have one, and reference your username
while requesting access to push packages.
```bash
twine upload dist/apache-superset-${SUPERSET_VERSION}.tar.gz
```
Set your username to `__token__`
Set your password to the token value, including the `pypi-` prefix
More information on https://pypi.org/help/#apitoken
### Announcing
Once it's all done, an [ANNOUNCE] thread announcing the release to the dev@ mailing list is the final step.
```bash
# Note use Superset's virtualenv
python generate_email.py announce
```
The script will generate the email text that should be sent to dev@superset.apache.org using an email client. The release version is fetched from the previously set environment variables.
### GitHub Release
Finally, so the GitHub UI reflects the latest release, you should create a release from the
tag corresponding with the new version. Go to https://github.com/apache/superset/tags,
click the 3-dot icon and select `Create Release`, paste the content of the ANNOUNCE thread in the
release notes, and publish the new release.
At this point, a GitHub action will run that will check whether this release's version number is higher than the current 'latest' release. If that condition is true, this release sha will automatically be tagged as `latest` so that the most recent release can be referenced simply by using the 'latest' tag instead of looking up the version number. The existing version number tag will still exist, and can also be used for reference.

View File

@@ -1,410 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import csv as lib_csv
import os
import re
import sys
from collections.abc import Iterator
from dataclasses import dataclass
from typing import Any, Optional, Union
import click
from click.core import Context
try:
from github import BadCredentialsException, Github, PullRequest, Repository
except ModuleNotFoundError:
print("PyGitHub is a required package for this script")
exit(1)
SUPERSET_REPO = "apache/superset"
SUPERSET_PULL_REQUEST_TYPES = r"^(fix|feat|chore|refactor|docs|build|ci|/gmi)"
SUPERSET_RISKY_LABELS = r"^(blocking|risk|hold|revert|security vulnerability)"
@dataclass
class GitLog:
"""
Represents a git log entry
"""
sha: str
author: str
time: str
message: str
pr_number: Union[int, None] = None
author_email: str = ""
def __eq__(self, other: object) -> bool:
"""A log entry is considered equal if it has the same PR number"""
if isinstance(other, self.__class__):
return other.pr_number == self.pr_number
return False
def __repr__(self) -> str:
return f"[{self.pr_number}]: {self.message} {self.time} {self.author}"
class GitChangeLog:
"""
Helper class to output a list of logs entries on a superset changelog format
We want to map a git author to a github login, for that we call github's API
"""
def __init__(
self,
version: str,
logs: list[GitLog],
access_token: Optional[str] = None,
risk: Optional[bool] = False,
) -> None:
self._version = version
self._logs = logs
self._pr_logs_with_details: dict[int, dict[str, Any]] = {}
self._github_login_cache: dict[str, Optional[str]] = {}
self._github_prs: dict[int, Any] = {}
self._wait = 10
github_token = access_token or os.environ.get("GITHUB_TOKEN")
self._github = Github(github_token)
self._show_risk = risk
self._superset_repo: Repository = None
def _fetch_github_pr(self, pr_number: int) -> PullRequest:
"""
Fetches a github PR info
"""
try:
github_repo = self._github.get_repo(SUPERSET_REPO)
self._superset_repo = github_repo
pull_request = self._github_prs.get(pr_number)
if not pull_request:
pull_request = github_repo.get_pull(pr_number)
self._github_prs[pr_number] = pull_request
except BadCredentialsException as ex:
print(
f"Bad credentials to github provided"
f" use access_token parameter or set GITHUB_TOKEN"
)
sys.exit(1)
return pull_request
def _get_github_login(self, git_log: GitLog) -> Optional[str]:
"""
Tries to fetch a github login (username) from a git author
"""
author_name = git_log.author
github_login = self._github_login_cache.get(author_name)
if github_login:
return github_login
if git_log.pr_number:
pr_info = self._fetch_github_pr(git_log.pr_number)
if pr_info:
github_login = pr_info.user.login
else:
github_login = author_name
# set cache
self._github_login_cache[author_name] = github_login
return github_login
def _has_commit_migrations(self, git_sha: str) -> bool:
commit = self._superset_repo.get_commit(sha=git_sha)
return any(
"superset/migrations/versions/" in file.filename for file in commit.files
)
def _get_pull_request_details(self, git_log: GitLog) -> dict[str, Any]:
pr_number = git_log.pr_number
if pr_number:
detail = self._pr_logs_with_details.get(pr_number)
if detail:
return detail
pr_info = self._fetch_github_pr(pr_number)
has_migrations = self._has_commit_migrations(git_log.sha)
title = pr_info.title if pr_info else git_log.message
pr_type = re.match(SUPERSET_PULL_REQUEST_TYPES, title)
if pr_type:
pr_type = pr_type.group().strip('"') # type: ignore
labels = (" | ").join([label.name for label in pr_info.labels])
is_risky = self._is_risk_pull_request(pr_info.labels)
detail = {
"id": pr_number,
"has_migrations": has_migrations,
"labels": labels,
"title": title,
"type": pr_type,
"is_risky": is_risky or has_migrations,
}
if pr_number:
self._pr_logs_with_details[pr_number] = detail
return detail
def _is_risk_pull_request(self, labels: list[Any]) -> bool:
for label in labels:
risk_label = re.match(SUPERSET_RISKY_LABELS, label.name)
if risk_label is not None:
return True
return False
def _get_changelog_version_head(self) -> str:
if not len(self._logs):
print(
f"No changes found between revisions. "
f"Make sure your branch is up to date."
)
sys.exit(1)
return f"### {self._version} ({self._logs[0].time})"
def _parse_change_log(
self,
changelog: dict[str, str],
pr_info: dict[str, str],
github_login: str,
) -> None:
formatted_pr = (
f"- [#{pr_info.get('id')}]"
f"(https://github.com/{SUPERSET_REPO}/pull/{pr_info.get('id')}) "
f"{pr_info.get('title')} (@{github_login})\n"
)
if pr_info.get("has_migrations"):
changelog["Database Migrations"] += formatted_pr
elif pr_info.get("type") == "fix":
changelog["Fixes"] += formatted_pr
elif pr_info.get("type") == "feat":
changelog["Features"] += formatted_pr
else:
changelog["Others"] += formatted_pr
def __repr__(self) -> str:
result = f"\n{self._get_changelog_version_head()}\n"
changelog = {
"Database Migrations": "\n",
"Features": "\n",
"Fixes": "\n",
"Others": "\n",
}
for i, log in enumerate(self._logs):
github_login = self._get_github_login(log)
pr_info = self._get_pull_request_details(log)
if not github_login:
github_login = log.author
if self._show_risk:
if pr_info.get("is_risky"):
result += (
f"- [#{log.pr_number}]"
f"(https://github.com/{SUPERSET_REPO}/pull/{log.pr_number}) "
f"{pr_info.get('title')} (@{github_login}) "
f"{pr_info.get('labels')} \n"
)
else:
self._parse_change_log(changelog, pr_info, github_login)
print(f"\r {i}/{len(self._logs)}", end="", flush=True)
if self._show_risk:
return result
for key in changelog:
result += f"**{key}** {changelog[key]}\n"
return result
def __iter__(self) -> Iterator[dict[str, Any]]:
for log in self._logs:
yield {
"pr_number": log.pr_number,
"pr_link": f"https://github.com/{SUPERSET_REPO}/pull/"
f"{log.pr_number}",
"message": log.message,
"time": log.time,
"author": log.author,
"email": log.author_email,
"sha": log.sha,
}
class GitLogs:
"""
Manages git log entries from a specific branch/tag
Can compare git log entries by PR number
"""
def __init__(self, git_ref: str) -> None:
self._git_ref = git_ref
self._logs: list[GitLog] = []
@property
def git_ref(self) -> str:
return self._git_ref
@property
def logs(self) -> list[GitLog]:
return self._logs
def fetch(self) -> None:
self._logs = list(map(self._parse_log, self._git_logs()))[::-1]
def diff(self, git_logs: "GitLogs") -> list[GitLog]:
return [log for log in git_logs.logs if log not in self._logs]
def __repr__(self) -> str:
return f"{self._git_ref}, Log count:{len(self._logs)}"
@staticmethod
def _git_get_current_head() -> str:
output = os.popen("git status | head -1").read()
match = re.match("(?:HEAD detached at|On branch) (.*)", output)
if not match:
return ""
return match.group(1)
def _git_checkout(self, git_ref: str) -> None:
os.popen(f"git checkout {git_ref}").read()
current_head = self._git_get_current_head()
if current_head != git_ref:
print(f"Could not checkout {git_ref}")
sys.exit(1)
def _git_logs(self) -> list[str]:
# let's get current git ref so we can revert it back
current_git_ref = self._git_get_current_head()
self._git_checkout(self._git_ref)
output = (
os.popen('git --no-pager log --pretty=format:"%h|%an|%ae|%ad|%s|"')
.read()
.split("\n")
)
# revert to git ref, let's be nice
self._git_checkout(current_git_ref)
return output
@staticmethod
def _parse_log(log_item: str) -> GitLog:
pr_number = None
split_log_item = log_item.split("|")
# parse the PR number from the log message
match = re.match(r".*\(\#(\d*)\)", split_log_item[4])
if match:
pr_number = int(match.group(1))
return GitLog(
sha=split_log_item[0],
author=split_log_item[1],
author_email=split_log_item[2],
time=split_log_item[3],
message=split_log_item[4],
pr_number=pr_number,
)
@dataclass
class BaseParameters:
previous_logs: GitLogs
current_logs: GitLogs
def print_title(message: str) -> None:
print(f"{50*'-'}")
print(message)
print(f"{50*'-'}")
@click.group()
@click.pass_context
@click.option("--previous_version", help="The previous release version", required=True)
@click.option("--current_version", help="The current release version", required=True)
def cli(ctx: Context, previous_version: str, current_version: str) -> None:
"""Welcome to change log generator"""
previous_logs = GitLogs(previous_version)
current_logs = GitLogs(current_version)
previous_logs.fetch()
current_logs.fetch()
base_parameters = BaseParameters(previous_logs, current_logs)
ctx.obj = base_parameters
@cli.command("compare")
@click.pass_obj
def compare(base_parameters: BaseParameters) -> None:
"""Compares both versions (by PR)"""
previous_logs = base_parameters.previous_logs
current_logs = base_parameters.current_logs
print_title(
f"Pull requests from " f"{current_logs.git_ref} not in {previous_logs.git_ref}"
)
previous_diff_logs = previous_logs.diff(current_logs)
for diff_log in previous_diff_logs:
print(f"{diff_log}")
print_title(
f"Pull requests from " f"{previous_logs.git_ref} not in {current_logs.git_ref}"
)
current_diff_logs = current_logs.diff(previous_logs)
for diff_log in current_diff_logs:
print(f"{diff_log}")
@cli.command("changelog")
@click.option(
"--csv",
help="The csv filename to export the changelog to",
)
@click.option(
"--access_token",
help="The github access token,"
" if not provided will try to fetch from GITHUB_TOKEN env var",
)
@click.option("--risk", is_flag=True, help="show all pull requests with risky labels")
@click.pass_obj
def change_log(
base_parameters: BaseParameters, csv: str, access_token: str, risk: bool
) -> None:
"""Outputs a changelog (by PR)"""
previous_logs = base_parameters.previous_logs
current_logs = base_parameters.current_logs
previous_diff_logs = previous_logs.diff(current_logs)
logs = GitChangeLog(
current_logs.git_ref,
previous_diff_logs[::-1],
access_token=access_token,
risk=risk,
)
if csv:
with open(csv, "w") as csv_file:
log_items = list(logs)
field_names = log_items[0].keys()
writer = lib_csv.DictWriter(
csv_file,
delimiter=",",
quotechar='"',
quoting=lib_csv.QUOTE_ALL,
fieldnames=field_names,
)
writer.writeheader()
for log in logs:
writer.writerow(log)
else:
print("Fetching github usernames, this may take a while:")
print(logs)
cli()

View File

@@ -1,45 +0,0 @@
{#
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-#}
To: {{ receiver_email }}
Subject: [ANNOUNCE] Apache {{ project_name }} version {{ version }} Released
Hello Community,
The Apache {{ project_name }} team is pleased to announce that {{ project_name }}
{{ version }} has just been released.
{{ project_description }}
The official source release:
https://www.apache.org/dist/{{ project_module }}/{{ version }}
The PyPI package:
https://pypi.org/project/apache-superset/
If you have any usage questions or have problems when upgrading or
find any issues with enhancements included in this release, please
don't hesitate to let us know by sending feedback to this mailing
list.
---------------------------------------------------------------------
To unsubscribe, e-mail: general-unsubscribe@apache.org
For additional commands, e-mail: general-help@apache.org

View File

@@ -1,60 +0,0 @@
{#
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-#}
To: {{ receiver_email }}
Subject: [RESULT] [VOTE] Release Apache {{ project_name }} {{ version }} based on Superset {{ version_rc }}
Thanks to everyone that participated. The vote to release
Apache {{ project_name }} version {{ version }} based on {{ version_rc }} is now closed.
{% if vote_negatives|length > 0 -%}
The vote did NOT PASS with {{vote_bindings|length}} binding +1, {{ vote_nonbindings|length}} non-binding +1, and {{vote_negatives|length}} -1 votes:
{% elif vote_bindings|length > 2 -%}
The vote PASSED with {{vote_bindings|length}} binding +1, {{ vote_nonbindings|length}} non-binding +1, and {{vote_negatives|length}} -1 votes:
{% else -%}
The vote is non conclusive with {{vote_bindings|length}} binding +1, {{ vote_nonbindings|length}} non-binding -1, and {{vote_negatives|length}} -1 votes:
{%- endif %}
{% if vote_bindings|length > 0 -%}
Binding votes:
{% for voter in vote_bindings -%}
- {{ voter }}
{% endfor -%}
{%- endif %}
{% if vote_nonbindings|length > 0 -%}
Non-binding votes:
{% for voter in vote_nonbindings -%}
- {{ voter }}
{% endfor -%}
{%- endif %}
{% if vote_negatives|length > 0 -%}
Negative votes:
{% for voter in vote_negatives -%}
- {{ voter }}
{% endfor -%}
{%- endif %}
Link to vote thread: {{ vote_thread }}
We will work to complete the release process.
Thanks,
The Apache {{ project_name }} Team

View File

@@ -1,52 +0,0 @@
{#
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-#}
To: {{ receiver_email }}
Subject: [VOTE] Release Apache {{ project_name }} {{ version }} based on Superset {{ version_rc }}
Hello {{ project_name }} Community,
This is a call for the vote to release Apache {{ project_name }} version {{ version }}.
The release candidate:
https://dist.apache.org/repos/dist/dev/{{ project_module }}/{{ version_rc }}/
Git tag for the release:
https://github.com/apache/{{ project_module }}/tree/{{ version_rc }}
The Change Log for the release:
https://github.com/apache/{{ project_module }}/blob/{{ version_rc }}/CHANGELOG.md
The Updating instructions for the release:
https://github.com/apache/{{ project_module }}/blob/{{ version_rc }}/UPDATING.md
Public keys are available at:
https://www.apache.org/dist/{{ project_module }}/KEYS
The vote will be open for at least 72 hours or until the necessary number
of votes are reached.
Please vote accordingly:
[ ] +1 approve
[ ] +0 no opinion
[ ] -1 disapprove with the reason
Thanks,
The Apache {{ project_name }} Team

View File

@@ -1,45 +0,0 @@
#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
set -ex
echo "[WARNING] this entrypoint creates an admin/admin user"
echo "[WARNING] it should only be used for lightweight testing/validation"
if [ -z "${SUPERSET_TESTENV}" ]; then
echo "SUPERSET IS RUNNING IN TEST MODE"
fi
# Create an admin user (you will be prompted to set username, first and last name before setting a password)
superset fab create-admin \
--username admin \
--firstname admin \
--lastname admin \
--email admin@admin.com \
--password admin
# Initialize the database
superset db upgrade
# Create default roles and permissions
superset init
# Loading examples
superset load-examples --force
SUPERSET_ENV=development FLASK_APP="superset.app:create_app()" \
flask run -p 8088 --with-threads --reload --debugger --host=0.0.0.0

View File

@@ -1,154 +0,0 @@
#!/usr/bin/python3
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from typing import Any
from click.core import Context
try:
import jinja2
except ModuleNotFoundError:
exit("Jinja2 is a required dependency for this script")
try:
import click
except ModuleNotFoundError:
exit("Click is a required dependency for this script")
RECEIVER_EMAIL = "dev@superset.apache.org"
PROJECT_NAME = "Superset"
PROJECT_MODULE = "superset"
PROJECT_DESCRIPTION = "Apache Superset is a modern, enterprise-ready business intelligence web application"
def string_comma_to_list(message: str) -> list[str]:
if not message:
return []
return [element.strip() for element in message.split(",")]
def render_template(template_file: str, **kwargs: Any) -> str:
"""
Simple render template based on named parameters
:param template_file: The template file location
:kwargs: Named parameters to use when rendering the template
:return: Rendered template
"""
template = jinja2.Template(open(template_file).read())
return template.render(kwargs)
class BaseParameters:
def __init__(
self,
version: str,
version_rc: str,
) -> None:
self.version = version
self.version_rc = version_rc
self.template_arguments: dict[str, Any] = {}
def __repr__(self) -> str:
return f"Apache Credentials: {self.version}/{self.version_rc}"
@click.group()
@click.pass_context
@click.option("--version", envvar="SUPERSET_VERSION")
@click.option("--version_rc", envvar="SUPERSET_VERSION_RC")
def cli(
ctx: Context,
version: str,
version_rc: str,
) -> None:
"""Welcome to releasing send email CLI interface!"""
base_parameters = BaseParameters(version, version_rc)
base_parameters.template_arguments["receiver_email"] = RECEIVER_EMAIL
base_parameters.template_arguments["project_name"] = PROJECT_NAME
base_parameters.template_arguments["project_module"] = PROJECT_MODULE
base_parameters.template_arguments["project_description"] = PROJECT_DESCRIPTION
base_parameters.template_arguments["version"] = base_parameters.version
base_parameters.template_arguments["version_rc"] = base_parameters.version_rc
ctx.obj = base_parameters
@cli.command("vote_pmc")
@click.pass_obj
def vote_pmc(base_parameters: BaseParameters) -> None:
template_file = "email_templates/vote_pmc.j2"
message = render_template(template_file, **base_parameters.template_arguments)
print(message)
@cli.command("result_pmc")
@click.option(
"--vote_bindings",
default="",
type=str,
prompt="A List of people with +1 binding vote (ex: Max,Grace,Krist)",
)
@click.option(
"--vote_nonbindings",
default="",
type=str,
prompt="A List of people with +1 non binding vote (ex: Ville)",
)
@click.option(
"--vote_negatives",
default="",
type=str,
prompt="A List of people with -1 vote (ex: John)",
)
@click.option(
"--vote_thread",
default="",
type=str,
prompt="Permalink to the vote thread "
"(see https://lists.apache.org/list.html?dev@superset.apache.org)",
)
@click.pass_obj
def result_pmc(
base_parameters: BaseParameters,
vote_bindings: str,
vote_nonbindings: str,
vote_negatives: str,
vote_thread: str,
) -> None:
template_file = "email_templates/result_pmc.j2"
base_parameters.template_arguments["vote_bindings"] = string_comma_to_list(
vote_bindings
)
base_parameters.template_arguments["vote_nonbindings"] = string_comma_to_list(
vote_nonbindings
)
base_parameters.template_arguments["vote_negatives"] = string_comma_to_list(
vote_negatives
)
base_parameters.template_arguments["vote_thread"] = vote_thread
message = render_template(template_file, **base_parameters.template_arguments)
print(message)
@cli.command("announce")
@click.pass_obj
def announce(base_parameters: BaseParameters) -> None:
template_file = "email_templates/announce.j2"
message = render_template(template_file, **base_parameters.template_arguments)
print(message)
cli()

View File

@@ -1,74 +0,0 @@
#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
set -e
usage() {
echo "usage: make_tarball.sh <SUPERSET_VERSION> <SUPERSET_RC> <PGP_KEY_FULLBANE>"
}
if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then
if [ -z "${SUPERSET_VERSION}" ] || [ -z "${SUPERSET_RC}" ] || [ -z "${SUPERSET_PGP_FULLNAME}" ] || [ -z "${SUPERSET_RELEASE_RC_TARBALL}" ]; then
echo "No parameters found and no required environment variables set"
echo "usage: make_tarball.sh <SUPERSET_VERSION> <SUPERSET_RC> <PGP_KEY_FULLBANE>"
usage;
exit 1
fi
else
SUPERSET_VERSION="${1}"
SUPERSET_RC="${2}"
SUPERSET_PGP_FULLNAME="${3}"
SUPERSET_RELEASE_RC_TARBALL="apache-superset-${SUPERSET_VERSION_RC}-source.tar.gz"
fi
SUPERSET_VERSION_RC="${SUPERSET_VERSION}rc${SUPERSET_RC}"
if [ -z "${SUPERSET_SVN_DEV_PATH}" ]; then
SUPERSET_SVN_DEV_PATH="$HOME/svn/superset_dev"
fi
if [[ ! -d "${SUPERSET_SVN_DEV_PATH}" ]]; then
echo "${SUPERSET_SVN_DEV_PATH} does not exist, you need to: svn checkout"
exit 1
fi
if [ -d "${SUPERSET_SVN_DEV_PATH}/${SUPERSET_VERSION_RC}" ]; then
echo "${SUPERSET_VERSION_RC} Already exists on svn, refusing to overwrite"
exit 1
fi
SUPERSET_RELEASE_RC_TARBALL_PATH="${SUPERSET_SVN_DEV_PATH}"/"${SUPERSET_VERSION_RC}"/"${SUPERSET_RELEASE_RC_TARBALL}"
DOCKER_SVN_PATH="/docker_svn"
# Building docker that will produce a tarball
docker build -t apache-builder -f Dockerfile.make_tarball .
# Running docker to produce a tarball
docker run \
-e SUPERSET_SVN_DEV_PATH="${DOCKER_SVN_PATH}" \
-e SUPERSET_VERSION="${SUPERSET_VERSION}" \
-e SUPERSET_VERSION_RC="${SUPERSET_VERSION_RC}" \
-e HOST_UID=${UID} \
-v "${SUPERSET_SVN_DEV_PATH}":"${DOCKER_SVN_PATH}":rw \
-ti apache-builder
gpg --armor --local-user "${SUPERSET_PGP_FULLNAME}" --output "${SUPERSET_RELEASE_RC_TARBALL_PATH}".asc --detach-sig "${SUPERSET_RELEASE_RC_TARBALL_PATH}"
gpg --print-md --local-user "${SUPERSET_PGP_FULLNAME}" SHA512 "${SUPERSET_RELEASE_RC_TARBALL_PATH}" > "${SUPERSET_RELEASE_RC_TARBALL_PATH}".sha512
echo ---------------------------------------
echo Release candidate tarball is ready
echo ---------------------------------------

View File

@@ -1,53 +0,0 @@
#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
set -e
if [ -z "${SUPERSET_VERSION_RC}" ] || [ -z "${SUPERSET_SVN_DEV_PATH}" ] || [ -z "${SUPERSET_VERSION}" ]; then
echo "SUPERSET_VERSION_RC, SUPERSET_SVN_DEV_PATH and SUPERSET_VERSION are required to run this container"
exit 1
fi
SUPERSET_RELEASE_RC=apache-superset-"${SUPERSET_VERSION_RC}"
SUPERSET_RELEASE_RC_TARBALL="${SUPERSET_RELEASE_RC}"-source.tar.gz
SUPERSET_RELEASE_RC_BASE_PATH="${SUPERSET_SVN_DEV_PATH}"/"${SUPERSET_VERSION_RC}"
SUPERSET_RELEASE_RC_TARBALL_PATH="${SUPERSET_RELEASE_RC_BASE_PATH}"/"${SUPERSET_RELEASE_RC_TARBALL}"
# Create directory release version
mkdir -p "${SUPERSET_SVN_DEV_PATH}"/"${SUPERSET_VERSION_RC}"
# Clone superset from tag to /tmp
cd /tmp
git clone --depth 1 --branch ${SUPERSET_VERSION_RC} https://github.com/apache/superset.git
mkdir -p "${HOME}/${SUPERSET_VERSION_RC}"
cd superset && \
# Check RC version
if ! jq -e --arg SUPERSET_VERSION $SUPERSET_VERSION '.version == $SUPERSET_VERSION' superset-frontend/package.json
then
SOURCE_VERSION=$(jq '.version' superset-frontend/package.json)
echo "Source package.json version is wrong, found: ${SOURCE_VERSION} should be: ${SUPERSET_VERSION}"
exit 1
fi
# Create source tarball
git archive \
--format=tar.gz "${SUPERSET_VERSION_RC}" \
--prefix="${SUPERSET_RELEASE_RC}/" \
-o "${SUPERSET_RELEASE_RC_TARBALL_PATH}"
chown -R ${HOST_UID}:${HOST_UID} "${SUPERSET_RELEASE_RC_BASE_PATH}"

View File

@@ -1,318 +0,0 @@
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
# Release Notes for Superset 0.38
## What's new
- [New features](#new-features)
- [Bugfixes](#bugfixes)
- [Breaking Changes](#breaking-changes)
- [Complete Changelog](#complete-changelog)
### New features
**DATABASES, DATASETS, QUERIES**
[SIP-40] Proposal for Custom Error Messages #9194 ([SIP](https://github.com/apache/superset/issues/9194))
- feat: improve presto query perf (#[11069](https://github.com/apache/superset/pull/11069))
- feat: more specific presto error messages (#[11099](https://github.com/apache/superset/pull/11099))
- feat: refactor error components and add database issue code (#[10473](https://github.com/apache/superset/pull/10473))
- feat: welcome presto to the suite of tested databases (#[10498](https://github.com/apache/superset/pull/10498))
Certification of Data Entities ([Roadmap](https://github.com/apache-superset/superset-roadmap/issues/73))
- feat: add certification to metrics (#[10630](https://github.com/apache/superset/pull/10630))
- feat: add extra column to tables and sql_metrics (#[10592](https://github.com/apache/superset/pull/10592))
- feat: bump superset-ui for certified tag (#[10650](https://github.com/apache/superset/pull/10650))
Database CRUD screen refactor/redesign ([Roadmap](https://github.com/apache-superset/superset-roadmap/issues/14))
- feat: add/edit database modal form sections UI (#[10745](https://github.com/apache/superset/pull/10745))
- feat: base tabbed modal for new database CRUD UI (#[10668](https://github.com/apache/superset/pull/10668))
Database list view refactor/redesign ([Roadmap](https://github.com/apache-superset/superset-roadmap/issues/55))
- feat: database delete warning (#[10800](https://github.com/apache/superset/pull/10800))\
<kbd><img alt="10800" src="media/10800.gif" width="400"/></kbd>
- feat: filters for database list view (#[10772](https://github.com/apache/superset/pull/10772))
- feat: SIP-34 table list view for databases (#[10705](https://github.com/apache/superset/pull/10705))
Database list view refactor/redesign #55 ([Roadmap](https://github.com/apache-superset/superset-roadmap/issues/55))
- feat(api): database schemas migration to new API (#[10436](https://github.com/apache/superset/pull/10436))
- feat(database): POST, PUT, DELETE API endpoints (#[10741](https://github.com/apache/superset/pull/10741))
- feat(databases): test connection api (#[10723](https://github.com/apache/superset/pull/10723))
Datasets CRUD screen refactor/redesign ([Roadmap](https://github.com/apache-superset/superset-roadmap/issues/13))
- feat: dataset editor improvements (#[10444](https://github.com/apache/superset/pull/10444))\
<kbd><img alt="10444" src="media/10444.gif" width="400"/></kbd>
Datasets CRUD screen refactor/redesign #13 ([Roadmap](https://github.com/apache-superset/superset-roadmap/issues/13))
- feat(datasource): remove deleted columns and update column type on metadata refresh (#[10619](https://github.com/apache/superset/pull/10619))\
<kbd><img alt="10619" src="media/10619.png" width="400"/></kbd>
Datasets list view refactor/redesign #12 ([Roadmap](https://github.com/apache-superset/superset-roadmap/issues/12))
- feat: update dataset editor modal (#[10347](https://github.com/apache/superset/pull/10347))\
<kbd><img alt="10347" src="media/10347.gif" width="400"/></kbd>
- feat(datasets): REST API bulk delete (#[11237](https://github.com/apache/superset/pull/11237))
Saved queries CRUD screen + list view refactor/redesign #15 ([Roadmap](https://github.com/apache-superset/superset-roadmap/issues/15))
- feat: CRUD REST API for saved queries (#[10777](https://github.com/apache/superset/pull/10777))
- feat: saved query list actions (#[11109](https://github.com/apache/superset/pull/11109))\
<kbd><img alt="11109" src="media/11109.gif" width="400"/></kbd>
- feat: saved query list view + sort/filters (#[11005](https://github.com/apache/superset/pull/11005))
- feat: SavedQuery REST API for bulk delete and new API fields (#[10793](https://github.com/apache/superset/pull/10793))
- feat: update saved query backend routing + add savedquery list (#[10922](https://github.com/apache/superset/pull/10922))
- feat(saved_queries): add custom api filter for all string & text fields (#[11031](https://github.com/apache/superset/pull/11031))
Other features
- feat: dataset REST API for distinct values (#[10595](https://github.com/apache/superset/pull/10595))
**EXPLORE, CHARTS, DASHBOARDS**
[SIP-40] Proposal for Custom Error Messages #9194 ([SIP](https://github.com/apache/superset/issues/9194))
- feat: add download as image button to explore (#[10297](https://github.com/apache/superset/pull/10297))\
<kbd><img alt="10297" src="media/10297.gif" width="400"/></kbd>
[SIP-34] Proposal to establish a new design direction, system, and process for Superset ([SIP](https://github.com/apache/superset/issues/8976))
- feat: SIP-34 explore save modal (#[10355](https://github.com/apache/superset/pull/10355))\
<kbd><img alt="10355" src="media/10355.gif" width="400"/></kbd>
Charts list view refactor/redesign ([Roadmap](https://github.com/apache-superset/superset-roadmap/issues/47))
- feat(charts): modify custom api filter to include more fields (#[11054](https://github.com/apache/superset/pull/11054))
Echarts integration ([Roadmap](https://github.com/apache-superset/superset-roadmap/issues/48))
- feat: add linear color scale to sunburst chart (#[10474](https://github.com/apache/superset/pull/10474))\
<kbd><img alt="10474" src="media/10474.gif" width="400"/></kbd>
- feat: add optional prophet forecasting functionality to chart data api (#[10324](https://github.com/apache/superset/pull/10324))
- feat(viz): add ECharts Timeseries chart (#[10752](https://github.com/apache/superset/pull/10752))\
<kbd><img alt="10752" src="media/10752.gif" width="400"/></kbd>
Improvements to cache handling #74 ([Roadmap](https://github.com/apache-superset/superset-roadmap/issues/74))
- feat: add ECharts Pie chart (#[10966](https://github.com/apache/superset/pull/10966))\
<kbd><img alt="10966" src="media/10966.gif" width="400"/></kbd>
- feat: implement cache invalidation api (#[10761](https://github.com/apache/superset/pull/10761))
- feat: Adding extra_filters to warm_up_cache (#[10675](https://github.com/apache/superset/pull/10675))
[SIP-34] Proposal to establish a new design direction, system, and process for Superset ([SIP](https://github.com/apache/superset/issues/8976))
- feat: add favorite star to dashboard and chart lists (#[10510](https://github.com/apache/superset/pull/10510))\
<kbd><img alt="10510" src="media/10510.png" width="400"/></kbd>
- feat: use svg for checkbox component (#[10799](https://github.com/apache/superset/pull/10799))\
<kbd><img alt="10799" src="media/10799.gif" width="400"/></kbd>
Other features
- feat(table-viz): translation and metric column header align right (#[10549](https://github.com/apache/superset/pull/10549))
- feat: adding dashboard toggle fullscreen button (#[10840](https://github.com/apache/superset/pull/10840))\
<kbd><img alt="10840" src="media/10840.png" width="400"/></kbd>
- feat: enable ETag header for dashboard GET requests (#[10963](https://github.com/apache/superset/pull/10963))
- feat: move ace-editor and mathjs to async modules (#[10837](https://github.com/apache/superset/pull/10837))
- feat: server side dashboard css for less repaint (#[10850](https://github.com/apache/superset/pull/10850))
- feat: use shorten url in standalone iframe (#[10651](https://github.com/apache/superset/pull/10651))
**SQL LAB**
- feat: Adding table comment and columns comment for SQLLab (#[10844](https://github.com/apache/superset/pull/10844))
**SYSTEM, OTHER**
- feat: add TXT as default CSV extension (#[10371](https://github.com/apache/superset/pull/10371))
[SIP-34] Proposal to establish a new design direction, system, and process for Superset ([SIP](https://github.com/apache/superset/issues/8976))
- feat: card view bulk select (#[10607](https://github.com/apache/superset/pull/10607))
- feat: custom favorite filter for dashboards, charts and saved queries (#[11083](https://github.com/apache/superset/pull/11083))
- feat: SIP-34 card/grid views for dashboards and charts (#[10526](https://github.com/apache/superset/pull/10526))
- feat(listviews): SIP-34 Bulk Select (#[10298](https://github.com/apache/superset/pull/10298))
- feat(listviews): SIP-34 filters for charts, dashboards, datasets (#[10335](https://github.com/apache/superset/pull/10335))
[SIP-40] Proposal for Custom Error Messages #9194 ([SIP](https://github.com/apache/superset/issues/9194))
- feat: update timeout error UX (#[10274](https://github.com/apache/superset/pull/10274))
Alerts (send notification when a condition is met) ([Roadmap](https://github.com/apache-superset/superset-roadmap/issues/54))
- feat: add test email functionality to SQL-based email alerts (#[10476](https://github.com/apache/superset/pull/10476))
- feat: refactored SQL-based alerting framework (#[10605](https://github.com/apache/superset/pull/10605))
[SIP-34] Proposal to establish a new design direction, system, and process for Superset ([SIP](https://github.com/apache/superset/issues/8976))
- feat: adding all icons from the design system to the codebase (#[11033](https://github.com/apache/superset/pull/11033))
- feat: storybook for Icon component (#[10515](https://github.com/apache/superset/pull/10515))
[SIP-48] Using Ant Design as our primary component library ([SIP](https://github.com/apache/superset/issues/10254))
- feat: Add antd to the codebase (#[10508](https://github.com/apache/superset/pull/10508))
Alerts (send notification when a condition is met) ([Roadmap](https://github.com/apache-superset/superset-roadmap/issues/54))
- feat: updated email format for SQL-based email alerts (#[10512](https://github.com/apache/superset/pull/10512))
Superset Component library - Phase 1 ([Roadmap](https://github.com/apache-superset/superset-roadmap/issues/23))
- feat: adding Storybook to Superset (#[10383](https://github.com/apache/superset/pull/10383))
Other
- feat: Allow tests files in /src (plus Label component tests) (#[10634](https://github.com/apache/superset/pull/10634))
- feat: Getting fancier with Storybook (#[10647](https://github.com/apache/superset/pull/10647))
### Bugfixes
- fix(permissions): alpha role has all full features (#[10241](https://github.com/apache/superset/pull/10241))
- fix: broken glyphicons used in react-json-schema (#[10267](https://github.com/apache/superset/pull/10267))
- fix: add additional ui tweaks (#[10275](https://github.com/apache/superset/pull/10275))
- fix: saving custom CSS correctly (#[10289](https://github.com/apache/superset/pull/10289))
- fix: fetch datasets list after dataset created successfully (#[10290](https://github.com/apache/superset/pull/10290))
- fix: update community Slack link (#[10360](https://github.com/apache/superset/pull/10360))
- fix: allow creating table option and remove schema requirement in dataset add modal (#[10369](https://github.com/apache/superset/pull/10369))
- fix(datasets): sort and humanized modified by (#[10380](https://github.com/apache/superset/pull/10380))
- fix(api): fixes perf on charts and introduces sorting by database on datasets (#[10392](https://github.com/apache/superset/pull/10392))
- fix(api): fixes openapi spec errors and adds a test to validate all spec (#[10393](https://github.com/apache/superset/pull/10393))
- fix(charts): disable CSRF for chart data endpoint (#[10397](https://github.com/apache/superset/pull/10397))
- fix: dataset list filters bug (#[10398](https://github.com/apache/superset/pull/10398))
- fix: remove FAB rendered menu in favor of react based one (#[10401](https://github.com/apache/superset/pull/10401))
- fix: show label for filters in filter box in explore (#[10412](https://github.com/apache/superset/pull/10412))
- fix: Implement updates to SQL-based email alerts (#[10454](https://github.com/apache/superset/pull/10454))
- fix(presto): Handle ROW data stored as string (#[10456](https://github.com/apache/superset/pull/10456))
- fix: change "add new slice" copy to "add new chart" (#[10457](https://github.com/apache/superset/pull/10457))
- fix(sqllab): button width isn't wide enough for 'Run Selection' (#[10461](https://github.com/apache/superset/pull/10461))
- fix: timeout error message (#[10478](https://github.com/apache/superset/pull/10478))
- fix: enforce mandatory chart name on save and edit (#[10482](https://github.com/apache/superset/pull/10482))
- fix: More tweaks needed after adding Doctype tag (#[10504](https://github.com/apache/superset/pull/10504))
- fix: explore panel missing padding (#[10505](https://github.com/apache/superset/pull/10505))
- fix: refactored SQL-based alerts to not pass sqlalchemy objects as args (#[10506](https://github.com/apache/superset/pull/10506))
- fix(sqllab): Handle long table names in SQL Lab (#[10518](https://github.com/apache/superset/pull/10518))
- fix: make SQL-based alert email links user friendly (#[10519](https://github.com/apache/superset/pull/10519))
- fix(dashboard): changing the chart title, except not (#[10527](https://github.com/apache/superset/pull/10527))
- fix: misaligned LimitControl buttons and port jsx->tsx (#[10529](https://github.com/apache/superset/pull/10529))
- fix: Resolves #10535 (#[10536](https://github.com/apache/superset/pull/10536))
- fix: add retry to SQL-based alerting celery task (#[10542](https://github.com/apache/superset/pull/10542))
- fix: Updating Dockerfile to work with updated python requirements. (#[10550](https://github.com/apache/superset/pull/10550))
- fix(thumbnails): missing field, logging and new config var (#[10562](https://github.com/apache/superset/pull/10562))
- fix: add translate for dropdown menu (#[10573](https://github.com/apache/superset/pull/10573))
- fix: error message modal overflow (#[10580](https://github.com/apache/superset/pull/10580))
- fix: add None checking to cast_to_num (#[10584](https://github.com/apache/superset/pull/10584))
- fix: removing unsupported modal sizes (#[10625](https://github.com/apache/superset/pull/10625))
- fix: remove duplicated params and cache_timeout from list_columns; add viz_type to list_columns (#[10643](https://github.com/apache/superset/pull/10643))
- fix: controls scroll issue (#[10644](https://github.com/apache/superset/pull/10644))
- fix(db_engine_specs): improve Presto column type matching (#[10658](https://github.com/apache/superset/pull/10658))
- fix(db_engine_specs): mysql longtext type should not be numeric (#[10661](https://github.com/apache/superset/pull/10661))
- fix: change public role like gamma procedure (#[10674](https://github.com/apache/superset/pull/10674))
- fix(sqllab): log exceptions caused by the user as debug and not error (#[10676](https://github.com/apache/superset/pull/10676))
- fix: only call signal if executing on the main thread (#[10677](https://github.com/apache/superset/pull/10677))
- fix: layout flexiness (#[10681](https://github.com/apache/superset/pull/10681))
- fix: SubMenu css (#[10682](https://github.com/apache/superset/pull/10682))
- fix: dashboard extra filters (#[10692](https://github.com/apache/superset/pull/10692))
- fix: shorten url with extra request parameters (#[10693](https://github.com/apache/superset/pull/10693))
- fix: card view failed cypress tests (#[10699](https://github.com/apache/superset/pull/10699))
- fix: deprecation warnings due to invalid escape sequences. (#[10710](https://github.com/apache/superset/pull/10710))
- fix: move menu reorg logic from crud app into Menu component (#[10717](https://github.com/apache/superset/pull/10717))
- fix: local docker deployment (#[10738](https://github.com/apache/superset/pull/10738))
- Fix: Rejiggering some dependencies, trying to get CI to pass (#[10747](https://github.com/apache/superset/pull/10747))
- fix(presto): default unknown types to string type (#[10753](https://github.com/apache/superset/pull/10753))
- fix: add validator information to email/slack alerts (#[10762](https://github.com/apache/superset/pull/10762))
- fix: re-installing local superset in cache image (#[10766](https://github.com/apache/superset/pull/10766))
- fix: can not type `0.05` in `TextControl` (#[10778](https://github.com/apache/superset/pull/10778))
- fix: MVC show saved query (#[10781](https://github.com/apache/superset/pull/10781))
- fix: disable domain sharding on explore view (#[10787](https://github.com/apache/superset/pull/10787))
- fix: Database API missing allow none on fields (#[10795](https://github.com/apache/superset/pull/10795))
- fix: bump node version on Dockerfile to be on par with docker-compose (#[10813](https://github.com/apache/superset/pull/10813))
- fix(tests): export dataset tests fails with presto (#[10818](https://github.com/apache/superset/pull/10818))
- fix: use nullpool in the celery workers (#[10819](https://github.com/apache/superset/pull/10819))
- fix: Making the database read-only (#[10823](https://github.com/apache/superset/pull/10823))
- fix(databases): test connection api endpoint (#[10824](https://github.com/apache/superset/pull/10824))
- fix: update the time filter for 'Last Year' option in explore (#[10829](https://github.com/apache/superset/pull/10829))
- fix(test): missing auth on tests (#[10842](https://github.com/apache/superset/pull/10842))
- fix(cypress): wait for filterValues request (#[10884](https://github.com/apache/superset/pull/10884))
- fix: superset alerting misc fixes (#[10891](https://github.com/apache/superset/pull/10891))
- fix(cypress): prevent CI failure on codecov failure (#[10892](https://github.com/apache/superset/pull/10892))
- fix: front end CI tests and test runner (#[10897](https://github.com/apache/superset/pull/10897))
- fix: babel script broken by format string (#[10902](https://github.com/apache/superset/pull/10902))
- fix: several disabled pylint rules in models/helpers.py (#[10909](https://github.com/apache/superset/pull/10909))
- fix: spelling in docs homepage (#[10912](https://github.com/apache/superset/pull/10912))
- fix: address all disabled pylint checks in charts/api.py (#[10932](https://github.com/apache/superset/pull/10932))
- fix: use nullpool even for user lookup in the celery (#[10938](https://github.com/apache/superset/pull/10938))
- fix: update pylint disabled checks in common/query_context.py (#[10941](https://github.com/apache/superset/pull/10941))
- fix: setting specific exceptions common/query_context.py (#[10942](https://github.com/apache/superset/pull/10942))
- fix: re-enabling several globally disabled lint rules (#[10957](https://github.com/apache/superset/pull/10957))
- fix: removed disabled lint rule `too-many-locals` in connectors/base/models.py (#[10958](https://github.com/apache/superset/pull/10958))
- fix: typo in prefer typescript (#[10959](https://github.com/apache/superset/pull/10959))
- fix: pylint checks in connectors/sqla/models.py (#[10974](https://github.com/apache/superset/pull/10974))
- fix: pylint disabled rules in dashboard/api.py (#[10976](https://github.com/apache/superset/pull/10976))
- fix: changes a pylint check in dashboard module (#[10978](https://github.com/apache/superset/pull/10978))
- fix: changed disabled rules in datasets module (#[10979](https://github.com/apache/superset/pull/10979))
- fix: Add Item Overflow on Dataset Editor (#[10983](https://github.com/apache/superset/pull/10983))
- fix: enable pylint rules in db_engine_specs module (#[10998](https://github.com/apache/superset/pull/10998))
- fix: enable several pylint rules partially in db_engines_specs module (#[11000](https://github.com/apache/superset/pull/11000))
- fix: unbreak ci (#[11003](https://github.com/apache/superset/pull/11003))
- fix: timer component, fixes #10849, closes #11002 (#[11004](https://github.com/apache/superset/pull/11004))
- fix: menu shows a 0 when there are not settings (#[11009](https://github.com/apache/superset/pull/11009))
- fix: reenable pylint rule `unused-import` in charts and connectors modules (#[11014](https://github.com/apache/superset/pull/11014))
- fix: query search low privileged user search access denied (#[11017](https://github.com/apache/superset/pull/11017))
- fix(api): unable to delete virtual dataset, wrong permission name (#[11019](https://github.com/apache/superset/pull/11019))
- fix: [dashboard] should not show edit button when user has no edit permit (#[11024](https://github.com/apache/superset/pull/11024))
- fix: dashboard edit button (again) (#[11029](https://github.com/apache/superset/pull/11029))
- fix: sql lab autocomplete width (#[11063](https://github.com/apache/superset/pull/11063))
- fix: fix table existence validation function (#[11066](https://github.com/apache/superset/pull/11066))
- fix: database list checkboxes (#[11068](https://github.com/apache/superset/pull/11068))
- fix: Adding timeout to flaky cypress test, to wait for animation to complete (#[11074](https://github.com/apache/superset/pull/11074))
- fix: surface connection error messages on the client (#[11077](https://github.com/apache/superset/pull/11077))
- fix(jest): using UTC mock date (#[11079](https://github.com/apache/superset/pull/11079))
- fix: double scroll bars on dataset editor (#[11095](https://github.com/apache/superset/pull/11095))
- fix: echarts timeseries groupby (#[11103](https://github.com/apache/superset/pull/11103))
- fix: Disabling timezone of dataframe before passing Prophet (#[11107](https://github.com/apache/superset/pull/11107))
- fix(chart-data-api): ignore missing filters (#[11112](https://github.com/apache/superset/pull/11112))
- fix: alembic migration error msg trying to delete constraint on tables (#[11115](https://github.com/apache/superset/pull/11115))
- fix: remove extra flash import (#[11121](https://github.com/apache/superset/pull/11121))
- fix: Revert "Replace reactable with DataTable from superset-ui in QueryTable (#10981)" (#[11125](https://github.com/apache/superset/pull/11125))
- fix: SpatialControl popover won't open (#[11127](https://github.com/apache/superset/pull/11127))
- fix: Alembic migration 18532d70ab98 (#[11136](https://github.com/apache/superset/pull/11136))
- fix(examples): missing expressions in birth_names (#[11141](https://github.com/apache/superset/pull/11141))
- fix: Fix Time Column dropdown for date filter (#[11149](https://github.com/apache/superset/pull/11149))
- fix(dataset): update user list endpoint (#[11221](https://github.com/apache/superset/pull/11221))
- fix(crud): set default extra value (#[11262](https://github.com/apache/superset/pull/11262))
- fix(sqla): allow 'unknown' type queries in explore view (#[11365](https://github.com/apache/superset/pull/11365))
- fix: prior npm font source had a glitch (#[11724](https://github.com/apache/superset/pull/11724))
## Breaking Changes
[List of backwards incompatible changes](https://github.com/apache/superset/blob/master/UPDATING.md#0380)
## Complete Changelog
For the complete changelog please see [apache/superset/CHANGELOG.md](https://github.com/apache/superset/blob/master/CHANGELOG.md)

Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.4 MiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.7 MiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.4 MiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 3.1 MiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.8 MiB

Some files were not shown because too many files have changed in this diff Show More