Compare commits

..
Author SHA1 Message Date
Beto Dealmeida 5f4c49a9bc feat: make sure to quote formulas on Excel export 2024-11-26 10:33:40 -05:00
Maxime BeaucheminandGitHub 97683ec052 fix: helm chart deploy to open PRs to now-protected gh-pages branch (#31155) 2024-11-25 18:38:12 -08:00
73164c61ad chore(🦾): bump python billiard 4.2.0 -> 4.2.1 (#31109)
Co-authored-by: GitHub Action <action@github.com>
2024-11-25 17:11:49 -08:00
564c168420 chore(🦾): bump python flask-limiter 3.7.0 -> 3.8.0 (#31138)
Co-authored-by: GitHub Action <action@github.com>
2024-11-25 17:11:21 -08:00
95f4fe0cb8 chore(🦾): bump python mako 1.3.5 -> 1.3.6 (#31140)
Co-authored-by: GitHub Action <action@github.com>
2024-11-25 17:10:58 -08:00
bbc6d374ea chore(🦾): bump python celery subpackage(s) (#31127)
Co-authored-by: GitHub Action <action@github.com>
2024-11-25 17:10:23 -08:00
316da5e5f5 chore(🦾): bump python humanize 4.9.0 -> 4.11.0 (#31128)
Co-authored-by: GitHub Action <action@github.com>
2024-11-25 17:10:05 -08:00
e2b9b8e9fd chore(🦾): bump python simplejson 3.19.2 -> 3.19.3 (#31129)
Co-authored-by: GitHub Action <action@github.com>
2024-11-25 17:09:49 -08:00
7154b8d40f chore(🦾): bump python numexpr 2.10.1 -> 2.10.2 (#31130)
Co-authored-by: GitHub Action <action@github.com>
2024-11-25 17:09:32 -08:00
fcb3ff3a41 chore(🦾): bump python slack-sdk 3.27.2 -> 3.33.4 (#31132)
Co-authored-by: GitHub Action <action@github.com>
2024-11-25 17:09:12 -08:00
342cfc41ec chore(🦾): bump python pyopenssl 24.1.0 -> 24.2.1 (#31133)
Co-authored-by: GitHub Action <action@github.com>
2024-11-25 17:08:54 -08:00
aa7d3b0f96 chore(🦾): bump python dnspython 2.6.1 -> 2.7.0 (#31135)
Co-authored-by: GitHub Action <action@github.com>
2024-11-25 17:06:53 -08:00
3e28bd2cfa chore(🦾): bump python zstandard 0.22.0 -> 0.23.0 (#31136)
Co-authored-by: GitHub Action <action@github.com>
2024-11-25 17:06:35 -08:00
cc1eec69df chore(🦾): bump python limits 3.12.0 -> 3.13.0 (#31137)
Co-authored-by: GitHub Action <action@github.com>
2024-11-25 17:06:18 -08:00
3fa0de4293 chore(🦾): bump python flask-jwt-extended 4.6.0 -> 4.7.1 (#31139)
Co-authored-by: GitHub Action <action@github.com>
2024-11-25 17:05:59 -08:00
2ad8af71b5 chore(🦾): bump python gunicorn 22.0.0 -> 23.0.0 (#31125)
Co-authored-by: GitHub Action <action@github.com>
2024-11-25 16:52:11 -08:00
b648cc1168 chore(🦾): bump python zipp 3.19.0 -> 3.21.0 (#31124)
Co-authored-by: GitHub Action <action@github.com>
2024-11-25 16:50:22 -08:00
f24bf873bf chore(🦾): bump python flask-compress 1.15 -> 1.17 (#31123)
Co-authored-by: GitHub Action <action@github.com>
2024-11-25 16:50:06 -08:00
e0a5033596 chore(🦾): bump python dill 0.3.8 -> 0.3.9 (#31108)
Co-authored-by: GitHub Action <action@github.com>
2024-11-25 16:49:31 -08:00
ef14d58c64 chore(🦾): bump python email-validator 2.1.1 -> 2.2.0 (#31116)
Co-authored-by: GitHub Action <action@github.com>
2024-11-25 16:48:15 -08:00
Michael S. MolinaandGitHub 547a4adef5 fix: Remove unwanted commit on Trino's handle_cursor (#31154) 2024-11-25 17:42:31 -03:00
Evan RusackasandGitHub 5256a2f194 chore(asf): add gh-pages to protected branches (#31153) 2024-11-25 13:31:52 -07:00
0560c2615d chore(🦾): bump python async-timeout 4.0.3 -> 5.0.1 (#31122)
Co-authored-by: GitHub Action <action@github.com>
2024-11-25 10:18:23 -08:00
Michael S. MolinaandGitHub ff282492a1 fix: Revert "feat(trino): Add functionality to upload data (#29164)" (#31151) 2024-11-25 15:16:28 -03:00
312dc1c749 chore(🦾): bump python prompt-toolkit 3.0.44 -> 3.0.48 (#31121)
Co-authored-by: GitHub Action <action@github.com>
2024-11-25 10:05:13 -08:00
1e26c34758 chore(🦾): bump python sqlparse 0.5.0 -> 0.5.2 (#31119)
Co-authored-by: GitHub Action <action@github.com>
2024-11-25 10:04:43 -08:00
Maxime BeaucheminandGitHub decaba72c3 fix: try to re-enable gh-pages (#31152) 2024-11-25 09:58:30 -08:00
Maxime BeaucheminandGitHub 7e8c77e636 fix: touch helm/ folder to trigger doc deploy in CI (#31148) 2024-11-25 09:08:34 -08:00
alexandrusoareandGitHub ba99980cf4 refactor(List): Upgrade List from antdesign4 to antdesign5 (#30963) 2024-11-25 16:44:17 +02:00
c62f722f99 chore(🦾): bump python mysqlclient 2.2.4 -> 2.2.6 (#31113)
Co-authored-by: GitHub Action <action@github.com>
2024-11-24 22:36:20 -08:00
3fd23508bc chore(🦾): bump python grpcio-status subpackage(s) (#31114)
Co-authored-by: GitHub Action <action@github.com>
2024-11-24 22:34:54 -08:00
9ff9e0299b chore(🦾): bump python cycler 0.11.0 -> 0.12.1 (#31112)
Co-authored-by: GitHub Action <action@github.com>
2024-11-24 22:34:35 -08:00
6488ced3d3 chore(🦾): bump python croniter 2.0.5 -> 5.0.1 (#31091)
Co-authored-by: GitHub Action <action@github.com>
2024-11-24 22:33:02 -08:00
9a2be95159 chore(🦾): bump python google-auth 2.29.0 -> 2.36.0 (#31107)
Co-authored-by: GitHub Action <action@github.com>
2024-11-24 22:25:23 -08:00
ef4e03c9fe chore(🦾): bump python psutil 6.0.0 -> 6.1.0 (#31106)
Co-authored-by: GitHub Action <action@github.com>
2024-11-24 22:25:05 -08:00
ca2f0288e5 chore(🦾): bump python dnspython 2.6.1 -> 2.7.0 (#31105)
Co-authored-by: GitHub Action <action@github.com>
2024-11-24 22:24:46 -08:00
ca63760a4b chore(🦾): bump python markdown 3.6 -> 3.7 (#31102)
Co-authored-by: GitHub Action <action@github.com>
2024-11-24 22:23:37 -08:00
83924f7e10 chore(🦾): bump python pluggy 1.4.0 -> 1.5.0 (#31101)
Co-authored-by: GitHub Action <action@github.com>
2024-11-24 22:23:18 -08:00
c4a56c3f6e chore(🦾): bump python sqloxide 0.1.43 -> 0.1.51 (#31100)
Co-authored-by: GitHub Action <action@github.com>
2024-11-24 22:22:51 -08:00
cf134ab3aa chore(🦾): bump python wheel 0.43.0 -> 0.45.1 (#31099)
Co-authored-by: GitHub Action <action@github.com>
2024-11-24 22:22:28 -08:00
043c585008 chore(🦾): bump python pyproject-api 1.6.1 -> 1.8.0 (#31098)
Co-authored-by: GitHub Action <action@github.com>
2024-11-24 22:22:08 -08:00
0d346d4414 chore(🦾): bump python pytest-cov 5.0.0 -> 6.0.0 (#31096)
Co-authored-by: GitHub Action <action@github.com>
2024-11-24 22:21:50 -08:00
9067371234 chore(🦾): bump python chardet 5.1.0 -> 5.2.0 (#31094)
Co-authored-by: GitHub Action <action@github.com>
2024-11-24 22:21:22 -08:00
40fe05c5e2 chore(🦾): bump python jsonpath-ng 1.6.1 -> 1.7.0 (#31093)
Co-authored-by: GitHub Action <action@github.com>
2024-11-24 22:20:26 -08:00
e3bdfb5def chore(🦾): bump python sshtunnel subpackage(s) (#31092)
Co-authored-by: GitHub Action <action@github.com>
2024-11-24 22:20:11 -08:00
55f0713a2f chore(🦾): bump python mako 1.3.5 -> 1.3.6 (#31097)
Co-authored-by: GitHub Action <action@github.com>
Co-authored-by: Maxime Beauchemin <maximebeauchemin@gmail.com>
2024-11-24 22:19:16 -08:00
5aee59cc3a chore(🦾): bump python tomlkit 0.12.5 -> 0.13.2 (#31090)
Co-authored-by: GitHub Action <action@github.com>
2024-11-24 21:28:45 -08:00
94d3774d9e chore(🦾): bump python isodate 0.6.1 -> 0.7.2 (#31087)
Co-authored-by: GitHub Action <action@github.com>
2024-11-24 21:28:28 -08:00
b665254f39 chore(🦾): bump python db-dtypes 1.2.0 -> 1.3.1 (#31082)
Co-authored-by: GitHub Action <action@github.com>
2024-11-24 21:28:11 -08:00
4dc8cce8e8 chore(🦾): bump python trino 0.328.0 -> 0.330.0 (#31081)
Co-authored-by: GitHub Action <action@github.com>
2024-11-24 21:27:52 -08:00
d206a20ce7 chore(🦾): bump python certifi 2024.2.2 -> 2024.8.30 (#31089)
Co-authored-by: GitHub Action <action@github.com>
2024-11-24 21:27:18 -08:00
6fcc282a4e chore(🦾): bump python pydata-google-auth 1.7.0 -> 1.9.0 (#31088)
Co-authored-by: GitHub Action <action@github.com>
2024-11-24 21:26:58 -08:00
93c35a7ba5 chore(🦾): bump python pyproject-hooks 1.0.0 -> 1.2.0 (#31086)
Co-authored-by: GitHub Action <action@github.com>
2024-11-24 21:26:36 -08:00
9dfa8d5f8f chore(🦾): bump python sqlalchemy-bigquery 1.11.0 -> 1.12.0 (#31085)
Co-authored-by: GitHub Action <action@github.com>
2024-11-24 21:26:14 -08:00
87504056fe chore(🦾): bump python kiwisolver 1.4.5 -> 1.4.7 (#31084)
Co-authored-by: GitHub Action <action@github.com>
2024-11-24 21:25:47 -08:00
429c18f9e8 chore(🦾): bump python coverage subpackage(s) (#31083)
Co-authored-by: GitHub Action <action@github.com>
2024-11-24 21:25:33 -08:00
5bddc81f60 chore(🦾): bump python cfgv 3.3.1 -> 3.4.0 (#31077)
Co-authored-by: GitHub Action <action@github.com>
2024-11-24 21:25:12 -08:00
9837b4a61e chore(🦾): bump python fonttools 4.51.0 -> 4.55.0 (#31075)
Co-authored-by: GitHub Action <action@github.com>
2024-11-24 21:24:49 -08:00
454f143661 chore(🦾): bump python pyasn1-modules 0.4.0 -> 0.4.1 (#31076)
Co-authored-by: GitHub Action <action@github.com>
2024-11-24 21:03:01 -08:00
7376dfc6e9 chore(🦾): bump python pyhive subpackage(s) (#31079)
Co-authored-by: GitHub Action <action@github.com>
2024-11-24 21:00:56 -08:00
838d47d578 chore(🦾): bump python google-cloud-core 2.3.2 -> 2.4.1 (#31078)
Co-authored-by: GitHub Action <action@github.com>
2024-11-24 20:57:49 -08:00
14e81d0a9a chore(🦾): bump python sqlalchemy-utils subpackage(s) (#31048)
Co-authored-by: GitHub Action <action@github.com>
Co-authored-by: Maxime Beauchemin <maximebeauchemin@gmail.com>
2024-11-24 20:45:06 -08:00
f68c2b2454 chore(🦾): bump python amqp 5.2.0 -> 5.3.1 (#31073)
Co-authored-by: GitHub Action <action@github.com>
2024-11-24 20:44:29 -08:00
814c3dfecc chore(🦾): bump python cachetools 5.3.3 -> 5.5.0 (#31071)
Co-authored-by: GitHub Action <action@github.com>
2024-11-24 20:44:05 -08:00
b8aade776b chore(🦾): bump python kombu 5.3.7 -> 5.4.2 (#31074)
Co-authored-by: GitHub Action <action@github.com>
2024-11-24 20:43:26 -08:00
e092e6002d chore(🦾): bump python pyyaml 6.0.1 -> 6.0.2 (#31066)
Co-authored-by: GitHub Action <action@github.com>
2024-11-24 18:10:24 -08:00
673754d16e chore(🦾): bump python tqdm 4.66.4 -> 4.67.1 (#31068)
Co-authored-by: GitHub Action <action@github.com>
2024-11-24 18:10:06 -08:00
27deeb2f51 chore(🦾): bump python proto-plus 1.22.2 -> 1.25.0 (#31069)
Co-authored-by: GitHub Action <action@github.com>
2024-11-24 18:09:38 -08:00
9a7a84c7a0 chore(🦾): bump python importlib-resources 6.4.0 -> 6.4.5 (#31067)
Co-authored-by: GitHub Action <action@github.com>
2024-11-24 18:09:13 -08:00
a3d2588313 chore(🦾): bump python apispec subpackage(s) (#31062)
Co-authored-by: GitHub Action <action@github.com>
2024-11-24 17:56:17 -08:00
5c87fee282 chore(🦾): bump python deprecated 1.2.14 -> 1.2.15 (#31056)
Co-authored-by: GitHub Action <action@github.com>
2024-11-24 17:55:07 -08:00
b24323d500 chore(🦾): bump python pre-commit 3.7.1 -> 4.0.1 (#31050)
Co-authored-by: GitHub Action <action@github.com>
Co-authored-by: Maxime Beauchemin <maximebeauchemin@gmail.com>
2024-11-24 17:34:36 -08:00
824aca85d0 chore(🦾): bump python charset-normalizer 3.3.2 -> 3.4.0 (#31064)
Co-authored-by: GitHub Action <action@github.com>
2024-11-24 17:31:13 -08:00
1e4098a29e chore(🦾): bump python ruff 0.4.5 -> 0.8.0 (#31001)
Co-authored-by: GitHub Action <action@github.com>
Co-authored-by: Maxime Beauchemin <maximebeauchemin@gmail.com>
2024-11-24 17:30:48 -08:00
3aa8f32ca9 chore(🦾): bump python googleapis-common-protos 1.63.0 -> 1.66.0 (#31049)
Co-authored-by: GitHub Action <action@github.com>
Co-authored-by: Maxime Beauchemin <maximebeauchemin@gmail.com>
2024-11-24 17:15:24 -08:00
bf42ea70ba chore(🦾): bump python cron-descriptor 1.4.3 -> 1.4.5 (#31046)
Co-authored-by: GitHub Action <action@github.com>
2024-11-24 16:31:38 -08:00
d69da5f0f5 chore(🦾): bump python flask-wtf 1.2.1 -> 1.2.2 (#31052)
Co-authored-by: GitHub Action <action@github.com>
2024-11-24 16:23:35 -08:00
078257dd1b docs: updated the install process in pypi.mdx (#31044)
Co-authored-by: Rida KEJJI <42012627+ShameGod@users.noreply.github.com>
2024-11-24 16:23:17 -08:00
8c1c2570b3 chore(🦾): bump python nh3 0.2.17 -> 0.2.18 (#31054)
Co-authored-by: GitHub Action <action@github.com>
2024-11-24 16:22:15 -08:00
a80803566d chore(🦾): bump python marshmallow 3.21.2 -> 3.23.1 (#31045)
Co-authored-by: GitHub Action <action@github.com>
2024-11-24 16:21:48 -08:00
f551f5b7b6 chore(🦾): bump python idna 3.7 -> 3.10 (#31041)
Co-authored-by: GitHub Action <action@github.com>
2024-11-24 13:38:14 -08:00
1978cde4f1 chore(🦾): bump python pyjwt 2.8.0 -> 2.10.0 (#31042)
Co-authored-by: GitHub Action <action@github.com>
2024-11-24 13:37:57 -08:00
c5f6cc6382 chore(🦾): bump python et-xmlfile 1.1.0 -> 2.0.0 & remove pyhive[hive] from requirements/development.in (#31040)
Co-authored-by: GitHub Action <action@github.com>
Co-authored-by: Maxime Beauchemin <maximebeauchemin@gmail.com>
2024-11-24 13:35:50 -08:00
Birk SkyumandGitHub e9e2c0bee8 chore(legacy-plugin-chart-map-box): replace viewport-mercator-project with @math.gl/web-mercator (#30651) 2024-11-24 20:00:26 +00:00
33a9817388 chore(🦾): bump python pandas subpackage(s) (#31004)
Co-authored-by: GitHub Action <action@github.com>
2024-11-24 10:05:59 -08:00
GeidoandGitHub 91301bcd5b fix(Dashboard): Ensure shared label colors are updated (#31031) 2024-11-23 16:39:40 +02:00
Maxime BeaucheminandGitHub 67ad7da5cc fix: ephemeral environments missing env var (#31035) 2024-11-22 17:39:34 -08:00
Maxime BeaucheminandGitHub e0deb704f9 feat: make ephemeral env use supersetbot + deprecate build_docker.py (#30870) 2024-11-22 14:19:08 -08:00
Kamil GabryjelskiandGitHub abf3790ea6 chore: Cleanup code related to MetadataBar, fix types (#31030) 2024-11-22 16:02:13 +01:00
85 changed files with 1226 additions and 1875 deletions
+10
View File
@@ -53,6 +53,9 @@ github:
merge: false
rebase: false
ghp_branch: gh-pages
ghp_path: /
protected_branches:
master:
required_status_checks:
@@ -88,3 +91,10 @@ github:
required_approving_review_count: 1
required_signatures: false
gh-pages:
required_pull_request_reviews:
dismiss_stale_reviews: false
require_code_owner_reviews: true
required_approving_review_count: 1
required_signatures: false
+1
View File
@@ -1,2 +1,3 @@
docker/**/*.sh text eol=lf
*.svg binary
*.ipynb binary
+1 -1
View File
@@ -42,7 +42,7 @@ runs:
- name: Install dependencies
run: |
if [ "${{ inputs.install-superset }}" = "true" ]; then
sudo apt-get update && sudo apt-get -y install libldap2-dev libsasl2-dev
sudo apt-get update && sudo apt-get -y install libldap2-dev libsasl2-dev build-essential
pip install --upgrade pip setuptools wheel
if [ "${{ inputs.requirements-type }}" = "dev" ]; then
pip install -r requirements/development.txt
+10 -1
View File
@@ -14,6 +14,12 @@ on:
required: true
description: Max number of PRs to open (0 for no limit)
default: 5
extra-flags:
required: false
default: --only-base
description: Additional flags to pass to the bump-python command
#schedule:
# - cron: '0 0 * * *' # Runs daily at midnight UTC
jobs:
bump-python-package:
@@ -59,10 +65,13 @@ jobs:
GROUP_OPT="-g ${{ github.event.inputs.group }}"
fi
EXTRA_FLAGS="${{ github.event.inputs.extra-flags }}"
supersetbot bump-python \
--verbose \
--use-current-repo \
--include-subpackages \
--limit ${{ github.event.inputs.limit }} \
$PACKAGE_OPT \
$GROUP_OPT
$GROUP_OPT \
$EXTRA_FLAGS
+76 -58
View File
@@ -1,30 +1,25 @@
name: Ephemeral env workflow
# Example manual trigger: gh workflow run ephemeral-env.yml --ref fix_ephemerals --field comment_body="/testenv up" --field issue_number=666
on:
issue_comment:
types: [created]
workflow_dispatch:
inputs:
comment_body:
description: 'Comment body to simulate /testenv command'
required: true
default: '/testenv up'
issue_number:
description: 'Issue or PR number'
required: true
jobs:
config:
runs-on: "ubuntu-22.04"
if: github.event.issue.pull_request
outputs:
has-secrets: ${{ steps.check.outputs.has-secrets }}
steps:
- name: "Check for secrets"
id: check
shell: bash
run: |
if [ -n "${{ (secrets.AWS_ACCESS_KEY_ID != '' && secrets.AWS_SECRET_ACCESS_KEY != '') || '' }}" ]; then
echo "has-secrets=1" >> "$GITHUB_OUTPUT"
fi
ephemeral-env-comment:
concurrency:
group: ${{ github.workflow }}-${{ github.event.issue.number || github.run_id }}-comment
group: ${{ github.workflow }}-${{ github.event.inputs.issue_number || github.event.issue.number || github.run_id }}-comment
cancel-in-progress: true
needs: config
if: needs.config.outputs.has-secrets
name: Evaluate ephemeral env comment trigger (/testenv)
runs-on: ubuntu-22.04
permissions:
@@ -44,18 +39,18 @@ jobs:
with:
result-encoding: string
script: |
const pattern = /^\/testenv (up|down)/
const result = pattern.exec(context.payload.comment.body)
return result === null ? 'noop' : result[1]
const pattern = /^\/testenv (up|down)/;
const result = pattern.exec('${{ github.event.inputs.comment_body || github.event.comment.body }}');
return result === null ? 'noop' : result[1];
- name: Eval comment body for feature flags
- name: Looking for feature flags
uses: actions/github-script@v7
id: eval-feature-flags
with:
script: |
const pattern = /FEATURE_(\w+)=(\w+)/g;
let results = [];
[...context.payload.comment.body.matchAll(pattern)].forEach(match => {
[...'${{ github.event.inputs.comment_body || github.event.comment.body }}'.matchAll(pattern)].forEach(match => {
const config = {
name: `SUPERSET_FEATURE_${match[1]}`,
value: match[2],
@@ -67,24 +62,47 @@ jobs:
- name: Limit to committers
if: >
steps.eval-body.outputs.result != 'noop' &&
github.event_name == 'issue_comment' &&
github.event.comment.author_association != 'MEMBER' &&
github.event.comment.author_association != 'OWNER'
uses: actions/github-script@v7
with:
github-token: ${{github.token}}
github-token: ${{ github.token }}
script: |
const errMsg = '@${{ github.event.comment.user.login }} Ephemeral environment creation is currently limited to committers.'
const errMsg = '@${{ github.event.comment.user.login }} Ephemeral environment creation is currently limited to committers.';
github.rest.issues.createComment({
issue_number: ${{ github.event.issue.number }},
owner: context.repo.owner,
repo: context.repo.repo,
body: errMsg
})
core.setFailed(errMsg)
});
core.setFailed(errMsg);
- name: Reply with confirmation comment
uses: actions/github-script@v7
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
const issueNumber = ${{ github.event.inputs.issue_number || github.event.issue.number }};
const user = '${{ github.event.comment.user.login || github.actor }}';
const action = '${{ steps.eval-body.outputs.result }}';
const runId = context.runId;
const workflowUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${runId}`;
const body = action === 'noop'
? `@${user} No ephemeral environment action detected. Please use '/testenv up' or '/testenv down'. [View workflow run](${workflowUrl}).`
: `@${user} Processing your ephemeral environment request [here](${workflowUrl}).`;
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: issueNumber,
body,
});
ephemeral-docker-build:
concurrency:
group: ${{ github.workflow }}-${{ github.event.issue.number || github.run_id }}-build
group: ${{ github.workflow }}-${{ github.event.inputs.issue_number || github.event.issue.number || github.run_id }}-build
cancel-in-progress: true
needs: ephemeral-env-comment
name: ephemeral-docker-build
@@ -98,9 +116,9 @@ jobs:
const request = {
owner: context.repo.owner,
repo: context.repo.repo,
pull_number: ${{ github.event.issue.number }},
}
core.info(`Getting PR #${request.pull_number} from ${request.owner}/${request.repo}`)
pull_number: ${{ github.event.inputs.issue_number || github.event.issue.number }},
};
core.info(`Getting PR #${request.pull_number} from ${request.owner}/${request.repo}`);
const pr = await github.rest.pulls.get(request);
return pr.data;
@@ -121,12 +139,17 @@ jobs:
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Setup supersetbot
uses: ./.github/actions/setup-supersetbot/
- name: Build ephemeral env image
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
./scripts/build_docker.py \
"ci" \
"pull_request" \
--build_context_ref ${{ github.event.issue.number }}
supersetbot docker \
--preset ci \
--platform linux/amd64 \
--context-ref "$RELEASE"
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v4
@@ -146,7 +169,7 @@ jobs:
ECR_REPOSITORY: superset-ci
IMAGE_TAG: apache/superset:${{ steps.get-sha.outputs.sha }}-ci
run: |
docker tag $IMAGE_TAG $ECR_REGISTRY/$ECR_REPOSITORY:pr-${{ github.event.issue.number }}-ci
docker tag $IMAGE_TAG $ECR_REGISTRY/$ECR_REPOSITORY:pr-${{ github.event.inputs.issue_number || github.event.issue.number }}-ci
docker push -a $ECR_REGISTRY/$ECR_REPOSITORY
ephemeral-env-up:
@@ -181,22 +204,22 @@ jobs:
aws ecr describe-images \
--registry-id $(echo "${{ steps.login-ecr.outputs.registry }}" | grep -Eo "^[0-9]+") \
--repository-name superset-ci \
--image-ids imageTag=pr-${{ github.event.issue.number }}-ci
--image-ids imageTag=pr-${{ github.event.inputs.issue_number || github.event.issue.number }}-ci
- name: Fail on missing container image
if: steps.check-image.outcome == 'failure'
uses: actions/github-script@v7
with:
github-token: ${{github.token}}
github-token: ${{ github.token }}
script: |
const errMsg = '@${{ github.event.comment.user.login }} Container image not yet published for this PR. Please try again when build is complete.'
const errMsg = '@${{ github.event.comment.user.login }} Container image not yet published for this PR. Please try again when build is complete.';
github.rest.issues.createComment({
issue_number: ${{ github.event.issue.number }},
issue_number: ${{ github.event.inputs.issue_number || github.event.issue.number }},
owner: context.repo.owner,
repo: context.repo.repo,
body: errMsg
})
core.setFailed(errMsg)
});
core.setFailed(errMsg);
- name: Fill in the new image ID in the Amazon ECS task definition
id: task-def
@@ -204,7 +227,7 @@ jobs:
with:
task-definition: .github/workflows/ecs-task-definition.json
container-name: superset-ci
image: ${{ steps.login-ecr.outputs.registry }}/superset-ci:pr-${{ github.event.issue.number }}-ci
image: ${{ steps.login-ecr.outputs.registry }}/superset-ci:pr-${{ github.event.inputs.issue_number || github.event.issue.number }}-ci
- name: Update env vars in the Amazon ECS task definition
run: |
@@ -213,30 +236,29 @@ jobs:
- name: Describe ECS service
id: describe-services
run: |
echo "active=$(aws ecs describe-services --cluster superset-ci --services pr-${{ github.event.issue.number }}-service | jq '.services[] | select(.status == "ACTIVE") | any')" >> $GITHUB_OUTPUT
echo "active=$(aws ecs describe-services --cluster superset-ci --services pr-${{ github.event.inputs.issue_number || github.event.issue.number }}-service | jq '.services[] | select(.status == "ACTIVE") | any')" >> $GITHUB_OUTPUT
- name: Create ECS service
if: steps.describe-services.outputs.active != 'true'
id: create-service
if: steps.describe-services.outputs.active != 'true'
env:
ECR_SUBNETS: subnet-0e15a5034b4121710,subnet-0e8efef4a72224974
ECR_SECURITY_GROUP: sg-092ff3a6ae0574d91
run: |
aws ecs create-service \
--cluster superset-ci \
--service-name pr-${{ github.event.issue.number }}-service \
--service-name pr-${{ github.event.inputs.issue_number || github.event.issue.number }}-service \
--task-definition superset-ci \
--launch-type FARGATE \
--desired-count 1 \
--platform-version LATEST \
--network-configuration "awsvpcConfiguration={subnets=[$ECR_SUBNETS],securityGroups=[$ECR_SECURITY_GROUP],assignPublicIp=ENABLED}" \
--tags key=pr,value=${{ github.event.issue.number }} key=github_user,value=${{ github.actor }}
--tags key=pr,value=${{ github.event.inputs.issue_number || github.event.issue.number }} key=github_user,value=${{ github.actor }}
- name: Deploy Amazon ECS task definition
id: deploy-task
uses: aws-actions/amazon-ecs-deploy-task-definition@v2
with:
task-definition: ${{ steps.task-def.outputs.task-definition }}
service: pr-${{ github.event.issue.number }}-service
service: pr-${{ github.event.inputs.issue_number || github.event.issue.number }}-service
cluster: superset-ci
wait-for-service-stability: true
wait-for-minutes: 10
@@ -244,18 +266,15 @@ jobs:
- name: List tasks
id: list-tasks
run: |
echo "task=$(aws ecs list-tasks --cluster superset-ci --service-name pr-${{ github.event.issue.number }}-service | jq '.taskArns | first')" >> $GITHUB_OUTPUT
echo "task=$(aws ecs list-tasks --cluster superset-ci --service-name pr-${{ github.event.inputs.issue_number || github.event.issue.number }}-service | jq '.taskArns | first')" >> $GITHUB_OUTPUT
- name: Get network interface
id: get-eni
run: |
echo "eni=$(aws ecs describe-tasks --cluster superset-ci --tasks ${{ steps.list-tasks.outputs.task }} | jq '.tasks | .[0] | .attachments | .[0] | .details | map(select(.name=="networkInterfaceId")) | .[0] | .value')" >> $GITHUB_OUTPUT
echo "eni=$(aws ecs describe-tasks --cluster superset-ci --tasks ${{ steps.list-tasks.outputs.task }} | jq '.tasks | .[0] | .attachments | .[0] | .details | map(select(.name==\"networkInterfaceId\")) | .[0] | .value')" >> $GITHUB_OUTPUT
- name: Get public IP
id: get-ip
run: |
echo "ip=$(aws ec2 describe-network-interfaces --network-interface-ids ${{ steps.get-eni.outputs.eni }} | jq -r '.NetworkInterfaces | first | .Association.PublicIp')" >> $GITHUB_OUTPUT
- name: Comment (success)
if: ${{ success() }}
uses: actions/github-script@v7
@@ -263,12 +282,11 @@ jobs:
github-token: ${{github.token}}
script: |
github.rest.issues.createComment({
issue_number: ${{ github.event.issue.number }},
issue_number: ${{ github.event.inputs.issue_number || github.event.issue.number }},
owner: context.repo.owner,
repo: context.repo.repo,
body: '@${{ github.event.comment.user.login }} Ephemeral environment spinning up at http://${{ steps.get-ip.outputs.ip }}:8080. Credentials are `admin`/`admin`. Please allow several minutes for bootstrapping and startup.'
body: '@${{ github.event.inputs.user_login || github.event.comment.user.login }} Ephemeral environment spinning up at http://${{ steps.get-ip.outputs.ip }}:8080. Credentials are `admin`/`admin`. Please allow several minutes for bootstrapping and startup.'
})
- name: Comment (failure)
if: ${{ failure() }}
uses: actions/github-script@v7
@@ -276,8 +294,8 @@ jobs:
github-token: ${{github.token}}
script: |
github.rest.issues.createComment({
issue_number: ${{ github.event.issue.number }},
issue_number: ${{ github.event.inputs.issue_number || github.event.issue.number }},
owner: context.repo.owner,
repo: context.repo.repo,
body: '@${{ github.event.comment.user.login }} Ephemeral environment creation failed. Please check the Actions logs for details.'
body: '@${{ github.event.inputs.user_login || github.event.comment.user.login }} Ephemeral environment creation failed. Please check the Actions logs for details.'
})
+1 -1
View File
@@ -1,4 +1,4 @@
name: Lint and Test Charts
name: "Helm: lint and test charts"
on:
pull_request:
+83 -3
View File
@@ -1,4 +1,8 @@
name: Release Charts
# This workflow automates the release process for Helm charts.
# The workflow creates a new branch for the release and opens a pull request against the 'gh-pages' branch,
# allowing the changes to be reviewed and merged manually.
name: "Helm: release charts"
on:
push:
@@ -7,18 +11,28 @@ on:
- "[0-9].[0-9]*"
paths:
- "helm/**"
workflow_dispatch:
inputs:
ref:
description: "The branch, tag, or commit SHA to check out"
required: false
default: "master"
jobs:
release:
runs-on: ubuntu-22.04
permissions:
contents: write
pull-requests: write
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- name: Checkout code
uses: actions/checkout@v4
with:
persist-credentials: false
ref: ${{ inputs.ref || github.ref_name }}
persist-credentials: true
submodules: recursive
fetch-depth: 0
@@ -35,11 +49,77 @@ jobs:
- name: Add bitnami repo dependency
run: helm repo add bitnami https://charts.bitnami.com/bitnami
- name: Fetch/list all tags
run: |
# Debugging tags
git fetch --tags --force
git tag -d superset-helm-chart-0.13.4 || true
echo "DEBUG TAGS"
git show-ref --tags
- name: Create unique pages branch name
id: vars
run: echo "branch_name=helm-publish-${GITHUB_SHA:0:7}" >> $GITHUB_ENV
- name: Force recreate branch from gh-pages
run: |
# Ensure a clean working directory
git reset --hard
git clean -fdx
git checkout -b local_gha_temp
git submodule update
# Fetch the latest gh-pages branch
git fetch origin gh-pages
# Check out and reset the target branch based on gh-pages
git checkout -B ${{ env.branch_name }} origin/gh-pages
# Remove submodules from the branch
git submodule deinit -f --all
# Force push to the remote branch
git push origin ${{ env.branch_name }} --force
# Return to the original branch
git checkout local_gha_temp
- name: Fetch/list all tags
run: |
git submodule update
cat .github/actions/chart-releaser-action/action.yml
- name: Run chart-releaser
uses: ./.github/actions/chart-releaser-action
with:
version: v1.6.0
charts_dir: helm
mark_as_latest: false
pages_branch: ${{ env.branch_name }}
env:
CR_TOKEN: "${{ github.token }}"
CR_RELEASE_NAME_TEMPLATE: "superset-helm-chart-{{ .Version }}"
- name: Open Pull Request
uses: actions/github-script@v7
with:
script: |
const branchName = '${{ env.branch_name }}';
const [owner, repo] = process.env.GITHUB_REPOSITORY.split('/');
if (!branchName) {
throw new Error("Branch name is not defined.");
}
const pr = await github.rest.pulls.create({
owner,
repo,
title: `Helm chart release for ${branchName}`,
head: branchName,
base: "gh-pages", // Adjust if the target branch is different
body: `This PR releases Helm charts to the gh-pages branch.`,
});
core.info(`Pull request created: ${pr.data.html_url}`);
env:
BRANCH_NAME: ${{ env.branch_name }}
@@ -142,6 +142,7 @@ jobs:
- name: Python unit tests (PostgreSQL)
if: steps.check.outputs.python
run: |
pip install -e .[hive]
./scripts/python_tests.sh -m 'chart_data_flow or sql_json_flow'
- name: Upload code coverage
uses: codecov/codecov-action@v4
+7 -7
View File
@@ -16,11 +16,11 @@
#
repos:
- repo: https://github.com/MarcoGorelli/auto-walrus
rev: v0.2.2
rev: 0.3.4
hooks:
- id: auto-walrus
- repo: https://github.com/pre-commit/mirrors-mypy
rev: v1.3.0
rev: v1.13.0
hooks:
- id: mypy
args: [--check-untyped-defs]
@@ -39,11 +39,11 @@ repos:
types-Markdown,
]
- repo: https://github.com/peterdemin/pip-compile-multi
rev: v2.6.2
rev: v2.6.4
hooks:
- id: pip-compile-multi-verify
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.4.0
rev: v5.0.0
hooks:
- id: check-docstring-first
- id: check-added-large-files
@@ -56,7 +56,7 @@ repos:
exclude: ^.*\.(snap)
args: ["--markdown-linebreak-ext=md"]
- repo: https://github.com/pre-commit/mirrors-prettier
rev: v3.1.0 # Use the sha or tag you want to point at
rev: v4.0.0-alpha.8 # Use the sha or tag you want to point at
hooks:
- id: prettier
additional_dependencies:
@@ -70,12 +70,12 @@ repos:
- id: blacklist
args: ["--blacklisted-names=make_url", "--ignore=tests/"]
- repo: https://github.com/norwoodj/helm-docs
rev: v1.11.0
rev: v1.14.2
hooks:
- id: helm-docs
files: helm
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.4.0
rev: v0.8.0
hooks:
- id: ruff
args: [ --fix ]
+6 -6
View File
@@ -115,7 +115,7 @@ RUN mkdir -p ${PYTHONPATH} superset/static requirements superset-frontend apache
libldap2-dev \
&& touch superset/static/version_info.json \
&& chown -R superset:superset ./* \
&& rm -rf /var/lib/apt/lists/*
&& rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
COPY --chown=superset:superset pyproject.toml setup.py MANIFEST.in README.md ./
# setup.py uses the version information in package.json
@@ -128,7 +128,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \
&& pip install --no-cache-dir --upgrade setuptools pip \
&& pip install --no-cache-dir -r requirements/base.txt \
&& apt-get autoremove -yqq --purge build-essential \
&& rm -rf /var/lib/apt/lists/*
&& rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
# Copy the compiled frontend assets
COPY --chown=superset:superset --from=superset-node /app/superset/static/assets superset/static/assets
@@ -177,7 +177,7 @@ RUN apt-get update -qq \
libxtst6 \
git \
pkg-config \
&& rm -rf /var/lib/apt/lists/*
&& rm -rf /var/cache/apt/archives/* /var/lib/apt/lists/*
RUN --mount=type=cache,target=/root/.cache/pip \
pip install --no-cache-dir playwright
@@ -199,13 +199,13 @@ RUN if [ "$INCLUDE_FIREFOX" = "true" ]; then \
&& wget -q https://github.com/mozilla/geckodriver/releases/download/${GECKODRIVER_VERSION}/geckodriver-${GECKODRIVER_VERSION}-linux64.tar.gz -O - | tar xfz - -C /usr/local/bin \
&& wget -q https://download-installer.cdn.mozilla.net/pub/firefox/releases/${FIREFOX_VERSION}/linux-x86_64/en-US/firefox-${FIREFOX_VERSION}.tar.bz2 -O - | tar xfj - -C /opt \
&& ln -s /opt/firefox/firefox /usr/local/bin/firefox \
&& apt-get autoremove -yqq --purge wget bzip2 && rm -rf /var/[log,tmp]/* /tmp/* /var/lib/apt/lists/*; \
&& apt-get autoremove -yqq --purge wget bzip2 && rm -rf /var/[log,tmp]/* /tmp/* /var/lib/apt/lists/* /var/cache/apt/archives/*; \
fi
# Installing mysql client os-level dependencies in dev image only because GPL
RUN apt-get install -yqq --no-install-recommends \
default-libmysqlclient-dev \
&& rm -rf /var/lib/apt/lists/*
&& rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
COPY --chown=superset:superset requirements/development.txt requirements/
RUN --mount=type=cache,target=/root/.cache/pip \
@@ -213,7 +213,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \
build-essential \
&& pip install --no-cache-dir -r requirements/development.txt \
&& apt-get autoremove -yqq --purge build-essential \
&& rm -rf /var/lib/apt/lists/*
&& rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
USER superset
######################################################################
+1 -18
View File
@@ -25,7 +25,6 @@ x-superset-user: &superset-user root
x-superset-depends-on: &superset-depends-on
- db
- redis
- superset-checks
x-superset-volumes: &superset-volumes
# /app/pythonpath_docker will be appended to the PYTHONPATH in the final container
- ./docker:/app/docker
@@ -131,23 +130,6 @@ services:
- REDIS_PORT=6379
- REDIS_SSL=false
superset-checks:
build:
context: .
target: python-base
cache_from:
- apache/superset-cache:3.10-slim-bookworm
container_name: superset_checks
command: ["/app/scripts/check-env.py"]
env_file:
- path: docker/.env # default
required: true
- path: docker/.env-local # optional override
required: false
user: *superset-user
healthcheck:
disable: true
superset-init:
build:
<<: *common-build
@@ -179,6 +161,7 @@ services:
# set this to false if you have perf issues running the npm i; npm run dev in-docker
# if you do so, you have to run this manually on the host, which should perform better!
BUILD_SUPERSET_FRONTEND_IN_DOCKER: true
NPM_RUN_PRUNE: false
SCARF_ANALYTICS: "${SCARF_ANALYTICS:-}"
container_name: superset_node
command: ["/app/docker/docker-frontend.sh"]
+5
View File
@@ -27,6 +27,11 @@ if [ "$BUILD_SUPERSET_FRONTEND_IN_DOCKER" = "true" ]; then
echo "Building Superset frontend in dev mode inside docker container"
cd /app/superset-frontend
if [ "$NPM_RUN_PRUNE" = "true" ]; then
echo "Running `npm run prune`"
npm run prune
fi
echo "Running `npm install`"
npm install
+3 -3
View File
@@ -29,7 +29,7 @@ We have a set of build "presets" that each represent a combination of
parameters for the build, mostly pointing to either different target layer
for the build, and/or base image.
Here are the build presets that are exposed through the `build_docker.py` script:
Here are the build presets that are exposed through the `supersetbot docker` utility:
- `lean`: The default Docker image, including both frontend and backend. Tags
without a build_preset are lean builds (ie: `latest`, `4.0.0`, `3.0.0`, ...). `lean`
@@ -62,8 +62,8 @@ Here are the build presets that are exposed through the `build_docker.py` script
For insights or modifications to the build matrix and tagging conventions,
check the [build_docker.py](https://github.com/apache/superset/blob/master/scripts/build_docker.py)
script and the [docker.yml](https://github.com/apache/superset/blob/master/.github/workflows/docker.yml)
check the [supersetbot docker](https://github.com/apache-superset/supersetbot)
subcommand and the [docker.yml](https://github.com/apache/superset/blob/master/.github/workflows/docker.yml)
GitHub action.
## Key ARGs in Dockerfile
@@ -95,6 +95,14 @@ perform those operations. In this case, we recommend you set the env var
Simply trigger `npm i && npm run dev`, this should be MUCH faster.
:::
:::tip
Sometimes, your npm-related state can get out-of-wack, running `npm run prune` from
the `superset-frontend/` folder will nuke the various' packages `node_module/` folders
and help you start fresh. In the context of `docker compose` setting
`export NPM_RUN_PRUNE=true` prior to running `docker compose up` will trigger that
from within docker. This will slow down the startup, but will fix various npm-related issues.
:::
### Option #2 - build a set of immutable images from the local branch
```bash
+6 -9
View File
@@ -77,10 +77,6 @@ versions officially supported by Superset. We'd recommend using a Python version
like [pyenv](https://github.com/pyenv/pyenv)
(and also [pyenv-virtualenv](https://github.com/pyenv/pyenv-virtualenv)).
:::tip
To identify the Python version used by the official docker image, see the [Dockerfile](https://github.com/apache/superset/blob/master/Dockerfile). Additional docker images published for newer versions of Python can be found in [this file](https://github.com/apache/superset/blob/master/scripts/build_docker.py).
:::
Let's also make sure we have the latest version of `pip` and `setuptools`:
```bash
@@ -134,21 +130,22 @@ First, start by installing `apache-superset`:
pip install apache-superset
```
Then, define mandatory configurations, SECRET_KEY and FLASK_APP:
```bash
export SUPERSET_SECRET_KEY=YOUR-SECRET-KEY
export FLASK_APP=superset
```
Then, you need to initialize the database:
```bash
superset db upgrade
```
:::tip
Note that some configuration is mandatory for production instances of Superset. In particular, Superset will not start without a user-specified value of SECRET_KEY. Please see [Configuring Superset](/docs/configuration/configuring-superset).
:::
Finish installing by running through the following commands:
```bash
# Create an admin user in your metadata database (use `admin` as username to be able to load the examples)
export FLASK_APP=superset
superset fab create-admin
# Load some data to play with
+1 -1
View File
@@ -29,7 +29,7 @@ maintainers:
- name: craig-rueda
email: craig@craigrueda.com
url: https://github.com/craig-rueda
version: 0.13.3
version: 0.13.4
dependencies:
- name: postgresql
version: 12.1.6
+1 -1
View File
@@ -23,7 +23,7 @@ NOTE: This file is generated by helm-docs: https://github.com/norwoodj/helm-docs
# superset
![Version: 0.13.3](https://img.shields.io/badge/Version-0.13.3-informational?style=flat-square)
![Version: 0.13.4](https://img.shields.io/badge/Version-0.13.4-informational?style=flat-square)
Apache Superset is a modern, enterprise-ready business intelligence web application
+2 -1
View File
@@ -19,7 +19,8 @@
# This is a YAML-formatted file.
# Declare variables to be passed into your templates.
# A README is automatically generated from this file to document it, using helm-docs (see https://github.com/norwoodj/helm-docs)
# A README is automatically generated from this file to document it,
# using helm-docs (see https://github.com/norwoodj/helm-docs)
# To update it, install helm-docs and run helm-docs from the root of this chart
# -- Provide a name to override the name of the chart
+3 -3
View File
@@ -69,7 +69,7 @@ dependencies = [
"nh3>=0.2.11, <0.3",
"numpy==1.23.5",
"packaging",
"pandas[performance]>=2.0.3, <2.1",
"pandas[excel,performance]>=2.0.3, <2.1",
"parsedatetime",
"paramiko>=3.4.0",
"pgsanity",
@@ -135,7 +135,6 @@ gevent = ["gevent>=23.9.1"]
gsheets = ["shillelagh[gsheetsapi]>=1.2.18, <2"]
hana = ["hdbcli==2.4.162", "sqlalchemy_hana==0.4.0"]
hive = [
"boto3",
"pyhive[hive]>=0.6.5;python_version<'3.11'",
"pyhive[hive_pure_sasl]>=0.7.0",
"tableschema",
@@ -158,7 +157,7 @@ pinot = ["pinotdb>=5.0.0, <6.0.0"]
playwright = ["playwright>=1.37.0, <2"]
postgres = ["psycopg2-binary==2.9.6"]
presto = ["pyhive[presto]>=0.6.5"]
trino = ["boto3", "trino>=0.328.0"]
trino = ["trino>=0.328.0"]
prophet = ["prophet>=1.1.5, <2"]
redshift = ["sqlalchemy-redshift>=0.8.1, <0.9"]
rockset = ["rockset-sqlalchemy>=0.0.1, <1"]
@@ -402,6 +401,7 @@ skipsdist = true
[tool.ruff]
# Exclude a variety of commonly ignored directories.
exclude = [
"**/*.ipynb",
".bzr",
".direnv",
".eggs",
+69 -54
View File
@@ -9,13 +9,13 @@
# via -r requirements/base.in
alembic==1.13.1
# via flask-migrate
amqp==5.2.0
amqp==5.3.1
# via kombu
apispec[yaml]==6.3.0
# via flask-appbuilder
apsw==3.46.0.0
# via shillelagh
async-timeout==4.0.3
async-timeout==5.0.1
# via redis
attrs==24.2.0
# via
@@ -26,13 +26,13 @@ babel==2.16.0
# via flask-babel
backoff==2.2.1
# via apache-superset
bcrypt==4.1.3
bcrypt==4.2.1
# via paramiko
billiard==4.2.0
billiard==4.2.1
# via celery
blinker==1.9.0
# via flask
bottleneck==1.3.8
bottleneck==1.4.2
# via pandas
brotli==1.1.0
# via flask-compress
@@ -40,19 +40,19 @@ cachelib==0.9.0
# via
# flask-caching
# flask-session
cachetools==5.3.3
cachetools==5.5.0
# via google-auth
cattrs==24.1.2
# via requests-cache
celery==5.4.0
# via apache-superset
certifi==2024.2.2
certifi==2024.8.30
# via requests
cffi==1.17.1
# via
# cryptography
# pynacl
charset-normalizer==3.3.2
charset-normalizer==3.4.0
# via requests
click==8.1.7
# via
@@ -76,23 +76,27 @@ colorama==0.4.6
# via
# apache-superset
# flask-appbuilder
cron-descriptor==1.4.3
cron-descriptor==1.4.5
# via apache-superset
croniter==2.0.5
croniter==5.0.1
# via apache-superset
cryptography==42.0.8
cryptography==43.0.3
# via
# apache-superset
# paramiko
# pyopenssl
deprecated==1.2.14
defusedxml==0.7.1
# via odfpy
deprecated==1.2.15
# via limits
deprecation==2.1.0
# via apache-superset
dnspython==2.6.1
dnspython==2.7.0
# via email-validator
email-validator==2.1.1
email-validator==2.2.0
# via flask-appbuilder
et-xmlfile==2.0.0
# via openpyxl
exceptiongroup==1.2.2
# via cattrs
flask==2.3.3
@@ -115,11 +119,11 @@ flask-babel==2.0.0
# via flask-appbuilder
flask-caching==2.3.0
# via apache-superset
flask-compress==1.15
flask-compress==1.17
# via apache-superset
flask-jwt-extended==4.6.0
flask-jwt-extended==4.7.1
# via flask-appbuilder
flask-limiter==3.7.0
flask-limiter==3.8.0
# via flask-appbuilder
flask-login==0.6.3
# via
@@ -135,7 +139,7 @@ flask-sqlalchemy==2.5.1
# flask-migrate
flask-talisman==1.1.0
# via apache-superset
flask-wtf==1.2.1
flask-wtf==1.2.2
# via
# apache-superset
# flask-appbuilder
@@ -145,29 +149,29 @@ geographiclib==2.0
# via geopy
geopy==2.4.1
# via apache-superset
google-auth==2.29.0
google-auth==2.36.0
# via shillelagh
greenlet==3.0.3
# via
# shillelagh
# sqlalchemy
gunicorn==22.0.0
gunicorn==23.0.0
# via apache-superset
hashids==1.3.1
# via apache-superset
holidays==0.25
# via apache-superset
humanize==4.9.0
humanize==4.11.0
# via apache-superset
idna==3.7
idna==3.10
# via
# email-validator
# requests
importlib-metadata==7.1.0
# via apache-superset
importlib-resources==6.4.0
importlib-resources==6.4.5
# via limits
isodate==0.6.1
isodate==0.7.2
# via apache-superset
itsdangerous==2.2.0
# via
@@ -177,23 +181,23 @@ jinja2==3.1.4
# via
# flask
# flask-babel
jsonpath-ng==1.6.1
jsonpath-ng==1.7.0
# via apache-superset
jsonschema==4.17.3
# via flask-appbuilder
kombu==5.3.7
kombu==5.4.2
# via celery
korean-lunar-calendar==0.3.1
# via holidays
limits==3.12.0
limits==3.13.0
# via flask-limiter
llvmlite==0.42.0
llvmlite==0.43.0
# via numba
mako==1.3.5
mako==1.3.6
# via
# alembic
# apache-superset
markdown==3.6
markdown==3.7
# via apache-superset
markdown-it-py==3.0.0
# via rich
@@ -203,7 +207,7 @@ markupsafe==3.0.2
# mako
# werkzeug
# wtforms
marshmallow==3.21.2
marshmallow==3.23.1
# via
# flask-appbuilder
# marshmallow-sqlalchemy
@@ -215,11 +219,11 @@ msgpack==1.0.8
# via apache-superset
msgspec==0.18.6
# via flask-session
nh3==0.2.17
nh3==0.2.18
# via apache-superset
numba==0.59.1
numba==0.60.0
# via pandas
numexpr==2.10.1
numexpr==2.10.2
# via
# -r requirements/base.in
# pandas
@@ -231,9 +235,13 @@ numpy==1.23.5
# numexpr
# pandas
# pyarrow
odfpy==1.4.1
# via pandas
openpyxl==3.1.5
# via pandas
ordered-set==4.1.0
# via flask-limiter
packaging==23.2
packaging==24.2
# via
# apache-superset
# apispec
@@ -243,9 +251,9 @@ packaging==23.2
# marshmallow
# marshmallow-sqlalchemy
# shillelagh
pandas[performance]==2.0.3
pandas[excel,performance]==2.0.3
# via apache-superset
paramiko==3.4.0
paramiko==3.5.0
# via
# apache-superset
# sshtunnel
@@ -261,28 +269,28 @@ polyline==2.0.2
# via apache-superset
prison==0.2.1
# via flask-appbuilder
prompt-toolkit==3.0.44
prompt-toolkit==3.0.48
# via click-repl
pyarrow==14.0.2
# via apache-superset
pyasn1==0.6.0
pyasn1==0.6.1
# via
# pyasn1-modules
# rsa
pyasn1-modules==0.4.0
pyasn1-modules==0.4.1
# via google-auth
pycparser==2.22
# via cffi
pygments==2.18.0
# via rich
pyjwt==2.8.0
pyjwt==2.10.0
# via
# apache-superset
# flask-appbuilder
# flask-jwt-extended
pynacl==1.5.0
# via paramiko
pyopenssl==24.1.0
pyopenssl==24.2.1
# via shillelagh
pyparsing==3.1.2
# via apache-superset
@@ -306,7 +314,9 @@ pytz==2024.2
# croniter
# flask-babel
# pandas
pyyaml==6.0.1
pyxlsb==1.0.10
# via pandas
pyyaml==6.0.2
# via
# apache-superset
# apispec
@@ -318,7 +328,7 @@ requests==2.32.2
# shillelagh
requests-cache==1.2.0
# via shillelagh
rich==13.7.1
rich==13.9.4
# via flask-limiter
rsa==4.9
# via google-auth
@@ -328,18 +338,17 @@ shillelagh[gsheetsapi]==1.2.18
# via apache-superset
shortid==0.1.2
# via apache-superset
simplejson==3.19.2
simplejson==3.19.3
# via apache-superset
six==1.16.0
# via
# isodate
# prison
# python-dateutil
# url-normalize
# wtforms-json
slack-sdk==3.27.2
slack-sdk==3.33.4
# via apache-superset
sqlalchemy==1.4.52
sqlalchemy==1.4.54
# via
# alembic
# apache-superset
@@ -354,7 +363,7 @@ sqlalchemy-utils==0.38.3
# flask-appbuilder
sqlglot==25.24.0
# via apache-superset
sqlparse==0.5.0
sqlparse==0.5.2
# via apache-superset
sshtunnel==0.4.0
# via apache-superset
@@ -367,10 +376,12 @@ typing-extensions==4.12.2
# cattrs
# flask-limiter
# limits
# rich
# shillelagh
tzdata==2024.1
tzdata==2024.2
# via
# celery
# kombu
# pandas
url-normalize==1.4.3
# via requests-cache
@@ -394,7 +405,7 @@ werkzeug==3.1.3
# flask-appbuilder
# flask-jwt-extended
# flask-login
wrapt==1.16.0
wrapt==1.17.0
# via deprecated
wtforms==3.2.1
# via
@@ -404,9 +415,13 @@ wtforms==3.2.1
# wtforms-json
wtforms-json==0.3.5
# via apache-superset
xlrd==2.0.1
# via pandas
xlsxwriter==3.0.9
# via apache-superset
zipp==3.19.0
# via
# apache-superset
# pandas
zipp==3.21.0
# via importlib-metadata
zstandard==0.22.0
zstandard==0.23.0
# via flask-compress
+1 -1
View File
@@ -17,4 +17,4 @@
# under the License.
#
-r base.in
-e .[development,bigquery,cors,druid,gevent,gsheets,hive,mysql,playwright,postgres,presto,prophet,trino,thumbnails]
-e .[development,bigquery,cors,druid,gevent,gsheets,mysql,playwright,postgres,presto,prophet,trino,thumbnails]
+41 -76
View File
@@ -1,4 +1,4 @@
# SHA1:c186006a3f82c8775e1039f37c52309f6c858197
# SHA1:dc767a7288b56c785b0cd3c38e95e7b5e66be1ac
#
# This file is autogenerated by pip-compile-multi
# To update, run:
@@ -13,43 +13,33 @@
astroid==3.1.0
# via pylint
boto3==1.34.112
# via
# apache-superset
# dataflows-tabulator
# via apache-superset
botocore==1.34.112
# via
# boto3
# s3transfer
build==1.2.1
# via pip-tools
cached-property==1.5.2
# via tableschema
cfgv==3.3.1
cfgv==3.4.0
# via pre-commit
chardet==5.1.0
# via
# dataflows-tabulator
# tox
chardet==5.2.0
# via tox
cmdstanpy==1.1.0
# via prophet
contourpy==1.0.7
# via matplotlib
coverage[toml]==7.2.5
coverage[toml]==7.6.8
# via pytest-cov
cycler==0.11.0
cycler==0.12.1
# via matplotlib
dataflows-tabulator==1.54.3
# via tableschema
db-dtypes==1.2.0
db-dtypes==1.3.1
# via pandas-gbq
dill==0.3.8
dill==0.3.9
# via pylint
distlib==0.3.8
# via virtualenv
docker==7.0.0
# via apache-superset
et-xmlfile==1.1.0
# via openpyxl
filelock==3.12.2
# via
# tox
@@ -58,43 +48,43 @@ flask-cors==4.0.0
# via apache-superset
flask-testing==0.8.1
# via apache-superset
fonttools==4.51.0
fonttools==4.55.0
# via matplotlib
freezegun==1.5.1
# via apache-superset
future==0.18.3
future==1.0.0
# via pyhive
gevent==24.2.1
# via apache-superset
google-api-core[grpc]==2.11.0
google-api-core[grpc]==2.23.0
# via
# google-cloud-bigquery
# google-cloud-bigquery-storage
# google-cloud-core
# pandas-gbq
# sqlalchemy-bigquery
google-auth-oauthlib==1.0.0
google-auth-oauthlib==1.2.1
# via
# pandas-gbq
# pydata-google-auth
google-cloud-bigquery==3.20.1
google-cloud-bigquery==3.27.0
# via
# apache-superset
# pandas-gbq
# sqlalchemy-bigquery
google-cloud-bigquery-storage==2.19.1
# via pandas-gbq
google-cloud-core==2.3.2
google-cloud-core==2.4.1
# via google-cloud-bigquery
google-crc32c==1.5.0
google-crc32c==1.6.0
# via google-resumable-media
google-resumable-media==2.7.0
google-resumable-media==2.7.2
# via google-cloud-bigquery
googleapis-common-protos==1.63.0
googleapis-common-protos==1.66.0
# via
# google-api-core
# grpcio-status
grpcio==1.62.1
grpcio==1.68.0
# via
# apache-superset
# google-api-core
@@ -103,8 +93,6 @@ grpcio-status==1.60.1
# via google-api-core
identify==2.5.36
# via pre-commit
ijson==3.2.3
# via dataflows-tabulator
iniconfig==2.0.0
# via pytest
isort==5.12.0
@@ -113,21 +101,17 @@ jmespath==1.0.1
# via
# boto3
# botocore
jsonlines==4.0.0
# via dataflows-tabulator
jsonschema-spec==0.1.6
# via openapi-spec-validator
kiwisolver==1.4.5
kiwisolver==1.4.7
# via matplotlib
lazy-object-proxy==1.10.0
# via openapi-spec-validator
linear-tsv==1.1.0
# via dataflows-tabulator
matplotlib==3.9.0
# via prophet
mccabe==0.7.0
# via pylint
mysqlclient==2.2.4
mysqlclient==2.2.6
# via apache-superset
nodeenv==1.8.0
# via pre-commit
@@ -137,8 +121,6 @@ openapi-schema-validator==0.4.4
# via openapi-spec-validator
openapi-spec-validator==0.5.6
# via apache-superset
openpyxl==3.1.2
# via dataflows-tabulator
pandas-gbq==0.19.1
# via apache-superset
parameterized==0.9.0
@@ -155,32 +137,32 @@ pip-tools==7.4.1
# via pip-compile-multi
playwright==1.42.0
# via apache-superset
pluggy==1.4.0
pluggy==1.5.0
# via
# pytest
# tox
pre-commit==3.7.1
pre-commit==4.0.1
# via apache-superset
progress==1.6
# via apache-superset
prophet==1.1.5
# via apache-superset
proto-plus==1.22.2
# via google-cloud-bigquery-storage
protobuf==4.23.0
proto-plus==1.25.0
# via
# google-api-core
# google-cloud-bigquery-storage
protobuf==4.25.5
# via
# google-api-core
# google-cloud-bigquery-storage
# googleapis-common-protos
# grpcio-status
# proto-plus
psutil==6.0.0
psutil==6.1.0
# via apache-superset
psycopg2-binary==2.9.6
# via apache-superset
pure-sasl==0.6.2
# via thrift-sasl
pydata-google-auth==1.7.0
pydata-google-auth==1.9.0
# via pandas-gbq
pydruid==0.6.9
# via apache-superset
@@ -194,9 +176,9 @@ pyinstrument==4.4.0
# via apache-superset
pylint==3.1.0
# via apache-superset
pyproject-api==1.6.1
pyproject-api==1.8.0
# via tox
pyproject-hooks==1.0.0
pyproject-hooks==1.2.0
# via
# build
# pip-tools
@@ -205,7 +187,7 @@ pytest==7.4.4
# apache-superset
# pytest-cov
# pytest-mock
pytest-cov==5.0.0
pytest-cov==6.0.0
# via apache-superset
pytest-mock==3.10.0
# via apache-superset
@@ -215,62 +197,45 @@ requests-oauthlib==2.0.0
# via google-auth-oauthlib
rfc3339-validator==0.1.4
# via openapi-schema-validator
rfc3986==2.0.0
# via tableschema
ruff==0.4.5
ruff==0.8.0
# via apache-superset
s3transfer==0.10.1
# via boto3
sqlalchemy-bigquery==1.11.0
sqlalchemy-bigquery==1.12.0
# via apache-superset
sqloxide==0.1.43
sqloxide==0.1.51
# via apache-superset
statsd==4.0.1
# via apache-superset
tableschema==1.20.10
# via apache-superset
thrift==0.16.0
# via
# apache-superset
# thrift-sasl
thrift-sasl==0.4.3
# via apache-superset
tomli==2.0.1
tomli==2.1.0
# via
# build
# coverage
# pip-tools
# pylint
# pyproject-api
# pyproject-hooks
# pytest
# tox
tomlkit==0.12.5
tomlkit==0.13.2
# via pylint
toposort==1.10
# via pip-compile-multi
tox==4.6.4
# via apache-superset
tqdm==4.66.4
tqdm==4.67.1
# via
# cmdstanpy
# prophet
trino==0.328.0
trino==0.330.0
# via apache-superset
tzlocal==5.2
# via trino
unicodecsv==0.14.1
# via
# dataflows-tabulator
# tableschema
virtualenv==20.23.1
# via
# pre-commit
# tox
wheel==0.43.0
wheel==0.45.1
# via pip-tools
xlrd==2.0.1
# via dataflows-tabulator
zope-event==5.0
# via gevent
zope-interface==5.4.0
-294
View File
@@ -1,294 +0,0 @@
#!/usr/bin/env python3
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import os
import re
import subprocess
from textwrap import dedent
import click
REPO = "apache/superset"
CACHE_REPO = f"{REPO}-cache"
BASE_PY_IMAGE = "3.10-slim-bookworm"
def run_cmd(command: str, raise_on_failure: bool = True) -> str:
process = subprocess.Popen(
command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True
)
output = ""
if process.stdout is not None:
for line in iter(process.stdout.readline, ""):
print(line.strip()) # Print the line to stdout in real-time
output += line
process.wait() # Wait for the subprocess to finish
if process.returncode != 0 and raise_on_failure:
raise subprocess.CalledProcessError(process.returncode, command, output)
return output
def get_git_sha() -> str:
return run_cmd("git rev-parse HEAD").strip()
def get_build_context_ref(build_context: str) -> str:
"""
Given a context, return a ref:
- if context is pull_request, return the PR's id
- if context is push, return the branch
- if context is release, return the release ref
"""
event = os.getenv("GITHUB_EVENT_NAME")
github_ref = os.getenv("GITHUB_REF", "")
if event == "pull_request":
github_head_ref = os.getenv("GITHUB_HEAD_REF", "")
return re.sub("[^a-zA-Z0-9]", "-", github_head_ref)[:40]
elif event == "release":
return re.sub("refs/tags/", "", github_ref)[:40]
elif event == "push":
return re.sub("[^a-zA-Z0-9]", "-", re.sub("refs/heads/", "", github_ref))[:40]
return ""
def is_latest_release(release: str) -> bool:
output = (
run_cmd(
f"./scripts/tag_latest_release.sh {release} --dry-run",
raise_on_failure=False,
)
or ""
)
return "SKIP_TAG::false" in output
def make_docker_tag(l: list[str]) -> str: # noqa: E741
return f"{REPO}:" + "-".join([o for o in l if o])
def get_docker_tags(
build_preset: str,
build_platforms: list[str],
sha: str,
build_context: str,
build_context_ref: str,
force_latest: bool = False,
) -> set[str]:
"""
Return a set of tags given a given build context
"""
tags: set[str] = set()
tag_chunks: list[str] = []
is_latest = is_latest_release(build_context_ref)
if build_preset != "lean":
# Always add the preset_build name if different from default (lean)
tag_chunks += [build_preset]
if len(build_platforms) == 1:
build_platform = build_platforms[0]
short_build_platform = build_platform.replace("linux/", "").replace("64", "")
if short_build_platform != "amd":
# Always a platform indicator if different from default (amd)
tag_chunks += [short_build_platform]
# Always craft a tag for the SHA
tags.add(make_docker_tag([sha] + tag_chunks))
# also a short SHA, cause it's nice
tags.add(make_docker_tag([sha[:7]] + tag_chunks))
if build_context == "release":
# add a release tag
tags.add(make_docker_tag([build_context_ref] + tag_chunks))
if is_latest or force_latest:
# add a latest tag
tags.add(make_docker_tag(["latest"] + tag_chunks))
elif build_context == "push" and build_context_ref == "master":
tags.add(make_docker_tag(["master"] + tag_chunks))
elif build_context == "pull_request":
tags.add(make_docker_tag([f"pr-{build_context_ref}"] + tag_chunks))
return tags
def get_docker_command(
build_preset: str,
build_platforms: list[str],
is_authenticated: bool,
sha: str,
build_context: str,
build_context_ref: str,
force_latest: bool = False,
) -> str:
tag = "" # noqa: F841
build_target = ""
py_ver = BASE_PY_IMAGE
docker_context = "."
if build_preset == "dev":
build_target = "dev"
elif build_preset == "lean":
build_target = "lean"
elif build_preset == "py311":
build_target = "lean"
py_ver = "3.11-slim-bookworm"
elif build_preset == "websocket":
build_target = ""
docker_context = "superset-websocket"
elif build_preset == "ci":
build_target = "ci"
elif build_preset == "dockerize":
build_target = ""
docker_context = "-f dockerize.Dockerfile ."
else:
print(f"Invalid build preset: {build_preset}")
exit(1)
# Try to get context reference if missing
if not build_context_ref:
build_context_ref = get_build_context_ref(build_context)
tags = get_docker_tags(
build_preset,
build_platforms,
sha,
build_context,
build_context_ref,
force_latest,
)
docker_tags = ("\\\n" + 8 * " ").join([f"-t {s} " for s in tags])
docker_args = "--load" if not is_authenticated else "--push"
target_argument = f"--target {build_target}" if build_target else ""
cache_ref = f"{CACHE_REPO}:{py_ver}"
if len(build_platforms) == 1:
build_platform = build_platforms[0]
short_build_platform = build_platform.replace("linux/", "").replace("64", "")
cache_ref = f"{CACHE_REPO}:{py_ver}-{short_build_platform}"
platform_arg = "--platform " + ",".join(build_platforms)
cache_from_arg = f"--cache-from=type=registry,ref={cache_ref}"
cache_to_arg = (
f"--cache-to=type=registry,mode=max,ref={cache_ref}" if is_authenticated else ""
)
build_arg = f"--build-arg PY_VER={py_ver}" if py_ver else ""
actor = os.getenv("GITHUB_ACTOR")
return dedent(
f"""\
docker buildx build \\
{docker_args} \\
{docker_tags} \\
{cache_from_arg} \\
{cache_to_arg} \\
{build_arg} \\
{platform_arg} \\
{target_argument} \\
--label sha={sha} \\
--label target={build_target} \\
--label build_trigger={build_context} \\
--label base={py_ver} \\
--label build_actor={actor} \\
{docker_context}"""
)
@click.command()
@click.argument(
"build_preset",
type=click.Choice(["lean", "dev", "dockerize", "websocket", "py311", "ci"]),
)
@click.argument("build_context", type=click.Choice(["push", "pull_request", "release"]))
@click.option(
"--platform",
type=click.Choice(["linux/arm64", "linux/amd64"]),
default=["linux/amd64"],
multiple=True,
)
@click.option("--build_context_ref", help="a reference to the pr, release or branch")
@click.option("--dry-run", is_flag=True, help="Run the command in dry-run mode.")
@click.option("--verbose", is_flag=True, help="Print more info")
@click.option(
"--force-latest", is_flag=True, help="Force the 'latest' tag on the release"
)
def main(
build_preset: str,
build_context: str,
build_context_ref: str,
platform: list[str],
dry_run: bool,
force_latest: bool,
verbose: bool,
) -> None:
"""
This script executes docker build and push commands based on given arguments.
"""
is_authenticated = (
True if os.getenv("DOCKERHUB_TOKEN") and os.getenv("DOCKERHUB_USER") else False
)
if force_latest and build_context != "release":
print(
"--force-latest can only be applied if the build context is set to 'release'"
)
exit(1)
if build_context == "release" and not build_context_ref.strip():
print("Release number has to be provided")
exit(1)
docker_build_command = get_docker_command(
build_preset,
platform,
is_authenticated,
get_git_sha(),
build_context,
build_context_ref,
force_latest,
)
if not dry_run:
print("Executing Docker Build Command:")
print(docker_build_command)
script = ""
if os.getenv("DOCKERHUB_USER"):
script = dedent(
f"""\
docker logout
docker login --username "{os.getenv("DOCKERHUB_USER")}" --password "{os.getenv("DOCKERHUB_TOKEN")}"
DOCKER_ARGS="--push"
"""
)
script = script + docker_build_command
if verbose:
run_cmd("cat Dockerfile")
stdout = run_cmd(script) # noqa: F841
else:
print("Dry Run - Docker Build Command:")
print(docker_build_command)
if __name__ == "__main__":
main()
+49 -19
View File
@@ -53067,14 +53067,6 @@
"node": ">=4"
}
},
"node_modules/viewport-mercator-project": {
"version": "6.2.3",
"license": "MIT",
"dependencies": {
"@babel/runtime": "^7.0.0",
"gl-matrix": "^3.0.0"
}
},
"node_modules/vlq": {
"version": "0.2.3",
"license": "MIT"
@@ -57951,10 +57943,10 @@
"version": "0.18.25",
"license": "Apache-2.0",
"dependencies": {
"@math.gl/web-mercator": "^4.1.0",
"prop-types": "^15.8.1",
"react-map-gl": "^6.1.19",
"supercluster": "^8.0.1",
"viewport-mercator-project": "^6.1.1"
"supercluster": "^8.0.1"
},
"peerDependencies": {
"@superset-ui/chart-controls": "*",
@@ -57963,6 +57955,30 @@
"react": "^15 || ^16"
}
},
"plugins/legacy-plugin-chart-map-box/node_modules/@math.gl/core": {
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/@math.gl/core/-/core-4.1.0.tgz",
"integrity": "sha512-FrdHBCVG3QdrworwrUSzXIaK+/9OCRLscxI2OUy6sLOHyHgBMyfnEGs99/m3KNvs+95BsnQLWklVfpKfQzfwKA==",
"license": "MIT",
"dependencies": {
"@math.gl/types": "4.1.0"
}
},
"plugins/legacy-plugin-chart-map-box/node_modules/@math.gl/types": {
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/@math.gl/types/-/types-4.1.0.tgz",
"integrity": "sha512-clYZdHcmRvMzVK5fjeDkQlHUzXQSNdZ7s4xOqC3nJPgz4C/TZkUecTo9YS4PruZqtDda/ag4erndP0MIn40dGA==",
"license": "MIT"
},
"plugins/legacy-plugin-chart-map-box/node_modules/@math.gl/web-mercator": {
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/@math.gl/web-mercator/-/web-mercator-4.1.0.tgz",
"integrity": "sha512-HZo3vO5GCMkXJThxRJ5/QYUYRr3XumfT8CzNNCwoJfinxy5NtKUd7dusNTXn7yJ40UoB8FMIwkVwNlqaiRZZAw==",
"license": "MIT",
"dependencies": {
"@math.gl/core": "4.1.0"
}
},
"plugins/legacy-plugin-chart-map-box/node_modules/kdbush": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/kdbush/-/kdbush-4.0.2.tgz",
@@ -68666,12 +68682,33 @@
"@superset-ui/legacy-plugin-chart-map-box": {
"version": "file:plugins/legacy-plugin-chart-map-box",
"requires": {
"@math.gl/web-mercator": "^4.1.0",
"prop-types": "^15.8.1",
"react-map-gl": "^6.1.19",
"supercluster": "^8.0.1",
"viewport-mercator-project": "^6.1.1"
"supercluster": "^8.0.1"
},
"dependencies": {
"@math.gl/core": {
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/@math.gl/core/-/core-4.1.0.tgz",
"integrity": "sha512-FrdHBCVG3QdrworwrUSzXIaK+/9OCRLscxI2OUy6sLOHyHgBMyfnEGs99/m3KNvs+95BsnQLWklVfpKfQzfwKA==",
"requires": {
"@math.gl/types": "4.1.0"
}
},
"@math.gl/types": {
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/@math.gl/types/-/types-4.1.0.tgz",
"integrity": "sha512-clYZdHcmRvMzVK5fjeDkQlHUzXQSNdZ7s4xOqC3nJPgz4C/TZkUecTo9YS4PruZqtDda/ag4erndP0MIn40dGA=="
},
"@math.gl/web-mercator": {
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/@math.gl/web-mercator/-/web-mercator-4.1.0.tgz",
"integrity": "sha512-HZo3vO5GCMkXJThxRJ5/QYUYRr3XumfT8CzNNCwoJfinxy5NtKUd7dusNTXn7yJ40UoB8FMIwkVwNlqaiRZZAw==",
"requires": {
"@math.gl/core": "4.1.0"
}
},
"kdbush": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/kdbush/-/kdbush-4.0.2.tgz",
@@ -95387,13 +95424,6 @@
"unist-util-stringify-position": "^3.0.0"
}
},
"viewport-mercator-project": {
"version": "6.2.3",
"requires": {
"@babel/runtime": "^7.0.0",
"gl-matrix": "^3.0.0"
}
},
"vlq": {
"version": "0.2.3"
},
@@ -1,13 +0,0 @@
{
"name": "@superset-ui/switchboard",
"version": "0.18.26-0",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "@superset-ui/switchboard",
"version": "0.18.26-0",
"license": "Apache-2.0"
}
}
}
@@ -26,10 +26,10 @@
"lib"
],
"dependencies": {
"@math.gl/web-mercator": "^4.1.0",
"prop-types": "^15.8.1",
"react-map-gl": "^6.1.19",
"supercluster": "^8.0.1",
"viewport-mercator-project": "^6.1.1"
"supercluster": "^8.0.1"
},
"peerDependencies": {
"@superset-ui/chart-controls": "*",
@@ -21,7 +21,7 @@
import { Component } from 'react';
import PropTypes from 'prop-types';
import MapGL from 'react-map-gl';
import ViewportMercator from 'viewport-mercator-project';
import { WebMercatorViewport } from '@math.gl/web-mercator';
import ScatterPlotGlowOverlay from './ScatterPlotGlowOverlay';
import './MapBox.css';
@@ -63,7 +63,7 @@ class MapBox extends Component {
// Get a viewport that fits the given bounds, which all marks to be clustered.
// Derive lat, lon and zoom from this viewport. This is only done on initial
// render as the bounds don't update as we pan/zoom in the current design.
const mercator = new ViewportMercator({
const mercator = new WebMercatorViewport({
width,
height,
}).fitBounds(bounds);
@@ -0,0 +1,80 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { List, ListProps } from '.';
export default {
title: 'List',
component: List,
};
const dataSource = ['Item 1', 'Item 2', 'Item 3'];
export const InteractiveList = (args: ListProps<any>) => (
<List
{...args}
dataSource={dataSource}
renderItem={item => <List.Item>{item}</List.Item>}
/>
);
InteractiveList.args = {
bordered: false,
split: true,
itemLayout: 'horizontal',
size: 'default',
loading: false,
};
InteractiveList.argTypes = {
bordered: {
control: { type: 'boolean' },
},
split: {
control: { type: 'boolean' },
},
loading: {
control: { type: 'boolean' },
},
itemLayout: {
control: { type: 'select' },
options: ['horizontal', 'vertical'],
},
size: {
control: { type: 'select' },
options: ['default', 'small', 'large'],
},
};
export const InteractiveListWithPagination = (args: ListProps<any>) => (
<List
{...args}
dataSource={dataSource}
renderItem={item => <List.Item>{item}</List.Item>}
pagination={{ pageSize: 2 }}
/>
);
InteractiveListWithPagination.args = {
...InteractiveList.args,
};
InteractiveListWithPagination.argTypes = {
...InteractiveList.argTypes,
};
@@ -0,0 +1,42 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { render, screen } from 'spec/helpers/testing-library';
import { ListProps } from 'antd-v5/lib/list';
import { List } from '.';
const mockedProps: ListProps<any> = {
dataSource: ['Item 1', 'Item 2', 'Item 3'],
renderItem: item => <div>{item}</div>,
};
test('should render', () => {
const { container } = render(<List {...mockedProps} />);
expect(container).toBeInTheDocument();
});
test('should render the correct number of items', () => {
render(<List {...mockedProps} />);
const listItemElements = screen.getAllByText(/Item \d/);
expect(listItemElements.length).toBe(3);
listItemElements.forEach((item, index) => {
expect(item).toHaveTextContent(`Item ${index + 1}`);
});
});
@@ -0,0 +1,27 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { ListProps, ListItemProps, ListItemMetaProps } from 'antd-v5/lib/list';
import { List as AntdList } from 'antd-v5';
export type { ListProps, ListItemProps, ListItemMetaProps };
export const List = Object.assign(AntdList, {
Item: AntdList.Item,
ItemMeta: AntdList.Item.Meta,
});
@@ -51,7 +51,7 @@ export type LastModified = {
export type Owner = {
type: MetadataType.Owner;
createdBy: string;
owners?: string[];
owners?: string[] | string;
createdOn: string;
onClick?: (type: string) => void;
};
@@ -23,27 +23,18 @@ import { styled } from '@superset-ui/core';
import { Tooltip, TooltipPlacement } from 'src/components/Tooltip';
import { ContentType } from './ContentType';
import { config } from './ContentConfig';
export const MIN_NUMBER_ITEMS = 2;
export const MAX_NUMBER_ITEMS = 6;
const HORIZONTAL_PADDING = 12;
const VERTICAL_PADDING = 8;
const ICON_PADDING = 8;
const SPACE_BETWEEN_ITEMS = 16;
const ICON_WIDTH = 16;
const TEXT_MIN_WIDTH = 70;
const TEXT_MAX_WIDTH = 150;
const ORDER = {
dashboards: 0,
table: 1,
sql: 2,
rows: 3,
tags: 4,
description: 5,
owner: 6,
lastModified: 7,
};
import {
HORIZONTAL_PADDING,
ICON_PADDING,
ICON_WIDTH,
VERTICAL_PADDING,
TEXT_MIN_WIDTH,
TEXT_MAX_WIDTH,
SPACE_BETWEEN_ITEMS,
ORDER,
MIN_NUMBER_ITEMS,
MAX_NUMBER_ITEMS,
} from './constants';
const Bar = styled.div<{ count: number }>`
${({ theme, count }) => `
@@ -0,0 +1,39 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
export const MIN_NUMBER_ITEMS = 2;
export const MAX_NUMBER_ITEMS = 6;
export const HORIZONTAL_PADDING = 12;
export const VERTICAL_PADDING = 8;
export const ICON_PADDING = 8;
export const SPACE_BETWEEN_ITEMS = 16;
export const ICON_WIDTH = 16;
export const TEXT_MIN_WIDTH = 70;
export const TEXT_MAX_WIDTH = 150;
export const ORDER = {
dashboards: 0,
table: 1,
sql: 2,
rows: 3,
tags: 4,
description: 5,
owner: 6,
lastModified: 7,
};
@@ -16,7 +16,8 @@
* specific language governing permissions and limitations
* under the License.
*/
import MetadataBar, { MIN_NUMBER_ITEMS, MAX_NUMBER_ITEMS } from './MetadataBar';
import MetadataBar from './MetadataBar';
import { MIN_NUMBER_ITEMS, MAX_NUMBER_ITEMS } from './constants';
export type { MetadataBarProps } from './MetadataBar';
@@ -37,7 +37,6 @@ export {
Divider,
Empty,
Grid,
List,
Row,
Skeleton,
Space,
@@ -72,7 +71,6 @@ export {
// Exported types
export type { FormInstance } from 'antd/lib/form';
export type { ListItemProps } from 'antd/lib/list';
export type { ModalProps as AntdModalProps } from 'antd/lib/modal';
export type { DropDownProps as AntdDropdownProps } from 'antd/lib/dropdown';
export type { RadioChangeEvent } from 'antd/lib/radio';
@@ -28,6 +28,7 @@ import {
t,
getClientErrorObject,
getCategoricalSchemeRegistry,
promiseTimeout,
} from '@superset-ui/core';
import {
addChart,
@@ -737,9 +738,9 @@ export const persistDashboardLabelsColor = () => async (dispatch, getState) => {
} = getState();
if (labelsColorMapMustSync || sharedLabelsColorsMustSync) {
storeDashboardMetadata(id, metadata);
dispatch(setDashboardLabelsColorMapSynced());
dispatch(setDashboardSharedLabelsColorsSynced());
storeDashboardMetadata(id, metadata);
}
};
@@ -755,16 +756,13 @@ export const applyDashboardLabelsColorOnLoad = metadata => async dispatch => {
try {
const updatedMetadata = { ...metadata };
const customLabelsColor = metadata.label_colors || {};
const sharedLabelsColor = metadata.shared_label_colors || [];
let hasChanged = false;
// backward compatibility of shared_label_colors
const sharedLabels = metadata.shared_label_colors || [];
if (!Array.isArray(sharedLabels) && Object.keys(sharedLabels).length > 0) {
hasChanged = true;
updatedMetadata.shared_label_colors = getFreshSharedLabels(
Object.keys(sharedLabelsColor),
);
updatedMetadata.shared_label_colors = [];
}
// backward compatibility of map_label_colors
const hasMapLabelColors =
@@ -828,27 +826,28 @@ export const applyDashboardLabelsColorOnLoad = metadata => async dispatch => {
* @returns void
*/
export const ensureSyncedLabelsColorMap = metadata => (dispatch, getState) => {
const {
dashboardState: { labelsColorMapMustSync },
} = getState();
const updatedMetadata = { ...metadata };
const customLabelsColor = metadata.label_colors || {};
const isMapSynced = isLabelsColorMapSynced(metadata);
const mustSync = !isMapSynced;
const syncLabelsColorMap = () => {
const {
dashboardState: { labelsColorMapMustSync },
} = getState();
const updatedMetadata = { ...metadata };
const customLabelsColor = metadata.label_colors || {};
const isMapSynced = isLabelsColorMapSynced(metadata);
const mustSync = !isMapSynced;
if (mustSync) {
const freshestColorMapEntries = getLabelsColorMapEntries(customLabelsColor);
updatedMetadata.map_label_colors = freshestColorMapEntries;
dispatch(setDashboardMetadata(updatedMetadata));
}
if (mustSync) {
const freshestColorMapEntries =
getLabelsColorMapEntries(customLabelsColor);
updatedMetadata.map_label_colors = freshestColorMapEntries;
dispatch(setDashboardMetadata(updatedMetadata));
}
if (mustSync && !labelsColorMapMustSync) {
// prepare to persist the just applied labels color map
dispatch(setDashboardLabelsColorMapSync());
}
if (!mustSync && labelsColorMapMustSync) {
dispatch(setDashboardLabelsColorMapSynced());
}
if (mustSync && !labelsColorMapMustSync) {
// prepare to persist the just applied labels color map
dispatch(setDashboardLabelsColorMapSync());
}
};
promiseTimeout(syncLabelsColorMap, 500);
};
/**
@@ -856,18 +855,21 @@ export const ensureSyncedLabelsColorMap = metadata => (dispatch, getState) => {
* Ensure that the stored shared labels colors match current.
*
* @param {*} metadata - the dashboard metadata
* @param {*} forceFresh - when true it will use the fresh shared labels ignoring stored ones
* @returns void
*/
export const ensureSyncedSharedLabelsColors =
metadata => (dispatch, getState) => {
// using a timeout to let the rendered charts finish processing labels
setTimeout(() => {
(metadata, forceFresh = false) =>
(dispatch, getState) => {
const syncSharedLabelsColors = () => {
const {
dashboardState: { sharedLabelsColorsMustSync },
} = getState();
const updatedMetadata = { ...metadata };
const sharedLabelsColors = metadata.shared_label_colors || [];
const freshLabelsColors = getFreshSharedLabels(sharedLabelsColors);
const freshLabelsColors = getFreshSharedLabels(
forceFresh ? [] : sharedLabelsColors,
);
const isSharedLabelsColorsSynced = isEqual(
sharedLabelsColors,
freshLabelsColors,
@@ -884,10 +886,8 @@ export const ensureSyncedSharedLabelsColors =
// prepare to persist the shared labels colors
dispatch(setDashboardSharedLabelsColorsSync());
}
if (!mustSync && sharedLabelsColorsMustSync) {
dispatch(setDashboardSharedLabelsColorsSynced());
}
}, 500);
};
promiseTimeout(syncSharedLabelsColors, 500);
};
/**
@@ -909,7 +909,6 @@ export const updateDashboardLabelsColor =
const fullLabelsColors = metadata.map_label_colors || {};
const sharedLabelsColors = metadata.shared_label_colors || [];
const customLabelsColors = metadata.label_colors || {};
const updatedMetadata = { ...metadata };
// for dashboards with no color scheme, the charts should always use their individual schemes
// this logic looks for unique labels (not shared across multiple charts) of each rendered chart
@@ -965,11 +964,7 @@ export const updateDashboardLabelsColor =
const shouldGoFresh = shouldReset.length > 0 ? shouldReset : false;
const shouldMerge = !shouldGoFresh;
// re-apply the color map first to get fresh maps accordingly
applyColors(updatedMetadata, shouldGoFresh, shouldMerge);
// new data may have appeared in the map (data changes)
// or new slices may have appeared while changing tabs
dispatch(ensureSyncedLabelsColorMap(updatedMetadata));
dispatch(ensureSyncedSharedLabelsColors(updatedMetadata));
applyColors(metadata, shouldGoFresh, shouldMerge);
} catch (e) {
console.error('Failed to update colors for new charts and labels:', e);
}
@@ -47,6 +47,8 @@ import {
applyDashboardLabelsColorOnLoad,
updateDashboardLabelsColor,
persistDashboardLabelsColor,
ensureSyncedSharedLabelsColors,
ensureSyncedLabelsColorMap,
} from 'src/dashboard/actions/dashboardState';
import { getColorNamespace, resetColors } from 'src/utils/colorScheme';
import { NATIVE_FILTER_DIVIDER_PREFIX } from '../nativeFilters/FiltersConfigModal/utils';
@@ -96,7 +98,6 @@ const DashboardContainer: FC<DashboardContainerProps> = ({ topLevelTabs }) => {
const [dashboardLabelsColorInitiated, setDashboardLabelsColorInitiated] =
useState(false);
const prevRenderedChartIds = useRef<number[]>([]);
const prevTabIndexRef = useRef();
const tabIndex = useMemo(() => {
const nextTabIndex = findTabIndexByComponentId({
@@ -110,6 +111,18 @@ const DashboardContainer: FC<DashboardContainerProps> = ({ topLevelTabs }) => {
prevTabIndexRef.current = nextTabIndex;
return nextTabIndex;
}, [dashboardLayout, directPathToChild]);
// when all charts have rendered, enforce fresh shared labels
const shouldForceFreshSharedLabelsColors =
dashboardLabelsColorInitiated &&
renderedChartIds.length > 0 &&
chartIds.length === renderedChartIds.length &&
prevRenderedChartIds.current.length < renderedChartIds.length;
const onBeforeUnload = useCallback(() => {
dispatch(persistDashboardLabelsColor());
resetColors(getColorNamespace(dashboardInfo?.metadata?.color_namespace));
prevRenderedChartIds.current = [];
}, [dashboardInfo?.metadata?.color_namespace, dispatch]);
useEffect(() => {
if (nativeFilterScopes.length === 0) {
@@ -148,11 +161,12 @@ const DashboardContainer: FC<DashboardContainerProps> = ({ topLevelTabs }) => {
const activeKey = min === 0 ? DASHBOARD_GRID_ID : min.toString();
const TOP_OF_PAGE_RANGE = 220;
const onBeforeUnload = useCallback(() => {
dispatch(persistDashboardLabelsColor());
resetColors(getColorNamespace(dashboardInfo?.metadata?.color_namespace));
prevRenderedChartIds.current = [];
}, [dashboardInfo?.metadata?.color_namespace, dispatch]);
useEffect(() => {
if (shouldForceFreshSharedLabelsColors) {
// all available charts have rendered, enforce freshest shared label colors
dispatch(ensureSyncedSharedLabelsColors(dashboardInfo.metadata, true));
}
}, [dashboardInfo.metadata, dispatch, shouldForceFreshSharedLabelsColors]);
useEffect(() => {
// verify freshness of color map
@@ -161,7 +175,6 @@ const DashboardContainer: FC<DashboardContainerProps> = ({ topLevelTabs }) => {
if (
dashboardLabelsColorInitiated &&
dashboardInfo?.id &&
numRenderedCharts > 0 &&
prevRenderedChartIds.current.length < numRenderedCharts
) {
@@ -170,12 +183,20 @@ const DashboardContainer: FC<DashboardContainerProps> = ({ topLevelTabs }) => {
);
prevRenderedChartIds.current = renderedChartIds;
dispatch(updateDashboardLabelsColor(newRenderedChartIds));
// new data may have appeared in the map (data changes)
// or new slices may have appeared while changing tabs
dispatch(ensureSyncedLabelsColorMap(dashboardInfo.metadata));
if (!shouldForceFreshSharedLabelsColors) {
dispatch(ensureSyncedSharedLabelsColors(dashboardInfo.metadata));
}
}
}, [
dashboardInfo?.id,
renderedChartIds,
dispatch,
dashboardLabelsColorInitiated,
dashboardInfo.metadata,
shouldForceFreshSharedLabelsColors,
]);
useEffect(() => {
@@ -183,9 +204,9 @@ const DashboardContainer: FC<DashboardContainerProps> = ({ topLevelTabs }) => {
labelsColorMap.source = LabelsColorMapSource.Dashboard;
if (dashboardInfo?.id && !dashboardLabelsColorInitiated) {
dispatch(applyDashboardLabelsColorOnLoad(dashboardInfo.metadata));
// apply labels color as dictated by stored metadata (if any)
setDashboardLabelsColorInitiated(true);
dispatch(applyDashboardLabelsColorOnLoad(dashboardInfo.metadata));
}
return () => {
@@ -44,7 +44,6 @@ import ConnectedHeaderActionsDropdown from 'src/dashboard/components/Header/Head
import PublishedStatus from 'src/dashboard/components/PublishedStatus';
import UndoRedoKeyListeners from 'src/dashboard/components/UndoRedoKeyListeners';
import PropertiesModal from 'src/dashboard/components/PropertiesModal';
import getOwnerName from 'src/utils/getOwnerName';
import {
UNDO_LIMIT,
SAVE_TYPE_OVERWRITE,
@@ -55,7 +54,6 @@ import setPeriodicRunner, {
stopPeriodicRender,
} from 'src/dashboard/util/setPeriodicRunner';
import { PageHeaderWithActions } from 'src/components/PageHeaderWithActions';
import MetadataBar, { MetadataType } from 'src/components/MetadataBar';
import DashboardEmbedModal from '../EmbeddedModal';
import OverwriteConfirm from '../OverwriteConfirm';
import {
@@ -88,6 +86,7 @@ import { logEvent } from '../../../logger/actions';
import { dashboardInfoChanged } from '../../actions/dashboardInfo';
import isDashboardLoading from '../../util/isDashboardLoading';
import { useChartIds } from '../../util/charts/useChartIds';
import { useDashboardMetadataBar } from './useDashboardMetadataBar';
const extensionsRegistry = getExtensionsRegistry();
@@ -472,32 +471,7 @@ const Header = () => {
setShowingEmbedModal(false);
}, []);
const getMetadataItems = useCallback(
() => [
{
type: MetadataType.LastModified,
value: dashboardInfo.changed_on_delta_humanized,
modifiedBy:
getOwnerName(dashboardInfo.changed_by) || t('Not available'),
},
{
type: MetadataType.Owner,
createdBy: getOwnerName(dashboardInfo.created_by) || t('Not available'),
owners:
dashboardInfo.owners.length > 0
? dashboardInfo.owners.map(getOwnerName)
: t('None'),
createdOn: dashboardInfo.created_on_delta_humanized,
},
],
[
dashboardInfo.changed_by,
dashboardInfo.changed_on_delta_humanized,
dashboardInfo.created_by,
dashboardInfo.created_on_delta_humanized,
dashboardInfo.owners,
],
);
const metadataBar = useDashboardMetadataBar(dashboardInfo);
const userCanEdit =
dashboardInfo.dash_edit_perm && !dashboardInfo.is_managed_externally;
@@ -579,15 +553,13 @@ const Header = () => {
visible={!editMode}
/>
),
!editMode && !isEmbedded && (
<MetadataBar items={getMetadataItems()} tooltipPlacement="bottom" />
),
!editMode && !isEmbedded && metadataBar,
],
[
boundActionCreators.savePublished,
dashboardInfo.id,
editMode,
getMetadataItems,
metadataBar,
isEmbedded,
isPublished,
userCanEdit,
@@ -0,0 +1,54 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { useMemo } from 'react';
import { t } from '@superset-ui/core';
import { DashboardInfo } from 'src/dashboard/types';
import MetadataBar, { MetadataType } from 'src/components/MetadataBar';
import getOwnerName from 'src/utils/getOwnerName';
export const useDashboardMetadataBar = (dashboardInfo: DashboardInfo) => {
const items = useMemo(
() => [
{
type: MetadataType.LastModified as const,
value: dashboardInfo.changed_on_delta_humanized,
modifiedBy:
getOwnerName(dashboardInfo.changed_by) || t('Not available'),
},
{
type: MetadataType.Owner as const,
createdBy: getOwnerName(dashboardInfo.created_by) || t('Not available'),
owners:
dashboardInfo.owners.length > 0
? dashboardInfo.owners.map(getOwnerName)
: t('None'),
createdOn: dashboardInfo.created_on_delta_humanized,
},
],
[
dashboardInfo.changed_by,
dashboardInfo.changed_on_delta_humanized,
dashboardInfo.created_by,
dashboardInfo.created_on_delta_humanized,
dashboardInfo.owners,
],
);
return <MetadataBar items={items} tooltipPlacement="bottom" />;
};
@@ -52,6 +52,9 @@ const initialState: { dashboardInfo: DashboardInfo } = {
conf: {},
},
crossFiltersEnabled: true,
created_on_delta_humanized: '',
changed_on_delta_humanized: '',
owners: [],
},
};
+6
View File
@@ -33,6 +33,7 @@ import Database from 'src/types/Database';
import { UrlParamEntries } from 'src/utils/urlUtils';
import { UserWithPermissionsAndRoles } from 'src/types/bootstrapTypes';
import Owner from 'src/types/Owner';
import { ChartState } from '../explore/types';
export type { Dashboard } from 'src/types/Dashboard';
@@ -139,6 +140,11 @@ export type DashboardInfo = {
};
crossFiltersEnabled: boolean;
filterBarOrientation: FilterBarOrientation;
created_on_delta_humanized: string;
changed_on_delta_humanized: string;
changed_by?: Owner;
created_by?: Owner;
owners: Owner[];
};
export type ChartsState = { [key: string]: Chart };
@@ -16,12 +16,12 @@
* specific language governing permissions and limitations
* under the License.
*/
import { useCallback, useEffect, useMemo, useState } from 'react';
import { useCallback, useEffect, useState } from 'react';
import { useHistory } from 'react-router-dom';
import { useDispatch } from 'react-redux';
import PropTypes from 'prop-types';
import { Tooltip } from 'src/components/Tooltip';
import { css, logging, SupersetClient, t, tn } from '@superset-ui/core';
import { css, logging, SupersetClient, t } from '@superset-ui/core';
import { chartPropShape } from 'src/dashboard/util/propShapes';
import AlteredSliceTag from 'src/components/AlteredSliceTag';
import Button from 'src/components/Button';
@@ -29,10 +29,10 @@ import Icons from 'src/components/Icons';
import PropertiesModal from 'src/explore/components/PropertiesModal';
import { sliceUpdated } from 'src/explore/actions/exploreActions';
import { PageHeaderWithActions } from 'src/components/PageHeaderWithActions';
import MetadataBar, { MetadataType } from 'src/components/MetadataBar';
import { setSaveChartModalVisibility } from 'src/explore/actions/saveModalActions';
import { applyColors, resetColors } from 'src/utils/colorScheme';
import { useExploreAdditionalActionsMenu } from '../useExploreAdditionalActionsMenu';
import { useExploreMetadataBar } from './useExploreMetadataBar';
const propTypes = {
actions: PropTypes.object.isRequired,
@@ -160,48 +160,7 @@ export const ExploreChartHeader = ({
metadata?.dashboards,
);
const metadataBar = useMemo(() => {
if (!metadata) {
return null;
}
const items = [];
items.push({
type: MetadataType.Dashboards,
title:
metadata.dashboards.length > 0
? tn(
'Added to 1 dashboard',
'Added to %s dashboards',
metadata.dashboards.length,
metadata.dashboards.length,
)
: t('Not added to any dashboard'),
description:
metadata.dashboards.length > 0
? t(
'You can preview the list of dashboards in the chart settings dropdown.',
)
: undefined,
});
items.push({
type: MetadataType.LastModified,
value: metadata.changed_on_humanized,
modifiedBy: metadata.changed_by || t('Not available'),
});
items.push({
type: MetadataType.Owner,
createdBy: metadata.created_by || t('Not available'),
owners: metadata.owners.length > 0 ? metadata.owners : t('None'),
createdOn: metadata.created_on_humanized,
});
if (slice?.description) {
items.push({
type: MetadataType.Description,
value: slice?.description,
});
}
return <MetadataBar items={items} tooltipPlacement="bottom" />;
}, [metadata, slice?.description]);
const metadataBar = useExploreMetadataBar(metadata, slice);
const oldSliceName = slice?.slice_name;
return (
@@ -0,0 +1,71 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { useMemo } from 'react';
import { t, tn } from '@superset-ui/core';
import MetadataBar, { MetadataType } from 'src/components/MetadataBar';
import { ExplorePageInitialData } from 'src/explore/types';
export const useExploreMetadataBar = (
metadata: ExplorePageInitialData['metadata'],
slice: ExplorePageInitialData['slice'],
) =>
useMemo(() => {
if (!metadata) {
return null;
}
const items = [];
if (metadata.dashboards) {
items.push({
type: MetadataType.Dashboards as const,
title:
metadata.dashboards.length > 0
? tn(
'Added to 1 dashboard',
'Added to %s dashboards',
metadata.dashboards.length,
metadata.dashboards.length,
)
: t('Not added to any dashboard'),
description:
metadata.dashboards.length > 0
? t(
'You can preview the list of dashboards in the chart settings dropdown.',
)
: undefined,
});
}
items.push({
type: MetadataType.LastModified as const,
value: metadata.changed_on_humanized,
modifiedBy: metadata.changed_by || t('Not available'),
});
items.push({
type: MetadataType.Owner as const,
createdBy: metadata.created_by || t('Not available'),
owners: metadata.owners.length > 0 ? metadata.owners : t('None'),
createdOn: metadata.created_on_humanized,
});
if (slice?.description) {
items.push({
type: MetadataType.Description as const,
value: slice?.description,
});
}
return <MetadataBar items={items} tooltipPlacement="bottom" />;
}, [metadata, slice?.description]);
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
import { List } from 'src/components';
import { List } from 'src/components/List';
import { connect } from 'react-redux';
import { PureComponent } from 'react';
import {
@@ -18,7 +18,7 @@
*/
import { Component } from 'react';
import PropTypes from 'prop-types';
import { List } from 'src/components';
import { List } from 'src/components/List';
import { nanoid } from 'nanoid';
import { t, withTheme } from '@superset-ui/core';
import {
@@ -118,7 +118,11 @@ class CollectionControl extends Component {
return (
<SortableListItem
className="clearfix"
css={{ justifyContent: 'flex-start' }}
css={theme => ({
justifyContent: 'flex-start',
display: '-webkit-flex',
paddingInline: theme.gridUnit * 3,
})}
key={this.props.keyAccessor(o)}
index={i}
>
@@ -17,7 +17,7 @@
* under the License.
*/
import { useTheme } from '@superset-ui/core';
import { List, ListItemProps } from 'src/components';
import { ListItemProps, List } from 'src/components/List';
export interface CustomListItemProps extends ListItemProps {
selectable: boolean;
@@ -27,8 +27,7 @@ export default function CustomListItem(props: CustomListItemProps) {
const { selectable, children, ...rest } = props;
const theme = useTheme();
const css = {
'&.ant-list-item': {
padding: `${theme.gridUnit + 2}px ${theme.gridUnit * 3}px`,
'&.antd5-list-item': {
':first-of-type': {
borderTopLeftRadius: theme.gridUnit,
borderTopRightRadius: theme.gridUnit,
+4
View File
@@ -82,6 +82,10 @@ export interface ExplorePageInitialData {
owners: string[];
created_by?: string;
changed_by?: string;
dashboards?: {
id: number;
dashboard_title: string;
}[];
};
saveAction?: SaveActionType | null;
}
+6
View File
@@ -81,6 +81,12 @@ const baseConfig: ThemeConfig = {
supersetTheme.colors.primary.light3
}`,
},
List: {
itemPadding: `${supersetTheme.gridUnit + 2}px ${supersetTheme.gridUnit * 3}px`,
paddingLG: supersetTheme.gridUnit * 3,
colorSplit: supersetTheme.colors.grayscale.light3,
colorText: supersetTheme.colors.grayscale.dark1,
},
Tag: {
borderRadiusSM: 2,
defaultBg: supersetTheme.colors.grayscale.light4,
+2 -1
View File
@@ -35,6 +35,7 @@ export const getColorNamespace = (namespace?: string) => namespace || undefined;
* Get labels shared across all charts in a dashboard.
* Merges a fresh instance of shared label colors with a stored one.
*
* @param currentSharedLabels - existing shared labels to merge with fresh
* @returns Record<string, string>
*/
export const getFreshSharedLabels = (
@@ -74,7 +75,7 @@ export const getSharedLabelsColorMapEntries = (
* @returns all color entries except custom label colors
*/
export const getLabelsColorMapEntries = (
customLabelsColor: Record<string, string>,
customLabelsColor: Record<string, string> = {},
): Record<string, string> => {
const labelsColorMapInstance = getLabelsColorMap();
const allEntries = Object.fromEntries(labelsColorMapInstance.getColorMap());
+5 -2
View File
@@ -36,7 +36,7 @@ from superset.errors import ErrorLevel, SupersetError, SupersetErrorType
from superset.exceptions import SupersetErrorException
from superset.extensions import db
from superset.models.core import Database
from superset.sql_parse import Table
from superset.sql_parse import ParsedQuery, Table
from superset.utils.decorators import on_error, transaction
logger = logging.getLogger(__name__)
@@ -70,7 +70,10 @@ class DuplicateDatasetCommand(CreateMixin, BaseCommand):
table.normalize_columns = self._base_model.normalize_columns
table.always_filter_main_dttm = self._base_model.always_filter_main_dttm
table.is_sqllab_view = True
table.sql = self._base_model.sql.strip().strip(";")
table.sql = ParsedQuery(
self._base_model.sql,
engine=database.db_engine_spec.engine,
).stripped()
db.session.add(table)
cols = []
for config_ in self._base_model.columns:
+1 -1
View File
@@ -1778,7 +1778,7 @@ GUEST_TOKEN_VALIDATOR_HOOK = None
# def DATASET_HEALTH_CHECK(datasource: SqlaTable) -> Optional[str]:
# if (
# datasource.sql and
# len(SQLScript(datasource.sql).tables) == 1
# len(sql_parse.ParsedQuery(datasource.sql, strip_comments=True).tables) == 1
# ):
# return (
# "This virtual dataset queries only one table and therefore could be "
+27 -6
View File
@@ -67,7 +67,7 @@ from sqlalchemy.orm.mapper import Mapper
from sqlalchemy.schema import UniqueConstraint
from sqlalchemy.sql import column, ColumnElement, literal_column, table
from sqlalchemy.sql.elements import ColumnClause, TextClause
from sqlalchemy.sql.expression import Label
from sqlalchemy.sql.expression import Label, TextAsFrom
from sqlalchemy.sql.selectable import Alias, TableClause
from superset import app, db, is_feature_enabled, security_manager
@@ -104,7 +104,7 @@ from superset.models.helpers import (
QueryResult,
)
from superset.models.slice import Slice
from superset.sql_parse import Table
from superset.sql_parse import ParsedQuery, Table
from superset.superset_typing import (
AdhocColumn,
AdhocMetric,
@@ -921,7 +921,7 @@ class TableColumn(AuditMixinNullable, ImportExportMixin, CertificationMixin, Mod
@property
def database(self) -> Database:
return self.table.database if self.table else self._database
return self.table.database if self.table else self._database # type: ignore
@property
def db_engine_spec(self) -> builtins.type[BaseEngineSpec]:
@@ -1469,13 +1469,34 @@ class SqlaTable(
return tbl
def get_from_clause(
self,
template_processor: BaseTemplateProcessor | None = None,
self, template_processor: BaseTemplateProcessor | None = None
) -> tuple[TableClause | Alias, str | None]:
"""
Return where to select the columns and metrics from. Either a physical table
or a virtual table with it's own subquery. If the FROM is referencing a
CTE, the CTE is returned as the second value in the return tuple.
"""
if not self.is_virtual:
return self.get_sqla_table(), None
return super().get_from_clause(template_processor)
from_sql = self.get_rendered_sql(template_processor) + "\n"
parsed_query = ParsedQuery(from_sql, engine=self.db_engine_spec.engine)
if not (
parsed_query.is_unknown()
or self.db_engine_spec.is_readonly_query(parsed_query)
):
raise QueryObjectValidationError(
_("Virtual dataset query must be read-only")
)
cte = self.db_engine_spec.get_cte_query(from_sql)
from_clause = (
table(self.db_engine_spec.cte_alias)
if cte
else TextAsFrom(self.text(from_sql), []).alias(VIRTUAL_TABLE_ALIAS)
)
return from_clause, cte
def adhoc_metric_to_sqla(
self,
+23 -10
View File
@@ -63,7 +63,7 @@ from superset.constants import QUERY_CANCEL_KEY, TimeGrain as TimeGrainConstants
from superset.databases.utils import get_table_metadata, make_url_safe
from superset.errors import ErrorLevel, SupersetError, SupersetErrorType
from superset.exceptions import DisallowedSQLFunction, OAuth2Error, OAuth2RedirectError
from superset.sql.parse import BaseSQLStatement, SQLScript, Table
from superset.sql.parse import SQLScript, Table
from superset.sql_parse import ParsedQuery
from superset.superset_typing import (
OAuth2ClientConfig,
@@ -1737,19 +1737,18 @@ class BaseEngineSpec: # pylint: disable=too-many-public-methods
)
@classmethod
def process_statement(
cls,
statement: BaseSQLStatement[Any],
database: Database,
) -> str:
def process_statement(cls, statement: str, database: Database) -> str:
"""
Process a SQL statement by mutating it.
Process a SQL statement by stripping and mutating it.
:param statement: A single SQL statement
:param database: Database instance
:return: Dictionary with different costs
"""
return database.mutate_sql_based_on_config(str(statement), is_split=True)
parsed_query = ParsedQuery(statement, engine=cls.engine)
sql = parsed_query.stripped()
return database.mutate_sql_based_on_config(sql, is_split=True)
@classmethod
def estimate_query_cost( # pylint: disable=too-many-arguments
@@ -1774,7 +1773,8 @@ class BaseEngineSpec: # pylint: disable=too-many-public-methods
"Database does not support cost estimation"
)
parsed_script = SQLScript(sql, engine=cls.engine)
parsed_query = sql_parse.ParsedQuery(sql, engine=cls.engine)
statements = parsed_query.get_statements()
with database.get_raw_connection(
catalog=catalog,
@@ -1788,7 +1788,7 @@ class BaseEngineSpec: # pylint: disable=too-many-public-methods
cls.process_statement(statement, database),
cursor,
)
for statement in parsed_script.statements
for statement in statements
]
@classmethod
@@ -2056,6 +2056,15 @@ class BaseEngineSpec: # pylint: disable=too-many-public-methods
logger.error(ex, exc_info=True)
raise
@classmethod
def is_readonly_query(cls, parsed_query: ParsedQuery) -> bool:
"""Pessimistic readonly, 100% sure statement won't mutate anything"""
return (
parsed_query.is_select()
or parsed_query.is_explain()
or parsed_query.is_show()
)
@classmethod
def is_select_query(cls, parsed_query: ParsedQuery) -> bool:
"""
@@ -2169,6 +2178,10 @@ class BaseEngineSpec: # pylint: disable=too-many-public-methods
return False
@classmethod
def parse_sql(cls, sql: str) -> list[str]:
return [str(s).strip(" ;") for s in sqlparse.parse(sql)]
@classmethod
def get_impersonation_key(cls, user: User | None) -> Any:
"""
+4 -3
View File
@@ -36,6 +36,7 @@ from sqlalchemy.engine.reflection import Inspector
from sqlalchemy.engine.url import URL
from sqlalchemy.sql import sqltypes
from superset import sql_parse
from superset.constants import TimeGrain
from superset.databases.schemas import encrypted_field_properties, EncryptedString
from superset.databases.utils import make_url_safe
@@ -43,7 +44,6 @@ from superset.db_engine_specs.base import BaseEngineSpec, BasicPropertiesType
from superset.db_engine_specs.exceptions import SupersetDBAPIConnectionError
from superset.errors import SupersetError, SupersetErrorType
from superset.exceptions import SupersetException
from superset.sql.parse import SQLScript
from superset.sql_parse import Table
from superset.superset_typing import ResultSetColumnType
from superset.utils import core as utils, json
@@ -449,7 +449,8 @@ class BigQueryEngineSpec(BaseEngineSpec): # pylint: disable=too-many-public-met
if not cls.get_allow_cost_estimate(extra):
raise SupersetException("Database does not support cost estimation")
parsed_script = SQLScript(sql, engine=cls.engine)
parsed_query = sql_parse.ParsedQuery(sql, engine=cls.engine)
statements = parsed_query.get_statements()
with cls.get_engine(
database,
@@ -462,7 +463,7 @@ class BigQueryEngineSpec(BaseEngineSpec): # pylint: disable=too-many-public-met
cls.process_statement(statement, database),
client,
)
for statement in parsed_script.statements
for statement in statements
]
@classmethod
+10 -7
View File
@@ -45,7 +45,7 @@ from superset.db_engine_specs.presto import PrestoEngineSpec
from superset.exceptions import SupersetException
from superset.extensions import cache_manager
from superset.models.sql_lab import Query
from superset.sql_parse import Table
from superset.sql_parse import ParsedQuery, Table
from superset.superset_typing import ResultSetColumnType
if TYPE_CHECKING:
@@ -79,12 +79,6 @@ def upload_to_s3(filename: str, upload_prefix: str, table: Table) -> str:
)
s3 = boto3.client("s3")
# The location is merely an S3 prefix and thus we first need to ensure that there is
# one and only one key associated with the table.
bucket = s3.Bucket(bucket_path)
bucket.objects.filter(Prefix=os.path.join(upload_prefix, table.table)).delete()
location = os.path.join("s3a://", bucket_path, upload_prefix, table.table)
s3.upload_file(
filename,
@@ -605,6 +599,15 @@ class HiveEngineSpec(PrestoEngineSpec):
# otherwise, return no function names to prevent errors
return []
@classmethod
def is_readonly_query(cls, parsed_query: ParsedQuery) -> bool:
"""Pessimistic readonly, 100% sure statement won't mutate anything"""
return (
super().is_readonly_query(parsed_query)
or parsed_query.is_set()
or parsed_query.is_show()
)
@classmethod
def has_implicit_cancel(cls) -> bool:
"""
+22
View File
@@ -104,6 +104,11 @@ class KustoSqlEngineSpec(BaseEngineSpec): # pylint: disable=abstract-method
return f"""CONVERT(DATETIME, '{datetime_formatted}', 126)"""
return None
@classmethod
def is_readonly_query(cls, parsed_query: ParsedQuery) -> bool:
"""Pessimistic readonly, 100% sure statement won't mutate anything"""
return parsed_query.sql.lower().startswith("select")
class KustoKqlEngineSpec(BaseEngineSpec): # pylint: disable=abstract-method
limit_method = LimitMethod.WRAP_SQL
@@ -153,6 +158,23 @@ class KustoKqlEngineSpec(BaseEngineSpec): # pylint: disable=abstract-method
return None
@classmethod
def is_readonly_query(cls, parsed_query: ParsedQuery) -> bool:
"""
Pessimistic readonly, 100% sure statement won't mutate anything.
"""
return KustoKqlEngineSpec.is_select_query(
parsed_query
) or parsed_query.sql.startswith(".show")
@classmethod
def is_select_query(cls, parsed_query: ParsedQuery) -> bool:
return not parsed_query.sql.startswith(".")
@classmethod
def parse_sql(cls, sql: str) -> list[str]:
"""
Kusto supports a single query statement, but it could include sub queries
and variables declared via let keyword.
"""
return [sql]
+6 -87
View File
@@ -21,19 +21,13 @@ import contextlib
import logging
import threading
import time
from tempfile import NamedTemporaryFile
from typing import Any, TYPE_CHECKING
import numpy as np
import pandas as pd
import pyarrow as pa
import requests
from flask import copy_current_request_context, ctx, current_app, Flask, g
from sqlalchemy import text
from sqlalchemy.engine.reflection import Inspector
from sqlalchemy.engine.url import URL
from sqlalchemy.exc import NoSuchTableError
from trino.exceptions import HttpError
from superset import db
from superset.constants import QUERY_CANCEL_KEY, QUERY_EARLY_CANCEL_KEY, USER_AGENT
@@ -45,9 +39,7 @@ from superset.db_engine_specs.exceptions import (
SupersetDBAPIOperationalError,
SupersetDBAPIProgrammingError,
)
from superset.db_engine_specs.hive import upload_to_s3
from superset.db_engine_specs.presto import PrestoBaseEngineSpec
from superset.exceptions import SupersetException
from superset.models.sql_lab import Query
from superset.sql_parse import Table
from superset.superset_typing import ResultSetColumnType
@@ -61,6 +53,12 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__)
try:
# since trino is an optional dependency, we need to handle the ImportError
from trino.exceptions import HttpError
except ImportError:
HttpError = Exception
class CustomTrinoAuthErrorMeta(type):
def __instancecheck__(cls, instance: object) -> bool:
@@ -218,8 +216,6 @@ class TrinoEngineSpec(PrestoBaseEngineSpec):
if tracking_url := cls.get_tracking_url(cursor):
query.tracking_url = tracking_url
db.session.commit()
# if query cancelation was requested prior to the handle_cursor call, but
# the query was still executed, trigger the actual query cancelation now
if query.extra.get(QUERY_EARLY_CANCEL_KEY):
@@ -512,80 +508,3 @@ class TrinoEngineSpec(PrestoBaseEngineSpec):
return super().get_indexes(database, inspector, table)
except NoSuchTableError:
return []
@classmethod
def df_to_sql(
cls,
database: Database,
table: Table,
df: pd.DataFrame,
to_sql_kwargs: dict[str, Any],
) -> None:
"""
Upload data from a Pandas DataFrame to a database.
The data is stored via the binary Parquet format which is both less problematic
and more performant than a text file.
Note this method does not create metadata for the table.
:param database: The database to upload the data to
:param table: The table to upload the data to
:param df: The Pandas Dataframe with data to be uploaded
:param to_sql_kwargs: The `pandas.DataFrame.to_sql` keyword arguments
:see: superset.db_engine_specs.HiveEngineSpec.df_to_sql
"""
if to_sql_kwargs["if_exists"] == "append":
raise SupersetException("Append operation not currently supported")
if to_sql_kwargs["if_exists"] == "fail":
if database.has_table_by_name(table.table, table.schema):
raise SupersetException("Table already exists")
elif to_sql_kwargs["if_exists"] == "replace":
with cls.get_engine(database) as engine:
engine.execute(f"DROP TABLE IF EXISTS {str(table)}")
def _get_trino_type(dtype: np.dtype[Any]) -> str:
return {
np.dtype("bool"): "BOOLEAN",
np.dtype("float64"): "DOUBLE",
np.dtype("int64"): "BIGINT",
np.dtype("object"): "VARCHAR",
}.get(dtype, "VARCHAR")
with NamedTemporaryFile(
dir=current_app.config["UPLOAD_FOLDER"],
suffix=".parquet",
) as file:
pa.parquet.write_table(pa.Table.from_pandas(df), where=file.name)
with cls.get_engine(database) as engine:
engine.execute(
# pylint: disable=consider-using-f-string
text(
"""
CREATE TABLE {table} ({schema})
WITH (
format = 'PARQUET',
external_location = '{location}'
)
""".format(
location=upload_to_s3(
filename=file.name,
upload_prefix=current_app.config[
"CSV_TO_HIVE_UPLOAD_DIRECTORY_FUNC"
](
database,
g.user,
table.schema,
),
table=table,
),
schema=", ".join(
f'"{name}" {_get_trino_type(dtype)}'
for name, dtype in df.dtypes.items()
),
table=str(table),
),
),
)
+1 -1
View File
@@ -85,7 +85,7 @@ class UIManifestProcessor:
return {
"js_manifest": lambda bundle: get_files(bundle, "js"),
"css_manifest": lambda bundle: get_files(bundle, "css"),
"assets_prefix": (
"assets_prefix": ( # type: ignore
self.app.config["STATIC_ASSETS_PREFIX"] if self.app else ""
),
}
+3 -1
View File
@@ -334,7 +334,9 @@ class SupersetShillelaghAdapter(Adapter):
primary_keys = [
column for column in list(self._table.primary_key) if column.primary_key
]
if len(primary_keys) == 1 and primary_keys[0].type.python_type == int:
if len(primary_keys) == 1 and isinstance(
primary_keys[0].type.python_type, type(int)
):
self._rowid = primary_keys[0].name
self.columns = {
+1 -2
View File
@@ -352,8 +352,7 @@ class ExtraCache:
for flt in form_data.get("adhoc_filters", []):
val: Union[Any, list[Any]] = flt.get("comparator")
op: str = flt["operator"].upper() if flt.get("operator") else None
# fltOpName: str = flt.get("filterOptionName")
op: str = flt["operator"].upper() if flt.get("operator") else None # type: ignore
if (
flt.get("expressionType") == "SIMPLE"
and flt.get("clause") == "WHERE"
+5 -7
View File
@@ -74,7 +74,6 @@ from superset.extensions import (
)
from superset.models.helpers import AuditMixinNullable, ImportExportMixin
from superset.result_set import SupersetResultSet
from superset.sql.parse import SQLScript
from superset.sql_parse import Table
from superset.superset_typing import OAuth2ClientConfig, ResultSetColumnType
from superset.utils import cache as cache_util, core as utils, json
@@ -675,7 +674,7 @@ class Database(Model, AuditMixinNullable, ImportExportMixin): # pylint: disable
schema: str | None = None,
mutator: Callable[[pd.DataFrame], None] | None = None,
) -> pd.DataFrame:
parsed_script = SQLScript(sql, engine=self.db_engine_spec.engine)
sqls = self.db_engine_spec.parse_sql(sql)
with self.get_sqla_engine(catalog=catalog, schema=schema) as engine:
engine_url = engine.url
@@ -692,9 +691,8 @@ class Database(Model, AuditMixinNullable, ImportExportMixin): # pylint: disable
with self.get_raw_connection(catalog=catalog, schema=schema) as conn:
cursor = conn.cursor()
df = None
for i, statement in enumerate(parsed_script.statements):
# pylint: disable=protected-access
sql_ = self.mutate_sql_based_on_config(statement._sql, is_split=True)
for i, sql_ in enumerate(sqls):
sql_ = self.mutate_sql_based_on_config(sql_, is_split=True)
_log_query(sql_)
with event_logger.log_context(
action="execute_sql",
@@ -702,7 +700,7 @@ class Database(Model, AuditMixinNullable, ImportExportMixin): # pylint: disable
object_ref=__name__,
):
self.db_engine_spec.execute(cursor, sql_, self)
if i < len(parsed_script.statements) - 1:
if i < len(sqls) - 1:
# If it's not the last, we don't keep the results
cursor.fetchall()
else:
@@ -1057,7 +1055,7 @@ class Database(Model, AuditMixinNullable, ImportExportMixin): # pylint: disable
)
def get_perm(self) -> str:
return self.perm # type: ignore
return self.perm
def has_table(self, table: Table) -> bool:
with self.get_sqla_engine(catalog=table.catalog, schema=table.schema) as engine:
+7 -5
View File
@@ -72,6 +72,7 @@ from superset.sql.parse import SQLScript
from superset.sql_parse import (
has_table_query,
insert_rls_in_predicate,
ParsedQuery,
sanitize_clause,
)
from superset.superset_typing import (
@@ -1038,9 +1039,6 @@ class ExploreMixin: # pylint: disable=too-many-public-methods
"""
Render sql with template engine (Jinja).
"""
if not self.sql:
return ""
sql = self.sql.strip("\t\r\n; ")
if template_processor:
try:
@@ -1074,9 +1072,13 @@ class ExploreMixin: # pylint: disable=too-many-public-methods
or a virtual table with it's own subquery. If the FROM is referencing a
CTE, the CTE is returned as the second value in the return tuple.
"""
from_sql = self.get_rendered_sql(template_processor) + "\n"
parsed_script = SQLScript(from_sql, engine=self.db_engine_spec.engine)
if parsed_script.has_mutation():
parsed_query = ParsedQuery(from_sql, engine=self.db_engine_spec.engine)
if not (
parsed_query.is_unknown()
or self.db_engine_spec.is_readonly_query(parsed_query)
):
raise QueryObjectValidationError(
_("Virtual dataset query must be read-only")
)
+1 -1
View File
@@ -477,7 +477,7 @@ class SupersetSecurityManager( # pylint: disable=too-many-public-methods
return (
self.can_access_all_datasources()
or self.can_access_all_databases()
or self.can_access("database_access", database.perm) # type: ignore
or self.can_access("database_access", database.perm)
)
def can_access_catalog(self, database: "Database", catalog: str) -> bool:
+41 -185
View File
@@ -20,11 +20,10 @@ from __future__ import annotations
import enum
import logging
import re
import string
import urllib.parse
from collections.abc import Iterable
from dataclasses import dataclass
from typing import Any, Generic, Iterator, TypeVar
from typing import Any, Generic, TypeVar
import sqlglot
import sqlparse
@@ -227,12 +226,6 @@ class BaseSQLStatement(Generic[InternalRepresentation]):
"""
raise NotImplementedError()
def is_select(self) -> bool:
"""
Check if the statement is a `SELECT` statement.
"""
raise NotImplementedError()
def __str__(self) -> str:
return self.format()
@@ -389,12 +382,6 @@ class SQLStatement(BaseSQLStatement[exp.Expression]):
return False
def is_select(self) -> bool:
"""
Check if the statement is a `SELECT` statement.
"""
return isinstance(self._parsed, exp.Select)
def format(self, comments: bool = True) -> str:
"""
Pretty-format the SQL statement.
@@ -444,115 +431,60 @@ class SQLStatement(BaseSQLStatement[exp.Expression]):
}
class KQLTokenizeState(enum.Enum):
class KQLSplitState(enum.Enum):
"""
State machine for tokenizing a KQL script.
State machine for splitting a KQL script.
The state machine keeps track of whether we're inside a string or not, so we
don't split the script in a semi-colon that's part of a string.
"""
OUTSIDE = enum.auto()
OUTSIDE_STRING = enum.auto()
INSIDE_SINGLE_QUOTED_STRING = enum.auto()
INSIDE_DOUBLE_QUOTED_STRING = enum.auto()
INSIDE_MULTILINE_STRING = enum.auto()
INSIDE_SINGLE_QUOTED_IDENTIFIER = enum.auto()
INSIDE_DOUBLE_QUOTED_IDENTIFIER = enum.auto()
def tokenize_kql(kql: str) -> Iterator[str]:
"""
Tokenize a KQL script.
"""
valid_identifier_chars = set(string.ascii_letters + string.digits + "_")
valid_quoted_identifier_chars = valid_identifier_chars | set(" .-")
script = kql if kql.endswith(";") else kql + ";"
cursor = 0
while cursor < len(script):
rest = script[cursor:]
# quoted identifiers
if rest[:2] in {"['", '["'}:
match = "']" if rest[:2] == "['" else '"]'
if match not in rest[2:]:
raise SupersetParseError(
script,
"kustokql",
message="Unclosed quoted identifier",
)
token = rest[: rest.index(match, 2) + 2]
if any(char not in valid_quoted_identifier_chars for char in token[2:-2]):
raise SupersetParseError(
script,
"kustokql",
message="Invalid quoted identifier",
)
yield token
cursor += len(token)
# multi-line strings
elif rest[:3] == "```":
if "```" not in rest[3:]:
raise SupersetParseError(
script,
"kustokql",
message="Unclosed multi-line string",
)
token = rest[: rest.index("```", 3) + 3]
yield token
cursor += len(token)
# single-quoted strings
elif rest[0] in {'"', "'"}:
match = rest[0]
# find first unescaped quote
start = 1
while True:
if match not in rest[start:]:
raise SupersetParseError(
script,
"kustokql",
message="Unclosed string",
)
index = rest.index(match, start)
if rest[index - 1] != "\\":
break
start = index + 1
token = rest[: index + 1]
yield token
cursor += len(token)
# identifiers and keywords
else:
for i, char in enumerate(rest):
if char not in valid_identifier_chars:
if i > 0:
yield rest[:i]
yield char
cursor += i + 1
break
def split_kql(kql: str) -> list[str]:
"""
Custom function for splitting KQL statements.
"""
statements: list[str] = []
statement: list[str] = []
for token in tokenize_kql(kql):
if token == ";":
statements.append("".join(statement))
statement = []
else:
statement.append(token)
statements = []
state = KQLSplitState.OUTSIDE_STRING
statement_start = 0
script = kql if kql.endswith(";") else kql + ";"
for i, character in enumerate(script):
if state == KQLSplitState.OUTSIDE_STRING:
if character == ";":
statements.append(script[statement_start:i])
statement_start = i + 1
elif character == "'":
state = KQLSplitState.INSIDE_SINGLE_QUOTED_STRING
elif character == '"':
state = KQLSplitState.INSIDE_DOUBLE_QUOTED_STRING
elif character == "`" and script[i - 2 : i] == "``":
state = KQLSplitState.INSIDE_MULTILINE_STRING
elif (
state == KQLSplitState.INSIDE_SINGLE_QUOTED_STRING
and character == "'"
and script[i - 1] != "\\"
):
state = KQLSplitState.OUTSIDE_STRING
elif (
state == KQLSplitState.INSIDE_DOUBLE_QUOTED_STRING
and character == '"'
and script[i - 1] != "\\"
):
state = KQLSplitState.OUTSIDE_STRING
elif (
state == KQLSplitState.INSIDE_MULTILINE_STRING
and character == "`"
and script[i - 2 : i] == "``"
):
state = KQLSplitState.OUTSIDE_STRING
return statements
@@ -574,14 +506,6 @@ class KustoKQLStatement(BaseSQLStatement[str]):
details about it.
"""
def __init__(
self,
statement: str,
engine: str = "kustokql",
ast: str | None = None,
):
super().__init__(statement, engine, ast)
@classmethod
def split_script(
cls,
@@ -664,56 +588,6 @@ class KustoKQLStatement(BaseSQLStatement[str]):
"""
return self._parsed.startswith(".") and not self._parsed.startswith(".show")
def is_select(self) -> bool:
"""
Check if the statement is a `SELECT` statement.
"""
if not self._parsed or self.is_mutating():
return False
# strip comments
kql = "\n".join(
line
for line in self._parsed.split("\n")
if not line.strip().startswith("//")
).strip()
first_token = next(tokenize_kql(kql), None)
if not first_token:
return False
return first_token == "|" or self._is_identifier(first_token)
@staticmethod
def _is_identifier(identifier: str) -> bool:
"""
Validates if a given string is a valid KQL identifier.
From the documentation:
Identifiers are case-sensitive. Database names are case-insensitive, and
therefore an exception to this rule.
Identifiers must be between 1 and 1024 characters long.
Identifiers may contain letters, digits, and underscores (_).
Identifiers may contain certain special characters: spaces, dots (.), and
dashes (-). For information on how to reference identifiers with special
characters, see Reference identifiers in queries.
"""
valid_chars = set(string.ascii_letters + string.digits + "_")
# Identifiers names that (1) include special character, (2) are language
# keywords, or (3) are literals must be enclosed using [' and '] or [" and "].
if (identifier.startswith("['") and identifier.endswith("']")) or (
identifier.startswith('["') and identifier.endswith('"]')
):
identifier = identifier[2:-2]
valid_chars.update(" .-")
return 1 <= len(identifier) <= 1024 and all(
char in valid_chars for char in identifier
)
class SQLScript:
"""
@@ -768,24 +642,6 @@ class SQLScript:
"""
return any(statement.is_mutating() for statement in self.statements)
def is_valid_ctas(self) -> bool:
"""
Check if the script contains a valid CTAS statement.
CTAS (`CREATE TABLE AS SELECT`) can only be run with scripts where the last
statement is a `SELECT`.
"""
return self.statements[-1].is_select()
def is_valid_cvas(self) -> bool:
"""
Check if the script contains a valid CVAS statement.
CVAS (`CREATE VIEW AS SELECT`) can only be run with scripts with a single
`SELECT` statement.
"""
return len(self.statements) == 1 and self.statements[0].is_select()
def extract_tables_from_statement(
statement: exp.Expression,
@@ -794,7 +650,7 @@ def extract_tables_from_statement(
"""
Extract all table references in a single statement.
Please note that this is not trivial; consider the following queries:
Please not that this is not trivial; consider the following queries:
DESCRIBE some_table;
SHOW PARTITIONS FROM some_table;
+11 -12
View File
@@ -20,11 +20,11 @@ from __future__ import annotations
import logging
import time
from contextlib import closing
from typing import Any, cast
from typing import Any
from superset import app
from superset.models.core import Database
from superset.sql.parse import SQLScript, SQLStatement
from superset.sql_parse import ParsedQuery
from superset.sql_validators.base import BaseSQLValidator, SQLValidationAnnotation
from superset.utils.core import QuerySource
@@ -46,15 +46,17 @@ class PrestoDBSQLValidator(BaseSQLValidator):
@classmethod
def validate_statement(
cls,
statement: SQLStatement,
statement: str,
database: Database,
cursor: Any,
) -> SQLValidationAnnotation | None:
# pylint: disable=too-many-locals
db_engine_spec = database.db_engine_spec
parsed_query = ParsedQuery(statement, engine=db_engine_spec.engine)
sql = parsed_query.stripped()
# Hook to allow environment-specific mutation (usually comments) to the SQL
sql = database.mutate_sql_based_on_config(str(statement))
sql = database.mutate_sql_based_on_config(sql)
# Transform the final statement to an explain call before sending it on
# to presto to validate
@@ -153,9 +155,10 @@ class PrestoDBSQLValidator(BaseSQLValidator):
For example, "SELECT 1 FROM default.mytable" becomes "EXPLAIN (TYPE
VALIDATE) SELECT 1 FROM default.mytable.
"""
parsed_script = SQLScript(sql, engine=database.db_engine_spec.engine)
parsed_query = ParsedQuery(sql, engine=database.db_engine_spec.engine)
statements = parsed_query.get_statements()
logger.info("Validating %i statement(s)", len(parsed_script.statements))
logger.info("Validating %i statement(s)", len(statements))
# todo(hughhh): update this to use new database.get_raw_connection()
# this function keeps stalling CI
with database.get_sqla_engine(
@@ -168,12 +171,8 @@ class PrestoDBSQLValidator(BaseSQLValidator):
annotations: list[SQLValidationAnnotation] = []
with closing(engine.raw_connection()) as conn:
cursor = conn.cursor()
for statement in parsed_script.statements:
annotation = cls.validate_statement(
cast(SQLStatement, statement),
database,
cursor,
)
for statement in parsed_query.get_statements():
annotation = cls.validate_statement(statement, database, cursor)
if annotation:
annotations.append(annotation)
logger.debug("Validation found %i error(s)", len(annotations))
+6 -2
View File
@@ -26,6 +26,7 @@ from jinja2.meta import find_undeclared_variables
from superset import is_feature_enabled
from superset.commands.sql_lab.execute import SqlQueryRender
from superset.errors import SupersetErrorType
from superset.sql_parse import ParsedQuery
from superset.sqllab.exceptions import SqlLabException
from superset.utils import core as utils
@@ -57,9 +58,12 @@ class SqlQueryRenderImpl(SqlQueryRender):
database=query_model.database, query=query_model
)
parsed_query = ParsedQuery(
query_model.sql,
engine=query_model.database.db_engine_spec.engine,
)
rendered_query = sql_template_processor.process_template(
query_model.sql.strip().strip(";"),
**execution_context.template_params,
parsed_query.stripped(), **execution_context.template_params
)
self._validate(execution_context, rendered_query, sql_template_processor)
return rendered_query
+1 -1
View File
@@ -174,7 +174,7 @@ def load_explore_json_into_cache( # pylint: disable=too-many-locals
errors = ex.errors
else:
error = ex.message if hasattr(ex, "message") else str(ex)
errors = [error]
errors = [error] # type: ignore
async_query_manager.update_job(
job_metadata, async_query_manager.STATUS_ERROR, errors=errors
+2 -3
View File
@@ -919,14 +919,13 @@ def merge_extra_form_data(form_data: dict[str, Any]) -> None:
"adhoc_filters", []
)
adhoc_filters.extend(
{"isExtra": True, **adhoc_filter} # type: ignore
for adhoc_filter in append_adhoc_filters
{"isExtra": True, **adhoc_filter} for adhoc_filter in append_adhoc_filters
)
if append_filters:
for key, value in form_data.items():
if re.match("adhoc_filter.*", key):
value.extend(
simple_filter_to_adhoc({"isExtra": True, **fltr}) # type: ignore
simple_filter_to_adhoc({"isExtra": True, **fltr})
for fltr in append_filters
if fltr
)
+21
View File
@@ -22,9 +22,30 @@ import pandas as pd
from superset.utils.core import GenericDataType
def quote_formulas(df: pd.DataFrame) -> pd.DataFrame:
"""
Make sure to quote any formulas for security reasons.
"""
formula_prefixes = {"=", "+", "-", "@"}
for col in df.select_dtypes(include="object").columns:
df[col] = df[col].apply(
lambda x: (
f"'{x}"
if isinstance(x, str) and len(x) and x[0] in formula_prefixes
else x
)
)
return df
def df_to_excel(df: pd.DataFrame, **kwargs: Any) -> Any:
output = io.BytesIO()
# make sure formulas are quoted, to prevent malicious injections
df = quote_formulas(df)
# pylint: disable=abstract-class-instantiated
with pd.ExcelWriter(output, engine="xlsxwriter") as writer:
df.to_excel(writer, **kwargs)
+1 -1
View File
@@ -2458,7 +2458,7 @@ class RoseViz(NVD3TimeSeriesViz):
result[timestamp].append(
{
"key": key,
"value": value,
"value": value, # type: ignore
"name": ", ".join(key) if isinstance(key, list) else key,
"time": val["x"],
}
+5 -2
View File
@@ -69,7 +69,7 @@ def login_as(test_client: FlaskClient[Any]):
@pytest.fixture
def login_as_admin(login_as: Callable[..., None]):
yield login_as("admin")
yield login_as("admin") # type: ignore
@pytest.fixture
@@ -122,7 +122,10 @@ def setup_sample_data() -> Any:
# relying on `tests.integration_tests.test_app.app` leveraging an `app` fixture
# which is purposely scoped to the function level to ensure tests remain idempotent.
with app.app_context():
setup_presto_if_needed()
try:
setup_presto_if_needed()
except Exception:
pass
from superset.examples.css_templates import load_css_templates
@@ -30,7 +30,7 @@ from superset.db_engine_specs.base import (
from superset.db_engine_specs.mysql import MySQLEngineSpec
from superset.db_engine_specs.sqlite import SqliteEngineSpec
from superset.errors import ErrorLevel, SupersetError, SupersetErrorType
from superset.sql_parse import Table
from superset.sql_parse import ParsedQuery, Table
from superset.utils.database import get_example_database
from tests.integration_tests.db_engine_specs.base_tests import TestDbEngineSpec
from tests.integration_tests.test_app import app
@@ -310,6 +310,20 @@ class TestDbEngineSpecs(TestDbEngineSpec):
)
def test_is_readonly():
def is_readonly(sql: str) -> bool:
return BaseEngineSpec.is_readonly_query(ParsedQuery(sql))
assert is_readonly("SHOW LOCKS test EXTENDED")
assert not is_readonly("SET hivevar:desc='Legislators'")
assert not is_readonly("UPDATE t1 SET col1 = NULL")
assert is_readonly("EXPLAIN SELECT 1")
assert is_readonly("SELECT 1")
assert is_readonly("WITH (SELECT 1) bla SELECT * from bla")
assert is_readonly("SHOW CATALOGS")
assert is_readonly("SHOW TABLES")
def test_time_grain_denylist():
config = app.config.copy()
app.config["TIME_GRAIN_DENYLIST"] = ["PT1M", "SQLITE_NONEXISTENT_GRAIN"]
@@ -16,6 +16,8 @@
# under the License.
# isort:skip_file
from unittest import mock
import unittest
from .base_tests import SupersetTestCase
import pytest
import pandas as pd
@@ -23,7 +25,7 @@ from sqlalchemy.sql import select
from superset.db_engine_specs.hive import HiveEngineSpec, upload_to_s3
from superset.exceptions import SupersetException
from superset.sql_parse import Table
from superset.sql_parse import ParsedQuery, Table
from tests.integration_tests.test_app import app
@@ -154,6 +156,9 @@ def test_df_to_sql_if_exists_fail(mock_g):
@mock.patch("superset.db_engine_specs.hive.g", spec={})
@unittest.skipUnless(
SupersetTestCase.is_module_installed("thrift"), "thrift not installed"
)
def test_df_to_sql_if_exists_fail_with_schema(mock_g):
mock_g.user = True
mock_database = mock.MagicMock()
@@ -222,6 +227,19 @@ def test_df_to_sql_if_exists_replace_with_schema(mock_upload_to_s3, mock_g):
app.config = config
def test_is_readonly():
def is_readonly(sql: str) -> bool:
return HiveEngineSpec.is_readonly_query(ParsedQuery(sql))
assert not is_readonly("UPDATE t1 SET col1 = NULL")
assert not is_readonly("INSERT OVERWRITE TABLE tabB SELECT a.Age FROM TableA")
assert is_readonly("SHOW LOCKS test EXTENDED")
assert is_readonly("SET hivevar:desc='Legislators'")
assert is_readonly("EXPLAIN SELECT 1")
assert is_readonly("SELECT 1")
assert is_readonly("WITH (SELECT 1) bla SELECT * from bla")
@pytest.mark.parametrize(
"schema,upload_prefix",
[("foo", "EXTERNAL_HIVE_TABLES/1/foo/"), (None, "EXTERNAL_HIVE_TABLES/1/")],
@@ -277,6 +295,9 @@ def test_upload_to_s3_success(client):
app.config = config
@unittest.skipUnless(
SupersetTestCase.is_module_installed("thrift"), "thrift not installed"
)
def test_fetch_data_query_error():
from TCLIService import ttypes
@@ -288,6 +309,9 @@ def test_fetch_data_query_error():
HiveEngineSpec.fetch_data(cursor)
@unittest.skipUnless(
SupersetTestCase.is_module_installed("thrift"), "thrift not installed"
)
@mock.patch("superset.db_engine_specs.base.BaseEngineSpec.fetch_data")
def test_fetch_data_programming_error(fetch_data_mock):
from pyhive.exc import ProgrammingError
@@ -297,6 +321,9 @@ def test_fetch_data_programming_error(fetch_data_mock):
assert HiveEngineSpec.fetch_data(cursor) == []
@unittest.skipUnless(
SupersetTestCase.is_module_installed("thrift"), "thrift not installed"
)
@mock.patch("superset.db_engine_specs.base.BaseEngineSpec.fetch_data")
def test_fetch_data_success(fetch_data_mock):
return_value = ["a", "b"]
@@ -25,7 +25,7 @@ from sqlalchemy.sql import select
from superset.db_engine_specs.presto import PrestoEngineSpec
from superset.errors import ErrorLevel, SupersetError, SupersetErrorType
from superset.sql_parse import Table
from superset.sql_parse import ParsedQuery, Table
from superset.utils.database import get_example_database
from tests.integration_tests.db_engine_specs.base_tests import TestDbEngineSpec
@@ -1172,6 +1172,19 @@ class TestPrestoDbEngineSpec(TestDbEngineSpec):
]
def test_is_readonly():
def is_readonly(sql: str) -> bool:
return PrestoEngineSpec.is_readonly_query(ParsedQuery(sql))
assert not is_readonly("SET hivevar:desc='Legislators'")
assert not is_readonly("UPDATE t1 SET col1 = NULL")
assert not is_readonly("INSERT OVERWRITE TABLE tabB SELECT a.Age FROM TableA")
assert is_readonly("SHOW LOCKS test EXTENDED")
assert is_readonly("EXPLAIN SELECT 1")
assert is_readonly("SELECT 1")
assert is_readonly("WITH (SELECT 1) bla SELECT * from bla")
def test_get_catalog_names(app_context: AppContext) -> None:
"""
Test the ``get_catalog_names`` method.
@@ -1,115 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
from unittest import mock
import pandas as pd
import pytest
from superset.db_engine_specs.trino import TrinoEngineSpec
from superset.exceptions import SupersetException
from superset.sql_parse import Table
from tests.integration_tests.test_app import app
def test_df_to_csv() -> None:
with pytest.raises(SupersetException):
TrinoEngineSpec.df_to_sql(
mock.MagicMock(),
Table("foobar"),
pd.DataFrame(),
{"if_exists": "append"},
)
@mock.patch("superset.db_engine_specs.trino.g", spec={})
def test_df_to_sql_if_exists_fail(mock_g):
mock_g.user = True
mock_database = mock.MagicMock()
mock_database.get_df.return_value.empty = False
with pytest.raises(SupersetException, match="Table already exists"):
TrinoEngineSpec.df_to_sql(
mock_database, Table("foobar"), pd.DataFrame(), {"if_exists": "fail"}
)
@mock.patch("superset.db_engine_specs.trino.g", spec={})
def test_df_to_sql_if_exists_fail_with_schema(mock_g):
mock_g.user = True
mock_database = mock.MagicMock()
mock_database.get_df.return_value.empty = False
with pytest.raises(SupersetException, match="Table already exists"):
TrinoEngineSpec.df_to_sql(
mock_database,
Table(table="foobar", schema="schema"),
pd.DataFrame(),
{"if_exists": "fail"},
)
@mock.patch("superset.db_engine_specs.trino.g", spec={})
@mock.patch("superset.db_engine_specs.trino.upload_to_s3")
def test_df_to_sql_if_exists_replace(mock_upload_to_s3, mock_g):
config = app.config.copy()
app.config["CSV_TO_HIVE_UPLOAD_DIRECTORY_FUNC"]: lambda *args: "" # noqa: F722
mock_upload_to_s3.return_value = "mock-location"
mock_g.user = True
mock_database = mock.MagicMock()
mock_database.get_df.return_value.empty = False
mock_execute = mock.MagicMock(return_value=True)
mock_database.get_sqla_engine.return_value.__enter__.return_value.execute = (
mock_execute
)
table_name = "foobar"
with app.app_context():
TrinoEngineSpec.df_to_sql(
mock_database,
Table(table=table_name),
pd.DataFrame(),
{"if_exists": "replace", "header": 1, "na_values": "mock", "sep": "mock"},
)
mock_execute.assert_any_call(f"DROP TABLE IF EXISTS {table_name}")
app.config = config
@mock.patch("superset.db_engine_specs.trino.g", spec={})
@mock.patch("superset.db_engine_specs.trino.upload_to_s3")
def test_df_to_sql_if_exists_replace_with_schema(mock_upload_to_s3, mock_g):
config = app.config.copy()
app.config["CSV_TO_HIVE_UPLOAD_DIRECTORY_FUNC"]: lambda *args: "" # noqa: F722
mock_upload_to_s3.return_value = "mock-location"
mock_g.user = True
mock_database = mock.MagicMock()
mock_database.get_df.return_value.empty = False
mock_execute = mock.MagicMock(return_value=True)
mock_database.get_sqla_engine.return_value.__enter__.return_value.execute = (
mock_execute
)
table_name = "foobar"
schema = "schema"
with app.app_context():
TrinoEngineSpec.df_to_sql(
mock_database,
Table(table=table_name, schema=schema),
pd.DataFrame(),
{"if_exists": "replace", "header": 1, "na_values": "mock", "sep": "mock"},
)
mock_execute.assert_any_call(f"DROP TABLE IF EXISTS {schema}.{table_name}")
app.config = config
+18 -3
View File
@@ -54,6 +54,9 @@ class TestDatabaseModel(SupersetTestCase):
@unittest.skipUnless(
SupersetTestCase.is_module_installed("requests"), "requests not installed"
)
@unittest.skipUnless(
SupersetTestCase.is_module_installed("pyhive"), "pyhive not installed"
)
def test_database_schema_presto(self):
sqlalchemy_uri = "presto://presto.airbnb.io:8080/hive/default"
model = Database(database_name="test_database", sqlalchemy_uri=sqlalchemy_uri)
@@ -108,7 +111,7 @@ class TestDatabaseModel(SupersetTestCase):
assert "core_db" == db
@unittest.skipUnless(
SupersetTestCase.is_module_installed("MySQLdb"), "mysqlclient not installed"
SupersetTestCase.is_module_installed("mysqlclient"), "mysqlclient not installed"
)
def test_database_schema_mysql(self):
sqlalchemy_uri = "mysql://root@localhost/superset"
@@ -123,7 +126,7 @@ class TestDatabaseModel(SupersetTestCase):
assert "staging" == db
@unittest.skipUnless(
SupersetTestCase.is_module_installed("MySQLdb"), "mysqlclient not installed"
SupersetTestCase.is_module_installed("mysqlclient"), "mysqlclient not installed"
)
def test_database_impersonate_user(self):
uri = "mysql://root@localhost"
@@ -142,6 +145,9 @@ class TestDatabaseModel(SupersetTestCase):
assert example_user.username != username
@mock.patch("superset.models.core.create_engine")
@unittest.skipUnless(
SupersetTestCase.is_module_installed("pyhive"), "pyhive not installed"
)
def test_impersonate_user_presto(self, mocked_create_engine):
uri = "presto://localhost"
principal_user = security_manager.find_user(username="gamma")
@@ -190,7 +196,7 @@ class TestDatabaseModel(SupersetTestCase):
}
@unittest.skipUnless(
SupersetTestCase.is_module_installed("MySQLdb"), "mysqlclient not installed"
SupersetTestCase.is_module_installed("mysqlclient"), "mysqlclient not installed"
)
@mock.patch("superset.models.core.create_engine")
def test_adjust_engine_params_mysql(self, mocked_create_engine):
@@ -245,6 +251,12 @@ class TestDatabaseModel(SupersetTestCase):
assert call_args[1]["connect_args"]["user"] == "gamma"
@mock.patch("superset.models.core.create_engine")
@unittest.skipUnless(
SupersetTestCase.is_module_installed("pyhive"), "pyhive not installed"
)
@unittest.skipUnless(
SupersetTestCase.is_module_installed("thrift"), "thrift not installed"
)
def test_impersonate_user_hive(self, mocked_create_engine):
uri = "hive://localhost"
principal_user = security_manager.find_user(username="gamma")
@@ -293,6 +305,9 @@ class TestDatabaseModel(SupersetTestCase):
}
@pytest.mark.usefixtures("load_energy_table_with_slice")
@unittest.skipUnless(
SupersetTestCase.is_module_installed("pyhive"), "pyhive not installed"
)
def test_select_star(self):
db = get_example_database()
table_name = "energy_usage"
+1 -1
View File
@@ -1091,7 +1091,7 @@ def test__normalize_prequery_result_type(
columns_by_name,
)
assert type(normalized) == type(result)
assert isinstance(normalized, type(result))
if isinstance(normalized, TextClause):
assert str(normalized) == str(result)
+1 -1
View File
@@ -91,7 +91,7 @@ class TestBaseViz(SupersetTestCase):
datasource = self.get_datasource_mock()
test_viz = viz.BaseViz(datasource, form_data)
result = test_viz.get_df(query_obj)
assert type(result) == pd.DataFrame
assert isinstance(result, pd.DataFrame)
assert result.empty
def test_get_df_handles_dttm_col(self):
+1 -1
View File
@@ -143,7 +143,7 @@ def test_create_pickle_entry(
found_entry = (
db.session.query(KeyValueEntry).filter_by(id=created_entry.id).one()
)
assert type(pickle.loads(found_entry.value)) == type(PICKLE_VALUE)
assert isinstance(pickle.loads(found_entry.value), type(PICKLE_VALUE))
assert found_entry.created_by_fk == admin_user.id
@@ -49,6 +49,32 @@ def test_get_text_clause_with_colon() -> None:
assert text_clause.text == "SELECT foo FROM tbl WHERE foo = '123\\:456')"
def test_parse_sql_single_statement() -> None:
"""
`parse_sql` should properly strip leading and trailing spaces and semicolons
"""
from superset.db_engine_specs.base import BaseEngineSpec
queries = BaseEngineSpec.parse_sql(" SELECT foo FROM tbl ; ")
assert queries == ["SELECT foo FROM tbl"]
def test_parse_sql_multi_statement() -> None:
"""
For string with multiple SQL-statements `parse_sql` method should return list
where each element represents the single SQL-statement
"""
from superset.db_engine_specs.base import BaseEngineSpec
queries = BaseEngineSpec.parse_sql("SELECT foo FROM tbl1; SELECT bar FROM tbl2;")
assert queries == [
"SELECT foo FROM tbl1",
"SELECT bar FROM tbl2",
]
def test_validate_db_uri(mocker: MockerFixture) -> None:
"""
Ensures that the `validate_database_uri` method invokes the validator correctly
@@ -24,6 +24,91 @@ from tests.unit_tests.db_engine_specs.utils import assert_convert_dttm
from tests.unit_tests.fixtures.common import dttm # noqa: F401
@pytest.mark.parametrize(
"sql,expected",
[
("SELECT foo FROM tbl", True),
("SHOW TABLES", False),
("EXPLAIN SELECT foo FROM tbl", False),
("INSERT INTO tbl (foo) VALUES (1)", False),
],
)
def test_sql_is_readonly_query(sql: str, expected: bool) -> None:
"""
Make sure that SQL dialect consider only SELECT statements as read-only
"""
from superset.db_engine_specs.kusto import KustoSqlEngineSpec
from superset.sql_parse import ParsedQuery
parsed_query = ParsedQuery(sql)
is_readonly = KustoSqlEngineSpec.is_readonly_query(parsed_query)
assert expected == is_readonly
@pytest.mark.parametrize(
"kql,expected",
[
("tbl | limit 100", True),
("let foo = 1; tbl | where bar == foo", True),
(".show tables", False),
],
)
def test_kql_is_select_query(kql: str, expected: bool) -> None:
"""
Make sure that KQL dialect consider only statements that do not start with "." (dot)
as a SELECT statements
"""
from superset.db_engine_specs.kusto import KustoKqlEngineSpec
from superset.sql_parse import ParsedQuery
parsed_query = ParsedQuery(kql)
is_select = KustoKqlEngineSpec.is_select_query(parsed_query)
assert expected == is_select
@pytest.mark.parametrize(
"kql,expected",
[
("tbl | limit 100", True),
("let foo = 1; tbl | where bar == foo", True),
(".show tables", True),
("print 1", True),
("set querytrace; Events | take 100", True),
(".drop table foo", False),
(".set-or-append table foo <| bar", False),
],
)
def test_kql_is_readonly_query(kql: str, expected: bool) -> None:
"""
Make sure that KQL dialect consider only SELECT statements as read-only
"""
from superset.db_engine_specs.kusto import KustoKqlEngineSpec
from superset.sql_parse import ParsedQuery
parsed_query = ParsedQuery(kql)
is_readonly = KustoKqlEngineSpec.is_readonly_query(parsed_query)
assert expected == is_readonly
def test_kql_parse_sql() -> None:
"""
parse_sql method should always return a list with a single element
which is an original query
"""
from superset.db_engine_specs.kusto import KustoKqlEngineSpec
queries = KustoKqlEngineSpec.parse_sql("let foo = 1; tbl | where bar == foo")
assert queries == ["let foo = 1; tbl | where bar == foo"]
@pytest.mark.parametrize(
"target_type,expected_result",
[
-290
View File
@@ -1,290 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import os
import sys
import pytest
SHA = "22e7c602b9aa321ec7e0df4bb0033048664dcdf0"
PR_ID = "666"
OLD_REL = "2.1.0"
NEW_REL = "2.1.1"
REPO = "apache/superset"
# Add the 'scripts' directory to sys.path
scripts_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), "../../../scripts")
)
sys.path.append(scripts_dir)
import build_docker as docker_utils # Replace with the actual function name # noqa: E402
@pytest.fixture(autouse=True)
def set_env_var():
os.environ["TEST_ENV"] = "true"
yield
del os.environ["TEST_ENV"]
@pytest.mark.parametrize(
"release, expected_bool",
[
("2.1.0", False),
("2.1.1", True),
("1.0.0", False),
("3.0.0", True),
],
)
def test_is_latest_release(release, expected_bool):
assert docker_utils.is_latest_release(release) == expected_bool
@pytest.mark.parametrize(
"build_preset, build_platforms, sha, build_context, build_context_ref, expected_tags",
[
# PRs
(
"lean",
["linux/arm64"],
SHA,
"pull_request",
PR_ID,
[f"{REPO}:22e7c60-arm", f"{REPO}:{SHA}-arm", f"{REPO}:pr-{PR_ID}-arm"],
),
(
"ci",
["linux/amd64"],
SHA,
"pull_request",
PR_ID,
[f"{REPO}:22e7c60-ci", f"{REPO}:{SHA}-ci", f"{REPO}:pr-{PR_ID}-ci"],
),
(
"lean",
["linux/amd64"],
SHA,
"pull_request",
PR_ID,
[f"{REPO}:22e7c60", f"{REPO}:{SHA}", f"{REPO}:pr-{PR_ID}"],
),
(
"dev",
["linux/arm64"],
SHA,
"pull_request",
PR_ID,
[
f"{REPO}:22e7c60-dev-arm",
f"{REPO}:{SHA}-dev-arm",
f"{REPO}:pr-{PR_ID}-dev-arm",
],
),
(
"dev",
["linux/amd64"],
SHA,
"pull_request",
PR_ID,
[f"{REPO}:22e7c60-dev", f"{REPO}:{SHA}-dev", f"{REPO}:pr-{PR_ID}-dev"],
),
# old releases
(
"lean",
["linux/arm64"],
SHA,
"release",
OLD_REL,
[f"{REPO}:22e7c60-arm", f"{REPO}:{SHA}-arm", f"{REPO}:{OLD_REL}-arm"],
),
(
"lean",
["linux/amd64"],
SHA,
"release",
OLD_REL,
[f"{REPO}:22e7c60", f"{REPO}:{SHA}", f"{REPO}:{OLD_REL}"],
),
(
"dev",
["linux/arm64"],
SHA,
"release",
OLD_REL,
[
f"{REPO}:22e7c60-dev-arm",
f"{REPO}:{SHA}-dev-arm",
f"{REPO}:{OLD_REL}-dev-arm",
],
),
(
"dev",
["linux/amd64"],
SHA,
"release",
OLD_REL,
[f"{REPO}:22e7c60-dev", f"{REPO}:{SHA}-dev", f"{REPO}:{OLD_REL}-dev"],
),
# new releases
(
"lean",
["linux/arm64"],
SHA,
"release",
NEW_REL,
[
f"{REPO}:22e7c60-arm",
f"{REPO}:{SHA}-arm",
f"{REPO}:{NEW_REL}-arm",
f"{REPO}:latest-arm",
],
),
(
"lean",
["linux/amd64"],
SHA,
"release",
NEW_REL,
[f"{REPO}:22e7c60", f"{REPO}:{SHA}", f"{REPO}:{NEW_REL}", f"{REPO}:latest"],
),
(
"dev",
["linux/arm64"],
SHA,
"release",
NEW_REL,
[
f"{REPO}:22e7c60-dev-arm",
f"{REPO}:{SHA}-dev-arm",
f"{REPO}:{NEW_REL}-dev-arm",
f"{REPO}:latest-dev-arm",
],
),
(
"dev",
["linux/amd64"],
SHA,
"release",
NEW_REL,
[
f"{REPO}:22e7c60-dev",
f"{REPO}:{SHA}-dev",
f"{REPO}:{NEW_REL}-dev",
f"{REPO}:latest-dev",
],
),
# merge on master
(
"lean",
["linux/arm64"],
SHA,
"push",
"master",
[f"{REPO}:22e7c60-arm", f"{REPO}:{SHA}-arm", f"{REPO}:master-arm"],
),
(
"lean",
["linux/amd64"],
SHA,
"push",
"master",
[f"{REPO}:22e7c60", f"{REPO}:{SHA}", f"{REPO}:master"],
),
(
"dev",
["linux/arm64"],
SHA,
"push",
"master",
[
f"{REPO}:22e7c60-dev-arm",
f"{REPO}:{SHA}-dev-arm",
f"{REPO}:master-dev-arm",
],
),
(
"dev",
["linux/amd64"],
SHA,
"push",
"master",
[f"{REPO}:22e7c60-dev", f"{REPO}:{SHA}-dev", f"{REPO}:master-dev"],
),
],
)
def test_get_docker_tags(
build_preset, build_platforms, sha, build_context, build_context_ref, expected_tags
):
tags = docker_utils.get_docker_tags(
build_preset, build_platforms, sha, build_context, build_context_ref
)
for tag in expected_tags:
assert tag in tags
@pytest.mark.parametrize(
"build_preset, build_platforms, is_authenticated, sha, build_context, build_context_ref, contains",
[
(
"lean",
["linux/amd64"],
True,
SHA,
"push",
"master",
["--push", f"-t {REPO}:master "],
),
(
"dev",
["linux/amd64"],
False,
SHA,
"push",
"master",
["--load", f"-t {REPO}:master-dev ", "--target dev"],
),
# multi-platform
(
"lean",
["linux/arm64", "linux/amd64"],
True,
SHA,
"push",
"master",
["--platform linux/arm64,linux/amd64"],
),
],
)
def test_get_docker_command(
build_preset,
build_platforms,
is_authenticated,
sha,
build_context,
build_context_ref,
contains,
):
cmd = docker_utils.get_docker_command(
build_preset,
build_platforms,
is_authenticated,
sha,
build_context,
build_context_ref,
)
for s in contains:
assert s in cmd
+1 -124
View File
@@ -945,32 +945,6 @@ on $left.Day1 == $right.Day
("kustokql", "set querytrace; Events | take 100", False),
("kustokql", ".drop table foo", True),
("kustokql", ".set-or-append table foo <| bar", True),
("kustosql", "SELECT foo FROM tbl", False),
("kustosql", "SHOW TABLES", False),
("kustosql", "EXPLAIN SELECT foo FROM tbl", False),
("kustosql", "INSERT INTO tbl (foo) VALUES (1)", True),
("base", "SHOW LOCKS test EXTENDED", False),
("base", "SET hivevar:desc='Legislators'", False),
("base", "UPDATE t1 SET col1 = NULL", True),
("base", "EXPLAIN SELECT 1", False),
("base", "SELECT 1", False),
("base", "WITH bla AS (SELECT 1) SELECT * FROM bla", False),
("base", "SHOW CATALOGS", False),
("base", "SHOW TABLES", False),
("hive", "UPDATE t1 SET col1 = NULL", True),
("hive", "INSERT OVERWRITE TABLE tabB SELECT a.Age FROM TableA", True),
("hive", "SHOW LOCKS test EXTENDED", False),
("hive", "SET hivevar:desc='Legislators'", False),
("hive", "EXPLAIN SELECT 1", False),
("hive", "SELECT 1", False),
("hive", "WITH bla AS (SELECT 1) SELECT * FROM bla", False),
("presto", "SET hivevar:desc='Legislators'", False),
("presto", "UPDATE t1 SET col1 = NULL", True),
("presto", "INSERT OVERWRITE TABLE tabB SELECT a.Age FROM TableA", True),
("presto", "SHOW LOCKS test EXTENDED", False),
("presto", "EXPLAIN SELECT 1", False),
("presto", "SELECT 1", False),
("presto", "WITH bla AS (SELECT 1) SELECT * FROM bla", False),
],
)
def test_has_mutation(engine: str, sql: str, expected: bool) -> None:
@@ -1068,106 +1042,9 @@ def test_custom_dialect(app: None) -> None:
)
def test_is_mutating(engine: str) -> None:
"""
Global tests for `is_mutating`, covering all supported engines.
Tests for `is_mutating`.
"""
assert not SQLStatement(
"with source as ( select 1 as one ) select * from source",
engine=engine,
).is_mutating()
@pytest.mark.parametrize(
"identifier,expected",
[
# Rule: Identifiers are case-sensitive
("myTable", True),
("MYTABLE", True),
("MyTable", True),
# Rule: Identifiers must be between 1 and 1024 characters long
("a", True),
("a" * 1024, True),
("a" * 1025, False),
("", False),
# Rule: Identifiers may contain letters, digits, and underscores
("My_Table_123", True),
("123Table", True),
# Rule: Identifiers may contain special characters: spaces, dots, dashes (when quoted)
("['My Table']", True),
("['My-Table']", True),
("['My.Table']", True),
("['Table-']", True),
("['My Table Name']", True),
("['My!Table']", False),
("['MyTable ']", True),
(" MyTable", False),
("MyTable ", False),
# Rule: Non-special identifiers don't require quoting
("MyTable", True),
("My-Table", False),
# Invalid quoting
("['Invalid]", False),
("['Invalid'Name']", False),
("['']", False),
# Rule: Literal identifiers or language keywords
("['select']", True),
("select", True),
],
)
def test_is_kql_identifier(identifier: str, expected: bool):
"""
Tests the _is_identifier method for various valid and invalid cases.
"""
assert KustoKQLStatement._is_identifier(identifier) == expected
@pytest.mark.parametrize(
"kql,expected",
[
# Simple SELECT-like statements (non-mutating queries)
("MyTable | count", True),
("MyTable", True),
("| count", True),
("tbl | limit 100", True),
(".show tables", False),
# With comments (ensure comments are stripped out)
("// Comment only", False),
("MyTable // trailing comment", True),
("// leading comment\nMyTable", True),
("MyTable\n// intermediate comment\n| count", True),
# Mutating query (should return False)
(".drop MyTable", False),
(
".update MyTable set Column1 = 100",
False,
),
(".alter MyTable", False),
# Edge cases for first token
("", False),
(" ", False),
(".command", False),
("['My Table']", True),
# Complex multi-line queries
(
"""
// Initial comment
MyTable
| where Column1 > 100
""",
True,
),
(
"""
MyTable
| where Column1 > 100
| summarize by Column2
// Final comment
""",
True,
),
],
)
def test_kql_is_select(kql: str, expected: bool):
"""
Tests the is_select method for various valid and invalid cases.
"""
assert KustoKQLStatement(kql).is_select() == expected
-268
View File
@@ -1,268 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import os
import sys
import pytest
SHA = "22e7c602b9aa321ec7e0df4bb0033048664dcdf0"
PR_ID = "666"
OLD_REL = "2.1.0"
NEW_REL = "2.1.1"
REPO = "apache/superset"
# Add the 'scripts' directory to sys.path
scripts_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), "../../../scripts")
)
sys.path.append(scripts_dir)
import build_docker as docker_utils # Replace with the actual function name # noqa: E402
@pytest.fixture(autouse=True)
def set_env_var():
os.environ["TEST_ENV"] = "true"
yield
del os.environ["TEST_ENV"]
@pytest.mark.parametrize(
"release, expected_bool",
[
("2.1.0", False),
("2.1.1", True),
("1.0.0", False),
("3.0.0", True),
],
)
def test_is_latest_release(release, expected_bool):
assert docker_utils.is_latest_release(release) == expected_bool
@pytest.mark.parametrize(
"build_preset, build_platform, sha, build_context, build_context_ref, expected_tags",
[
# PRs
(
"lean",
"linux/arm64",
SHA,
"pull_request",
PR_ID,
[f"{REPO}:22e7c60-arm", f"{REPO}:{SHA}-arm"],
),
(
"lean",
"linux/amd64",
SHA,
"pull_request",
PR_ID,
[f"{REPO}:22e7c60", f"{REPO}:{SHA}"],
),
(
"dev",
"linux/arm64",
SHA,
"pull_request",
PR_ID,
[f"{REPO}:22e7c60-dev-arm", f"{REPO}:{SHA}-dev-arm"],
),
(
"dev",
"linux/amd64",
SHA,
"pull_request",
PR_ID,
[f"{REPO}:22e7c60-dev", f"{REPO}:{SHA}-dev"],
),
# old releases
(
"lean",
"linux/arm64",
SHA,
"release",
OLD_REL,
[f"{REPO}:22e7c60-arm", f"{REPO}:{SHA}-arm", f"{REPO}:{OLD_REL}-arm"],
),
(
"lean",
"linux/amd64",
SHA,
"release",
OLD_REL,
[f"{REPO}:22e7c60", f"{REPO}:{SHA}", f"{REPO}:{OLD_REL}"],
),
(
"dev",
"linux/arm64",
SHA,
"release",
OLD_REL,
[
f"{REPO}:22e7c60-dev-arm",
f"{REPO}:{SHA}-dev-arm",
f"{REPO}:{OLD_REL}-dev-arm",
],
),
(
"dev",
"linux/amd64",
SHA,
"release",
OLD_REL,
[f"{REPO}:22e7c60-dev", f"{REPO}:{SHA}-dev", f"{REPO}:{OLD_REL}-dev"],
),
# new releases
(
"lean",
"linux/arm64",
SHA,
"release",
NEW_REL,
[
f"{REPO}:22e7c60-arm",
f"{REPO}:{SHA}-arm",
f"{REPO}:{NEW_REL}-arm",
f"{REPO}:latest-arm",
],
),
(
"lean",
"linux/amd64",
SHA,
"release",
NEW_REL,
[f"{REPO}:22e7c60", f"{REPO}:{SHA}", f"{REPO}:{NEW_REL}", f"{REPO}:latest"],
),
(
"dev",
"linux/arm64",
SHA,
"release",
NEW_REL,
[
f"{REPO}:22e7c60-dev-arm",
f"{REPO}:{SHA}-dev-arm",
f"{REPO}:{NEW_REL}-dev-arm",
f"{REPO}:latest-dev-arm",
],
),
(
"dev",
"linux/amd64",
SHA,
"release",
NEW_REL,
[
f"{REPO}:22e7c60-dev",
f"{REPO}:{SHA}-dev",
f"{REPO}:{NEW_REL}-dev",
f"{REPO}:latest-dev",
],
),
# merge on master
(
"lean",
"linux/arm64",
SHA,
"push",
"master",
[f"{REPO}:22e7c60-arm", f"{REPO}:{SHA}-arm", f"{REPO}:master-arm"],
),
(
"lean",
"linux/amd64",
SHA,
"push",
"master",
[f"{REPO}:22e7c60", f"{REPO}:{SHA}", f"{REPO}:master"],
),
(
"dev",
"linux/arm64",
SHA,
"push",
"master",
[
f"{REPO}:22e7c60-dev-arm",
f"{REPO}:{SHA}-dev-arm",
f"{REPO}:master-dev-arm",
],
),
(
"dev",
"linux/amd64",
SHA,
"push",
"master",
[f"{REPO}:22e7c60-dev", f"{REPO}:{SHA}-dev", f"{REPO}:master-dev"],
),
],
)
def test_get_docker_tags(
build_preset, build_platform, sha, build_context, build_context_ref, expected_tags
):
tags = docker_utils.get_docker_tags(
build_preset, build_platform, sha, build_context, build_context_ref
)
for tag in expected_tags:
assert tag in tags
@pytest.mark.parametrize(
"build_preset, build_platform, is_authenticated, sha, build_context, build_context_ref, contains",
[
(
"lean",
"linux/amd64",
True,
SHA,
"push",
"master",
["--push", f"-t {REPO}:master "],
),
(
"dev",
"linux/amd64",
False,
SHA,
"push",
"master",
["--load", f"-t {REPO}:master-dev "],
),
],
)
def test_get_docker_command(
build_preset,
build_platform,
is_authenticated,
sha,
build_context,
build_context_ref,
contains,
):
cmd = docker_utils.get_docker_command(
build_preset,
build_platform,
is_authenticated,
sha,
build_context,
build_context_ref,
)
for s in contains:
assert s in cmd
+13
View File
@@ -34,6 +34,19 @@ def test_timezone_conversion() -> None:
assert pd.read_excel(contents)["dt"][0] == "2023-01-01 00:00:00+00:00"
def test_quote_formulas() -> None:
"""
Test that formulas are quoted in Excel.
"""
df = pd.DataFrame({"formula": ["=SUM(A1:A2)", "normal", "@SUM(A1:A2)"]})
contents = df_to_excel(df)
assert pd.read_excel(contents)["formula"].tolist() == [
"'=SUM(A1:A2)",
"normal",
"'@SUM(A1:A2)",
]
def test_column_data_types_with_one_numeric_column():
df = pd.DataFrame(
{