Compare commits

..

3 Commits

Author SHA1 Message Date
Maxime Beauchemin
e5369ab0f0 chore: ran 'npm audit fix' 2024-11-22 17:01:50 -08:00
Maxime Beauchemin
e0deb704f9 feat: make ephemeral env use supersetbot + deprecate build_docker.py (#30870) 2024-11-22 14:19:08 -08:00
Kamil Gabryjelski
abf3790ea6 chore: Cleanup code related to MetadataBar, fix types (#31030) 2024-11-22 16:02:13 +01:00
44 changed files with 845 additions and 1542 deletions

View File

@@ -1,30 +1,25 @@
name: Ephemeral env workflow
# Example manual trigger: gh workflow run ephemeral-env.yml --ref fix_ephemerals --field comment_body="/testenv up" --field issue_number=666
on:
issue_comment:
types: [created]
workflow_dispatch:
inputs:
comment_body:
description: 'Comment body to simulate /testenv command'
required: true
default: '/testenv up'
issue_number:
description: 'Issue or PR number'
required: true
jobs:
config:
runs-on: "ubuntu-22.04"
if: github.event.issue.pull_request
outputs:
has-secrets: ${{ steps.check.outputs.has-secrets }}
steps:
- name: "Check for secrets"
id: check
shell: bash
run: |
if [ -n "${{ (secrets.AWS_ACCESS_KEY_ID != '' && secrets.AWS_SECRET_ACCESS_KEY != '') || '' }}" ]; then
echo "has-secrets=1" >> "$GITHUB_OUTPUT"
fi
ephemeral-env-comment:
concurrency:
group: ${{ github.workflow }}-${{ github.event.issue.number || github.run_id }}-comment
group: ${{ github.workflow }}-${{ github.event.inputs.issue_number || github.event.issue.number || github.run_id }}-comment
cancel-in-progress: true
needs: config
if: needs.config.outputs.has-secrets
name: Evaluate ephemeral env comment trigger (/testenv)
runs-on: ubuntu-22.04
permissions:
@@ -44,18 +39,18 @@ jobs:
with:
result-encoding: string
script: |
const pattern = /^\/testenv (up|down)/
const result = pattern.exec(context.payload.comment.body)
return result === null ? 'noop' : result[1]
const pattern = /^\/testenv (up|down)/;
const result = pattern.exec('${{ github.event.inputs.comment_body || github.event.comment.body }}');
return result === null ? 'noop' : result[1];
- name: Eval comment body for feature flags
- name: Looking for feature flags
uses: actions/github-script@v7
id: eval-feature-flags
with:
script: |
const pattern = /FEATURE_(\w+)=(\w+)/g;
let results = [];
[...context.payload.comment.body.matchAll(pattern)].forEach(match => {
[...'${{ github.event.inputs.comment_body || github.event.comment.body }}'.matchAll(pattern)].forEach(match => {
const config = {
name: `SUPERSET_FEATURE_${match[1]}`,
value: match[2],
@@ -67,24 +62,25 @@ jobs:
- name: Limit to committers
if: >
steps.eval-body.outputs.result != 'noop' &&
github.event_name == 'issue_comment' &&
github.event.comment.author_association != 'MEMBER' &&
github.event.comment.author_association != 'OWNER'
uses: actions/github-script@v7
with:
github-token: ${{github.token}}
github-token: ${{ github.token }}
script: |
const errMsg = '@${{ github.event.comment.user.login }} Ephemeral environment creation is currently limited to committers.'
const errMsg = '@${{ github.event.comment.user.login }} Ephemeral environment creation is currently limited to committers.';
github.rest.issues.createComment({
issue_number: ${{ github.event.issue.number }},
owner: context.repo.owner,
repo: context.repo.repo,
body: errMsg
})
core.setFailed(errMsg)
});
core.setFailed(errMsg);
ephemeral-docker-build:
concurrency:
group: ${{ github.workflow }}-${{ github.event.issue.number || github.run_id }}-build
group: ${{ github.workflow }}-${{ github.event.inputs.issue_number || github.event.issue.number || github.run_id }}-build
cancel-in-progress: true
needs: ephemeral-env-comment
name: ephemeral-docker-build
@@ -98,9 +94,9 @@ jobs:
const request = {
owner: context.repo.owner,
repo: context.repo.repo,
pull_number: ${{ github.event.issue.number }},
}
core.info(`Getting PR #${request.pull_number} from ${request.owner}/${request.repo}`)
pull_number: ${{ github.event.inputs.issue_number || github.event.issue.number }},
};
core.info(`Getting PR #${request.pull_number} from ${request.owner}/${request.repo}`);
const pr = await github.rest.pulls.get(request);
return pr.data;
@@ -121,12 +117,17 @@ jobs:
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Setup supersetbot
uses: ./.github/actions/setup-supersetbot/
- name: Build ephemeral env image
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
./scripts/build_docker.py \
"ci" \
"pull_request" \
--build_context_ref ${{ github.event.issue.number }}
supersetbot docker \
--preset ci \
--platform linux/amd64 \
--context-ref "$RELEASE"
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v4
@@ -146,7 +147,7 @@ jobs:
ECR_REPOSITORY: superset-ci
IMAGE_TAG: apache/superset:${{ steps.get-sha.outputs.sha }}-ci
run: |
docker tag $IMAGE_TAG $ECR_REGISTRY/$ECR_REPOSITORY:pr-${{ github.event.issue.number }}-ci
docker tag $IMAGE_TAG $ECR_REGISTRY/$ECR_REPOSITORY:pr-${{ github.event.inputs.issue_number || github.event.issue.number }}-ci
docker push -a $ECR_REGISTRY/$ECR_REPOSITORY
ephemeral-env-up:
@@ -181,22 +182,22 @@ jobs:
aws ecr describe-images \
--registry-id $(echo "${{ steps.login-ecr.outputs.registry }}" | grep -Eo "^[0-9]+") \
--repository-name superset-ci \
--image-ids imageTag=pr-${{ github.event.issue.number }}-ci
--image-ids imageTag=pr-${{ github.event.inputs.issue_number || github.event.issue.number }}-ci
- name: Fail on missing container image
if: steps.check-image.outcome == 'failure'
uses: actions/github-script@v7
with:
github-token: ${{github.token}}
github-token: ${{ github.token }}
script: |
const errMsg = '@${{ github.event.comment.user.login }} Container image not yet published for this PR. Please try again when build is complete.'
const errMsg = '@${{ github.event.comment.user.login }} Container image not yet published for this PR. Please try again when build is complete.';
github.rest.issues.createComment({
issue_number: ${{ github.event.issue.number }},
issue_number: ${{ github.event.inputs.issue_number || github.event.issue.number }},
owner: context.repo.owner,
repo: context.repo.repo,
body: errMsg
})
core.setFailed(errMsg)
});
core.setFailed(errMsg);
- name: Fill in the new image ID in the Amazon ECS task definition
id: task-def
@@ -204,7 +205,7 @@ jobs:
with:
task-definition: .github/workflows/ecs-task-definition.json
container-name: superset-ci
image: ${{ steps.login-ecr.outputs.registry }}/superset-ci:pr-${{ github.event.issue.number }}-ci
image: ${{ steps.login-ecr.outputs.registry }}/superset-ci:pr-${{ github.event.inputs.issue_number || github.event.issue.number }}-ci
- name: Update env vars in the Amazon ECS task definition
run: |
@@ -213,13 +214,10 @@ jobs:
- name: Describe ECS service
id: describe-services
run: |
echo "active=$(aws ecs describe-services --cluster superset-ci --services pr-${{ github.event.issue.number }}-service | jq '.services[] | select(.status == "ACTIVE") | any')" >> $GITHUB_OUTPUT
echo "active=$(aws ecs describe-services --cluster superset-ci --services pr-${{ github.event.inputs.issue_number || github.event.issue.number }}-service | jq '.services[] | select(.status == "ACTIVE") | any')" >> $GITHUB_OUTPUT
- name: Create ECS service
if: steps.describe-services.outputs.active != 'true'
id: create-service
env:
ECR_SUBNETS: subnet-0e15a5034b4121710,subnet-0e8efef4a72224974
ECR_SECURITY_GROUP: sg-092ff3a6ae0574d91
if: steps.describe-services.outputs.active != 'true'
run: |
aws ecs create-service \
--cluster superset-ci \
@@ -230,7 +228,6 @@ jobs:
--platform-version LATEST \
--network-configuration "awsvpcConfiguration={subnets=[$ECR_SUBNETS],securityGroups=[$ECR_SECURITY_GROUP],assignPublicIp=ENABLED}" \
--tags key=pr,value=${{ github.event.issue.number }} key=github_user,value=${{ github.actor }}
- name: Deploy Amazon ECS task definition
id: deploy-task
uses: aws-actions/amazon-ecs-deploy-task-definition@v2
@@ -245,17 +242,14 @@ jobs:
id: list-tasks
run: |
echo "task=$(aws ecs list-tasks --cluster superset-ci --service-name pr-${{ github.event.issue.number }}-service | jq '.taskArns | first')" >> $GITHUB_OUTPUT
- name: Get network interface
id: get-eni
run: |
echo "eni=$(aws ecs describe-tasks --cluster superset-ci --tasks ${{ steps.list-tasks.outputs.task }} | jq '.tasks | .[0] | .attachments | .[0] | .details | map(select(.name=="networkInterfaceId")) | .[0] | .value')" >> $GITHUB_OUTPUT
echo "eni=$(aws ecs describe-tasks --cluster superset-ci --tasks ${{ steps.list-tasks.outputs.task }} | jq '.tasks | .[0] | .attachments | .[0] | .details | map(select(.name==\"networkInterfaceId\")) | .[0] | .value')" >> $GITHUB_OUTPUT
- name: Get public IP
id: get-ip
run: |
echo "ip=$(aws ec2 describe-network-interfaces --network-interface-ids ${{ steps.get-eni.outputs.eni }} | jq -r '.NetworkInterfaces | first | .Association.PublicIp')" >> $GITHUB_OUTPUT
- name: Comment (success)
if: ${{ success() }}
uses: actions/github-script@v7
@@ -263,12 +257,11 @@ jobs:
github-token: ${{github.token}}
script: |
github.rest.issues.createComment({
issue_number: ${{ github.event.issue.number }},
issue_number: ${{ github.event.inputs.issue_number || github.event.issue.number }},
owner: context.repo.owner,
repo: context.repo.repo,
body: '@${{ github.event.comment.user.login }} Ephemeral environment spinning up at http://${{ steps.get-ip.outputs.ip }}:8080. Credentials are `admin`/`admin`. Please allow several minutes for bootstrapping and startup.'
body: '@${{ github.event.inputs.user_login || github.event.comment.user.login }} Ephemeral environment spinning up at http://${{ steps.get-ip.outputs.ip }}:8080. Credentials are `admin`/`admin`. Please allow several minutes for bootstrapping and startup.'
})
- name: Comment (failure)
if: ${{ failure() }}
uses: actions/github-script@v7
@@ -276,8 +269,8 @@ jobs:
github-token: ${{github.token}}
script: |
github.rest.issues.createComment({
issue_number: ${{ github.event.issue.number }},
issue_number: ${{ github.event.inputs.issue_number || github.event.issue.number }},
owner: context.repo.owner,
repo: context.repo.repo,
body: '@${{ github.event.comment.user.login }} Ephemeral environment creation failed. Please check the Actions logs for details.'
body: '@${{ github.event.inputs.user_login || github.event.comment.user.login }} Ephemeral environment creation failed. Please check the Actions logs for details.'
})

View File

@@ -115,7 +115,7 @@ RUN mkdir -p ${PYTHONPATH} superset/static requirements superset-frontend apache
libldap2-dev \
&& touch superset/static/version_info.json \
&& chown -R superset:superset ./* \
&& rm -rf /var/lib/apt/lists/*
&& rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
COPY --chown=superset:superset pyproject.toml setup.py MANIFEST.in README.md ./
# setup.py uses the version information in package.json
@@ -128,7 +128,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \
&& pip install --no-cache-dir --upgrade setuptools pip \
&& pip install --no-cache-dir -r requirements/base.txt \
&& apt-get autoremove -yqq --purge build-essential \
&& rm -rf /var/lib/apt/lists/*
&& rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
# Copy the compiled frontend assets
COPY --chown=superset:superset --from=superset-node /app/superset/static/assets superset/static/assets
@@ -177,7 +177,7 @@ RUN apt-get update -qq \
libxtst6 \
git \
pkg-config \
&& rm -rf /var/lib/apt/lists/*
&& rm -rf /var/cache/apt/archives/* /var/lib/apt/lists/*
RUN --mount=type=cache,target=/root/.cache/pip \
pip install --no-cache-dir playwright
@@ -199,13 +199,13 @@ RUN if [ "$INCLUDE_FIREFOX" = "true" ]; then \
&& wget -q https://github.com/mozilla/geckodriver/releases/download/${GECKODRIVER_VERSION}/geckodriver-${GECKODRIVER_VERSION}-linux64.tar.gz -O - | tar xfz - -C /usr/local/bin \
&& wget -q https://download-installer.cdn.mozilla.net/pub/firefox/releases/${FIREFOX_VERSION}/linux-x86_64/en-US/firefox-${FIREFOX_VERSION}.tar.bz2 -O - | tar xfj - -C /opt \
&& ln -s /opt/firefox/firefox /usr/local/bin/firefox \
&& apt-get autoremove -yqq --purge wget bzip2 && rm -rf /var/[log,tmp]/* /tmp/* /var/lib/apt/lists/*; \
&& apt-get autoremove -yqq --purge wget bzip2 && rm -rf /var/[log,tmp]/* /tmp/* /var/lib/apt/lists/* /var/cache/apt/archives/*; \
fi
# Installing mysql client os-level dependencies in dev image only because GPL
RUN apt-get install -yqq --no-install-recommends \
default-libmysqlclient-dev \
&& rm -rf /var/lib/apt/lists/*
&& rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
COPY --chown=superset:superset requirements/development.txt requirements/
RUN --mount=type=cache,target=/root/.cache/pip \
@@ -213,7 +213,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \
build-essential \
&& pip install --no-cache-dir -r requirements/development.txt \
&& apt-get autoremove -yqq --purge build-essential \
&& rm -rf /var/lib/apt/lists/*
&& rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
USER superset
######################################################################

View File

@@ -25,7 +25,6 @@ x-superset-user: &superset-user root
x-superset-depends-on: &superset-depends-on
- db
- redis
- superset-checks
x-superset-volumes: &superset-volumes
# /app/pythonpath_docker will be appended to the PYTHONPATH in the final container
- ./docker:/app/docker
@@ -131,23 +130,6 @@ services:
- REDIS_PORT=6379
- REDIS_SSL=false
superset-checks:
build:
context: .
target: python-base
cache_from:
- apache/superset-cache:3.10-slim-bookworm
container_name: superset_checks
command: ["/app/scripts/check-env.py"]
env_file:
- path: docker/.env # default
required: true
- path: docker/.env-local # optional override
required: false
user: *superset-user
healthcheck:
disable: true
superset-init:
build:
<<: *common-build
@@ -179,6 +161,7 @@ services:
# set this to false if you have perf issues running the npm i; npm run dev in-docker
# if you do so, you have to run this manually on the host, which should perform better!
BUILD_SUPERSET_FRONTEND_IN_DOCKER: true
NPM_RUN_PRUNE: false
SCARF_ANALYTICS: "${SCARF_ANALYTICS:-}"
container_name: superset_node
command: ["/app/docker/docker-frontend.sh"]

View File

@@ -27,6 +27,11 @@ if [ "$BUILD_SUPERSET_FRONTEND_IN_DOCKER" = "true" ]; then
echo "Building Superset frontend in dev mode inside docker container"
cd /app/superset-frontend
if [ "$NPM_RUN_PRUNE" = "true" ]; then
echo "Running `npm run prune`"
npm run prune
fi
echo "Running `npm install`"
npm install

View File

@@ -29,7 +29,7 @@ We have a set of build "presets" that each represent a combination of
parameters for the build, mostly pointing to either different target layer
for the build, and/or base image.
Here are the build presets that are exposed through the `build_docker.py` script:
Here are the build presets that are exposed through the `supersetbot docker` utility:
- `lean`: The default Docker image, including both frontend and backend. Tags
without a build_preset are lean builds (ie: `latest`, `4.0.0`, `3.0.0`, ...). `lean`
@@ -62,8 +62,8 @@ Here are the build presets that are exposed through the `build_docker.py` script
For insights or modifications to the build matrix and tagging conventions,
check the [build_docker.py](https://github.com/apache/superset/blob/master/scripts/build_docker.py)
script and the [docker.yml](https://github.com/apache/superset/blob/master/.github/workflows/docker.yml)
check the [supersetbot docker](https://github.com/apache-superset/supersetbot)
subcommand and the [docker.yml](https://github.com/apache/superset/blob/master/.github/workflows/docker.yml)
GitHub action.
## Key ARGs in Dockerfile

View File

@@ -95,6 +95,14 @@ perform those operations. In this case, we recommend you set the env var
Simply trigger `npm i && npm run dev`, this should be MUCH faster.
:::
:::tip
Sometimes, your npm-related state can get out-of-wack, running `npm run prune` from
the `superset-frontend/` folder will nuke the various' packages `node_module/` folders
and help you start fresh. In the context of `docker compose` setting
`export NPM_RUN_PRUNE=true` prior to running `docker compose up` will trigger that
from within docker. This will slow down the startup, but will fix various npm-related issues.
:::
### Option #2 - build a set of immutable images from the local branch
```bash

View File

@@ -77,10 +77,6 @@ versions officially supported by Superset. We'd recommend using a Python version
like [pyenv](https://github.com/pyenv/pyenv)
(and also [pyenv-virtualenv](https://github.com/pyenv/pyenv-virtualenv)).
:::tip
To identify the Python version used by the official docker image, see the [Dockerfile](https://github.com/apache/superset/blob/master/Dockerfile). Additional docker images published for newer versions of Python can be found in [this file](https://github.com/apache/superset/blob/master/scripts/build_docker.py).
:::
Let's also make sure we have the latest version of `pip` and `setuptools`:
```bash

View File

@@ -32,7 +32,7 @@ billiard==4.2.0
# via celery
blinker==1.9.0
# via flask
bottleneck==1.3.8
bottleneck==1.4.2
# via pandas
brotli==1.1.0
# via flask-compress
@@ -148,9 +148,7 @@ geopy==2.4.1
google-auth==2.29.0
# via shillelagh
greenlet==3.0.3
# via
# shillelagh
# sqlalchemy
# via shillelagh
gunicorn==22.0.0
# via apache-superset
hashids==1.3.1

View File

@@ -1,294 +0,0 @@
#!/usr/bin/env python3
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import os
import re
import subprocess
from textwrap import dedent
import click
REPO = "apache/superset"
CACHE_REPO = f"{REPO}-cache"
BASE_PY_IMAGE = "3.10-slim-bookworm"
def run_cmd(command: str, raise_on_failure: bool = True) -> str:
process = subprocess.Popen(
command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True
)
output = ""
if process.stdout is not None:
for line in iter(process.stdout.readline, ""):
print(line.strip()) # Print the line to stdout in real-time
output += line
process.wait() # Wait for the subprocess to finish
if process.returncode != 0 and raise_on_failure:
raise subprocess.CalledProcessError(process.returncode, command, output)
return output
def get_git_sha() -> str:
return run_cmd("git rev-parse HEAD").strip()
def get_build_context_ref(build_context: str) -> str:
"""
Given a context, return a ref:
- if context is pull_request, return the PR's id
- if context is push, return the branch
- if context is release, return the release ref
"""
event = os.getenv("GITHUB_EVENT_NAME")
github_ref = os.getenv("GITHUB_REF", "")
if event == "pull_request":
github_head_ref = os.getenv("GITHUB_HEAD_REF", "")
return re.sub("[^a-zA-Z0-9]", "-", github_head_ref)[:40]
elif event == "release":
return re.sub("refs/tags/", "", github_ref)[:40]
elif event == "push":
return re.sub("[^a-zA-Z0-9]", "-", re.sub("refs/heads/", "", github_ref))[:40]
return ""
def is_latest_release(release: str) -> bool:
output = (
run_cmd(
f"./scripts/tag_latest_release.sh {release} --dry-run",
raise_on_failure=False,
)
or ""
)
return "SKIP_TAG::false" in output
def make_docker_tag(l: list[str]) -> str: # noqa: E741
return f"{REPO}:" + "-".join([o for o in l if o])
def get_docker_tags(
build_preset: str,
build_platforms: list[str],
sha: str,
build_context: str,
build_context_ref: str,
force_latest: bool = False,
) -> set[str]:
"""
Return a set of tags given a given build context
"""
tags: set[str] = set()
tag_chunks: list[str] = []
is_latest = is_latest_release(build_context_ref)
if build_preset != "lean":
# Always add the preset_build name if different from default (lean)
tag_chunks += [build_preset]
if len(build_platforms) == 1:
build_platform = build_platforms[0]
short_build_platform = build_platform.replace("linux/", "").replace("64", "")
if short_build_platform != "amd":
# Always a platform indicator if different from default (amd)
tag_chunks += [short_build_platform]
# Always craft a tag for the SHA
tags.add(make_docker_tag([sha] + tag_chunks))
# also a short SHA, cause it's nice
tags.add(make_docker_tag([sha[:7]] + tag_chunks))
if build_context == "release":
# add a release tag
tags.add(make_docker_tag([build_context_ref] + tag_chunks))
if is_latest or force_latest:
# add a latest tag
tags.add(make_docker_tag(["latest"] + tag_chunks))
elif build_context == "push" and build_context_ref == "master":
tags.add(make_docker_tag(["master"] + tag_chunks))
elif build_context == "pull_request":
tags.add(make_docker_tag([f"pr-{build_context_ref}"] + tag_chunks))
return tags
def get_docker_command(
build_preset: str,
build_platforms: list[str],
is_authenticated: bool,
sha: str,
build_context: str,
build_context_ref: str,
force_latest: bool = False,
) -> str:
tag = "" # noqa: F841
build_target = ""
py_ver = BASE_PY_IMAGE
docker_context = "."
if build_preset == "dev":
build_target = "dev"
elif build_preset == "lean":
build_target = "lean"
elif build_preset == "py311":
build_target = "lean"
py_ver = "3.11-slim-bookworm"
elif build_preset == "websocket":
build_target = ""
docker_context = "superset-websocket"
elif build_preset == "ci":
build_target = "ci"
elif build_preset == "dockerize":
build_target = ""
docker_context = "-f dockerize.Dockerfile ."
else:
print(f"Invalid build preset: {build_preset}")
exit(1)
# Try to get context reference if missing
if not build_context_ref:
build_context_ref = get_build_context_ref(build_context)
tags = get_docker_tags(
build_preset,
build_platforms,
sha,
build_context,
build_context_ref,
force_latest,
)
docker_tags = ("\\\n" + 8 * " ").join([f"-t {s} " for s in tags])
docker_args = "--load" if not is_authenticated else "--push"
target_argument = f"--target {build_target}" if build_target else ""
cache_ref = f"{CACHE_REPO}:{py_ver}"
if len(build_platforms) == 1:
build_platform = build_platforms[0]
short_build_platform = build_platform.replace("linux/", "").replace("64", "")
cache_ref = f"{CACHE_REPO}:{py_ver}-{short_build_platform}"
platform_arg = "--platform " + ",".join(build_platforms)
cache_from_arg = f"--cache-from=type=registry,ref={cache_ref}"
cache_to_arg = (
f"--cache-to=type=registry,mode=max,ref={cache_ref}" if is_authenticated else ""
)
build_arg = f"--build-arg PY_VER={py_ver}" if py_ver else ""
actor = os.getenv("GITHUB_ACTOR")
return dedent(
f"""\
docker buildx build \\
{docker_args} \\
{docker_tags} \\
{cache_from_arg} \\
{cache_to_arg} \\
{build_arg} \\
{platform_arg} \\
{target_argument} \\
--label sha={sha} \\
--label target={build_target} \\
--label build_trigger={build_context} \\
--label base={py_ver} \\
--label build_actor={actor} \\
{docker_context}"""
)
@click.command()
@click.argument(
"build_preset",
type=click.Choice(["lean", "dev", "dockerize", "websocket", "py311", "ci"]),
)
@click.argument("build_context", type=click.Choice(["push", "pull_request", "release"]))
@click.option(
"--platform",
type=click.Choice(["linux/arm64", "linux/amd64"]),
default=["linux/amd64"],
multiple=True,
)
@click.option("--build_context_ref", help="a reference to the pr, release or branch")
@click.option("--dry-run", is_flag=True, help="Run the command in dry-run mode.")
@click.option("--verbose", is_flag=True, help="Print more info")
@click.option(
"--force-latest", is_flag=True, help="Force the 'latest' tag on the release"
)
def main(
build_preset: str,
build_context: str,
build_context_ref: str,
platform: list[str],
dry_run: bool,
force_latest: bool,
verbose: bool,
) -> None:
"""
This script executes docker build and push commands based on given arguments.
"""
is_authenticated = (
True if os.getenv("DOCKERHUB_TOKEN") and os.getenv("DOCKERHUB_USER") else False
)
if force_latest and build_context != "release":
print(
"--force-latest can only be applied if the build context is set to 'release'"
)
exit(1)
if build_context == "release" and not build_context_ref.strip():
print("Release number has to be provided")
exit(1)
docker_build_command = get_docker_command(
build_preset,
platform,
is_authenticated,
get_git_sha(),
build_context,
build_context_ref,
force_latest,
)
if not dry_run:
print("Executing Docker Build Command:")
print(docker_build_command)
script = ""
if os.getenv("DOCKERHUB_USER"):
script = dedent(
f"""\
docker logout
docker login --username "{os.getenv("DOCKERHUB_USER")}" --password "{os.getenv("DOCKERHUB_TOKEN")}"
DOCKER_ARGS="--push"
"""
)
script = script + docker_build_command
if verbose:
run_cmd("cat Dockerfile")
stdout = run_cmd(script) # noqa: F841
else:
print("Dry Run - Docker Build Command:")
print(docker_build_command)
if __name__ == "__main__":
main()

View File

@@ -5920,7 +5920,9 @@
}
},
"node_modules/@jest/core/node_modules/micromatch": {
"version": "4.0.7",
"version": "4.0.8",
"resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz",
"integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==",
"dev": true,
"license": "MIT",
"dependencies": {
@@ -6245,10 +6247,11 @@
}
},
"node_modules/@jest/transform/node_modules/micromatch": {
"version": "4.0.7",
"resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.7.tgz",
"integrity": "sha512-LPP/3KorzCwBxfeUuZmaR6bG2kdeHSbe0P2tY3FLRU4vYrjYz5hI4QZwV0njUx3jeuKe67YukQ1LSPZBKDqO/Q==",
"version": "4.0.8",
"resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz",
"integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==",
"dev": true,
"license": "MIT",
"dependencies": {
"braces": "^3.0.3",
"picomatch": "^2.3.1"
@@ -6686,10 +6689,11 @@
}
},
"node_modules/@lerna/create/node_modules/cross-spawn": {
"version": "7.0.3",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
"integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==",
"version": "7.0.6",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
"integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==",
"dev": true,
"license": "MIT",
"dependencies": {
"path-key": "^3.1.0",
"shebang-command": "^2.0.0",
@@ -8001,10 +8005,11 @@
}
},
"node_modules/@npmcli/map-workspaces/node_modules/cross-spawn": {
"version": "7.0.3",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
"integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==",
"version": "7.0.6",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
"integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==",
"devOptional": true,
"license": "MIT",
"dependencies": {
"path-key": "^3.1.0",
"shebang-command": "^2.0.0",
@@ -8214,10 +8219,11 @@
}
},
"node_modules/@npmcli/package-json/node_modules/cross-spawn": {
"version": "7.0.3",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
"integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==",
"version": "7.0.6",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
"integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==",
"devOptional": true,
"license": "MIT",
"dependencies": {
"path-key": "^3.1.0",
"shebang-command": "^2.0.0",
@@ -10415,7 +10421,9 @@
}
},
"node_modules/@storybook/cli/node_modules/cross-spawn": {
"version": "7.0.3",
"version": "7.0.6",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
"integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==",
"dev": true,
"license": "MIT",
"dependencies": {
@@ -10705,7 +10713,9 @@
}
},
"node_modules/@storybook/codemod/node_modules/cross-spawn": {
"version": "7.0.3",
"version": "7.0.6",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
"integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==",
"dev": true,
"license": "MIT",
"dependencies": {
@@ -10905,7 +10915,9 @@
}
},
"node_modules/@storybook/core-common/node_modules/cross-spawn": {
"version": "7.0.3",
"version": "7.0.6",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
"integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==",
"license": "MIT",
"dependencies": {
"path-key": "^3.1.0",
@@ -12005,7 +12017,9 @@
}
},
"node_modules/@storybook/react-docgen-typescript-plugin/node_modules/micromatch": {
"version": "4.0.7",
"version": "4.0.8",
"resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz",
"integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==",
"dev": true,
"license": "MIT",
"dependencies": {
@@ -18730,7 +18744,9 @@
}
},
"node_modules/cacache/node_modules/cross-spawn": {
"version": "7.0.3",
"version": "7.0.6",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
"integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==",
"devOptional": true,
"license": "MIT",
"dependencies": {
@@ -20586,9 +20602,9 @@
}
},
"node_modules/cross-env/node_modules/cross-spawn": {
"version": "7.0.3",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
"integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==",
"version": "7.0.6",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
"integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==",
"dev": true,
"license": "MIT",
"dependencies": {
@@ -20650,7 +20666,9 @@
}
},
"node_modules/cross-spawn": {
"version": "6.0.5",
"version": "6.0.6",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-6.0.6.tgz",
"integrity": "sha512-VqCUuhcd1iB+dsv8gxPttb5iZh/D0iubSP21g36KXdEuf6I5JiioesUVjpCdHV9MZRUfVFlvwtIUyPfxo5trtw==",
"dev": true,
"license": "MIT",
"dependencies": {
@@ -21083,7 +21101,9 @@
}
},
"node_modules/cypress/node_modules/cross-spawn": {
"version": "7.0.3",
"version": "7.0.6",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
"integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==",
"dev": true,
"license": "MIT",
"peer": true,
@@ -22061,7 +22081,9 @@
}
},
"node_modules/default-gateway/node_modules/cross-spawn": {
"version": "7.0.3",
"version": "7.0.6",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
"integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==",
"dev": true,
"license": "MIT",
"dependencies": {
@@ -22490,7 +22512,9 @@
}
},
"node_modules/detect-package-manager/node_modules/cross-spawn": {
"version": "7.0.3",
"version": "7.0.6",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
"integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==",
"dev": true,
"license": "MIT",
"dependencies": {
@@ -24823,7 +24847,9 @@
"license": "Python-2.0"
},
"node_modules/eslint/node_modules/cross-spawn": {
"version": "7.0.3",
"version": "7.0.6",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
"integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==",
"dev": true,
"license": "MIT",
"dependencies": {
@@ -25824,9 +25850,10 @@
}
},
"node_modules/fast-glob/node_modules/micromatch": {
"version": "4.0.7",
"resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.7.tgz",
"integrity": "sha512-LPP/3KorzCwBxfeUuZmaR6bG2kdeHSbe0P2tY3FLRU4vYrjYz5hI4QZwV0njUx3jeuKe67YukQ1LSPZBKDqO/Q==",
"version": "4.0.8",
"resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz",
"integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==",
"license": "MIT",
"dependencies": {
"braces": "^3.0.3",
"picomatch": "^2.3.1"
@@ -26546,7 +26573,9 @@
}
},
"node_modules/foreground-child/node_modules/cross-spawn": {
"version": "7.0.3",
"version": "7.0.6",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
"integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==",
"dev": true,
"license": "MIT",
"peer": true,
@@ -29053,11 +29082,13 @@
}
},
"node_modules/http-proxy-middleware/node_modules/micromatch": {
"version": "4.0.5",
"version": "4.0.8",
"resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz",
"integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==",
"dev": true,
"license": "MIT",
"dependencies": {
"braces": "^3.0.2",
"braces": "^3.0.3",
"picomatch": "^2.3.1"
},
"engines": {
@@ -30432,7 +30463,9 @@
}
},
"node_modules/istanbul-lib-processinfo/node_modules/cross-spawn": {
"version": "7.0.3",
"version": "7.0.6",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
"integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==",
"dev": true,
"license": "MIT",
"peer": true,
@@ -30718,7 +30751,9 @@
}
},
"node_modules/jest-changed-files/node_modules/cross-spawn": {
"version": "7.0.3",
"version": "7.0.6",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
"integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==",
"dev": true,
"license": "MIT",
"dependencies": {
@@ -31113,7 +31148,9 @@
}
},
"node_modules/jest-config/node_modules/micromatch": {
"version": "4.0.7",
"version": "4.0.8",
"resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz",
"integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==",
"dev": true,
"license": "MIT",
"dependencies": {
@@ -32276,10 +32313,11 @@
}
},
"node_modules/jest-haste-map/node_modules/micromatch": {
"version": "4.0.7",
"resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.7.tgz",
"integrity": "sha512-LPP/3KorzCwBxfeUuZmaR6bG2kdeHSbe0P2tY3FLRU4vYrjYz5hI4QZwV0njUx3jeuKe67YukQ1LSPZBKDqO/Q==",
"version": "4.0.8",
"resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz",
"integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==",
"dev": true,
"license": "MIT",
"dependencies": {
"braces": "^3.0.3",
"picomatch": "^2.3.1"
@@ -32498,7 +32536,9 @@
}
},
"node_modules/jest-message-util/node_modules/micromatch": {
"version": "4.0.7",
"version": "4.0.8",
"resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz",
"integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==",
"dev": true,
"license": "MIT",
"dependencies": {
@@ -33082,7 +33122,9 @@
}
},
"node_modules/jscodeshift/node_modules/micromatch": {
"version": "4.0.7",
"version": "4.0.8",
"resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz",
"integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==",
"dev": true,
"license": "MIT",
"dependencies": {
@@ -34052,7 +34094,9 @@
}
},
"node_modules/lerna/node_modules/cross-spawn": {
"version": "7.0.3",
"version": "7.0.6",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
"integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==",
"dev": true,
"license": "MIT",
"dependencies": {
@@ -40872,10 +40916,11 @@
}
},
"node_modules/nise/node_modules/path-to-regexp": {
"version": "6.2.2",
"resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.2.2.tgz",
"integrity": "sha512-GQX3SSMokngb36+whdpRXE+3f9V8UzyAorlYvOGx87ufGHehNTn5lCxrKtLyZ4Yl/wEKnNnr98ZzOwwDZV5ogw==",
"dev": true
"version": "6.3.0",
"resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.3.0.tgz",
"integrity": "sha512-Yhpw4T9C6hPpgPeA28us07OJeqZ5EzQTkbfwuhsUg0c237RomFoETJgmp2sa3F/41gfLE6G5cqcYwznmeEeOlQ==",
"dev": true,
"license": "MIT"
},
"node_modules/no-case": {
"version": "3.0.4",
@@ -41019,10 +41064,11 @@
}
},
"node_modules/node-gyp/node_modules/cross-spawn": {
"version": "7.0.3",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
"integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==",
"version": "7.0.6",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
"integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==",
"devOptional": true,
"license": "MIT",
"dependencies": {
"path-key": "^3.1.0",
"shebang-command": "^2.0.0",
@@ -42067,7 +42113,9 @@
}
},
"node_modules/nypm/node_modules/cross-spawn": {
"version": "7.0.3",
"version": "7.0.6",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
"integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==",
"dev": true,
"license": "MIT",
"dependencies": {
@@ -46425,7 +46473,9 @@
"license": "MIT"
},
"node_modules/react-router/node_modules/path-to-regexp": {
"version": "1.8.0",
"version": "1.9.0",
"resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-1.9.0.tgz",
"integrity": "sha512-xIp7/apCFJuUHdDLWe8O1HIkb0kQrOMb/0u6FXQjemHn/ii5LrIzU6bdECnsiTF/GjZkMEKg1xdiZwNqDYlZ6g==",
"license": "MIT",
"dependencies": {
"isarray": "0.0.1"
@@ -48702,7 +48752,9 @@
}
},
"node_modules/rimraf/node_modules/cross-spawn": {
"version": "7.0.3",
"version": "7.0.6",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
"integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==",
"license": "MIT",
"dependencies": {
"path-key": "^3.1.0",
@@ -51905,15 +51957,17 @@
}
},
"node_modules/ts-loader/node_modules/micromatch": {
"version": "4.0.2",
"version": "4.0.8",
"resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz",
"integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==",
"dev": true,
"license": "MIT",
"dependencies": {
"braces": "^3.0.1",
"picomatch": "^2.0.5"
"braces": "^3.0.3",
"picomatch": "^2.3.1"
},
"engines": {
"node": ">=8"
"node": ">=8.6"
}
},
"node_modules/ts-loader/node_modules/semver": {
@@ -53400,7 +53454,9 @@
}
},
"node_modules/webpack-cli/node_modules/cross-spawn": {
"version": "7.0.3",
"version": "7.0.6",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
"integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==",
"dev": true,
"license": "MIT",
"dependencies": {
@@ -54046,7 +54102,9 @@
}
},
"node_modules/which-package-manager/node_modules/cross-spawn": {
"version": "7.0.3",
"version": "7.0.6",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
"integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==",
"license": "MIT",
"optional": true,
"peer": true,
@@ -54168,7 +54226,9 @@
}
},
"node_modules/which-package-manager/node_modules/micromatch": {
"version": "4.0.7",
"version": "4.0.8",
"resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz",
"integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==",
"license": "MIT",
"optional": true,
"peer": true,
@@ -55144,7 +55204,9 @@
}
},
"packages/generator-superset/node_modules/cross-spawn": {
"version": "7.0.3",
"version": "7.0.6",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
"integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==",
"license": "MIT",
"dependencies": {
"path-key": "^3.1.0",
@@ -55639,7 +55701,9 @@
}
},
"packages/generator-superset/node_modules/path-to-regexp": {
"version": "6.2.2",
"version": "6.3.0",
"resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.3.0.tgz",
"integrity": "sha512-Yhpw4T9C6hPpgPeA28us07OJeqZ5EzQTkbfwuhsUg0c237RomFoETJgmp2sa3F/41gfLE6G5cqcYwznmeEeOlQ==",
"dev": true,
"license": "MIT"
},
@@ -57504,12 +57568,14 @@
"license": "MIT"
},
"packages/superset-ui-demo/node_modules/micromatch": {
"version": "4.0.4",
"version": "4.0.8",
"resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz",
"integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==",
"dev": true,
"license": "MIT",
"dependencies": {
"braces": "^3.0.1",
"picomatch": "^2.2.3"
"braces": "^3.0.3",
"picomatch": "^2.3.1"
},
"engines": {
"node": ">=8.6"
@@ -62473,7 +62539,9 @@
"dev": true
},
"micromatch": {
"version": "4.0.7",
"version": "4.0.8",
"resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz",
"integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==",
"dev": true,
"requires": {
"braces": "^3.0.3",
@@ -62709,9 +62777,9 @@
"dev": true
},
"micromatch": {
"version": "4.0.7",
"resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.7.tgz",
"integrity": "sha512-LPP/3KorzCwBxfeUuZmaR6bG2kdeHSbe0P2tY3FLRU4vYrjYz5hI4QZwV0njUx3jeuKe67YukQ1LSPZBKDqO/Q==",
"version": "4.0.8",
"resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz",
"integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==",
"dev": true,
"requires": {
"braces": "^3.0.3",
@@ -63053,9 +63121,9 @@
}
},
"cross-spawn": {
"version": "7.0.3",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
"integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==",
"version": "7.0.6",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
"integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==",
"dev": true,
"requires": {
"path-key": "^3.1.0",
@@ -64047,9 +64115,9 @@
}
},
"cross-spawn": {
"version": "7.0.3",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
"integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==",
"version": "7.0.6",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
"integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==",
"devOptional": true,
"requires": {
"path-key": "^3.1.0",
@@ -64198,9 +64266,9 @@
}
},
"cross-spawn": {
"version": "7.0.3",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
"integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==",
"version": "7.0.6",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
"integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==",
"devOptional": true,
"requires": {
"path-key": "^3.1.0",
@@ -65617,7 +65685,9 @@
"dev": true
},
"cross-spawn": {
"version": "7.0.3",
"version": "7.0.6",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
"integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==",
"dev": true,
"requires": {
"path-key": "^3.1.0",
@@ -65776,7 +65846,9 @@
"dev": true
},
"cross-spawn": {
"version": "7.0.3",
"version": "7.0.6",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
"integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==",
"dev": true,
"requires": {
"path-key": "^3.1.0",
@@ -65923,7 +65995,9 @@
}
},
"cross-spawn": {
"version": "7.0.3",
"version": "7.0.6",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
"integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==",
"requires": {
"path-key": "^3.1.0",
"shebang-command": "^2.0.0",
@@ -66599,7 +66673,9 @@
}
},
"micromatch": {
"version": "4.0.7",
"version": "4.0.8",
"resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz",
"integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==",
"dev": true,
"requires": {
"braces": "^3.0.3",
@@ -67473,11 +67549,13 @@
"version": "6.0.0"
},
"micromatch": {
"version": "4.0.4",
"version": "4.0.8",
"resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz",
"integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==",
"dev": true,
"requires": {
"braces": "^3.0.1",
"picomatch": "^2.2.3"
"braces": "^3.0.3",
"picomatch": "^2.3.1"
}
},
"p-limit": {
@@ -67848,7 +67926,9 @@
"peer": true
},
"cross-spawn": {
"version": "7.0.3",
"version": "7.0.6",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
"integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==",
"requires": {
"path-key": "^3.1.0",
"shebang-command": "^2.0.0",
@@ -68124,7 +68204,9 @@
"version": "3.1.1"
},
"path-to-regexp": {
"version": "6.2.2",
"version": "6.3.0",
"resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.3.0.tgz",
"integrity": "sha512-Yhpw4T9C6hPpgPeA28us07OJeqZ5EzQTkbfwuhsUg0c237RomFoETJgmp2sa3F/41gfLE6G5cqcYwznmeEeOlQ==",
"dev": true
},
"path-type": {
@@ -73623,7 +73705,9 @@
}
},
"cross-spawn": {
"version": "7.0.3",
"version": "7.0.6",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
"integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==",
"devOptional": true,
"requires": {
"path-key": "^3.1.0",
@@ -74872,9 +74956,9 @@
},
"dependencies": {
"cross-spawn": {
"version": "7.0.3",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
"integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==",
"version": "7.0.6",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
"integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==",
"dev": true,
"requires": {
"path-key": "^3.1.0",
@@ -74915,7 +74999,9 @@
}
},
"cross-spawn": {
"version": "6.0.5",
"version": "6.0.6",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-6.0.6.tgz",
"integrity": "sha512-VqCUuhcd1iB+dsv8gxPttb5iZh/D0iubSP21g36KXdEuf6I5JiioesUVjpCdHV9MZRUfVFlvwtIUyPfxo5trtw==",
"dev": true,
"requires": {
"nice-try": "^1.0.4",
@@ -75195,7 +75281,9 @@
"peer": true
},
"cross-spawn": {
"version": "7.0.3",
"version": "7.0.6",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
"integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==",
"dev": true,
"peer": true,
"requires": {
@@ -75882,7 +75970,9 @@
},
"dependencies": {
"cross-spawn": {
"version": "7.0.3",
"version": "7.0.6",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
"integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==",
"dev": true,
"requires": {
"path-key": "^3.1.0",
@@ -76152,7 +76242,9 @@
},
"dependencies": {
"cross-spawn": {
"version": "7.0.3",
"version": "7.0.6",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
"integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==",
"dev": true,
"requires": {
"path-key": "^3.1.0",
@@ -77222,7 +77314,9 @@
"dev": true
},
"cross-spawn": {
"version": "7.0.3",
"version": "7.0.6",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
"integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==",
"dev": true,
"requires": {
"path-key": "^3.1.0",
@@ -78326,9 +78420,9 @@
"integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng=="
},
"micromatch": {
"version": "4.0.7",
"resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.7.tgz",
"integrity": "sha512-LPP/3KorzCwBxfeUuZmaR6bG2kdeHSbe0P2tY3FLRU4vYrjYz5hI4QZwV0njUx3jeuKe67YukQ1LSPZBKDqO/Q==",
"version": "4.0.8",
"resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz",
"integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==",
"requires": {
"braces": "^3.0.3",
"picomatch": "^2.3.1"
@@ -78827,7 +78921,9 @@
},
"dependencies": {
"cross-spawn": {
"version": "7.0.3",
"version": "7.0.6",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
"integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==",
"dev": true,
"peer": true,
"requires": {
@@ -80440,10 +80536,12 @@
"dev": true
},
"micromatch": {
"version": "4.0.5",
"version": "4.0.8",
"resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz",
"integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==",
"dev": true,
"requires": {
"braces": "^3.0.2",
"braces": "^3.0.3",
"picomatch": "^2.3.1"
}
},
@@ -81290,7 +81388,9 @@
},
"dependencies": {
"cross-spawn": {
"version": "7.0.3",
"version": "7.0.6",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
"integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==",
"dev": true,
"peer": true,
"requires": {
@@ -81464,7 +81564,9 @@
},
"dependencies": {
"cross-spawn": {
"version": "7.0.3",
"version": "7.0.6",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
"integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==",
"dev": true,
"requires": {
"path-key": "^3.1.0",
@@ -81696,7 +81798,9 @@
"dev": true
},
"micromatch": {
"version": "4.0.7",
"version": "4.0.8",
"resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz",
"integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==",
"dev": true,
"requires": {
"braces": "^3.0.3",
@@ -82489,9 +82593,9 @@
"dev": true
},
"micromatch": {
"version": "4.0.7",
"resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.7.tgz",
"integrity": "sha512-LPP/3KorzCwBxfeUuZmaR6bG2kdeHSbe0P2tY3FLRU4vYrjYz5hI4QZwV0njUx3jeuKe67YukQ1LSPZBKDqO/Q==",
"version": "4.0.8",
"resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz",
"integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==",
"dev": true,
"requires": {
"braces": "^3.0.3",
@@ -82635,7 +82739,9 @@
"dev": true
},
"micromatch": {
"version": "4.0.7",
"version": "4.0.8",
"resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz",
"integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==",
"dev": true,
"requires": {
"braces": "^3.0.3",
@@ -83042,7 +83148,9 @@
"dev": true
},
"micromatch": {
"version": "4.0.7",
"version": "4.0.8",
"resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz",
"integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==",
"dev": true,
"requires": {
"braces": "^3.0.3",
@@ -83730,7 +83838,9 @@
}
},
"cross-spawn": {
"version": "7.0.3",
"version": "7.0.6",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
"integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==",
"dev": true,
"requires": {
"path-key": "^3.1.0",
@@ -87511,9 +87621,9 @@
},
"dependencies": {
"path-to-regexp": {
"version": "6.2.2",
"resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.2.2.tgz",
"integrity": "sha512-GQX3SSMokngb36+whdpRXE+3f9V8UzyAorlYvOGx87ufGHehNTn5lCxrKtLyZ4Yl/wEKnNnr98ZzOwwDZV5ogw==",
"version": "6.3.0",
"resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.3.0.tgz",
"integrity": "sha512-Yhpw4T9C6hPpgPeA28us07OJeqZ5EzQTkbfwuhsUg0c237RomFoETJgmp2sa3F/41gfLE6G5cqcYwznmeEeOlQ==",
"dev": true
}
}
@@ -87616,9 +87726,9 @@
}
},
"cross-spawn": {
"version": "7.0.3",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
"integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==",
"version": "7.0.6",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
"integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==",
"devOptional": true,
"requires": {
"path-key": "^3.1.0",
@@ -88350,7 +88460,9 @@
},
"dependencies": {
"cross-spawn": {
"version": "7.0.3",
"version": "7.0.6",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
"integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==",
"dev": true,
"requires": {
"path-key": "^3.1.0",
@@ -91062,7 +91174,9 @@
"version": "0.0.1"
},
"path-to-regexp": {
"version": "1.8.0",
"version": "1.9.0",
"resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-1.9.0.tgz",
"integrity": "sha512-xIp7/apCFJuUHdDLWe8O1HIkb0kQrOMb/0u6FXQjemHn/ii5LrIzU6bdECnsiTF/GjZkMEKg1xdiZwNqDYlZ6g==",
"requires": {
"isarray": "0.0.1"
}
@@ -92514,7 +92628,9 @@
}
},
"cross-spawn": {
"version": "7.0.3",
"version": "7.0.6",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
"integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==",
"requires": {
"path-key": "^3.1.0",
"shebang-command": "^2.0.0",
@@ -94657,11 +94773,13 @@
}
},
"micromatch": {
"version": "4.0.2",
"version": "4.0.8",
"resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz",
"integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==",
"dev": true,
"requires": {
"braces": "^3.0.1",
"picomatch": "^2.0.5"
"braces": "^3.0.3",
"picomatch": "^2.3.1"
}
},
"semver": {
@@ -95625,7 +95743,9 @@
"dev": true
},
"cross-spawn": {
"version": "7.0.3",
"version": "7.0.6",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
"integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==",
"dev": true,
"requires": {
"path-key": "^3.1.0",
@@ -96042,7 +96162,9 @@
}
},
"cross-spawn": {
"version": "7.0.3",
"version": "7.0.6",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
"integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==",
"optional": true,
"peer": true,
"requires": {
@@ -96113,7 +96235,9 @@
}
},
"micromatch": {
"version": "4.0.7",
"version": "4.0.8",
"resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz",
"integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==",
"optional": true,
"peer": true,
"requires": {

View File

@@ -1,13 +0,0 @@
{
"name": "@superset-ui/switchboard",
"version": "0.18.26-0",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "@superset-ui/switchboard",
"version": "0.18.26-0",
"license": "Apache-2.0"
}
}
}

View File

@@ -51,7 +51,7 @@ export type LastModified = {
export type Owner = {
type: MetadataType.Owner;
createdBy: string;
owners?: string[];
owners?: string[] | string;
createdOn: string;
onClick?: (type: string) => void;
};

View File

@@ -23,27 +23,18 @@ import { styled } from '@superset-ui/core';
import { Tooltip, TooltipPlacement } from 'src/components/Tooltip';
import { ContentType } from './ContentType';
import { config } from './ContentConfig';
export const MIN_NUMBER_ITEMS = 2;
export const MAX_NUMBER_ITEMS = 6;
const HORIZONTAL_PADDING = 12;
const VERTICAL_PADDING = 8;
const ICON_PADDING = 8;
const SPACE_BETWEEN_ITEMS = 16;
const ICON_WIDTH = 16;
const TEXT_MIN_WIDTH = 70;
const TEXT_MAX_WIDTH = 150;
const ORDER = {
dashboards: 0,
table: 1,
sql: 2,
rows: 3,
tags: 4,
description: 5,
owner: 6,
lastModified: 7,
};
import {
HORIZONTAL_PADDING,
ICON_PADDING,
ICON_WIDTH,
VERTICAL_PADDING,
TEXT_MIN_WIDTH,
TEXT_MAX_WIDTH,
SPACE_BETWEEN_ITEMS,
ORDER,
MIN_NUMBER_ITEMS,
MAX_NUMBER_ITEMS,
} from './constants';
const Bar = styled.div<{ count: number }>`
${({ theme, count }) => `

View File

@@ -0,0 +1,39 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
export const MIN_NUMBER_ITEMS = 2;
export const MAX_NUMBER_ITEMS = 6;
export const HORIZONTAL_PADDING = 12;
export const VERTICAL_PADDING = 8;
export const ICON_PADDING = 8;
export const SPACE_BETWEEN_ITEMS = 16;
export const ICON_WIDTH = 16;
export const TEXT_MIN_WIDTH = 70;
export const TEXT_MAX_WIDTH = 150;
export const ORDER = {
dashboards: 0,
table: 1,
sql: 2,
rows: 3,
tags: 4,
description: 5,
owner: 6,
lastModified: 7,
};

View File

@@ -16,7 +16,8 @@
* specific language governing permissions and limitations
* under the License.
*/
import MetadataBar, { MIN_NUMBER_ITEMS, MAX_NUMBER_ITEMS } from './MetadataBar';
import MetadataBar from './MetadataBar';
import { MIN_NUMBER_ITEMS, MAX_NUMBER_ITEMS } from './constants';
export type { MetadataBarProps } from './MetadataBar';

View File

@@ -44,7 +44,6 @@ import ConnectedHeaderActionsDropdown from 'src/dashboard/components/Header/Head
import PublishedStatus from 'src/dashboard/components/PublishedStatus';
import UndoRedoKeyListeners from 'src/dashboard/components/UndoRedoKeyListeners';
import PropertiesModal from 'src/dashboard/components/PropertiesModal';
import getOwnerName from 'src/utils/getOwnerName';
import {
UNDO_LIMIT,
SAVE_TYPE_OVERWRITE,
@@ -55,7 +54,6 @@ import setPeriodicRunner, {
stopPeriodicRender,
} from 'src/dashboard/util/setPeriodicRunner';
import { PageHeaderWithActions } from 'src/components/PageHeaderWithActions';
import MetadataBar, { MetadataType } from 'src/components/MetadataBar';
import DashboardEmbedModal from '../EmbeddedModal';
import OverwriteConfirm from '../OverwriteConfirm';
import {
@@ -88,6 +86,7 @@ import { logEvent } from '../../../logger/actions';
import { dashboardInfoChanged } from '../../actions/dashboardInfo';
import isDashboardLoading from '../../util/isDashboardLoading';
import { useChartIds } from '../../util/charts/useChartIds';
import { useDashboardMetadataBar } from './useDashboardMetadataBar';
const extensionsRegistry = getExtensionsRegistry();
@@ -472,32 +471,7 @@ const Header = () => {
setShowingEmbedModal(false);
}, []);
const getMetadataItems = useCallback(
() => [
{
type: MetadataType.LastModified,
value: dashboardInfo.changed_on_delta_humanized,
modifiedBy:
getOwnerName(dashboardInfo.changed_by) || t('Not available'),
},
{
type: MetadataType.Owner,
createdBy: getOwnerName(dashboardInfo.created_by) || t('Not available'),
owners:
dashboardInfo.owners.length > 0
? dashboardInfo.owners.map(getOwnerName)
: t('None'),
createdOn: dashboardInfo.created_on_delta_humanized,
},
],
[
dashboardInfo.changed_by,
dashboardInfo.changed_on_delta_humanized,
dashboardInfo.created_by,
dashboardInfo.created_on_delta_humanized,
dashboardInfo.owners,
],
);
const metadataBar = useDashboardMetadataBar(dashboardInfo);
const userCanEdit =
dashboardInfo.dash_edit_perm && !dashboardInfo.is_managed_externally;
@@ -579,15 +553,13 @@ const Header = () => {
visible={!editMode}
/>
),
!editMode && !isEmbedded && (
<MetadataBar items={getMetadataItems()} tooltipPlacement="bottom" />
),
!editMode && !isEmbedded && metadataBar,
],
[
boundActionCreators.savePublished,
dashboardInfo.id,
editMode,
getMetadataItems,
metadataBar,
isEmbedded,
isPublished,
userCanEdit,

View File

@@ -0,0 +1,54 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { useMemo } from 'react';
import { t } from '@superset-ui/core';
import { DashboardInfo } from 'src/dashboard/types';
import MetadataBar, { MetadataType } from 'src/components/MetadataBar';
import getOwnerName from 'src/utils/getOwnerName';
export const useDashboardMetadataBar = (dashboardInfo: DashboardInfo) => {
const items = useMemo(
() => [
{
type: MetadataType.LastModified as const,
value: dashboardInfo.changed_on_delta_humanized,
modifiedBy:
getOwnerName(dashboardInfo.changed_by) || t('Not available'),
},
{
type: MetadataType.Owner as const,
createdBy: getOwnerName(dashboardInfo.created_by) || t('Not available'),
owners:
dashboardInfo.owners.length > 0
? dashboardInfo.owners.map(getOwnerName)
: t('None'),
createdOn: dashboardInfo.created_on_delta_humanized,
},
],
[
dashboardInfo.changed_by,
dashboardInfo.changed_on_delta_humanized,
dashboardInfo.created_by,
dashboardInfo.created_on_delta_humanized,
dashboardInfo.owners,
],
);
return <MetadataBar items={items} tooltipPlacement="bottom" />;
};

View File

@@ -52,6 +52,9 @@ const initialState: { dashboardInfo: DashboardInfo } = {
conf: {},
},
crossFiltersEnabled: true,
created_on_delta_humanized: '',
changed_on_delta_humanized: '',
owners: [],
},
};

View File

@@ -33,6 +33,7 @@ import Database from 'src/types/Database';
import { UrlParamEntries } from 'src/utils/urlUtils';
import { UserWithPermissionsAndRoles } from 'src/types/bootstrapTypes';
import Owner from 'src/types/Owner';
import { ChartState } from '../explore/types';
export type { Dashboard } from 'src/types/Dashboard';
@@ -139,6 +140,11 @@ export type DashboardInfo = {
};
crossFiltersEnabled: boolean;
filterBarOrientation: FilterBarOrientation;
created_on_delta_humanized: string;
changed_on_delta_humanized: string;
changed_by?: Owner;
created_by?: Owner;
owners: Owner[];
};
export type ChartsState = { [key: string]: Chart };

View File

@@ -16,12 +16,12 @@
* specific language governing permissions and limitations
* under the License.
*/
import { useCallback, useEffect, useMemo, useState } from 'react';
import { useCallback, useEffect, useState } from 'react';
import { useHistory } from 'react-router-dom';
import { useDispatch } from 'react-redux';
import PropTypes from 'prop-types';
import { Tooltip } from 'src/components/Tooltip';
import { css, logging, SupersetClient, t, tn } from '@superset-ui/core';
import { css, logging, SupersetClient, t } from '@superset-ui/core';
import { chartPropShape } from 'src/dashboard/util/propShapes';
import AlteredSliceTag from 'src/components/AlteredSliceTag';
import Button from 'src/components/Button';
@@ -29,10 +29,10 @@ import Icons from 'src/components/Icons';
import PropertiesModal from 'src/explore/components/PropertiesModal';
import { sliceUpdated } from 'src/explore/actions/exploreActions';
import { PageHeaderWithActions } from 'src/components/PageHeaderWithActions';
import MetadataBar, { MetadataType } from 'src/components/MetadataBar';
import { setSaveChartModalVisibility } from 'src/explore/actions/saveModalActions';
import { applyColors, resetColors } from 'src/utils/colorScheme';
import { useExploreAdditionalActionsMenu } from '../useExploreAdditionalActionsMenu';
import { useExploreMetadataBar } from './useExploreMetadataBar';
const propTypes = {
actions: PropTypes.object.isRequired,
@@ -160,48 +160,7 @@ export const ExploreChartHeader = ({
metadata?.dashboards,
);
const metadataBar = useMemo(() => {
if (!metadata) {
return null;
}
const items = [];
items.push({
type: MetadataType.Dashboards,
title:
metadata.dashboards.length > 0
? tn(
'Added to 1 dashboard',
'Added to %s dashboards',
metadata.dashboards.length,
metadata.dashboards.length,
)
: t('Not added to any dashboard'),
description:
metadata.dashboards.length > 0
? t(
'You can preview the list of dashboards in the chart settings dropdown.',
)
: undefined,
});
items.push({
type: MetadataType.LastModified,
value: metadata.changed_on_humanized,
modifiedBy: metadata.changed_by || t('Not available'),
});
items.push({
type: MetadataType.Owner,
createdBy: metadata.created_by || t('Not available'),
owners: metadata.owners.length > 0 ? metadata.owners : t('None'),
createdOn: metadata.created_on_humanized,
});
if (slice?.description) {
items.push({
type: MetadataType.Description,
value: slice?.description,
});
}
return <MetadataBar items={items} tooltipPlacement="bottom" />;
}, [metadata, slice?.description]);
const metadataBar = useExploreMetadataBar(metadata, slice);
const oldSliceName = slice?.slice_name;
return (

View File

@@ -0,0 +1,71 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { useMemo } from 'react';
import { t, tn } from '@superset-ui/core';
import MetadataBar, { MetadataType } from 'src/components/MetadataBar';
import { ExplorePageInitialData } from 'src/explore/types';
export const useExploreMetadataBar = (
metadata: ExplorePageInitialData['metadata'],
slice: ExplorePageInitialData['slice'],
) =>
useMemo(() => {
if (!metadata) {
return null;
}
const items = [];
if (metadata.dashboards) {
items.push({
type: MetadataType.Dashboards as const,
title:
metadata.dashboards.length > 0
? tn(
'Added to 1 dashboard',
'Added to %s dashboards',
metadata.dashboards.length,
metadata.dashboards.length,
)
: t('Not added to any dashboard'),
description:
metadata.dashboards.length > 0
? t(
'You can preview the list of dashboards in the chart settings dropdown.',
)
: undefined,
});
}
items.push({
type: MetadataType.LastModified as const,
value: metadata.changed_on_humanized,
modifiedBy: metadata.changed_by || t('Not available'),
});
items.push({
type: MetadataType.Owner as const,
createdBy: metadata.created_by || t('Not available'),
owners: metadata.owners.length > 0 ? metadata.owners : t('None'),
createdOn: metadata.created_on_humanized,
});
if (slice?.description) {
items.push({
type: MetadataType.Description as const,
value: slice?.description,
});
}
return <MetadataBar items={items} tooltipPlacement="bottom" />;
}, [metadata, slice?.description]);

View File

@@ -82,6 +82,10 @@ export interface ExplorePageInitialData {
owners: string[];
created_by?: string;
changed_by?: string;
dashboards?: {
id: number;
dashboard_title: string;
}[];
};
saveAction?: SaveActionType | null;
}

View File

@@ -36,7 +36,7 @@ from superset.errors import ErrorLevel, SupersetError, SupersetErrorType
from superset.exceptions import SupersetErrorException
from superset.extensions import db
from superset.models.core import Database
from superset.sql_parse import Table
from superset.sql_parse import ParsedQuery, Table
from superset.utils.decorators import on_error, transaction
logger = logging.getLogger(__name__)
@@ -70,7 +70,10 @@ class DuplicateDatasetCommand(CreateMixin, BaseCommand):
table.normalize_columns = self._base_model.normalize_columns
table.always_filter_main_dttm = self._base_model.always_filter_main_dttm
table.is_sqllab_view = True
table.sql = self._base_model.sql.strip().strip(";")
table.sql = ParsedQuery(
self._base_model.sql,
engine=database.db_engine_spec.engine,
).stripped()
db.session.add(table)
cols = []
for config_ in self._base_model.columns:

View File

@@ -1778,7 +1778,7 @@ GUEST_TOKEN_VALIDATOR_HOOK = None
# def DATASET_HEALTH_CHECK(datasource: SqlaTable) -> Optional[str]:
# if (
# datasource.sql and
# len(SQLScript(datasource.sql).tables) == 1
# len(sql_parse.ParsedQuery(datasource.sql, strip_comments=True).tables) == 1
# ):
# return (
# "This virtual dataset queries only one table and therefore could be "

View File

@@ -67,7 +67,7 @@ from sqlalchemy.orm.mapper import Mapper
from sqlalchemy.schema import UniqueConstraint
from sqlalchemy.sql import column, ColumnElement, literal_column, table
from sqlalchemy.sql.elements import ColumnClause, TextClause
from sqlalchemy.sql.expression import Label
from sqlalchemy.sql.expression import Label, TextAsFrom
from sqlalchemy.sql.selectable import Alias, TableClause
from superset import app, db, is_feature_enabled, security_manager
@@ -104,7 +104,7 @@ from superset.models.helpers import (
QueryResult,
)
from superset.models.slice import Slice
from superset.sql_parse import Table
from superset.sql_parse import ParsedQuery, Table
from superset.superset_typing import (
AdhocColumn,
AdhocMetric,
@@ -1469,13 +1469,34 @@ class SqlaTable(
return tbl
def get_from_clause(
self,
template_processor: BaseTemplateProcessor | None = None,
self, template_processor: BaseTemplateProcessor | None = None
) -> tuple[TableClause | Alias, str | None]:
"""
Return where to select the columns and metrics from. Either a physical table
or a virtual table with it's own subquery. If the FROM is referencing a
CTE, the CTE is returned as the second value in the return tuple.
"""
if not self.is_virtual:
return self.get_sqla_table(), None
return super().get_from_clause(template_processor)
from_sql = self.get_rendered_sql(template_processor) + "\n"
parsed_query = ParsedQuery(from_sql, engine=self.db_engine_spec.engine)
if not (
parsed_query.is_unknown()
or self.db_engine_spec.is_readonly_query(parsed_query)
):
raise QueryObjectValidationError(
_("Virtual dataset query must be read-only")
)
cte = self.db_engine_spec.get_cte_query(from_sql)
from_clause = (
table(self.db_engine_spec.cte_alias)
if cte
else TextAsFrom(self.text(from_sql), []).alias(VIRTUAL_TABLE_ALIAS)
)
return from_clause, cte
def adhoc_metric_to_sqla(
self,

View File

@@ -63,7 +63,7 @@ from superset.constants import QUERY_CANCEL_KEY, TimeGrain as TimeGrainConstants
from superset.databases.utils import get_table_metadata, make_url_safe
from superset.errors import ErrorLevel, SupersetError, SupersetErrorType
from superset.exceptions import DisallowedSQLFunction, OAuth2Error, OAuth2RedirectError
from superset.sql.parse import BaseSQLStatement, SQLScript, Table
from superset.sql.parse import SQLScript, Table
from superset.sql_parse import ParsedQuery
from superset.superset_typing import (
OAuth2ClientConfig,
@@ -1737,19 +1737,18 @@ class BaseEngineSpec: # pylint: disable=too-many-public-methods
)
@classmethod
def process_statement(
cls,
statement: BaseSQLStatement[Any],
database: Database,
) -> str:
def process_statement(cls, statement: str, database: Database) -> str:
"""
Process a SQL statement by mutating it.
Process a SQL statement by stripping and mutating it.
:param statement: A single SQL statement
:param database: Database instance
:return: Dictionary with different costs
"""
return database.mutate_sql_based_on_config(str(statement), is_split=True)
parsed_query = ParsedQuery(statement, engine=cls.engine)
sql = parsed_query.stripped()
return database.mutate_sql_based_on_config(sql, is_split=True)
@classmethod
def estimate_query_cost( # pylint: disable=too-many-arguments
@@ -1774,7 +1773,8 @@ class BaseEngineSpec: # pylint: disable=too-many-public-methods
"Database does not support cost estimation"
)
parsed_script = SQLScript(sql, engine=cls.engine)
parsed_query = sql_parse.ParsedQuery(sql, engine=cls.engine)
statements = parsed_query.get_statements()
with database.get_raw_connection(
catalog=catalog,
@@ -1788,7 +1788,7 @@ class BaseEngineSpec: # pylint: disable=too-many-public-methods
cls.process_statement(statement, database),
cursor,
)
for statement in parsed_script.statements
for statement in statements
]
@classmethod
@@ -2056,6 +2056,15 @@ class BaseEngineSpec: # pylint: disable=too-many-public-methods
logger.error(ex, exc_info=True)
raise
@classmethod
def is_readonly_query(cls, parsed_query: ParsedQuery) -> bool:
"""Pessimistic readonly, 100% sure statement won't mutate anything"""
return (
parsed_query.is_select()
or parsed_query.is_explain()
or parsed_query.is_show()
)
@classmethod
def is_select_query(cls, parsed_query: ParsedQuery) -> bool:
"""
@@ -2169,6 +2178,10 @@ class BaseEngineSpec: # pylint: disable=too-many-public-methods
return False
@classmethod
def parse_sql(cls, sql: str) -> list[str]:
return [str(s).strip(" ;") for s in sqlparse.parse(sql)]
@classmethod
def get_impersonation_key(cls, user: User | None) -> Any:
"""

View File

@@ -36,6 +36,7 @@ from sqlalchemy.engine.reflection import Inspector
from sqlalchemy.engine.url import URL
from sqlalchemy.sql import sqltypes
from superset import sql_parse
from superset.constants import TimeGrain
from superset.databases.schemas import encrypted_field_properties, EncryptedString
from superset.databases.utils import make_url_safe
@@ -43,7 +44,6 @@ from superset.db_engine_specs.base import BaseEngineSpec, BasicPropertiesType
from superset.db_engine_specs.exceptions import SupersetDBAPIConnectionError
from superset.errors import SupersetError, SupersetErrorType
from superset.exceptions import SupersetException
from superset.sql.parse import SQLScript
from superset.sql_parse import Table
from superset.superset_typing import ResultSetColumnType
from superset.utils import core as utils, json
@@ -449,7 +449,8 @@ class BigQueryEngineSpec(BaseEngineSpec): # pylint: disable=too-many-public-met
if not cls.get_allow_cost_estimate(extra):
raise SupersetException("Database does not support cost estimation")
parsed_script = SQLScript(sql, engine=cls.engine)
parsed_query = sql_parse.ParsedQuery(sql, engine=cls.engine)
statements = parsed_query.get_statements()
with cls.get_engine(
database,
@@ -462,7 +463,7 @@ class BigQueryEngineSpec(BaseEngineSpec): # pylint: disable=too-many-public-met
cls.process_statement(statement, database),
client,
)
for statement in parsed_script.statements
for statement in statements
]
@classmethod

View File

@@ -45,7 +45,7 @@ from superset.db_engine_specs.presto import PrestoEngineSpec
from superset.exceptions import SupersetException
from superset.extensions import cache_manager
from superset.models.sql_lab import Query
from superset.sql_parse import Table
from superset.sql_parse import ParsedQuery, Table
from superset.superset_typing import ResultSetColumnType
if TYPE_CHECKING:
@@ -605,6 +605,15 @@ class HiveEngineSpec(PrestoEngineSpec):
# otherwise, return no function names to prevent errors
return []
@classmethod
def is_readonly_query(cls, parsed_query: ParsedQuery) -> bool:
"""Pessimistic readonly, 100% sure statement won't mutate anything"""
return (
super().is_readonly_query(parsed_query)
or parsed_query.is_set()
or parsed_query.is_show()
)
@classmethod
def has_implicit_cancel(cls) -> bool:
"""

View File

@@ -104,6 +104,11 @@ class KustoSqlEngineSpec(BaseEngineSpec): # pylint: disable=abstract-method
return f"""CONVERT(DATETIME, '{datetime_formatted}', 126)"""
return None
@classmethod
def is_readonly_query(cls, parsed_query: ParsedQuery) -> bool:
"""Pessimistic readonly, 100% sure statement won't mutate anything"""
return parsed_query.sql.lower().startswith("select")
class KustoKqlEngineSpec(BaseEngineSpec): # pylint: disable=abstract-method
limit_method = LimitMethod.WRAP_SQL
@@ -153,6 +158,23 @@ class KustoKqlEngineSpec(BaseEngineSpec): # pylint: disable=abstract-method
return None
@classmethod
def is_readonly_query(cls, parsed_query: ParsedQuery) -> bool:
"""
Pessimistic readonly, 100% sure statement won't mutate anything.
"""
return KustoKqlEngineSpec.is_select_query(
parsed_query
) or parsed_query.sql.startswith(".show")
@classmethod
def is_select_query(cls, parsed_query: ParsedQuery) -> bool:
return not parsed_query.sql.startswith(".")
@classmethod
def parse_sql(cls, sql: str) -> list[str]:
"""
Kusto supports a single query statement, but it could include sub queries
and variables declared via let keyword.
"""
return [sql]

View File

@@ -33,7 +33,6 @@ from sqlalchemy import text
from sqlalchemy.engine.reflection import Inspector
from sqlalchemy.engine.url import URL
from sqlalchemy.exc import NoSuchTableError
from trino.exceptions import HttpError
from superset import db
from superset.constants import QUERY_CANCEL_KEY, QUERY_EARLY_CANCEL_KEY, USER_AGENT
@@ -61,6 +60,12 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__)
try:
# since trino is an optional dependency, we need to handle the ImportError
from trino.exceptions import HttpError
except ImportError:
HttpError = Exception
class CustomTrinoAuthErrorMeta(type):
def __instancecheck__(cls, instance: object) -> bool:

View File

@@ -74,7 +74,6 @@ from superset.extensions import (
)
from superset.models.helpers import AuditMixinNullable, ImportExportMixin
from superset.result_set import SupersetResultSet
from superset.sql.parse import SQLScript
from superset.sql_parse import Table
from superset.superset_typing import OAuth2ClientConfig, ResultSetColumnType
from superset.utils import cache as cache_util, core as utils, json
@@ -675,7 +674,7 @@ class Database(Model, AuditMixinNullable, ImportExportMixin): # pylint: disable
schema: str | None = None,
mutator: Callable[[pd.DataFrame], None] | None = None,
) -> pd.DataFrame:
parsed_script = SQLScript(sql, engine=self.db_engine_spec.engine)
sqls = self.db_engine_spec.parse_sql(sql)
with self.get_sqla_engine(catalog=catalog, schema=schema) as engine:
engine_url = engine.url
@@ -692,9 +691,8 @@ class Database(Model, AuditMixinNullable, ImportExportMixin): # pylint: disable
with self.get_raw_connection(catalog=catalog, schema=schema) as conn:
cursor = conn.cursor()
df = None
for i, statement in enumerate(parsed_script.statements):
# pylint: disable=protected-access
sql_ = self.mutate_sql_based_on_config(statement._sql, is_split=True)
for i, sql_ in enumerate(sqls):
sql_ = self.mutate_sql_based_on_config(sql_, is_split=True)
_log_query(sql_)
with event_logger.log_context(
action="execute_sql",
@@ -702,7 +700,7 @@ class Database(Model, AuditMixinNullable, ImportExportMixin): # pylint: disable
object_ref=__name__,
):
self.db_engine_spec.execute(cursor, sql_, self)
if i < len(parsed_script.statements) - 1:
if i < len(sqls) - 1:
# If it's not the last, we don't keep the results
cursor.fetchall()
else:

View File

@@ -72,6 +72,7 @@ from superset.sql.parse import SQLScript
from superset.sql_parse import (
has_table_query,
insert_rls_in_predicate,
ParsedQuery,
sanitize_clause,
)
from superset.superset_typing import (
@@ -1038,9 +1039,6 @@ class ExploreMixin: # pylint: disable=too-many-public-methods
"""
Render sql with template engine (Jinja).
"""
if not self.sql:
return ""
sql = self.sql.strip("\t\r\n; ")
if template_processor:
try:
@@ -1074,9 +1072,13 @@ class ExploreMixin: # pylint: disable=too-many-public-methods
or a virtual table with it's own subquery. If the FROM is referencing a
CTE, the CTE is returned as the second value in the return tuple.
"""
from_sql = self.get_rendered_sql(template_processor) + "\n"
parsed_script = SQLScript(from_sql, engine=self.db_engine_spec.engine)
if parsed_script.has_mutation():
parsed_query = ParsedQuery(from_sql, engine=self.db_engine_spec.engine)
if not (
parsed_query.is_unknown()
or self.db_engine_spec.is_readonly_query(parsed_query)
):
raise QueryObjectValidationError(
_("Virtual dataset query must be read-only")
)

View File

@@ -20,11 +20,10 @@ from __future__ import annotations
import enum
import logging
import re
import string
import urllib.parse
from collections.abc import Iterable
from dataclasses import dataclass
from typing import Any, Generic, Iterator, TypeVar
from typing import Any, Generic, TypeVar
import sqlglot
import sqlparse
@@ -227,12 +226,6 @@ class BaseSQLStatement(Generic[InternalRepresentation]):
"""
raise NotImplementedError()
def is_select(self) -> bool:
"""
Check if the statement is a `SELECT` statement.
"""
raise NotImplementedError()
def __str__(self) -> str:
return self.format()
@@ -389,12 +382,6 @@ class SQLStatement(BaseSQLStatement[exp.Expression]):
return False
def is_select(self) -> bool:
"""
Check if the statement is a `SELECT` statement.
"""
return isinstance(self._parsed, exp.Select)
def format(self, comments: bool = True) -> str:
"""
Pretty-format the SQL statement.
@@ -444,115 +431,60 @@ class SQLStatement(BaseSQLStatement[exp.Expression]):
}
class KQLTokenizeState(enum.Enum):
class KQLSplitState(enum.Enum):
"""
State machine for tokenizing a KQL script.
State machine for splitting a KQL script.
The state machine keeps track of whether we're inside a string or not, so we
don't split the script in a semi-colon that's part of a string.
"""
OUTSIDE = enum.auto()
OUTSIDE_STRING = enum.auto()
INSIDE_SINGLE_QUOTED_STRING = enum.auto()
INSIDE_DOUBLE_QUOTED_STRING = enum.auto()
INSIDE_MULTILINE_STRING = enum.auto()
INSIDE_SINGLE_QUOTED_IDENTIFIER = enum.auto()
INSIDE_DOUBLE_QUOTED_IDENTIFIER = enum.auto()
def tokenize_kql(kql: str) -> Iterator[str]:
"""
Tokenize a KQL script.
"""
valid_identifier_chars = set(string.ascii_letters + string.digits + "_")
valid_quoted_identifier_chars = valid_identifier_chars | set(" .-")
script = kql if kql.endswith(";") else kql + ";"
cursor = 0
while cursor < len(script):
rest = script[cursor:]
# quoted identifiers
if rest[:2] in {"['", '["'}:
match = "']" if rest[:2] == "['" else '"]'
if match not in rest[2:]:
raise SupersetParseError(
script,
"kustokql",
message="Unclosed quoted identifier",
)
token = rest[: rest.index(match, 2) + 2]
if any(char not in valid_quoted_identifier_chars for char in token[2:-2]):
raise SupersetParseError(
script,
"kustokql",
message="Invalid quoted identifier",
)
yield token
cursor += len(token)
# multi-line strings
elif rest[:3] == "```":
if "```" not in rest[3:]:
raise SupersetParseError(
script,
"kustokql",
message="Unclosed multi-line string",
)
token = rest[: rest.index("```", 3) + 3]
yield token
cursor += len(token)
# single-quoted strings
elif rest[0] in {'"', "'"}:
match = rest[0]
# find first unescaped quote
start = 1
while True:
if match not in rest[start:]:
raise SupersetParseError(
script,
"kustokql",
message="Unclosed string",
)
index = rest.index(match, start)
if rest[index - 1] != "\\":
break
start = index + 1
token = rest[: index + 1]
yield token
cursor += len(token)
# identifiers and keywords
else:
for i, char in enumerate(rest):
if char not in valid_identifier_chars:
if i > 0:
yield rest[:i]
yield char
cursor += i + 1
break
def split_kql(kql: str) -> list[str]:
"""
Custom function for splitting KQL statements.
"""
statements: list[str] = []
statement: list[str] = []
for token in tokenize_kql(kql):
if token == ";":
statements.append("".join(statement))
statement = []
else:
statement.append(token)
statements = []
state = KQLSplitState.OUTSIDE_STRING
statement_start = 0
script = kql if kql.endswith(";") else kql + ";"
for i, character in enumerate(script):
if state == KQLSplitState.OUTSIDE_STRING:
if character == ";":
statements.append(script[statement_start:i])
statement_start = i + 1
elif character == "'":
state = KQLSplitState.INSIDE_SINGLE_QUOTED_STRING
elif character == '"':
state = KQLSplitState.INSIDE_DOUBLE_QUOTED_STRING
elif character == "`" and script[i - 2 : i] == "``":
state = KQLSplitState.INSIDE_MULTILINE_STRING
elif (
state == KQLSplitState.INSIDE_SINGLE_QUOTED_STRING
and character == "'"
and script[i - 1] != "\\"
):
state = KQLSplitState.OUTSIDE_STRING
elif (
state == KQLSplitState.INSIDE_DOUBLE_QUOTED_STRING
and character == '"'
and script[i - 1] != "\\"
):
state = KQLSplitState.OUTSIDE_STRING
elif (
state == KQLSplitState.INSIDE_MULTILINE_STRING
and character == "`"
and script[i - 2 : i] == "``"
):
state = KQLSplitState.OUTSIDE_STRING
return statements
@@ -574,14 +506,6 @@ class KustoKQLStatement(BaseSQLStatement[str]):
details about it.
"""
def __init__(
self,
statement: str,
engine: str = "kustokql",
ast: str | None = None,
):
super().__init__(statement, engine, ast)
@classmethod
def split_script(
cls,
@@ -664,56 +588,6 @@ class KustoKQLStatement(BaseSQLStatement[str]):
"""
return self._parsed.startswith(".") and not self._parsed.startswith(".show")
def is_select(self) -> bool:
"""
Check if the statement is a `SELECT` statement.
"""
if not self._parsed or self.is_mutating():
return False
# strip comments
kql = "\n".join(
line
for line in self._parsed.split("\n")
if not line.strip().startswith("//")
).strip()
first_token = next(tokenize_kql(kql), None)
if not first_token:
return False
return first_token == "|" or self._is_identifier(first_token)
@staticmethod
def _is_identifier(identifier: str) -> bool:
"""
Validates if a given string is a valid KQL identifier.
From the documentation:
Identifiers are case-sensitive. Database names are case-insensitive, and
therefore an exception to this rule.
Identifiers must be between 1 and 1024 characters long.
Identifiers may contain letters, digits, and underscores (_).
Identifiers may contain certain special characters: spaces, dots (.), and
dashes (-). For information on how to reference identifiers with special
characters, see Reference identifiers in queries.
"""
valid_chars = set(string.ascii_letters + string.digits + "_")
# Identifiers names that (1) include special character, (2) are language
# keywords, or (3) are literals must be enclosed using [' and '] or [" and "].
if (identifier.startswith("['") and identifier.endswith("']")) or (
identifier.startswith('["') and identifier.endswith('"]')
):
identifier = identifier[2:-2]
valid_chars.update(" .-")
return 1 <= len(identifier) <= 1024 and all(
char in valid_chars for char in identifier
)
class SQLScript:
"""
@@ -768,24 +642,6 @@ class SQLScript:
"""
return any(statement.is_mutating() for statement in self.statements)
def is_valid_ctas(self) -> bool:
"""
Check if the script contains a valid CTAS statement.
CTAS (`CREATE TABLE AS SELECT`) can only be run with scripts where the last
statement is a `SELECT`.
"""
return self.statements[-1].is_select()
def is_valid_cvas(self) -> bool:
"""
Check if the script contains a valid CVAS statement.
CVAS (`CREATE VIEW AS SELECT`) can only be run with scripts with a single
`SELECT` statement.
"""
return len(self.statements) == 1 and self.statements[0].is_select()
def extract_tables_from_statement(
statement: exp.Expression,
@@ -794,7 +650,7 @@ def extract_tables_from_statement(
"""
Extract all table references in a single statement.
Please note that this is not trivial; consider the following queries:
Please not that this is not trivial; consider the following queries:
DESCRIBE some_table;
SHOW PARTITIONS FROM some_table;

View File

@@ -20,11 +20,11 @@ from __future__ import annotations
import logging
import time
from contextlib import closing
from typing import Any, cast
from typing import Any
from superset import app
from superset.models.core import Database
from superset.sql.parse import SQLScript, SQLStatement
from superset.sql_parse import ParsedQuery
from superset.sql_validators.base import BaseSQLValidator, SQLValidationAnnotation
from superset.utils.core import QuerySource
@@ -46,15 +46,17 @@ class PrestoDBSQLValidator(BaseSQLValidator):
@classmethod
def validate_statement(
cls,
statement: SQLStatement,
statement: str,
database: Database,
cursor: Any,
) -> SQLValidationAnnotation | None:
# pylint: disable=too-many-locals
db_engine_spec = database.db_engine_spec
parsed_query = ParsedQuery(statement, engine=db_engine_spec.engine)
sql = parsed_query.stripped()
# Hook to allow environment-specific mutation (usually comments) to the SQL
sql = database.mutate_sql_based_on_config(str(statement))
sql = database.mutate_sql_based_on_config(sql)
# Transform the final statement to an explain call before sending it on
# to presto to validate
@@ -153,9 +155,10 @@ class PrestoDBSQLValidator(BaseSQLValidator):
For example, "SELECT 1 FROM default.mytable" becomes "EXPLAIN (TYPE
VALIDATE) SELECT 1 FROM default.mytable.
"""
parsed_script = SQLScript(sql, engine=database.db_engine_spec.engine)
parsed_query = ParsedQuery(sql, engine=database.db_engine_spec.engine)
statements = parsed_query.get_statements()
logger.info("Validating %i statement(s)", len(parsed_script.statements))
logger.info("Validating %i statement(s)", len(statements))
# todo(hughhh): update this to use new database.get_raw_connection()
# this function keeps stalling CI
with database.get_sqla_engine(
@@ -168,12 +171,8 @@ class PrestoDBSQLValidator(BaseSQLValidator):
annotations: list[SQLValidationAnnotation] = []
with closing(engine.raw_connection()) as conn:
cursor = conn.cursor()
for statement in parsed_script.statements:
annotation = cls.validate_statement(
cast(SQLStatement, statement),
database,
cursor,
)
for statement in parsed_query.get_statements():
annotation = cls.validate_statement(statement, database, cursor)
if annotation:
annotations.append(annotation)
logger.debug("Validation found %i error(s)", len(annotations))

View File

@@ -26,6 +26,7 @@ from jinja2.meta import find_undeclared_variables
from superset import is_feature_enabled
from superset.commands.sql_lab.execute import SqlQueryRender
from superset.errors import SupersetErrorType
from superset.sql_parse import ParsedQuery
from superset.sqllab.exceptions import SqlLabException
from superset.utils import core as utils
@@ -57,9 +58,12 @@ class SqlQueryRenderImpl(SqlQueryRender):
database=query_model.database, query=query_model
)
parsed_query = ParsedQuery(
query_model.sql,
engine=query_model.database.db_engine_spec.engine,
)
rendered_query = sql_template_processor.process_template(
query_model.sql.strip().strip(";"),
**execution_context.template_params,
parsed_query.stripped(), **execution_context.template_params
)
self._validate(execution_context, rendered_query, sql_template_processor)
return rendered_query

View File

@@ -30,7 +30,7 @@ from superset.db_engine_specs.base import (
from superset.db_engine_specs.mysql import MySQLEngineSpec
from superset.db_engine_specs.sqlite import SqliteEngineSpec
from superset.errors import ErrorLevel, SupersetError, SupersetErrorType
from superset.sql_parse import Table
from superset.sql_parse import ParsedQuery, Table
from superset.utils.database import get_example_database
from tests.integration_tests.db_engine_specs.base_tests import TestDbEngineSpec
from tests.integration_tests.test_app import app
@@ -310,6 +310,20 @@ class TestDbEngineSpecs(TestDbEngineSpec):
)
def test_is_readonly():
def is_readonly(sql: str) -> bool:
return BaseEngineSpec.is_readonly_query(ParsedQuery(sql))
assert is_readonly("SHOW LOCKS test EXTENDED")
assert not is_readonly("SET hivevar:desc='Legislators'")
assert not is_readonly("UPDATE t1 SET col1 = NULL")
assert is_readonly("EXPLAIN SELECT 1")
assert is_readonly("SELECT 1")
assert is_readonly("WITH (SELECT 1) bla SELECT * from bla")
assert is_readonly("SHOW CATALOGS")
assert is_readonly("SHOW TABLES")
def test_time_grain_denylist():
config = app.config.copy()
app.config["TIME_GRAIN_DENYLIST"] = ["PT1M", "SQLITE_NONEXISTENT_GRAIN"]

View File

@@ -23,7 +23,7 @@ from sqlalchemy.sql import select
from superset.db_engine_specs.hive import HiveEngineSpec, upload_to_s3
from superset.exceptions import SupersetException
from superset.sql_parse import Table
from superset.sql_parse import ParsedQuery, Table
from tests.integration_tests.test_app import app
@@ -222,6 +222,19 @@ def test_df_to_sql_if_exists_replace_with_schema(mock_upload_to_s3, mock_g):
app.config = config
def test_is_readonly():
def is_readonly(sql: str) -> bool:
return HiveEngineSpec.is_readonly_query(ParsedQuery(sql))
assert not is_readonly("UPDATE t1 SET col1 = NULL")
assert not is_readonly("INSERT OVERWRITE TABLE tabB SELECT a.Age FROM TableA")
assert is_readonly("SHOW LOCKS test EXTENDED")
assert is_readonly("SET hivevar:desc='Legislators'")
assert is_readonly("EXPLAIN SELECT 1")
assert is_readonly("SELECT 1")
assert is_readonly("WITH (SELECT 1) bla SELECT * from bla")
@pytest.mark.parametrize(
"schema,upload_prefix",
[("foo", "EXTERNAL_HIVE_TABLES/1/foo/"), (None, "EXTERNAL_HIVE_TABLES/1/")],

View File

@@ -25,7 +25,7 @@ from sqlalchemy.sql import select
from superset.db_engine_specs.presto import PrestoEngineSpec
from superset.errors import ErrorLevel, SupersetError, SupersetErrorType
from superset.sql_parse import Table
from superset.sql_parse import ParsedQuery, Table
from superset.utils.database import get_example_database
from tests.integration_tests.db_engine_specs.base_tests import TestDbEngineSpec
@@ -1172,6 +1172,19 @@ class TestPrestoDbEngineSpec(TestDbEngineSpec):
]
def test_is_readonly():
def is_readonly(sql: str) -> bool:
return PrestoEngineSpec.is_readonly_query(ParsedQuery(sql))
assert not is_readonly("SET hivevar:desc='Legislators'")
assert not is_readonly("UPDATE t1 SET col1 = NULL")
assert not is_readonly("INSERT OVERWRITE TABLE tabB SELECT a.Age FROM TableA")
assert is_readonly("SHOW LOCKS test EXTENDED")
assert is_readonly("EXPLAIN SELECT 1")
assert is_readonly("SELECT 1")
assert is_readonly("WITH (SELECT 1) bla SELECT * from bla")
def test_get_catalog_names(app_context: AppContext) -> None:
"""
Test the ``get_catalog_names`` method.

View File

@@ -49,6 +49,32 @@ def test_get_text_clause_with_colon() -> None:
assert text_clause.text == "SELECT foo FROM tbl WHERE foo = '123\\:456')"
def test_parse_sql_single_statement() -> None:
"""
`parse_sql` should properly strip leading and trailing spaces and semicolons
"""
from superset.db_engine_specs.base import BaseEngineSpec
queries = BaseEngineSpec.parse_sql(" SELECT foo FROM tbl ; ")
assert queries == ["SELECT foo FROM tbl"]
def test_parse_sql_multi_statement() -> None:
"""
For string with multiple SQL-statements `parse_sql` method should return list
where each element represents the single SQL-statement
"""
from superset.db_engine_specs.base import BaseEngineSpec
queries = BaseEngineSpec.parse_sql("SELECT foo FROM tbl1; SELECT bar FROM tbl2;")
assert queries == [
"SELECT foo FROM tbl1",
"SELECT bar FROM tbl2",
]
def test_validate_db_uri(mocker: MockerFixture) -> None:
"""
Ensures that the `validate_database_uri` method invokes the validator correctly

View File

@@ -24,6 +24,91 @@ from tests.unit_tests.db_engine_specs.utils import assert_convert_dttm
from tests.unit_tests.fixtures.common import dttm # noqa: F401
@pytest.mark.parametrize(
"sql,expected",
[
("SELECT foo FROM tbl", True),
("SHOW TABLES", False),
("EXPLAIN SELECT foo FROM tbl", False),
("INSERT INTO tbl (foo) VALUES (1)", False),
],
)
def test_sql_is_readonly_query(sql: str, expected: bool) -> None:
"""
Make sure that SQL dialect consider only SELECT statements as read-only
"""
from superset.db_engine_specs.kusto import KustoSqlEngineSpec
from superset.sql_parse import ParsedQuery
parsed_query = ParsedQuery(sql)
is_readonly = KustoSqlEngineSpec.is_readonly_query(parsed_query)
assert expected == is_readonly
@pytest.mark.parametrize(
"kql,expected",
[
("tbl | limit 100", True),
("let foo = 1; tbl | where bar == foo", True),
(".show tables", False),
],
)
def test_kql_is_select_query(kql: str, expected: bool) -> None:
"""
Make sure that KQL dialect consider only statements that do not start with "." (dot)
as a SELECT statements
"""
from superset.db_engine_specs.kusto import KustoKqlEngineSpec
from superset.sql_parse import ParsedQuery
parsed_query = ParsedQuery(kql)
is_select = KustoKqlEngineSpec.is_select_query(parsed_query)
assert expected == is_select
@pytest.mark.parametrize(
"kql,expected",
[
("tbl | limit 100", True),
("let foo = 1; tbl | where bar == foo", True),
(".show tables", True),
("print 1", True),
("set querytrace; Events | take 100", True),
(".drop table foo", False),
(".set-or-append table foo <| bar", False),
],
)
def test_kql_is_readonly_query(kql: str, expected: bool) -> None:
"""
Make sure that KQL dialect consider only SELECT statements as read-only
"""
from superset.db_engine_specs.kusto import KustoKqlEngineSpec
from superset.sql_parse import ParsedQuery
parsed_query = ParsedQuery(kql)
is_readonly = KustoKqlEngineSpec.is_readonly_query(parsed_query)
assert expected == is_readonly
def test_kql_parse_sql() -> None:
"""
parse_sql method should always return a list with a single element
which is an original query
"""
from superset.db_engine_specs.kusto import KustoKqlEngineSpec
queries = KustoKqlEngineSpec.parse_sql("let foo = 1; tbl | where bar == foo")
assert queries == ["let foo = 1; tbl | where bar == foo"]
@pytest.mark.parametrize(
"target_type,expected_result",
[

View File

@@ -1,290 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import os
import sys
import pytest
SHA = "22e7c602b9aa321ec7e0df4bb0033048664dcdf0"
PR_ID = "666"
OLD_REL = "2.1.0"
NEW_REL = "2.1.1"
REPO = "apache/superset"
# Add the 'scripts' directory to sys.path
scripts_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), "../../../scripts")
)
sys.path.append(scripts_dir)
import build_docker as docker_utils # Replace with the actual function name # noqa: E402
@pytest.fixture(autouse=True)
def set_env_var():
os.environ["TEST_ENV"] = "true"
yield
del os.environ["TEST_ENV"]
@pytest.mark.parametrize(
"release, expected_bool",
[
("2.1.0", False),
("2.1.1", True),
("1.0.0", False),
("3.0.0", True),
],
)
def test_is_latest_release(release, expected_bool):
assert docker_utils.is_latest_release(release) == expected_bool
@pytest.mark.parametrize(
"build_preset, build_platforms, sha, build_context, build_context_ref, expected_tags",
[
# PRs
(
"lean",
["linux/arm64"],
SHA,
"pull_request",
PR_ID,
[f"{REPO}:22e7c60-arm", f"{REPO}:{SHA}-arm", f"{REPO}:pr-{PR_ID}-arm"],
),
(
"ci",
["linux/amd64"],
SHA,
"pull_request",
PR_ID,
[f"{REPO}:22e7c60-ci", f"{REPO}:{SHA}-ci", f"{REPO}:pr-{PR_ID}-ci"],
),
(
"lean",
["linux/amd64"],
SHA,
"pull_request",
PR_ID,
[f"{REPO}:22e7c60", f"{REPO}:{SHA}", f"{REPO}:pr-{PR_ID}"],
),
(
"dev",
["linux/arm64"],
SHA,
"pull_request",
PR_ID,
[
f"{REPO}:22e7c60-dev-arm",
f"{REPO}:{SHA}-dev-arm",
f"{REPO}:pr-{PR_ID}-dev-arm",
],
),
(
"dev",
["linux/amd64"],
SHA,
"pull_request",
PR_ID,
[f"{REPO}:22e7c60-dev", f"{REPO}:{SHA}-dev", f"{REPO}:pr-{PR_ID}-dev"],
),
# old releases
(
"lean",
["linux/arm64"],
SHA,
"release",
OLD_REL,
[f"{REPO}:22e7c60-arm", f"{REPO}:{SHA}-arm", f"{REPO}:{OLD_REL}-arm"],
),
(
"lean",
["linux/amd64"],
SHA,
"release",
OLD_REL,
[f"{REPO}:22e7c60", f"{REPO}:{SHA}", f"{REPO}:{OLD_REL}"],
),
(
"dev",
["linux/arm64"],
SHA,
"release",
OLD_REL,
[
f"{REPO}:22e7c60-dev-arm",
f"{REPO}:{SHA}-dev-arm",
f"{REPO}:{OLD_REL}-dev-arm",
],
),
(
"dev",
["linux/amd64"],
SHA,
"release",
OLD_REL,
[f"{REPO}:22e7c60-dev", f"{REPO}:{SHA}-dev", f"{REPO}:{OLD_REL}-dev"],
),
# new releases
(
"lean",
["linux/arm64"],
SHA,
"release",
NEW_REL,
[
f"{REPO}:22e7c60-arm",
f"{REPO}:{SHA}-arm",
f"{REPO}:{NEW_REL}-arm",
f"{REPO}:latest-arm",
],
),
(
"lean",
["linux/amd64"],
SHA,
"release",
NEW_REL,
[f"{REPO}:22e7c60", f"{REPO}:{SHA}", f"{REPO}:{NEW_REL}", f"{REPO}:latest"],
),
(
"dev",
["linux/arm64"],
SHA,
"release",
NEW_REL,
[
f"{REPO}:22e7c60-dev-arm",
f"{REPO}:{SHA}-dev-arm",
f"{REPO}:{NEW_REL}-dev-arm",
f"{REPO}:latest-dev-arm",
],
),
(
"dev",
["linux/amd64"],
SHA,
"release",
NEW_REL,
[
f"{REPO}:22e7c60-dev",
f"{REPO}:{SHA}-dev",
f"{REPO}:{NEW_REL}-dev",
f"{REPO}:latest-dev",
],
),
# merge on master
(
"lean",
["linux/arm64"],
SHA,
"push",
"master",
[f"{REPO}:22e7c60-arm", f"{REPO}:{SHA}-arm", f"{REPO}:master-arm"],
),
(
"lean",
["linux/amd64"],
SHA,
"push",
"master",
[f"{REPO}:22e7c60", f"{REPO}:{SHA}", f"{REPO}:master"],
),
(
"dev",
["linux/arm64"],
SHA,
"push",
"master",
[
f"{REPO}:22e7c60-dev-arm",
f"{REPO}:{SHA}-dev-arm",
f"{REPO}:master-dev-arm",
],
),
(
"dev",
["linux/amd64"],
SHA,
"push",
"master",
[f"{REPO}:22e7c60-dev", f"{REPO}:{SHA}-dev", f"{REPO}:master-dev"],
),
],
)
def test_get_docker_tags(
build_preset, build_platforms, sha, build_context, build_context_ref, expected_tags
):
tags = docker_utils.get_docker_tags(
build_preset, build_platforms, sha, build_context, build_context_ref
)
for tag in expected_tags:
assert tag in tags
@pytest.mark.parametrize(
"build_preset, build_platforms, is_authenticated, sha, build_context, build_context_ref, contains",
[
(
"lean",
["linux/amd64"],
True,
SHA,
"push",
"master",
["--push", f"-t {REPO}:master "],
),
(
"dev",
["linux/amd64"],
False,
SHA,
"push",
"master",
["--load", f"-t {REPO}:master-dev ", "--target dev"],
),
# multi-platform
(
"lean",
["linux/arm64", "linux/amd64"],
True,
SHA,
"push",
"master",
["--platform linux/arm64,linux/amd64"],
),
],
)
def test_get_docker_command(
build_preset,
build_platforms,
is_authenticated,
sha,
build_context,
build_context_ref,
contains,
):
cmd = docker_utils.get_docker_command(
build_preset,
build_platforms,
is_authenticated,
sha,
build_context,
build_context_ref,
)
for s in contains:
assert s in cmd

View File

@@ -945,32 +945,6 @@ on $left.Day1 == $right.Day
("kustokql", "set querytrace; Events | take 100", False),
("kustokql", ".drop table foo", True),
("kustokql", ".set-or-append table foo <| bar", True),
("kustosql", "SELECT foo FROM tbl", False),
("kustosql", "SHOW TABLES", False),
("kustosql", "EXPLAIN SELECT foo FROM tbl", False),
("kustosql", "INSERT INTO tbl (foo) VALUES (1)", True),
("base", "SHOW LOCKS test EXTENDED", False),
("base", "SET hivevar:desc='Legislators'", False),
("base", "UPDATE t1 SET col1 = NULL", True),
("base", "EXPLAIN SELECT 1", False),
("base", "SELECT 1", False),
("base", "WITH bla AS (SELECT 1) SELECT * FROM bla", False),
("base", "SHOW CATALOGS", False),
("base", "SHOW TABLES", False),
("hive", "UPDATE t1 SET col1 = NULL", True),
("hive", "INSERT OVERWRITE TABLE tabB SELECT a.Age FROM TableA", True),
("hive", "SHOW LOCKS test EXTENDED", False),
("hive", "SET hivevar:desc='Legislators'", False),
("hive", "EXPLAIN SELECT 1", False),
("hive", "SELECT 1", False),
("hive", "WITH bla AS (SELECT 1) SELECT * FROM bla", False),
("presto", "SET hivevar:desc='Legislators'", False),
("presto", "UPDATE t1 SET col1 = NULL", True),
("presto", "INSERT OVERWRITE TABLE tabB SELECT a.Age FROM TableA", True),
("presto", "SHOW LOCKS test EXTENDED", False),
("presto", "EXPLAIN SELECT 1", False),
("presto", "SELECT 1", False),
("presto", "WITH bla AS (SELECT 1) SELECT * FROM bla", False),
],
)
def test_has_mutation(engine: str, sql: str, expected: bool) -> None:
@@ -1068,106 +1042,9 @@ def test_custom_dialect(app: None) -> None:
)
def test_is_mutating(engine: str) -> None:
"""
Global tests for `is_mutating`, covering all supported engines.
Tests for `is_mutating`.
"""
assert not SQLStatement(
"with source as ( select 1 as one ) select * from source",
engine=engine,
).is_mutating()
@pytest.mark.parametrize(
"identifier,expected",
[
# Rule: Identifiers are case-sensitive
("myTable", True),
("MYTABLE", True),
("MyTable", True),
# Rule: Identifiers must be between 1 and 1024 characters long
("a", True),
("a" * 1024, True),
("a" * 1025, False),
("", False),
# Rule: Identifiers may contain letters, digits, and underscores
("My_Table_123", True),
("123Table", True),
# Rule: Identifiers may contain special characters: spaces, dots, dashes (when quoted)
("['My Table']", True),
("['My-Table']", True),
("['My.Table']", True),
("['Table-']", True),
("['My Table Name']", True),
("['My!Table']", False),
("['MyTable ']", True),
(" MyTable", False),
("MyTable ", False),
# Rule: Non-special identifiers don't require quoting
("MyTable", True),
("My-Table", False),
# Invalid quoting
("['Invalid]", False),
("['Invalid'Name']", False),
("['']", False),
# Rule: Literal identifiers or language keywords
("['select']", True),
("select", True),
],
)
def test_is_kql_identifier(identifier: str, expected: bool):
"""
Tests the _is_identifier method for various valid and invalid cases.
"""
assert KustoKQLStatement._is_identifier(identifier) == expected
@pytest.mark.parametrize(
"kql,expected",
[
# Simple SELECT-like statements (non-mutating queries)
("MyTable | count", True),
("MyTable", True),
("| count", True),
("tbl | limit 100", True),
(".show tables", False),
# With comments (ensure comments are stripped out)
("// Comment only", False),
("MyTable // trailing comment", True),
("// leading comment\nMyTable", True),
("MyTable\n// intermediate comment\n| count", True),
# Mutating query (should return False)
(".drop MyTable", False),
(
".update MyTable set Column1 = 100",
False,
),
(".alter MyTable", False),
# Edge cases for first token
("", False),
(" ", False),
(".command", False),
("['My Table']", True),
# Complex multi-line queries
(
"""
// Initial comment
MyTable
| where Column1 > 100
""",
True,
),
(
"""
MyTable
| where Column1 > 100
| summarize by Column2
// Final comment
""",
True,
),
],
)
def test_kql_is_select(kql: str, expected: bool):
"""
Tests the is_select method for various valid and invalid cases.
"""
assert KustoKQLStatement(kql).is_select() == expected

View File

@@ -1,268 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import os
import sys
import pytest
SHA = "22e7c602b9aa321ec7e0df4bb0033048664dcdf0"
PR_ID = "666"
OLD_REL = "2.1.0"
NEW_REL = "2.1.1"
REPO = "apache/superset"
# Add the 'scripts' directory to sys.path
scripts_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), "../../../scripts")
)
sys.path.append(scripts_dir)
import build_docker as docker_utils # Replace with the actual function name # noqa: E402
@pytest.fixture(autouse=True)
def set_env_var():
os.environ["TEST_ENV"] = "true"
yield
del os.environ["TEST_ENV"]
@pytest.mark.parametrize(
"release, expected_bool",
[
("2.1.0", False),
("2.1.1", True),
("1.0.0", False),
("3.0.0", True),
],
)
def test_is_latest_release(release, expected_bool):
assert docker_utils.is_latest_release(release) == expected_bool
@pytest.mark.parametrize(
"build_preset, build_platform, sha, build_context, build_context_ref, expected_tags",
[
# PRs
(
"lean",
"linux/arm64",
SHA,
"pull_request",
PR_ID,
[f"{REPO}:22e7c60-arm", f"{REPO}:{SHA}-arm"],
),
(
"lean",
"linux/amd64",
SHA,
"pull_request",
PR_ID,
[f"{REPO}:22e7c60", f"{REPO}:{SHA}"],
),
(
"dev",
"linux/arm64",
SHA,
"pull_request",
PR_ID,
[f"{REPO}:22e7c60-dev-arm", f"{REPO}:{SHA}-dev-arm"],
),
(
"dev",
"linux/amd64",
SHA,
"pull_request",
PR_ID,
[f"{REPO}:22e7c60-dev", f"{REPO}:{SHA}-dev"],
),
# old releases
(
"lean",
"linux/arm64",
SHA,
"release",
OLD_REL,
[f"{REPO}:22e7c60-arm", f"{REPO}:{SHA}-arm", f"{REPO}:{OLD_REL}-arm"],
),
(
"lean",
"linux/amd64",
SHA,
"release",
OLD_REL,
[f"{REPO}:22e7c60", f"{REPO}:{SHA}", f"{REPO}:{OLD_REL}"],
),
(
"dev",
"linux/arm64",
SHA,
"release",
OLD_REL,
[
f"{REPO}:22e7c60-dev-arm",
f"{REPO}:{SHA}-dev-arm",
f"{REPO}:{OLD_REL}-dev-arm",
],
),
(
"dev",
"linux/amd64",
SHA,
"release",
OLD_REL,
[f"{REPO}:22e7c60-dev", f"{REPO}:{SHA}-dev", f"{REPO}:{OLD_REL}-dev"],
),
# new releases
(
"lean",
"linux/arm64",
SHA,
"release",
NEW_REL,
[
f"{REPO}:22e7c60-arm",
f"{REPO}:{SHA}-arm",
f"{REPO}:{NEW_REL}-arm",
f"{REPO}:latest-arm",
],
),
(
"lean",
"linux/amd64",
SHA,
"release",
NEW_REL,
[f"{REPO}:22e7c60", f"{REPO}:{SHA}", f"{REPO}:{NEW_REL}", f"{REPO}:latest"],
),
(
"dev",
"linux/arm64",
SHA,
"release",
NEW_REL,
[
f"{REPO}:22e7c60-dev-arm",
f"{REPO}:{SHA}-dev-arm",
f"{REPO}:{NEW_REL}-dev-arm",
f"{REPO}:latest-dev-arm",
],
),
(
"dev",
"linux/amd64",
SHA,
"release",
NEW_REL,
[
f"{REPO}:22e7c60-dev",
f"{REPO}:{SHA}-dev",
f"{REPO}:{NEW_REL}-dev",
f"{REPO}:latest-dev",
],
),
# merge on master
(
"lean",
"linux/arm64",
SHA,
"push",
"master",
[f"{REPO}:22e7c60-arm", f"{REPO}:{SHA}-arm", f"{REPO}:master-arm"],
),
(
"lean",
"linux/amd64",
SHA,
"push",
"master",
[f"{REPO}:22e7c60", f"{REPO}:{SHA}", f"{REPO}:master"],
),
(
"dev",
"linux/arm64",
SHA,
"push",
"master",
[
f"{REPO}:22e7c60-dev-arm",
f"{REPO}:{SHA}-dev-arm",
f"{REPO}:master-dev-arm",
],
),
(
"dev",
"linux/amd64",
SHA,
"push",
"master",
[f"{REPO}:22e7c60-dev", f"{REPO}:{SHA}-dev", f"{REPO}:master-dev"],
),
],
)
def test_get_docker_tags(
build_preset, build_platform, sha, build_context, build_context_ref, expected_tags
):
tags = docker_utils.get_docker_tags(
build_preset, build_platform, sha, build_context, build_context_ref
)
for tag in expected_tags:
assert tag in tags
@pytest.mark.parametrize(
"build_preset, build_platform, is_authenticated, sha, build_context, build_context_ref, contains",
[
(
"lean",
"linux/amd64",
True,
SHA,
"push",
"master",
["--push", f"-t {REPO}:master "],
),
(
"dev",
"linux/amd64",
False,
SHA,
"push",
"master",
["--load", f"-t {REPO}:master-dev "],
),
],
)
def test_get_docker_command(
build_preset,
build_platform,
is_authenticated,
sha,
build_context,
build_context_ref,
contains,
):
cmd = docker_utils.get_docker_command(
build_preset,
build_platform,
is_authenticated,
sha,
build_context,
build_context_ref,
)
for s in contains:
assert s in cmd