diff --git a/.github/workflows/cancel_duplicates.yml b/.github/workflows/cancel_duplicates.yml deleted file mode 100644 index 76525767ad0..00000000000 --- a/.github/workflows/cancel_duplicates.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: Cancel Duplicates -on: - workflow_run: - workflows: - - "Miscellaneous" - types: - - requested - -jobs: - cancel-duplicate-runs: - name: Cancel duplicate workflow runs - runs-on: ubuntu-24.04 - permissions: - actions: write - contents: read - steps: - - name: Check number of queued tasks - id: check_queued - env: - GITHUB_TOKEN: ${{ github.token }} - GITHUB_REPO: ${{ github.repository }} - run: | - get_count() { - echo $(curl -s -H "Authorization: token $GITHUB_TOKEN" \ - "https://api.github.com/repos/$GITHUB_REPO/actions/runs?status=$1" | \ - jq ".total_count") - } - count=$(( `get_count queued` + `get_count in_progress` )) - echo "Found $count unfinished jobs." - echo "count=$count" >> $GITHUB_OUTPUT - - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - if: steps.check_queued.outputs.count >= 20 - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - - - name: Cancel duplicate workflow runs - if: steps.check_queued.outputs.count >= 20 - env: - GITHUB_TOKEN: ${{ github.token }} - GITHUB_REPOSITORY: ${{ github.repository }} - run: | - pip install click requests typing_extensions python-dateutil - python ./scripts/cancel_github_workflows.py diff --git a/.github/workflows/ephemeral-env-pr-close.yml b/.github/workflows/ephemeral-env-pr-close.yml deleted file mode 100644 index b7f79c69283..00000000000 --- a/.github/workflows/ephemeral-env-pr-close.yml +++ /dev/null @@ -1,83 +0,0 @@ -name: Cleanup ephemeral envs (PR close) [DEPRECATED] - -# ⚠️ DEPRECATION NOTICE ⚠️ -# This workflow is deprecated and will be removed in a future version. -# The new Superset Showtime workflow handles cleanup automatically. -# See .github/workflows/showtime.yml and showtime-cleanup.yml for replacements. -# Migration guide: https://github.com/mistercrunch/superset-showtime - -on: - pull_request_target: - types: [closed] - -jobs: - config: - runs-on: ubuntu-24.04 - outputs: - has-secrets: ${{ steps.check.outputs.has-secrets }} - steps: - - name: "Check for secrets" - id: check - shell: bash - run: | - if [ -n "${AWS_ACCESS_KEY_ID}" ]; then - echo "has-secrets=1" >> "$GITHUB_OUTPUT" - fi - - env: - AWS_ACCESS_KEY_ID: ${{ (secrets.AWS_ACCESS_KEY_ID != '' && secrets.AWS_SECRET_ACCESS_KEY != '') || '' }} - ephemeral-env-cleanup: - needs: config - if: needs.config.outputs.has-secrets - name: Cleanup ephemeral envs - runs-on: ubuntu-24.04 - permissions: - pull-requests: write - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: us-west-2 - - - name: Describe ECS service - id: describe-services - run: | - echo "active=$(aws ecs describe-services --cluster superset-ci --services pr-${{ github.event.number }}-service | jq '.services[] | select(.status == "ACTIVE") | any')" >> $GITHUB_OUTPUT - - - name: Delete ECS service - if: steps.describe-services.outputs.active == 'true' - id: delete-service - run: | - aws ecs delete-service \ - --cluster superset-ci \ - --service pr-${{ github.event.number }}-service \ - --force - - - name: Login to Amazon ECR - if: steps.describe-services.outputs.active == 'true' - id: login-ecr - uses: aws-actions/amazon-ecr-login@fa648b43de3d4d023bcb3f89ed6940096949c419 # v2 - - - name: Delete ECR image tag - if: steps.describe-services.outputs.active == 'true' - id: delete-image-tag - run: | - aws ecr batch-delete-image \ - --registry-id $(echo "${{ steps.login-ecr.outputs.registry }}" | grep -Eo "^[0-9]+") \ - --repository-name superset-ci \ - --image-ids imageTag=pr-${{ github.event.number }} - - - name: Comment (success) - if: steps.describe-services.outputs.active == 'true' - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - with: - github-token: ${{github.token}} - script: | - github.rest.issues.createComment({ - issue_number: ${{ github.event.number }}, - owner: context.repo.owner, - repo: context.repo.repo, - body: '⚠️ **DEPRECATED WORKFLOW** - Ephemeral environment shutdown and build artifacts deleted. Please migrate to the new Superset Showtime system for future PRs.' - }) diff --git a/.github/workflows/ephemeral-env.yml b/.github/workflows/ephemeral-env.yml deleted file mode 100644 index 8dd91b80c98..00000000000 --- a/.github/workflows/ephemeral-env.yml +++ /dev/null @@ -1,350 +0,0 @@ -name: Ephemeral env workflow [DEPRECATED] - -# ⚠️ DEPRECATION NOTICE ⚠️ -# This workflow is deprecated and will be removed in a future version. -# Please use the new Superset Showtime workflow instead: -# - Use label "🎪 trigger-start" instead of "testenv-up" -# - Showtime provides better reliability and easier management -# - See .github/workflows/showtime.yml for the replacement -# - Migration guide: https://github.com/mistercrunch/superset-showtime - -# Example manual trigger: -# gh workflow run ephemeral-env.yml --ref fix_ephemerals --field label_name="testenv-up" --field issue_number=666 - -on: - pull_request_target: - types: - - labeled - workflow_dispatch: - inputs: - label_name: - description: 'Label name to simulate label-based /testenv trigger' - required: true - default: 'testenv-up' - issue_number: - description: 'Issue or PR number' - required: true - -jobs: - ephemeral-env-label: - concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}-label - cancel-in-progress: true - name: Evaluate ephemeral env label trigger - runs-on: ubuntu-24.04 - permissions: - pull-requests: write - outputs: - slash-command: ${{ steps.eval-label.outputs.result }} - feature-flags: ${{ steps.eval-feature-flags.outputs.result }} - sha: ${{ steps.get-sha.outputs.sha }} - env: - DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - - steps: - - name: Check for the "testenv-up" label - id: eval-label - run: | - if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then - LABEL_NAME="${INPUT_LABEL_NAME}" - else - LABEL_NAME="${{ github.event.label.name }}" - fi - - echo "Evaluating label: $LABEL_NAME" - - if [[ "$LABEL_NAME" == "testenv-up" ]]; then - echo "result=up" >> $GITHUB_OUTPUT - else - echo "result=noop" >> $GITHUB_OUTPUT - fi - - env: - INPUT_LABEL_NAME: ${{ github.event.inputs.label_name }} - - name: Get event SHA - id: get-sha - if: steps.eval-label.outputs.result == 'up' - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - let prSha; - - // If event is workflow_dispatch, use the issue_number from inputs - if (context.eventName === "workflow_dispatch") { - const prNumber = "${{ github.event.inputs.issue_number }}"; - if (!prNumber) { - console.log("No PR number found."); - return; - } - - // Fetch PR details using the provided issue_number - const { data: pr } = await github.rest.pulls.get({ - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: prNumber - }); - - prSha = pr.head.sha; - } else { - // If it's not workflow_dispatch, use the PR head sha from the event - prSha = context.payload.pull_request.head.sha; - } - - console.log(`PR SHA: ${prSha}`); - core.setOutput("sha", prSha); - - - name: Looking for feature flags in PR description - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - id: eval-feature-flags - if: steps.eval-label.outputs.result == 'up' - with: - script: | - const description = context.payload.pull_request - ? context.payload.pull_request.body || '' - : context.payload.inputs.pr_description || ''; - - const pattern = /FEATURE_(\w+)=(\w+)/g; - let results = []; - [...description.matchAll(pattern)].forEach(match => { - const config = { - name: `SUPERSET_FEATURE_${match[1]}`, - value: match[2], - }; - results.push(config); - }); - - return results; - - - name: Reply with confirmation comment - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - if: steps.eval-label.outputs.result == 'up' - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const action = '${{ steps.eval-label.outputs.result }}'; - const user = context.actor; - const runId = context.runId; - const workflowUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${runId}`; - - const issueNumber = context.payload.pull_request - ? context.payload.pull_request.number - : context.payload.inputs.issue_number; - - if (!issueNumber) { - throw new Error("Issue number is not available."); - } - - const body = `⚠️ **DEPRECATED WORKFLOW** ⚠️\n\n@${user} This workflow is deprecated! Please use the new **Superset Showtime** system instead:\n\n` + - `- Replace "testenv-up" label with "🎪 trigger-start"\n` + - `- Better reliability and easier management\n` + - `- See https://github.com/mistercrunch/superset-showtime for details\n\n` + - `Processing your ephemeral environment request [here](${workflowUrl}). Action: **${action}**.` + - ` More information on [how to use or configure ephemeral environments]` + - `(https://superset.apache.org/docs/contributing/howtos/#github-ephemeral-environments)`; - - - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: issueNumber, - body, - }); - - ephemeral-docker-build: - concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}-build - cancel-in-progress: true - needs: ephemeral-env-label - if: needs.ephemeral-env-label.outputs.slash-command == 'up' - name: ephemeral-docker-build - runs-on: ubuntu-24.04 - steps: - - name: "Checkout ${{ github.ref }} ( ${{ needs.ephemeral-env-label.outputs.sha }} : ${{steps.get-sha.outputs.sha}} )" - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - with: - ref: ${{ needs.ephemeral-env-label.outputs.sha }} - persist-credentials: false - - - name: Setup Docker Environment - uses: ./.github/actions/setup-docker - with: - dockerhub-user: ${{ secrets.DOCKERHUB_USER }} - dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }} - build: "true" - install-docker-compose: "false" - - - name: Setup supersetbot - uses: ./.github/actions/setup-supersetbot/ - - - name: Build ephemeral env image - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - supersetbot docker \ - --push \ - --load \ - --preset ci \ - --platform linux/amd64 \ - --context-ref "$RELEASE" \ - --extra-flags "--build-arg INCLUDE_CHROMIUM=false" - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: us-west-2 - - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@fa648b43de3d4d023bcb3f89ed6940096949c419 # v2 - - - name: Load, tag and push image to ECR - id: push-image - env: - ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - ECR_REPOSITORY: superset-ci - IMAGE_TAG: apache/superset:${{ needs.ephemeral-env-label.outputs.sha }}-ci - PR_NUMBER: ${{ github.event.inputs.issue_number || github.event.pull_request.number }} - run: | - docker tag $IMAGE_TAG $ECR_REGISTRY/$ECR_REPOSITORY:pr-$PR_NUMBER-ci - docker push -a $ECR_REGISTRY/$ECR_REPOSITORY - - ephemeral-env-up: - needs: [ephemeral-env-label, ephemeral-docker-build] - if: needs.ephemeral-env-label.outputs.slash-command == 'up' - name: Spin up an ephemeral environment - runs-on: ubuntu-24.04 - permissions: - contents: read - pull-requests: write - - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - with: - persist-credentials: false - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: us-west-2 - - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@fa648b43de3d4d023bcb3f89ed6940096949c419 # v2 - - - name: Check target image exists in ECR - id: check-image - continue-on-error: true - env: - PR_NUMBER: ${{ github.event.inputs.issue_number || github.event.pull_request.number }} - run: | - aws ecr describe-images \ - --registry-id $(echo "${{ steps.login-ecr.outputs.registry }}" | grep -Eo "^[0-9]+") \ - --repository-name superset-ci \ - --image-ids imageTag=pr-$PR_NUMBER-ci - - - name: Fail on missing container image - if: steps.check-image.outcome == 'failure' - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - with: - github-token: ${{ github.token }} - script: | - const errMsg = '@${{ github.event.comment.user.login }} Container image not yet published for this PR. Please try again when build is complete.'; - github.rest.issues.createComment({ - issue_number: ${{ github.event.inputs.issue_number || github.event.pull_request.number }}, - owner: context.repo.owner, - repo: context.repo.repo, - body: errMsg - }); - core.setFailed(errMsg); - - - name: Fill in the new image ID in the Amazon ECS task definition - id: task-def - uses: aws-actions/amazon-ecs-render-task-definition@6853cfae8c3a7d978fbf68b5a55453395541dfbb # v1 - with: - task-definition: .github/workflows/ecs-task-definition.json - container-name: superset-ci - image: ${{ steps.login-ecr.outputs.registry }}/superset-ci:pr-${{ github.event.inputs.issue_number || github.event.pull_request.number }}-ci - - - name: Update env vars in the Amazon ECS task definition - run: | - cat <<< "$(jq '.containerDefinitions[0].environment += ${{ needs.ephemeral-env-label.outputs.feature-flags }}' < ${{ steps.task-def.outputs.task-definition }})" > ${{ steps.task-def.outputs.task-definition }} - - - name: Describe ECS service - id: describe-services - run: | - echo "active=$(aws ecs describe-services --cluster superset-ci --services pr-${INPUT_ISSUE_NUMBER}-service | jq '.services[] | select(.status == "ACTIVE") | any')" >> $GITHUB_OUTPUT - env: - INPUT_ISSUE_NUMBER: ${{ github.event.inputs.issue_number || github.event.pull_request.number }} - - name: Create ECS service - id: create-service - if: steps.describe-services.outputs.active != 'true' - env: - ECR_SUBNETS: subnet-0e15a5034b4121710,subnet-0e8efef4a72224974 - ECR_SECURITY_GROUP: sg-092ff3a6ae0574d91 - PR_NUMBER: ${{ github.event.inputs.issue_number || github.event.pull_request.number }} - run: | - aws ecs create-service \ - --cluster superset-ci \ - --service-name pr-$PR_NUMBER-service \ - --task-definition superset-ci \ - --launch-type FARGATE \ - --desired-count 1 \ - --platform-version LATEST \ - --network-configuration "awsvpcConfiguration={subnets=[$ECR_SUBNETS],securityGroups=[$ECR_SECURITY_GROUP],assignPublicIp=ENABLED}" \ - --tags key=pr,value=$PR_NUMBER key=github_user,value=${{ github.actor }} - - name: Deploy Amazon ECS task definition - id: deploy-task - uses: aws-actions/amazon-ecs-deploy-task-definition@a310a830f5c14e583e35d84e4e1ec7dd177c3c9c # v2 - with: - task-definition: ${{ steps.task-def.outputs.task-definition }} - service: pr-${{ github.event.inputs.issue_number || github.event.pull_request.number }}-service - cluster: superset-ci - wait-for-service-stability: true - wait-for-minutes: 10 - - - name: List tasks - id: list-tasks - run: | - echo "task=$(aws ecs list-tasks --cluster superset-ci --service-name pr-${INPUT_ISSUE_NUMBER}-service | jq '.taskArns | first')" >> $GITHUB_OUTPUT - env: - INPUT_ISSUE_NUMBER: ${{ github.event.inputs.issue_number || github.event.pull_request.number }} - - name: Get network interface - id: get-eni - run: | - echo "eni=$(aws ecs describe-tasks --cluster superset-ci --tasks ${{ steps.list-tasks.outputs.task }} | jq '.tasks[0].attachments[0].details | map(select(.name=="networkInterfaceId"))[0].value')" >> $GITHUB_OUTPUT - - name: Get public IP - id: get-ip - run: | - echo "ip=$(aws ec2 describe-network-interfaces --network-interface-ids ${{ steps.get-eni.outputs.eni }} | jq -r '.NetworkInterfaces | first | .Association.PublicIp')" >> $GITHUB_OUTPUT - - name: Comment (success) - if: ${{ success() }} - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - with: - github-token: ${{github.token}} - script: | - const issue_number = context.payload.inputs?.issue_number || context.issue.number; - github.rest.issues.createComment({ - issue_number: issue_number, - owner: context.repo.owner, - repo: context.repo.repo, - body: `@${{ github.actor }} Ephemeral environment spinning up at http://${{ steps.get-ip.outputs.ip }}:8080. Credentials are 'admin'/'admin'. Please allow several minutes for bootstrapping and startup.` - }); - - name: Comment (failure) - if: ${{ failure() }} - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - with: - github-token: ${{github.token}} - script: | - const issue_number = context.payload.inputs?.issue_number || context.issue.number; - github.rest.issues.createComment({ - issue_number: issue_number, - owner: context.repo.owner, - repo: context.repo.repo, - body: '@${{ github.event.inputs.user_login || github.event.comment.user.login }} Ephemeral environment creation failed. Please check the Actions logs for details.' - })