mirror of
https://github.com/apache/superset.git
synced 2026-07-18 04:35:40 +00:00
Compare commits
1 Commits
chore/drop
...
supersetbo
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f44a58e1ea |
43
.asf.yaml
43
.asf.yaml
@@ -17,16 +17,7 @@
|
||||
|
||||
# https://cwiki.apache.org/confluence/display/INFRA/.asf.yaml+features+for+git+repositories
|
||||
---
|
||||
notifications:
|
||||
commits: commits@superset.apache.org
|
||||
issues: notifications@superset.apache.org
|
||||
pullrequests: notifications@superset.apache.org
|
||||
discussions: notifications@superset.apache.org
|
||||
|
||||
github:
|
||||
pull_requests:
|
||||
del_branch_on_merge: true
|
||||
allow_update_branch: true
|
||||
description: "Apache Superset is a Data Visualization and Data Exploration Platform"
|
||||
homepage: https://superset.apache.org/
|
||||
labels:
|
||||
@@ -56,17 +47,12 @@ github:
|
||||
projects: true
|
||||
# Enable wiki for documentation
|
||||
wiki: true
|
||||
# Enable discussions
|
||||
discussions: true
|
||||
|
||||
enabled_merge_buttons:
|
||||
squash: true
|
||||
merge: false
|
||||
rebase: false
|
||||
|
||||
ghp_branch: gh-pages
|
||||
ghp_path: /
|
||||
|
||||
protected_branches:
|
||||
master:
|
||||
required_status_checks:
|
||||
@@ -77,17 +63,19 @@ github:
|
||||
# combination here.
|
||||
contexts:
|
||||
- lint-check
|
||||
- cypress-matrix-required
|
||||
- dependency-review
|
||||
- cypress-matrix (1, chrome)
|
||||
- cypress-matrix (2, chrome)
|
||||
- cypress-matrix (3, chrome)
|
||||
- frontend-build
|
||||
- playwright-tests-required
|
||||
- pre-commit (current)
|
||||
- test-mysql
|
||||
- test-postgres-required
|
||||
- test-postgres-hive
|
||||
- test-postgres-presto
|
||||
- test-sqlite
|
||||
- unit-tests-required
|
||||
- pre-commit (3.9)
|
||||
- python-lint (3.9)
|
||||
- test-mysql (3.9)
|
||||
- test-postgres (3.9)
|
||||
- test-postgres (3.10)
|
||||
- test-postgres-hive (3.9)
|
||||
- test-postgres-presto (3.9)
|
||||
- test-sqlite (3.9)
|
||||
- unit-tests (3.9)
|
||||
|
||||
required_pull_request_reviews:
|
||||
dismiss_stale_reviews: false
|
||||
@@ -95,10 +83,3 @@ github:
|
||||
required_approving_review_count: 1
|
||||
|
||||
required_signatures: false
|
||||
gh-pages:
|
||||
required_pull_request_reviews:
|
||||
dismiss_stale_reviews: false
|
||||
require_code_owner_reviews: true
|
||||
required_approving_review_count: 1
|
||||
|
||||
required_signatures: false
|
||||
|
||||
@@ -1,10 +0,0 @@
|
||||
# JavaScript to TypeScript Migration Command
|
||||
|
||||
## Usage
|
||||
```
|
||||
/js-to-ts <core-filename>
|
||||
```
|
||||
- `<core-filename>` - Path to CORE file relative to `superset-frontend/` (e.g., `src/utils/common.js`, `src/middleware/loggerMiddleware.js`)
|
||||
|
||||
## Agent Instructions
|
||||
**See:** [../projects/js-to-ts/AGENT.md](../projects/js-to-ts/AGENT.md) for complete migration guide.
|
||||
@@ -1,684 +0,0 @@
|
||||
# JavaScript to TypeScript Migration Agent Guide
|
||||
|
||||
**Complete technical reference for converting JavaScript/JSX files to TypeScript/TSX in Apache Superset frontend.**
|
||||
|
||||
**Agent Role:** Atomic migration unit - migrate the core file + ALL related tests/mocks as one cohesive unit. Use `git mv` to preserve history, NO `git commit`. NO global import changes. Report results upon completion.
|
||||
|
||||
---
|
||||
|
||||
## 🎯 Migration Principles
|
||||
|
||||
1. **Atomic migration units** - Core file + all related tests/mocks migrate together
|
||||
2. **Zero `any` types** - Use proper TypeScript throughout
|
||||
3. **Leverage existing types** - Reuse established definitions
|
||||
4. **Type inheritance** - Derivatives extend base component types
|
||||
5. **Strategic placement** - File types for maximum discoverability
|
||||
6. **Surgical improvements** - Enhance existing types during migration
|
||||
|
||||
---
|
||||
|
||||
## Step 0: Dependency Check (MANDATORY)
|
||||
|
||||
**Command:**
|
||||
```bash
|
||||
grep -E "from '\.\./.*\.jsx?'|from '\./.*\.jsx?'|from 'src/.*\.jsx?'" superset-frontend/{filename}
|
||||
```
|
||||
|
||||
**Decision:**
|
||||
- ✅ No matches → Proceed with atomic migration (core + tests + mocks)
|
||||
- ❌ Matches found → EXIT with dependency report (see format below)
|
||||
|
||||
---
|
||||
|
||||
## Step 1: Identify Related Files (REQUIRED)
|
||||
|
||||
**Atomic Migration Scope:**
|
||||
For core file `src/utils/example.js`, also migrate:
|
||||
- `src/utils/example.test.js` / `src/utils/example.test.jsx`
|
||||
- `src/utils/example.spec.js` / `src/utils/example.spec.jsx`
|
||||
- `src/utils/__mocks__/example.js`
|
||||
- Any other related test/mock files found by pattern matching
|
||||
|
||||
**Find all related test and mock files:**
|
||||
```bash
|
||||
# Pattern-based search for related files
|
||||
basename=$(basename {filename} .js)
|
||||
dirname=$(dirname superset-frontend/{filename})
|
||||
|
||||
# Find test files
|
||||
find "$dirname" -name "${basename}.test.js" -o -name "${basename}.test.jsx"
|
||||
find "$dirname" -name "${basename}.spec.js" -o -name "${basename}.spec.jsx"
|
||||
|
||||
# Find mock files
|
||||
find "$dirname" -name "__mocks__/${basename}.js"
|
||||
find "$dirname" -name "${basename}.mock.js"
|
||||
```
|
||||
|
||||
**Migration Requirement:** All discovered related files MUST be migrated together as one atomic unit.
|
||||
|
||||
**Test File Creation:** If NO test files exist for the core file, CREATE a minimal test file using the following pattern:
|
||||
- Location: Same directory as core file
|
||||
- Name: `{basename}.test.ts` (e.g., `DebouncedMessageQueue.test.ts`)
|
||||
- Content: Basic test structure importing and testing the main functionality
|
||||
- Use proper TypeScript types in test file
|
||||
|
||||
---
|
||||
|
||||
## 🗺️ Type Reference Map
|
||||
|
||||
### From `@superset-ui/core`
|
||||
```typescript
|
||||
// Data & Query
|
||||
QueryFormData, QueryData, JsonObject, AnnotationData, AdhocMetric
|
||||
LatestQueryFormData, GenericDataType, DatasourceType, ExtraFormData
|
||||
DataMaskStateWithId, NativeFilterScope, NativeFiltersState, NativeFilterTarget
|
||||
|
||||
// UI & Theme
|
||||
FeatureFlagMap, LanguagePack, ColorSchemeConfig, SequentialSchemeConfig
|
||||
```
|
||||
|
||||
### From `@superset-ui/chart-controls`
|
||||
```typescript
|
||||
Dataset, ColumnMeta, ControlStateMapping
|
||||
```
|
||||
|
||||
### From Local Types (`src/types/`)
|
||||
```typescript
|
||||
// Authentication
|
||||
User, UserWithPermissionsAndRoles, BootstrapUser, PermissionsAndRoles
|
||||
|
||||
// Dashboard
|
||||
Dashboard, DashboardState, DashboardInfo, DashboardLayout, LayoutItem
|
||||
ComponentType, ChartConfiguration, ActiveFilters
|
||||
|
||||
// Charts
|
||||
Chart, ChartState, ChartStatus, ChartLinkedDashboard, Slice, SaveActionType
|
||||
|
||||
// Data
|
||||
Datasource, Database, Owner, Role
|
||||
|
||||
// UI Components
|
||||
TagType, FavoriteStatus, Filter, ImportResourceName
|
||||
```
|
||||
|
||||
### From Domain Types
|
||||
```typescript
|
||||
// src/dashboard/types.ts
|
||||
RootState, ChartsState, DatasourcesState, FilterBarOrientation
|
||||
ChartCrossFiltersConfig, ActiveTabs, MenuKeys
|
||||
|
||||
// src/explore/types.ts
|
||||
ExplorePageInitialData, ExplorePageState, ExploreResponsePayload, OptionSortType
|
||||
|
||||
// src/SqlLab/types.ts
|
||||
[SQL Lab specific types]
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 🏗️ Type Organization Strategy
|
||||
|
||||
### Type Placement Hierarchy
|
||||
|
||||
1. **Component-Colocated** (90% of cases)
|
||||
```typescript
|
||||
// Same file as component
|
||||
interface MyComponentProps {
|
||||
title: string;
|
||||
onClick: () => void;
|
||||
}
|
||||
```
|
||||
|
||||
2. **Feature-Shared**
|
||||
```typescript
|
||||
// src/[domain]/components/[Feature]/types.ts
|
||||
export interface FilterConfiguration {
|
||||
filterId: string;
|
||||
targets: NativeFilterTarget[];
|
||||
}
|
||||
```
|
||||
|
||||
3. **Domain-Wide**
|
||||
```typescript
|
||||
// src/[domain]/types.ts
|
||||
export interface ExploreFormData extends QueryFormData {
|
||||
viz_type: string;
|
||||
}
|
||||
```
|
||||
|
||||
4. **Global**
|
||||
```typescript
|
||||
// src/types/[TypeName].ts
|
||||
export interface ApiResponse<T> {
|
||||
result: T;
|
||||
count?: number;
|
||||
}
|
||||
```
|
||||
|
||||
### Type Discovery Commands
|
||||
```bash
|
||||
# Search existing types before creating
|
||||
find superset-frontend/src -name "types.ts" -exec grep -l "[TypeConcept]" {} \;
|
||||
grep -r "interface.*Props\|type.*Props" superset-frontend/src/
|
||||
```
|
||||
|
||||
### Derivative Component Patterns
|
||||
|
||||
**Rule:** Components that extend others should extend their type interfaces.
|
||||
|
||||
```typescript
|
||||
// ✅ Base component type
|
||||
interface SelectProps {
|
||||
value: string | number;
|
||||
options: SelectOption[];
|
||||
onChange: (value: string | number) => void;
|
||||
disabled?: boolean;
|
||||
}
|
||||
|
||||
// ✅ Derivative extends base
|
||||
interface ChartSelectProps extends SelectProps {
|
||||
charts: Chart[];
|
||||
onChartSelect: (chart: Chart) => void;
|
||||
}
|
||||
|
||||
// ✅ Derivative with modified props
|
||||
interface DatabaseSelectProps extends Omit<SelectProps, 'value' | 'onChange'> {
|
||||
value: number; // Narrowed type
|
||||
onChange: (databaseId: number) => void; // Specific signature
|
||||
}
|
||||
```
|
||||
|
||||
**Common Patterns:**
|
||||
- **Extension:** `extends BaseProps` - adds new props
|
||||
- **Omission:** `Omit<BaseProps, 'prop'>` - removes props
|
||||
- **Modification:** `Omit<BaseProps, 'prop'> & { prop: NewType }` - changes prop type
|
||||
- **Restriction:** Override with narrower types (union → specific)
|
||||
|
||||
---
|
||||
|
||||
## 📋 Migration Recipe
|
||||
|
||||
### Step 2: File Conversion
|
||||
```bash
|
||||
# Use git mv to preserve history
|
||||
git mv component.js component.ts
|
||||
git mv Component.jsx Component.tsx
|
||||
```
|
||||
|
||||
### Step 3: Import & Type Setup
|
||||
```typescript
|
||||
// Import order (enforced by linting)
|
||||
import { FC, ReactNode } from 'react';
|
||||
import { JsonObject, QueryFormData } from '@superset-ui/core';
|
||||
import { Dataset } from '@superset-ui/chart-controls';
|
||||
import type { Dashboard } from 'src/types/Dashboard';
|
||||
```
|
||||
|
||||
### Step 4: Function & Component Typing
|
||||
```typescript
|
||||
// Functions with proper parameter/return types
|
||||
export function processData(
|
||||
data: Dataset[],
|
||||
config: JsonObject
|
||||
): ProcessedData[] {
|
||||
// implementation
|
||||
}
|
||||
|
||||
// Component props with inheritance
|
||||
interface ComponentProps extends BaseProps {
|
||||
data: Chart[];
|
||||
onSelect: (id: number) => void;
|
||||
}
|
||||
|
||||
const Component: FC<ComponentProps> = ({ data, onSelect }) => {
|
||||
// implementation
|
||||
};
|
||||
```
|
||||
|
||||
### Step 5: State & Redux Typing
|
||||
```typescript
|
||||
// Hooks with specific types
|
||||
const [data, setData] = useState<Chart[]>([]);
|
||||
const [selected, setSelected] = useState<number | null>(null);
|
||||
|
||||
// Redux with existing RootState
|
||||
const mapStateToProps = (state: RootState) => ({
|
||||
charts: state.charts,
|
||||
user: state.user,
|
||||
});
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 🧠 Type Debugging Strategies (Real-World Learnings)
|
||||
|
||||
### The Evolution of Type Approaches
|
||||
When you hit type errors, follow this debugging evolution:
|
||||
|
||||
#### 1. ❌ Idealized Union Types (First Attempt)
|
||||
```typescript
|
||||
// Looks clean but doesn't match reality
|
||||
type DatasourceInput = Datasource | QueryEditor;
|
||||
```
|
||||
**Problem**: Real calling sites pass variations, not exact types.
|
||||
|
||||
#### 2. ❌ Overly Precise Types (Second Attempt)
|
||||
```typescript
|
||||
// Tried to match exact calling signatures
|
||||
type DatasourceInput =
|
||||
| IDatasource // From DatasourcePanel
|
||||
| (QueryEditor & { columns: ColumnMeta[] }); // From SaveQuery
|
||||
```
|
||||
**Problem**: Too rigid, doesn't handle legacy variations.
|
||||
|
||||
#### 3. ✅ Flexible Interface (Final Solution)
|
||||
```typescript
|
||||
// Captures what the function actually needs
|
||||
interface DatasourceInput {
|
||||
name?: string | null; // Allow null for compatibility
|
||||
datasource_name?: string | null; // Legacy variations
|
||||
columns?: any[]; // Multiple column types accepted
|
||||
database?: { id?: number };
|
||||
// ... other optional properties
|
||||
}
|
||||
```
|
||||
**Success**: Works with all calling sites, focuses on function needs.
|
||||
|
||||
### Type Debugging Process
|
||||
1. **Start with compilation errors** - they show exact mismatches
|
||||
2. **Examine actual usage** - look at calling sites, not idealized types
|
||||
3. **Build flexible interfaces** - capture what functions need, not rigid contracts
|
||||
4. **Iterate based on downstream validation** - let calling sites guide your types
|
||||
|
||||
---
|
||||
|
||||
## 🚨 Anti-Patterns to Avoid
|
||||
|
||||
```typescript
|
||||
// ❌ Never use any
|
||||
const obj: any = {};
|
||||
|
||||
// ✅ Use proper types
|
||||
const obj: Record<string, JsonObject> = {};
|
||||
|
||||
// ❌ Don't recreate base component props
|
||||
interface ChartSelectProps {
|
||||
value: string; // Duplicated from SelectProps
|
||||
onChange: () => void; // Duplicated from SelectProps
|
||||
charts: Chart[]; // New prop
|
||||
}
|
||||
|
||||
// ✅ Inherit and extend
|
||||
interface ChartSelectProps extends SelectProps {
|
||||
charts: Chart[]; // Only new props
|
||||
}
|
||||
|
||||
// ❌ Don't create ad-hoc type variations
|
||||
interface UserInfo {
|
||||
name: string;
|
||||
email: string;
|
||||
}
|
||||
|
||||
// ✅ Extend existing types (DRY principle)
|
||||
import { User } from 'src/types/bootstrapTypes';
|
||||
type UserDisplayInfo = Pick<User, 'firstName' | 'lastName' | 'email'>;
|
||||
|
||||
// ❌ Don't create overly rigid unions
|
||||
type StrictInput = ExactTypeA | ExactTypeB;
|
||||
|
||||
// ✅ Create flexible interfaces for function parameters
|
||||
interface FlexibleInput {
|
||||
// Focus on what the function actually needs
|
||||
commonProperty: string;
|
||||
optionalVariations?: any; // Allow for legacy variations
|
||||
}
|
||||
```
|
||||
|
||||
## 📍 DRY Type Guidelines (WHERE TYPES BELONG)
|
||||
|
||||
### Type Placement Rules
|
||||
**CRITICAL**: Type variations must live close to where they belong, not scattered across files.
|
||||
|
||||
#### ✅ Proper Type Organization
|
||||
```typescript
|
||||
// ❌ Don't create one-off interfaces in utility files
|
||||
// src/utils/datasourceUtils.ts
|
||||
interface DatasourceInput { /* custom interface */ } // Wrong!
|
||||
|
||||
// ✅ Use existing types or extend them in their proper domain
|
||||
// src/utils/datasourceUtils.ts
|
||||
import { IDatasource } from 'src/explore/components/DatasourcePanel';
|
||||
import { QueryEditor } from 'src/SqlLab/types';
|
||||
|
||||
// Create flexible interface that references existing types
|
||||
interface FlexibleDatasourceInput {
|
||||
// Properties that actually exist across variations
|
||||
}
|
||||
```
|
||||
|
||||
#### Type Location Hierarchy
|
||||
1. **Domain Types**: `src/{domain}/types.ts` (dashboard, explore, SqlLab)
|
||||
2. **Component Types**: Co-located with components
|
||||
3. **Global Types**: `src/types/` directory
|
||||
4. **Utility Types**: Only when they truly don't belong elsewhere
|
||||
|
||||
#### ✅ DRY Type Patterns
|
||||
```typescript
|
||||
// ✅ Extend existing domain types
|
||||
interface SaveQueryData extends Pick<QueryEditor, 'sql' | 'dbId' | 'catalog'> {
|
||||
columns: ColumnMeta[]; // Add what's needed
|
||||
}
|
||||
|
||||
// ✅ Create flexible interfaces for cross-domain utilities
|
||||
interface CrossDomainInput {
|
||||
// Common properties that exist across different source types
|
||||
name?: string | null; // Accommodate legacy null values
|
||||
// Only include properties the function actually uses
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 🎯 PropTypes Auto-Generation (Elegant Approach)
|
||||
|
||||
**IMPORTANT**: Superset has `babel-plugin-typescript-to-proptypes` configured to automatically generate PropTypes from TypeScript interfaces. Use this instead of manual PropTypes duplication!
|
||||
|
||||
### ❌ Manual PropTypes Duplication (Avoid This)
|
||||
```typescript
|
||||
export interface MyComponentProps {
|
||||
title: string;
|
||||
count?: number;
|
||||
}
|
||||
|
||||
// 8+ lines of manual PropTypes duplication 😱
|
||||
const propTypes = PropTypes.shape({
|
||||
title: PropTypes.string.isRequired,
|
||||
count: PropTypes.number,
|
||||
});
|
||||
|
||||
export default propTypes;
|
||||
```
|
||||
|
||||
### ✅ Auto-Generated PropTypes (Use This)
|
||||
```typescript
|
||||
import { InferProps } from 'prop-types';
|
||||
|
||||
export interface MyComponentProps {
|
||||
title: string;
|
||||
count?: number;
|
||||
}
|
||||
|
||||
// Single validator function - babel plugin auto-generates PropTypes! ✨
|
||||
export default function MyComponentValidator(props: MyComponentProps) {
|
||||
return null; // PropTypes auto-assigned by babel-plugin-typescript-to-proptypes
|
||||
}
|
||||
|
||||
// Optional: For consumers needing PropTypes type inference
|
||||
export type MyComponentPropsInferred = InferProps<typeof MyComponentValidator>;
|
||||
```
|
||||
|
||||
### Migration Pattern for Type-Only Files
|
||||
|
||||
**When migrating type-only files with manual PropTypes:**
|
||||
|
||||
1. **Keep the TypeScript interfaces** (single source of truth)
|
||||
2. **Replace manual PropTypes** with validator function
|
||||
3. **Remove PropTypes imports** and manual shape definitions
|
||||
4. **Add InferProps import** if type inference needed
|
||||
|
||||
**Example Migration:**
|
||||
```typescript
|
||||
// Before: 25+ lines with manual PropTypes duplication
|
||||
export interface AdhocFilterType { /* ... */ }
|
||||
const adhocFilterTypePropTypes = PropTypes.oneOfType([...]);
|
||||
|
||||
// After: 3 lines with auto-generation
|
||||
export interface AdhocFilterType { /* ... */ }
|
||||
export default function AdhocFilterValidator(props: { filter: AdhocFilterType }) {
|
||||
return null; // Auto-generated PropTypes by babel plugin
|
||||
}
|
||||
```
|
||||
|
||||
### Component PropTypes Pattern
|
||||
|
||||
**For React components, the babel plugin works automatically:**
|
||||
|
||||
```typescript
|
||||
interface ComponentProps {
|
||||
title: string;
|
||||
onClick: () => void;
|
||||
}
|
||||
|
||||
const MyComponent: FC<ComponentProps> = ({ title, onClick }) => {
|
||||
// Component implementation
|
||||
};
|
||||
|
||||
// PropTypes automatically generated by babel plugin - no manual work needed!
|
||||
export default MyComponent;
|
||||
```
|
||||
|
||||
### Auto-Generation Benefits
|
||||
|
||||
- ✅ **Single source of truth**: TypeScript interfaces drive PropTypes
|
||||
- ✅ **No duplication**: Eliminate 15-20 lines of manual PropTypes code
|
||||
- ✅ **Automatic updates**: Changes to TypeScript automatically update PropTypes
|
||||
- ✅ **Type safety**: Compile-time checking ensures PropTypes match interfaces
|
||||
- ✅ **Backward compatibility**: Existing JavaScript components continue working
|
||||
|
||||
### Babel Plugin Configuration
|
||||
|
||||
The plugin is already configured in `babel.config.js`:
|
||||
```javascript
|
||||
['babel-plugin-typescript-to-proptypes', { loose: true }]
|
||||
```
|
||||
|
||||
**No additional setup required** - just use TypeScript interfaces and the plugin handles the rest!
|
||||
|
||||
---
|
||||
|
||||
## 🧪 Test File Migration Patterns
|
||||
|
||||
### Test File Priority
|
||||
- **Always migrate test files** alongside production files
|
||||
- **Test files are often leaf nodes** - good starting candidates
|
||||
- **Create tests if missing** - Leverage new TypeScript types for better test coverage
|
||||
|
||||
### Test-Specific Type Patterns
|
||||
```typescript
|
||||
// Mock interfaces for testing
|
||||
interface MockStore {
|
||||
getState: () => Partial<RootState>; // Partial allows minimal mocking
|
||||
}
|
||||
|
||||
// Type-safe mocking for complex objects
|
||||
const mockDashboardInfo: Partial<DashboardInfo> as DashboardInfo = {
|
||||
id: 123,
|
||||
json_metadata: '{}',
|
||||
};
|
||||
|
||||
// Sinon stub typing
|
||||
let postStub: sinon.SinonStub;
|
||||
beforeEach(() => {
|
||||
postStub = sinon.stub(SupersetClient, 'post');
|
||||
});
|
||||
|
||||
// Use stub reference instead of original method
|
||||
expect(postStub.callCount).toBe(1);
|
||||
expect(postStub.getCall(0).args[0].endpoint).toMatch('/api/');
|
||||
```
|
||||
|
||||
### Test Migration Recipe
|
||||
1. **Migrate production file first** (if both need migration)
|
||||
2. **Update test imports** to point to `.ts/.tsx` files
|
||||
3. **Add proper mock typing** using `Partial<T> as T` pattern
|
||||
4. **Fix stub typing** - Use stub references, not original methods
|
||||
5. **Verify all tests pass** with TypeScript compilation
|
||||
|
||||
---
|
||||
|
||||
## 🔧 Type Conflict Resolution
|
||||
|
||||
### Multiple Type Definitions Issue
|
||||
**Problem**: Same type name defined in multiple files causes compilation errors.
|
||||
|
||||
**Example**: `DashboardInfo` defined in both:
|
||||
- `src/dashboard/reducers/types.ts` (minimal)
|
||||
- `src/dashboard/components/Header/types.ts` (different shape)
|
||||
- `src/dashboard/types.ts` (complete - used by RootState)
|
||||
|
||||
### Resolution Strategy
|
||||
1. **Identify the authoritative type**:
|
||||
```bash
|
||||
# Find which type is used by RootState/main interfaces
|
||||
grep -r "DashboardInfo" src/dashboard/types.ts
|
||||
```
|
||||
|
||||
2. **Use import from authoritative source**:
|
||||
```typescript
|
||||
// ✅ Import from main domain types
|
||||
import { RootState, DashboardInfo } from 'src/dashboard/types';
|
||||
|
||||
// ❌ Don't import from component-specific files
|
||||
import { DashboardInfo } from 'src/dashboard/components/Header/types';
|
||||
```
|
||||
|
||||
3. **Mock complex types in tests**:
|
||||
```typescript
|
||||
// For testing - provide minimal required fields
|
||||
const mockInfo: Partial<DashboardInfo> as DashboardInfo = {
|
||||
id: 123,
|
||||
json_metadata: '{}',
|
||||
// Only provide fields actually used in test
|
||||
};
|
||||
```
|
||||
|
||||
### Type Hierarchy Discovery Commands
|
||||
```bash
|
||||
# Find all definitions of a type
|
||||
grep -r "interface.*TypeName\|type.*TypeName" src/
|
||||
|
||||
# Find import usage patterns
|
||||
grep -r "import.*TypeName" src/
|
||||
|
||||
# Check what RootState uses
|
||||
grep -A 10 -B 10 "TypeName" src/*/types.ts
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Agent Constraints (CRITICAL)
|
||||
|
||||
1. **Use git mv** - Run `git mv file.js file.ts` to preserve git history, but NO `git commit`
|
||||
2. **NO global import changes** - Don't update imports across codebase
|
||||
3. **Type files OK** - Can modify existing type files to improve/align types
|
||||
4. **Single-File TypeScript Validation** (CRITICAL) - tsc has known issues with multi-file compilation:
|
||||
- **Core Issue**: TypeScript's `tsc` has documented problems validating multiple files simultaneously in complex projects
|
||||
- **Solution**: ALWAYS validate files one at a time using individual `tsc` calls
|
||||
- **Command Pattern**: `cd superset-frontend && npx tscw --noEmit --allowJs --composite false --project tsconfig.json {single-file-path}`
|
||||
- **Why**: Multi-file validation can produce false positives, miss real errors, and conflict during parallel agent execution
|
||||
5. **Downstream Impact Validation** (CRITICAL) - Your migration affects calling sites:
|
||||
- **Find downstream files**: `find superset-frontend/src -name "*.tsx" -o -name "*.ts" | xargs grep -l "your-core-filename" 2>/dev/null || echo "No files found"`
|
||||
- **Validate each downstream file individually**: `cd superset-frontend && npx tscw --noEmit --allowJs --composite false --project tsconfig.json {each-downstream-file}`
|
||||
- **Fix type mismatches** you introduced in calling sites
|
||||
- **NEVER ignore downstream errors** - they indicate your types don't match reality
|
||||
6. **Avoid Project-Wide Validation During Migration**:
|
||||
- **NEVER use `npm run type`** during parallel agent execution - produces unreliable results
|
||||
- **Single-file validation is authoritative** - trust individual file checks over project-wide scans
|
||||
6. **ESLint validation** - Run `npm run eslint -- --fix {file}` for each migrated file to auto-fix formatting/linting issues
|
||||
6. Zero `any` types - use proper TypeScript types
|
||||
7. Search existing types before creating new ones
|
||||
8. Follow patterns from this guide
|
||||
|
||||
---
|
||||
|
||||
## Success Report Format
|
||||
|
||||
```
|
||||
SUCCESS: Atomic Migration of {core-filename}
|
||||
|
||||
## Files Migrated (Atomic Unit)
|
||||
- Core: {core-filename} → {core-filename.ts/tsx}
|
||||
- Tests: {list-of-test-files} → {list-of-test-files.ts/tsx} OR "CREATED: {basename}.test.ts"
|
||||
- Mocks: {list-of-mock-files} → {list-of-mock-files.ts}
|
||||
- Type files modified: {list-of-type-files}
|
||||
|
||||
## Types Created/Improved
|
||||
- {TypeName}: {location} ({scope}) - {rationale}
|
||||
- {ExistingType}: enhanced in {location} - {improvement-description}
|
||||
|
||||
## Documentation Recommendations
|
||||
- ADD_TO_DIRECTORY: {TypeName} - {reason}
|
||||
- NO_DOCUMENTATION: {TypeName} - {reason}
|
||||
|
||||
## Quality Validation
|
||||
- **Single-File TypeScript Validation**: ✅ PASS - Core files individually validated
|
||||
- Core file: `npx tscw --noEmit --allowJs --composite false --project tsconfig.json {core-file}`
|
||||
- Test files: `npx tscw --noEmit --allowJs --composite false --project tsconfig.json {test-file}` (if exists)
|
||||
- **Downstream Impact Check**: ✅ PASS - Found {N} files importing this module, all validate individually
|
||||
- Downstream files: {list-of-files-that-import-your-module}
|
||||
- Individual validation: `npx tscw --noEmit --allowJs --composite false --project tsconfig.json {each-downstream-file}`
|
||||
- **ESLint validation**: ✅ PASS (using `npm run eslint -- --fix {files}` to auto-fix formatting)
|
||||
- **Zero any types**: ✅ PASS
|
||||
- **Local imports resolved**: ✅ PASS
|
||||
- **Functionality preserved**: ✅ PASS
|
||||
- **Tests pass** (if test file): ✅ PASS
|
||||
- **Follow-up action required**: {YES/NO}
|
||||
|
||||
## Validation Strategy Notes
|
||||
- **Single-file approach used**: Avoided multi-file tsc validation due to known TypeScript compilation issues
|
||||
- **Project-wide validation skipped**: `npm run type` not used during parallel migration to prevent false positives
|
||||
|
||||
## Migration Learnings
|
||||
- Type conflicts encountered: {describe any multiple type definitions}
|
||||
- Mock patterns used: {describe test mocking approaches}
|
||||
- Import hierarchy decisions: {note authoritative type sources used}
|
||||
- PropTypes strategy: {AUTO_GENERATED via babel plugin | MANUAL_DUPLICATION_REMOVED | N/A}
|
||||
|
||||
## Improvement Suggestions for Documentation
|
||||
- AGENT.md enhancement: {suggest additions to migration guide}
|
||||
- Common pattern identified: {note reusable patterns for future migrations}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Dependency Block Report Format
|
||||
|
||||
```
|
||||
DEPENDENCY_BLOCK: Cannot migrate {filename}
|
||||
|
||||
## Blocking Dependencies
|
||||
- {path}: {type} - {usage} - {priority}
|
||||
|
||||
## Impact Analysis
|
||||
- Estimated types: {number}
|
||||
- Expected locations: {list}
|
||||
- Cross-domain: {YES/NO}
|
||||
|
||||
## Recommended Order
|
||||
{ordered-list}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 📚 Quick Reference
|
||||
|
||||
**Type Utilities:**
|
||||
- `Record<K, V>` - Object with specific key/value types
|
||||
- `Partial<T>` - All properties optional
|
||||
- `Pick<T, K>` - Subset of properties
|
||||
- `Omit<T, K>` - Exclude specific properties
|
||||
- `NonNullable<T>` - Exclude null/undefined
|
||||
|
||||
**Event Types:**
|
||||
- `MouseEvent<HTMLButtonElement>`
|
||||
- `ChangeEvent<HTMLInputElement>`
|
||||
- `FormEvent<HTMLFormElement>`
|
||||
|
||||
**React Types:**
|
||||
- `FC<Props>` - Functional component
|
||||
- `ReactNode` - Any renderable content
|
||||
- `CSSProperties` - Style objects
|
||||
|
||||
---
|
||||
|
||||
**Remember:** Every type should add value and clarity. The goal is meaningful type safety that catches bugs and improves developer experience.
|
||||
@@ -1,199 +0,0 @@
|
||||
# JS-to-TS Coordinator Workflow
|
||||
|
||||
**Role:** Strategic migration coordination - select leaf-node files, trigger agents, review results, handle integration, manage dependencies.
|
||||
|
||||
---
|
||||
|
||||
## 1. Core File Selection Strategy
|
||||
|
||||
**Target ONLY Core Files**: Coordinators identify core files (production code), agents handle related tests/mocks atomically.
|
||||
|
||||
**File Analysis Commands**:
|
||||
```bash
|
||||
# Find CORE files with no JS/JSX dependencies (exclude tests/mocks) - SIZE PRIORITIZED
|
||||
find superset-frontend/src -name "*.js" -o -name "*.jsx" | grep -v "test\|spec\|mock" | xargs wc -l | sort -n | head -20
|
||||
|
||||
# Alternative: Get file sizes in lines with paths
|
||||
find superset-frontend/src -name "*.js" -o -name "*.jsx" | grep -v "test\|spec\|mock" | while read file; do
|
||||
lines=$(wc -l < "$file")
|
||||
echo "$lines $file"
|
||||
done | sort -n | head -20
|
||||
|
||||
# Check dependencies for core files only (start with smallest)
|
||||
for file in <core-files-sorted-by-size>; do
|
||||
echo "=== $file ($(wc -l < "$file") lines) ==="
|
||||
grep -E "from '\.\./.*\.jsx?'|from '\./.*\.jsx?'|from 'src/.*\.jsx?'" "$file" || echo "✅ LEAF CANDIDATE"
|
||||
done
|
||||
|
||||
# Identify heavily imported files (migrate last)
|
||||
grep -r "from.*utils/common" superset-frontend/src/ | wc -l
|
||||
|
||||
# Quick leaf analysis with size priority
|
||||
find superset-frontend/src -name "*.js" -o -name "*.jsx" | grep -v "test\|spec\|mock" | head -30 | while read file; do
|
||||
deps=$(grep -E "from '\.\./.*\.jsx?'|from '\./.*\.jsx?'|from 'src/.*\.jsx?'" "$file" | wc -l)
|
||||
lines=$(wc -l < "$file")
|
||||
if [ "$deps" -eq 0 ]; then
|
||||
echo "✅ LEAF: $lines lines - $file"
|
||||
fi
|
||||
done | sort -n
|
||||
```
|
||||
|
||||
**Priority Order** (Smallest files first for easier wins):
|
||||
1. **Small leaf files** (<50 lines) - No JS/JSX imports, quick TypeScript conversion
|
||||
2. **Medium leaf files** (50-200 lines) - Self-contained utilities and helpers
|
||||
3. **Small dependency files** (<100 lines) - Import only already-migrated files
|
||||
4. **Larger components** (200+ lines) - Complex but well-contained functionality
|
||||
5. **Core foundational files** (utils/common.js, controls.jsx) - migrate last regardless of size
|
||||
|
||||
**Size-First Benefits**:
|
||||
- Faster completion builds momentum
|
||||
- Earlier validation of migration patterns
|
||||
- Easier rollback if issues arise
|
||||
- Better success rate for agent learning
|
||||
|
||||
**Migration Unit**: Each agent call migrates:
|
||||
- 1 core file (primary target)
|
||||
- All related `*.test.js/jsx` files
|
||||
- All related `*.mock.js` files
|
||||
- All related `__mocks__/` files
|
||||
|
||||
---
|
||||
|
||||
## 2. Task Creation & Agent Control
|
||||
|
||||
### Task Triggering
|
||||
When triggering the `/js-to-ts` command:
|
||||
- **Task Title**: Use the core filename as the task title (e.g., "DebouncedMessageQueue.js migration", "hostNamesConfig.js migration")
|
||||
- **Task Description**: Include the full relative path to help agent locate the file
|
||||
- **Reference**: Point agent to [AGENT.md](./AGENT.md) for technical instructions
|
||||
|
||||
### Post-Processing Workflow
|
||||
After each agent completes:
|
||||
|
||||
1. **Review Agent Report**: Always read and analyze the complete agent report
|
||||
2. **Share Summary**: Provide user with key highlights from agent's work:
|
||||
- Files migrated (core + tests/mocks)
|
||||
- Types created or improved
|
||||
- Any validation issues or coordinator actions needed
|
||||
3. **Quality Assessment**: Evaluate agent's TypeScript implementation against criteria:
|
||||
- ✅ **Type Usage**: Proper types used, no `any` types
|
||||
- ✅ **Type Filing**: Types placed in correct hierarchy (component → feature → domain → global)
|
||||
- ✅ **Side Effects**: No unintended changes to other files
|
||||
- ✅ **Import Alignment**: Proper .ts/.tsx import extensions
|
||||
4. **Integration Decision**:
|
||||
- **COMMIT**: If agent work is complete and high quality
|
||||
- **FIX & COMMIT**: If minor issues need coordinator fixes
|
||||
- **ROLLBACK**: If major issues require complete rework
|
||||
5. **Next Action**: Ask user preference - commit this work or trigger next migration
|
||||
|
||||
---
|
||||
|
||||
## 3. Integration Decision Framework
|
||||
|
||||
**Automatic Integration** ✅:
|
||||
- `npm run type` passes without errors
|
||||
- Agent created clean TypeScript with proper types
|
||||
- Types appropriately filed in hierarchy
|
||||
|
||||
**Coordinator Integration** (Fix Side-Effects) 🔧:
|
||||
- `npm run type` fails BUT agent's work is high quality
|
||||
- Good type usage, proper patterns, well-organized
|
||||
- Side-effects are manageable TypeScript compilation errors
|
||||
- **Coordinator Action**: Integrate the change, then fix global compilation issues
|
||||
|
||||
**Rollback Only** ❌:
|
||||
- Agent introduced `any` types or poor type choices
|
||||
- Types poorly organized or conflicting with existing patterns
|
||||
- Fundamental approach issues requiring complete rework
|
||||
|
||||
**Integration Process**:
|
||||
1. **Review**: Agent already used `git mv` to preserve history
|
||||
2. **Fix Side-Effects**: Update dependent files with proper import extensions
|
||||
3. **Resolve Types**: Fix any cascading type issues across codebase
|
||||
4. **Validate**: Ensure `npm run type` passes after fixes
|
||||
|
||||
---
|
||||
|
||||
## 4. Common Integration Patterns
|
||||
|
||||
**Common Side-Effects (Expect These)**:
|
||||
- **Type import conflicts**: Multiple definitions of same type name
|
||||
- **Mock object typing**: Tests need complete type satisfaction
|
||||
- **Stub method references**: Use stub vars instead of original methods
|
||||
|
||||
**Coordinator Fixes (Standard Process)**:
|
||||
1. **Import Resolution**:
|
||||
```bash
|
||||
# Find authoritative type source
|
||||
grep -r "TypeName" src/*/types.ts
|
||||
# Import from domain types (src/dashboard/types.ts) not component types
|
||||
```
|
||||
|
||||
2. **Test Mock Completion**:
|
||||
```typescript
|
||||
// Use Partial<T> as T pattern for minimal mocking
|
||||
const mockDashboard: Partial<DashboardInfo> as DashboardInfo = {
|
||||
id: 123,
|
||||
json_metadata: '{}',
|
||||
};
|
||||
```
|
||||
|
||||
3. **Stub Reference Fixes**:
|
||||
```typescript
|
||||
// ✅ Use stub variable
|
||||
expect(postStub.callCount).toBe(1);
|
||||
// ❌ Don't use original method
|
||||
expect(SupersetClient.post.callCount).toBe(1);
|
||||
```
|
||||
|
||||
4. **Validation Commands**:
|
||||
```bash
|
||||
npm run type # TypeScript compilation
|
||||
npm test -- filename # Test functionality
|
||||
git status # Should show rename, not add/delete
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 5. File Categories for Planning
|
||||
|
||||
### Leaf Files (Start Here)
|
||||
**Self-contained files with minimal JS/JSX dependencies**:
|
||||
- Test files (80 files) - Usually only import the file being tested
|
||||
- Utility files without internal dependencies
|
||||
- Components importing only external libraries
|
||||
|
||||
### Heavily Imported Files (Migrate Last)
|
||||
**Core files that many others depend on**:
|
||||
- `utils/common.js` - Core utility functions
|
||||
- `utils/reducerUtils.js` - Redux helpers
|
||||
- `@superset-ui/core` equivalent files
|
||||
- Major state management files (`explore/store.js`, `dashboard/actions/`)
|
||||
|
||||
### Complex Components (Middle Priority)
|
||||
**Large files requiring careful type analysis**:
|
||||
- `components/Datasource/DatasourceEditor.jsx` (1,809 lines)
|
||||
- `explore/components/controls/AnnotationLayerControl/AnnotationLayer.jsx` (1,031 lines)
|
||||
- `explore/components/ExploreViewContainer/index.jsx` (911 lines)
|
||||
|
||||
---
|
||||
|
||||
## 6. Success Metrics & Continuous Improvement
|
||||
|
||||
**Per-File Gates**:
|
||||
- ✅ `npm run type` passes after each migration
|
||||
- ✅ Zero `any` types introduced
|
||||
- ✅ All imports properly typed
|
||||
- ✅ Types filed in correct hierarchy
|
||||
|
||||
**Linear Scheduling**:
|
||||
When agents report `DEPENDENCY_BLOCK`:
|
||||
- Queue dependencies in linear order
|
||||
- Process one file at a time to avoid conflicts
|
||||
- Handle cascading type changes between files
|
||||
|
||||
**After Each Migration**:
|
||||
1. **Update guides** with new patterns discovered
|
||||
2. **Document coordinator fixes** that become common
|
||||
3. **Enhance agent instructions** based on recurring issues
|
||||
4. **Track success metrics** - automatic vs coordinator integration rates
|
||||
@@ -1,76 +0,0 @@
|
||||
# JavaScript to TypeScript Migration Project
|
||||
|
||||
Progressive migration of 219 JS/JSX files to TypeScript in Apache Superset frontend.
|
||||
|
||||
## 📁 Project Documentation
|
||||
|
||||
- **[AGENT.md](./AGENT.md)** - Complete technical migration guide for agents (includes type reference, patterns, validation)
|
||||
- **[COORDINATOR.md](./COORDINATOR.md)** - Strategic workflow for coordinators (file selection, task management, integration)
|
||||
|
||||
## 🎯 Quick Start
|
||||
|
||||
**For Agents:** Read [AGENT.md](./AGENT.md) for complete migration instructions
|
||||
**For Coordinators:** Read [COORDINATOR.md](./COORDINATOR.md) for workflow and [AGENT.md](./AGENT.md) for supervision
|
||||
|
||||
**Command:** `/js-to-ts <filename>` - See [../../commands/js-to-ts.md](../../commands/js-to-ts.md)
|
||||
|
||||
## 📊 Migration Progress
|
||||
|
||||
**Scope**: 219 files total (112 JS + 107 JSX)
|
||||
- Production files: 139 (63%)
|
||||
- Test files: 80 (37%)
|
||||
|
||||
**Strategy**: Leaf-first migration with dependency-aware coordination
|
||||
|
||||
### Completed Migrations ✅
|
||||
|
||||
1. **roundDecimal** - `plugins/legacy-plugin-chart-map-box/src/utils/roundDecimal.js`
|
||||
- Migrated core + test files
|
||||
- Added proper TypeScript function signature with optional precision parameter
|
||||
- All tests pass
|
||||
|
||||
2. **timeGrainSqlaAnimationOverrides** - `src/explore/controlPanels/timeGrainSqlaAnimationOverrides.js`
|
||||
- Migrated to TypeScript with ControlPanelState and Dataset types
|
||||
- Added TimeGrainOverrideState interface for return type
|
||||
- Used type guards for safe property access
|
||||
|
||||
3. **DebouncedMessageQueue** - `src/utils/DebouncedMessageQueue.js`
|
||||
- Migrated to TypeScript with proper generics
|
||||
- Created DebouncedMessageQueueOptions interface
|
||||
- **CREATED test file** with 4 comprehensive test cases
|
||||
- Excellent class property typing with private/readonly modifiers
|
||||
|
||||
**Files Migrated**: 3/219 (1.4%)
|
||||
**Tests Created**: 2 (roundDecimal had existing, DebouncedMessageQueue created)
|
||||
|
||||
### Next Candidates (Leaf Nodes) 🎯
|
||||
|
||||
**Identified leaf files with no JS/JSX dependencies:**
|
||||
- `src/utils/hostNamesConfig.js` - Domain configuration utility
|
||||
- `src/explore/controlPanels/Separator.js` - Control panel configuration
|
||||
- `src/middleware/loggerMiddleware.js` - Logging middleware
|
||||
|
||||
**Migration Quality**: All completed migrations have:
|
||||
- ✅ Zero `any` types
|
||||
- ✅ Proper TypeScript compilation
|
||||
- ✅ ESLint validation passed
|
||||
- ✅ Test coverage (created where missing)
|
||||
|
||||
---
|
||||
|
||||
## 📈 Success Metrics
|
||||
|
||||
**Per-File Gates**:
|
||||
- ✅ `npm run type` passes after each migration
|
||||
- ✅ Zero `any` types introduced
|
||||
- ✅ All imports properly typed
|
||||
- ✅ Types filed in correct hierarchy
|
||||
|
||||
**Overall Progress**:
|
||||
- **Automatic Integration Rate**: 100% (3/3 migrations required no coordinator fixes)
|
||||
- **Test Coverage**: Improved (1 new test file created)
|
||||
- **Type Safety**: Enhanced with proper interfaces and generics
|
||||
|
||||
---
|
||||
|
||||
*This is a claudette-managed progressive refactor. All documentation and coordination resources are organized under `.claude/projects/js-to-ts/`*
|
||||
@@ -1,15 +0,0 @@
|
||||
{
|
||||
"hooks": {
|
||||
"PreToolUse": [
|
||||
{
|
||||
"matcher": "Bash",
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "jq -r '.tool_input.command // \"\"' | grep -qE '^git commit' && cd \"$CLAUDE_PROJECT_DIR\" && echo '🔍 Running pre-commit before commit...' && pre-commit run || true"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
36
.coveragerc
36
.coveragerc
@@ -1,36 +0,0 @@
|
||||
# .coveragerc to control coverage.py
|
||||
[run]
|
||||
branch = True
|
||||
source = superset
|
||||
# omit = bad_file.py
|
||||
|
||||
[paths]
|
||||
source =
|
||||
superset/
|
||||
*/site-packages/
|
||||
|
||||
[report]
|
||||
# Regexes for lines to exclude from consideration
|
||||
exclude_lines =
|
||||
# Have to re-enable the standard pragma
|
||||
pragma: no cover
|
||||
|
||||
# Don't complain about missing debug-only code:
|
||||
def __repr__
|
||||
if self\.debug
|
||||
|
||||
# Don't complain if tests don't hit defensive assertion code:
|
||||
raise AssertionError
|
||||
raise NotImplementedError
|
||||
|
||||
# Don't complain if non-runnable code isn't run:
|
||||
if 0:
|
||||
if __name__ == .__main__.:
|
||||
|
||||
# Ignore importlib backport
|
||||
from importlib
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
||||
#fail_under = 100
|
||||
show_missing = True
|
||||
@@ -1,125 +0,0 @@
|
||||
---
|
||||
description: Apache Superset development standards and guidelines for Cursor IDE
|
||||
globs: ["**/*.py", "**/*.ts", "**/*.tsx", "**/*.js", "**/*.jsx", "**/*.sql", "**/*.md"]
|
||||
alwaysApply: true
|
||||
---
|
||||
|
||||
# Apache Superset Development Standards for Cursor IDE
|
||||
|
||||
Apache Superset is a data visualization platform with Flask/Python backend and React/TypeScript frontend.
|
||||
|
||||
## ⚠️ CRITICAL: Ongoing Refactors (What NOT to Do)
|
||||
|
||||
**These migrations are actively happening - avoid deprecated patterns:**
|
||||
|
||||
### Frontend Modernization
|
||||
- **NO `any` types** - Use proper TypeScript types
|
||||
- **NO JavaScript files** - Convert to TypeScript (.ts/.tsx)
|
||||
- **NO Enzyme** - Use React Testing Library/Jest (Enzyme fully removed)
|
||||
- **Use @superset-ui/core** - Don't import Ant Design directly
|
||||
|
||||
### Testing Strategy Migration
|
||||
- **Prefer unit tests** over integration tests
|
||||
- **Prefer integration tests** over Cypress end-to-end tests
|
||||
- **Cypress is last resort** - Actively moving away from Cypress
|
||||
- **Use Jest + React Testing Library** for component testing
|
||||
|
||||
### Backend Type Safety
|
||||
- **Add type hints** - All new Python code needs proper typing
|
||||
- **MyPy compliance** - Run `pre-commit run mypy` to validate
|
||||
- **SQLAlchemy typing** - Use proper model annotations
|
||||
|
||||
## Code Standards
|
||||
|
||||
### TypeScript Frontend
|
||||
- **NO `any` types** - Use proper TypeScript
|
||||
- **Functional components** with hooks
|
||||
- **@superset-ui/core** for UI components (not direct antd)
|
||||
- **Jest** for testing (NO Enzyme)
|
||||
- **Redux** for global state, hooks for local
|
||||
|
||||
### Python Backend
|
||||
- **Type hints required** for all new code
|
||||
- **MyPy compliant** - run `pre-commit run mypy`
|
||||
- **SQLAlchemy models** with proper typing
|
||||
- **pytest** for testing
|
||||
|
||||
### Apache License Headers
|
||||
- **New files require ASF license headers** - When creating new code files, include the standard Apache Software Foundation license header
|
||||
- **LLM instruction files are excluded** - Files like LLMS.md, CLAUDE.md, etc. are in `.rat-excludes` to avoid header token overhead
|
||||
|
||||
## Key Directory Structure
|
||||
|
||||
```
|
||||
superset/
|
||||
├── superset/ # Python backend (Flask, SQLAlchemy)
|
||||
│ ├── views/api/ # REST API endpoints
|
||||
│ ├── models/ # Database models
|
||||
│ └── connectors/ # Database connections
|
||||
├── superset-frontend/src/ # React TypeScript frontend
|
||||
│ ├── components/ # Reusable components
|
||||
│ ├── explore/ # Chart builder
|
||||
│ ├── dashboard/ # Dashboard interface
|
||||
│ └── SqlLab/ # SQL editor
|
||||
├── superset-frontend/packages/
|
||||
│ └── superset-ui-core/ # UI component library (USE THIS)
|
||||
├── tests/ # Python/integration tests
|
||||
├── docs/ # Documentation (UPDATE FOR CHANGES)
|
||||
└── UPDATING.md # Breaking changes log
|
||||
```
|
||||
|
||||
## Architecture Patterns
|
||||
|
||||
### Dataset-Centric Approach
|
||||
Charts built from enriched datasets containing:
|
||||
- Dimension columns with labels/descriptions
|
||||
- Predefined metrics as SQL expressions
|
||||
- Self-service analytics within defined contexts
|
||||
|
||||
### Security & Features
|
||||
- **RBAC**: Role-based access via Flask-AppBuilder
|
||||
- **Feature flags**: Control feature rollouts
|
||||
- **Row-level security**: SQL-based data access control
|
||||
|
||||
## Test Utilities
|
||||
|
||||
### Python Test Helpers
|
||||
- **`SupersetTestCase`** - Base class in `tests/integration_tests/base_tests.py`
|
||||
- **`@with_config`** - Config mocking decorator
|
||||
- **`@with_feature_flags`** - Feature flag testing
|
||||
- **`login_as()`, `login_as_admin()`** - Authentication helpers
|
||||
- **`create_dashboard()`, `create_slice()`** - Data setup utilities
|
||||
|
||||
### TypeScript Test Helpers
|
||||
- **`superset-frontend/spec/helpers/testing-library.tsx`** - Custom render() with providers
|
||||
- **`createWrapper()`** - Redux/Router/Theme wrapper
|
||||
- **`selectOption()`** - Select component helper
|
||||
- **React Testing Library** - NO Enzyme (removed)
|
||||
|
||||
## Pre-commit Validation
|
||||
|
||||
**Use pre-commit hooks for quality validation:**
|
||||
|
||||
```bash
|
||||
# Install hooks
|
||||
pre-commit install
|
||||
|
||||
# Quick validation (faster than --all-files)
|
||||
pre-commit run # Staged files only
|
||||
pre-commit run mypy # Python type checking
|
||||
pre-commit run prettier # Code formatting
|
||||
pre-commit run eslint # Frontend linting
|
||||
```
|
||||
|
||||
## Development Guidelines
|
||||
|
||||
- **Documentation**: Update docs/ for any user-facing changes
|
||||
- **Breaking Changes**: Add to UPDATING.md
|
||||
- **Docstrings**: Required for new functions/classes
|
||||
- **Follow existing patterns**: Mimic code style, use existing libraries and utilities
|
||||
- **Type Safety**: This codebase is actively modernizing toward full TypeScript and type safety
|
||||
- **Always run `pre-commit run`** to validate changes before committing
|
||||
|
||||
---
|
||||
|
||||
**Note**: This codebase is actively modernizing toward full TypeScript and type safety. Always run `pre-commit run` to validate changes. Follow the ongoing refactors section to avoid deprecated patterns.
|
||||
@@ -1,20 +0,0 @@
|
||||
# Keep this in sync with the base image in the main Dockerfile (ARG PY_VER)
|
||||
FROM python:3.11.13-trixie AS base
|
||||
|
||||
# Install system dependencies that Superset needs
|
||||
# This layer will be cached across Codespace sessions
|
||||
RUN apt-get update && apt-get install -y \
|
||||
libsasl2-dev \
|
||||
libldap2-dev \
|
||||
libpq-dev \
|
||||
tmux \
|
||||
gh \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Install uv for fast Python package management
|
||||
# This will also be cached in the image
|
||||
RUN curl -LsSf https://astral.sh/uv/install.sh | sh && \
|
||||
echo 'export PATH="/root/.cargo/bin:$PATH"' >> /etc/bash.bashrc
|
||||
|
||||
# Set the cargo/bin directory in PATH for all users
|
||||
ENV PATH="/root/.cargo/bin:${PATH}"
|
||||
@@ -1,5 +0,0 @@
|
||||
# Superset Development with GitHub Codespaces
|
||||
|
||||
For complete documentation on using GitHub Codespaces with Apache Superset, please see:
|
||||
|
||||
**[Setting up a Development Environment - GitHub Codespaces](https://superset.apache.org/docs/contributing/development#github-codespaces-cloud-development)**
|
||||
@@ -1,62 +0,0 @@
|
||||
# Superset Codespaces environment setup
|
||||
# This file is appended to ~/.bashrc during Codespace setup
|
||||
|
||||
# Find the workspace directory (handles both 'superset' and 'superset-2' names)
|
||||
WORKSPACE_DIR=$(find /workspaces -maxdepth 1 -name "superset*" -type d | head -1)
|
||||
|
||||
if [ -n "$WORKSPACE_DIR" ]; then
|
||||
# Check if virtual environment exists
|
||||
if [ -d "$WORKSPACE_DIR/.venv" ]; then
|
||||
# Activate the virtual environment
|
||||
source "$WORKSPACE_DIR/.venv/bin/activate"
|
||||
echo "✅ Python virtual environment activated"
|
||||
|
||||
# Verify pre-commit is installed and set up
|
||||
if command -v pre-commit &> /dev/null; then
|
||||
echo "✅ pre-commit is available ($(pre-commit --version))"
|
||||
# Install git hooks if not already installed
|
||||
if [ -d "$WORKSPACE_DIR/.git" ] && [ ! -f "$WORKSPACE_DIR/.git/hooks/pre-commit" ]; then
|
||||
echo "🪝 Installing pre-commit hooks..."
|
||||
cd "$WORKSPACE_DIR" && pre-commit install
|
||||
fi
|
||||
else
|
||||
echo "⚠️ pre-commit not found. Run: pip install pre-commit"
|
||||
fi
|
||||
else
|
||||
echo "⚠️ Python virtual environment not found at $WORKSPACE_DIR/.venv"
|
||||
echo " Run: cd $WORKSPACE_DIR && .devcontainer/setup-dev.sh"
|
||||
fi
|
||||
|
||||
# Always cd to the workspace directory for convenience
|
||||
cd "$WORKSPACE_DIR"
|
||||
fi
|
||||
|
||||
# Add helpful aliases for Superset development
|
||||
alias start-superset="$WORKSPACE_DIR/.devcontainer/start-superset.sh"
|
||||
alias setup-dev="$WORKSPACE_DIR/.devcontainer/setup-dev.sh"
|
||||
|
||||
# Show helpful message on login
|
||||
echo ""
|
||||
echo "🚀 Superset Codespaces Environment"
|
||||
echo "=================================="
|
||||
|
||||
# Check if Superset is running
|
||||
if docker ps 2>/dev/null | grep -q "superset"; then
|
||||
echo "✅ Superset is running!"
|
||||
echo " - Check the 'Ports' tab for your live Superset URL"
|
||||
echo " - Initial startup takes 10-20 minutes"
|
||||
echo " - Login: admin/admin"
|
||||
else
|
||||
echo "⚠️ Superset is not running. Use: start-superset"
|
||||
# Check if there's a startup log
|
||||
if [ -f "/tmp/superset-startup.log" ]; then
|
||||
echo " 📋 Startup log found: cat /tmp/superset-startup.log"
|
||||
fi
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Quick commands:"
|
||||
echo " start-superset - Start Superset with Docker Compose"
|
||||
echo " setup-dev - Set up Python environment (if not already done)"
|
||||
echo " pre-commit run - Run pre-commit checks on staged files"
|
||||
echo ""
|
||||
@@ -1,20 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Script to build and push the devcontainer image to GitHub Container Registry
|
||||
# This allows caching the image between Codespace sessions
|
||||
|
||||
# You'll need to run this with appropriate GitHub permissions
|
||||
# gh auth login --scopes write:packages
|
||||
|
||||
REGISTRY="ghcr.io"
|
||||
OWNER="apache"
|
||||
REPO="superset"
|
||||
TAG="devcontainer-base"
|
||||
|
||||
echo "Building devcontainer image..."
|
||||
docker build -t $REGISTRY/$OWNER/$REPO:$TAG .devcontainer/
|
||||
|
||||
echo "Pushing to GitHub Container Registry..."
|
||||
docker push $REGISTRY/$OWNER/$REPO:$TAG
|
||||
|
||||
echo "Done! Update .devcontainer/devcontainer.json to use:"
|
||||
echo " \"image\": \"$REGISTRY/$OWNER/$REPO:$TAG\""
|
||||
@@ -1,19 +0,0 @@
|
||||
{
|
||||
// Extend the base configuration
|
||||
"extends": "../devcontainer-base.json",
|
||||
|
||||
"name": "Apache Superset Development (Default)",
|
||||
|
||||
// Forward ports for development
|
||||
"forwardPorts": [9001],
|
||||
"portsAttributes": {
|
||||
"9001": {
|
||||
"label": "Superset (via Webpack Dev Server)",
|
||||
"onAutoForward": "notify",
|
||||
"visibility": "public"
|
||||
}
|
||||
},
|
||||
|
||||
// Auto-start Superset on Codespace resume
|
||||
"postStartCommand": ".devcontainer/start-superset.sh"
|
||||
}
|
||||
@@ -1,39 +0,0 @@
|
||||
{
|
||||
"name": "Apache Superset Development",
|
||||
// Keep this in sync with the base image in Dockerfile (ARG PY_VER)
|
||||
// Using the same base as Dockerfile, but non-slim for dev tools
|
||||
"image": "python:3.11.13-bookworm",
|
||||
|
||||
"features": {
|
||||
"ghcr.io/devcontainers/features/docker-in-docker:2": {
|
||||
"moby": true,
|
||||
"dockerDashComposeVersion": "v2"
|
||||
},
|
||||
"ghcr.io/devcontainers/features/node:1": {
|
||||
"version": "20"
|
||||
},
|
||||
"ghcr.io/devcontainers/features/git:1": {},
|
||||
"ghcr.io/devcontainers/features/common-utils:2": {
|
||||
"configureZshAsDefaultShell": true
|
||||
},
|
||||
"ghcr.io/devcontainers/features/sshd:1": {
|
||||
"version": "latest"
|
||||
}
|
||||
},
|
||||
|
||||
// Run commands after container is created
|
||||
"postCreateCommand": "chmod +x .devcontainer/setup-dev.sh && .devcontainer/setup-dev.sh",
|
||||
|
||||
// VS Code customizations
|
||||
"customizations": {
|
||||
"vscode": {
|
||||
"extensions": [
|
||||
"ms-python.python",
|
||||
"ms-python.vscode-pylance",
|
||||
"charliermarsh.ruff",
|
||||
"dbaeumer.vscode-eslint",
|
||||
"esbenp.prettier-vscode"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,66 +0,0 @@
|
||||
{
|
||||
"name": "Apache Superset Development",
|
||||
// Option 1: Use pre-built image directly
|
||||
// "image": "ghcr.io/apache/superset:devcontainer-base",
|
||||
|
||||
// Option 2: Build from Dockerfile with cache (current approach)
|
||||
"build": {
|
||||
"dockerfile": "Dockerfile",
|
||||
"context": ".",
|
||||
// Cache from the Apache registry image
|
||||
"cacheFrom": ["ghcr.io/apache/superset:devcontainer-base"]
|
||||
},
|
||||
|
||||
"features": {
|
||||
"ghcr.io/devcontainers/features/docker-in-docker:2": {
|
||||
"moby": false,
|
||||
"dockerDashComposeVersion": "v2"
|
||||
},
|
||||
"ghcr.io/devcontainers/features/node:1": {
|
||||
"version": "20"
|
||||
},
|
||||
"ghcr.io/devcontainers/features/git:1": {},
|
||||
"ghcr.io/devcontainers/features/common-utils:2": {
|
||||
"configureZshAsDefaultShell": true
|
||||
},
|
||||
"ghcr.io/devcontainers/features/sshd:1": {
|
||||
"version": "latest"
|
||||
}
|
||||
},
|
||||
|
||||
// Forward ports for development
|
||||
"forwardPorts": [9001],
|
||||
"portsAttributes": {
|
||||
"9001": {
|
||||
"label": "Superset (via Webpack Dev Server)",
|
||||
"onAutoForward": "notify",
|
||||
"visibility": "public"
|
||||
}
|
||||
},
|
||||
|
||||
// Run commands after container is created
|
||||
"postCreateCommand": "bash .devcontainer/setup-dev.sh || echo '⚠️ Setup had issues - run .devcontainer/setup-dev.sh manually'",
|
||||
|
||||
// Auto-start Superset after ensuring Docker is ready
|
||||
// Run in foreground to see any errors, but don't block on failures
|
||||
"postStartCommand": "bash -c 'echo \"Waiting 30s for services to initialize...\"; sleep 30; .devcontainer/start-superset.sh || echo \"⚠️ Auto-start failed - run start-superset manually\"'",
|
||||
|
||||
// Set environment variables
|
||||
"remoteEnv": {
|
||||
// Removed automatic venv activation to prevent startup issues
|
||||
// The setup script will handle this
|
||||
},
|
||||
|
||||
// VS Code customizations
|
||||
"customizations": {
|
||||
"vscode": {
|
||||
"extensions": [
|
||||
"ms-python.python",
|
||||
"ms-python.vscode-pylance",
|
||||
"charliermarsh.ruff",
|
||||
"dbaeumer.vscode-eslint",
|
||||
"esbenp.prettier-vscode"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,32 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Setup script for Superset Codespaces development environment
|
||||
|
||||
echo "🔧 Setting up Superset development environment..."
|
||||
|
||||
# The universal image has most tools, just need Superset-specific libs
|
||||
echo "📦 Installing Superset-specific dependencies..."
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y \
|
||||
libsasl2-dev \
|
||||
libldap2-dev \
|
||||
libpq-dev \
|
||||
tmux \
|
||||
gh
|
||||
|
||||
# Install uv for fast Python package management
|
||||
echo "📦 Installing uv..."
|
||||
curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
|
||||
# Add cargo/bin to PATH for uv
|
||||
echo 'export PATH="$HOME/.cargo/bin:$PATH"' >> ~/.bashrc
|
||||
echo 'export PATH="$HOME/.cargo/bin:$PATH"' >> ~/.zshrc
|
||||
|
||||
# Install Claude Code CLI via npm
|
||||
echo "🤖 Installing Claude Code..."
|
||||
npm install -g @anthropic-ai/claude-code
|
||||
|
||||
# Make the start script executable
|
||||
chmod +x .devcontainer/start-superset.sh
|
||||
|
||||
echo "✅ Development environment setup complete!"
|
||||
echo "🚀 Run '.devcontainer/start-superset.sh' to start Superset"
|
||||
@@ -1,69 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Startup script for Superset in Codespaces
|
||||
|
||||
echo "🚀 Starting Superset in Codespaces..."
|
||||
echo "🌐 Frontend will be available at port 9001"
|
||||
|
||||
# Check if MCP is enabled
|
||||
if [ "$ENABLE_MCP" = "true" ]; then
|
||||
echo "🤖 MCP Service will be available at port 5008"
|
||||
fi
|
||||
|
||||
# Find the workspace directory (Codespaces clones as 'superset', not 'superset-2')
|
||||
WORKSPACE_DIR=$(find /workspaces -maxdepth 1 -name "superset*" -type d | head -1)
|
||||
if [ -n "$WORKSPACE_DIR" ]; then
|
||||
cd "$WORKSPACE_DIR"
|
||||
echo "📁 Working in: $WORKSPACE_DIR"
|
||||
else
|
||||
echo "📁 Using current directory: $(pwd)"
|
||||
fi
|
||||
|
||||
# Check if docker is running
|
||||
if ! docker info > /dev/null 2>&1; then
|
||||
echo "⏳ Waiting for Docker to start..."
|
||||
sleep 5
|
||||
fi
|
||||
|
||||
# Clean up any existing containers
|
||||
echo "🧹 Cleaning up existing containers..."
|
||||
docker-compose -f docker-compose-light.yml --profile mcp down
|
||||
|
||||
# Start services
|
||||
echo "🏗️ Building and starting services..."
|
||||
echo ""
|
||||
echo "📝 Once started, login with:"
|
||||
echo " Username: admin"
|
||||
echo " Password: admin"
|
||||
echo ""
|
||||
echo "📋 Running in foreground with live logs (Ctrl+C to stop)..."
|
||||
|
||||
# Run docker-compose and capture exit code
|
||||
if [ "$ENABLE_MCP" = "true" ]; then
|
||||
echo "🤖 Starting with MCP Service enabled..."
|
||||
docker-compose -f docker-compose-light.yml --profile mcp up
|
||||
else
|
||||
docker-compose -f docker-compose-light.yml up
|
||||
fi
|
||||
EXIT_CODE=$?
|
||||
|
||||
# If it failed, provide helpful instructions
|
||||
if [ $EXIT_CODE -ne 0 ] && [ $EXIT_CODE -ne 130 ]; then # 130 is Ctrl+C
|
||||
echo ""
|
||||
echo "❌ Superset startup failed (exit code: $EXIT_CODE)"
|
||||
echo ""
|
||||
echo "🔄 To restart Superset, run:"
|
||||
echo " .devcontainer/start-superset.sh"
|
||||
echo ""
|
||||
echo "🔧 For troubleshooting:"
|
||||
echo " # View logs:"
|
||||
echo " docker-compose -f docker-compose-light.yml logs"
|
||||
echo ""
|
||||
echo " # Clean restart (removes volumes):"
|
||||
echo " docker-compose -f docker-compose-light.yml down -v"
|
||||
echo " .devcontainer/start-superset.sh"
|
||||
echo ""
|
||||
echo " # Common issues:"
|
||||
echo " - Network timeouts: Just retry, often transient"
|
||||
echo " - Port conflicts: Check 'docker ps'"
|
||||
echo " - Database issues: Try clean restart with -v"
|
||||
fi
|
||||
@@ -1,29 +0,0 @@
|
||||
{
|
||||
// Extend the base configuration
|
||||
"extends": "../devcontainer-base.json",
|
||||
|
||||
"name": "Apache Superset Development with MCP",
|
||||
|
||||
// Forward ports for development
|
||||
"forwardPorts": [9001, 5008],
|
||||
"portsAttributes": {
|
||||
"9001": {
|
||||
"label": "Superset (via Webpack Dev Server)",
|
||||
"onAutoForward": "notify",
|
||||
"visibility": "public"
|
||||
},
|
||||
"5008": {
|
||||
"label": "MCP Service (Model Context Protocol)",
|
||||
"onAutoForward": "notify",
|
||||
"visibility": "private"
|
||||
}
|
||||
},
|
||||
|
||||
// Auto-start Superset with MCP on Codespace resume
|
||||
"postStartCommand": "ENABLE_MCP=true .devcontainer/start-superset.sh",
|
||||
|
||||
// Environment variables
|
||||
"containerEnv": {
|
||||
"ENABLE_MCP": "true"
|
||||
}
|
||||
}
|
||||
@@ -34,6 +34,7 @@
|
||||
**/*.sqllite
|
||||
**/*.swp
|
||||
**/.terser-plugin-cache/
|
||||
**/.storybook/
|
||||
**/node_modules/
|
||||
|
||||
tests/
|
||||
@@ -41,8 +42,6 @@ docs/
|
||||
install/
|
||||
superset-frontend/cypress-base/
|
||||
superset-frontend/coverage/
|
||||
superset-frontend/.temp_cache/
|
||||
superset/static/assets/
|
||||
superset-websocket/dist/
|
||||
venv
|
||||
.venv
|
||||
|
||||
@@ -1,41 +0,0 @@
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# Auto-configure Docker Compose for multi-instance support
|
||||
# Requires direnv: https://direnv.net/
|
||||
#
|
||||
# Install: brew install direnv (or apt install direnv)
|
||||
# Setup: Add 'eval "$(direnv hook bash)"' to ~/.bashrc (or ~/.zshrc)
|
||||
# Allow: Run 'direnv allow' in this directory once
|
||||
|
||||
# Generate unique project name from directory
|
||||
export COMPOSE_PROJECT_NAME=$(basename "$PWD" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9]/-/g')
|
||||
|
||||
# Find available ports sequentially to avoid collisions
|
||||
_is_free() { ! lsof -i ":$1" &>/dev/null 2>&1; }
|
||||
|
||||
_p=80; while ! _is_free $_p; do ((_p++)); done; export NGINX_PORT=$_p
|
||||
_p=8088; while ! _is_free $_p; do ((_p++)); done; export SUPERSET_PORT=$_p
|
||||
_p=9000; while ! _is_free $_p; do ((_p++)); done; export NODE_PORT=$_p
|
||||
_p=8080; while ! _is_free $_p || [ $_p -eq $NGINX_PORT ]; do ((_p++)); done; export WEBSOCKET_PORT=$_p
|
||||
_p=8081; while ! _is_free $_p || [ $_p -eq $WEBSOCKET_PORT ]; do ((_p++)); done; export CYPRESS_PORT=$_p
|
||||
_p=5432; while ! _is_free $_p; do ((_p++)); done; export DATABASE_PORT=$_p
|
||||
_p=6379; while ! _is_free $_p; do ((_p++)); done; export REDIS_PORT=$_p
|
||||
|
||||
unset _p _is_free
|
||||
|
||||
echo "🐳 Superset configured: http://localhost:$SUPERSET_PORT (dev: localhost:$NODE_PORT)"
|
||||
3
.gitattributes
vendored
3
.gitattributes
vendored
@@ -1,4 +1 @@
|
||||
docker/**/*.sh text eol=lf
|
||||
*.svg binary
|
||||
*.ipynb binary
|
||||
*.geojson binary
|
||||
|
||||
40
.github/CODEOWNERS
vendored
40
.github/CODEOWNERS
vendored
@@ -1,6 +1,8 @@
|
||||
# Notify all committers of DB migration changes, per SIP-59
|
||||
|
||||
/superset/migrations/ @mistercrunch @michael-s-molina @betodealmeida @eschutho @sadpandajoe
|
||||
# https://github.com/apache/superset/issues/13351
|
||||
|
||||
/superset/migrations/ @apache/superset-committers
|
||||
|
||||
# Notify some committers of changes in the components
|
||||
|
||||
@@ -10,43 +12,23 @@
|
||||
|
||||
# Notify Helm Chart maintainers about changes in it
|
||||
|
||||
/helm/superset/ @dpgaspar @villebro @nytai @michael-s-molina @mistercrunch @rusackas @Antonio-RiveroMartnez @hainenber
|
||||
/helm/superset/ @craig-rueda @dpgaspar @villebro
|
||||
|
||||
# Notify E2E test maintainers of changes
|
||||
|
||||
/superset-frontend/playwright/ @sadpandajoe @geido @eschutho @rusackas @mistercrunch
|
||||
/superset-frontend/cypress-base/ @sadpandajoe @geido @eschutho @rusackas @mistercrunch
|
||||
/superset-frontend/cypress-base/ @jinghua-qa @geido @eschutho @rusackas @betodealmeida
|
||||
|
||||
# Notify PMC members of changes to GitHub Actions
|
||||
|
||||
/.github/ @villebro @geido @eschutho @rusackas @betodealmeida @nytai @mistercrunch @kgabryje @sha174n @dpgaspar @sadpandajoe @hainenber
|
||||
/.github/ @villebro @geido @eschutho @rusackas @betodealmeida @nytai @mistercrunch @craig-rueda @john-bodley @kgabryje @dpgaspar
|
||||
|
||||
# Notify PMC members of changes to CI-executed scripts (supply-chain risk:
|
||||
# scripts/ files run directly in CI workflows and can execute arbitrary code)
|
||||
# Notify PMC members of changes to required Github Actions
|
||||
|
||||
/scripts/ @villebro @geido @eschutho @rusackas @betodealmeida @nytai @mistercrunch @kgabryje @dpgaspar @sha174n @sadpandajoe @hainenber
|
||||
|
||||
# Notify PMC members of changes to required GitHub Actions
|
||||
|
||||
/.asf.yaml @villebro @geido @eschutho @rusackas @betodealmeida @nytai @mistercrunch @kgabryje @dpgaspar @sha174n @Antonio-RiveroMartnez
|
||||
|
||||
# Maps are a finicky contribution process we care about
|
||||
/.asf.yaml @villebro @geido @eschutho @rusackas @betodealmeida @nytai @mistercrunch @craig-rueda @john-bodley @kgabryje @dpgaspar
|
||||
|
||||
# Maps are a finnicky contribution process we care about
|
||||
**/*.geojson @villebro @rusackas
|
||||
**/*.ipynb @villebro @rusackas
|
||||
/superset-frontend/plugins/legacy-plugin-chart-country-map/ @villebro @rusackas
|
||||
|
||||
# Notify translation maintainers of changes to translations
|
||||
|
||||
/superset/translations/ @sfirke @rusackas @villebro @sadpandajoe @hainenber
|
||||
|
||||
# Notify PMC members of changes to extension-related files
|
||||
|
||||
/docs/developer_portal/extensions/ @michael-s-molina @villebro @rusackas
|
||||
/superset-core/ @michael-s-molina @villebro @geido @eschutho @rusackas @kgabryje
|
||||
/superset-extensions-cli/ @michael-s-molina @villebro @geido @eschutho @rusackas @kgabryje
|
||||
/superset/core/ @michael-s-molina @villebro @geido @eschutho @rusackas @kgabryje
|
||||
/superset/extensions/ @michael-s-molina @villebro @geido @eschutho @rusackas @kgabryje
|
||||
/superset-frontend/src/packages/superset-core/ @michael-s-molina @villebro @geido @eschutho @rusackas @kgabryje
|
||||
/superset-frontend/src/core/ @michael-s-molina @villebro @geido @eschutho @rusackas @kgabryje
|
||||
/superset-frontend/src/extensions/ @michael-s-molina @villebro @geido @eschutho @rusackas @kgabryje
|
||||
# Translations are a finnicky contribution that we care about
|
||||
/superset/translations/ @villebro @rusackas @michael-s-molina
|
||||
|
||||
17
.github/ISSUE_TEMPLATE/bug-report.yml
vendored
17
.github/ISSUE_TEMPLATE/bug-report.yml
vendored
@@ -7,7 +7,6 @@ body:
|
||||
value: |
|
||||
Hello Superset Community member! Please keep things tidy by putting your post in the proper place:
|
||||
|
||||
🚨 Reporting a security issue: send an email to security@superset.apache.org. DO NOT USE GITHUB ISSUES TO REPORT SECURITY PROBLEMS.
|
||||
🐛 Reporting a bug: use this form.
|
||||
🙏 Asking a question or getting help: post in the [Superset Slack chat](http://bit.ly/join-superset-slack) or [GitHub Discussions](https://github.com/apache/superset/discussions) under "Q&A / Help".
|
||||
💡 Requesting a new feature: Search [GitHub Discussions](https://github.com/apache/superset/discussions) to see if it exists already. If not, add a new post there under "Ideas".
|
||||
@@ -15,9 +14,14 @@ body:
|
||||
id: bug-description
|
||||
attributes:
|
||||
label: Bug description
|
||||
description: A clear description of what the bug is, including reproduction steps and expected behavior.
|
||||
description: A clear and concise description of what the bug is.
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: repro-steps
|
||||
attributes:
|
||||
label: How to reproduce the bug
|
||||
placeholder: |
|
||||
The bug is that...
|
||||
1. Go to '...'
|
||||
2. Click on '....'
|
||||
3. Scroll down to '....'
|
||||
@@ -41,8 +45,8 @@ body:
|
||||
label: Superset version
|
||||
options:
|
||||
- master / latest-dev
|
||||
- "6.1.0"
|
||||
- "6.0.0"
|
||||
- "3.1.1"
|
||||
- "3.0.4"
|
||||
validations:
|
||||
required: true
|
||||
- type: dropdown
|
||||
@@ -50,8 +54,9 @@ body:
|
||||
attributes:
|
||||
label: Python version
|
||||
options:
|
||||
- "3.9"
|
||||
- "3.10"
|
||||
- "3.11"
|
||||
- "3.12"
|
||||
- Not applicable
|
||||
- I don't know
|
||||
validations:
|
||||
|
||||
38
.github/SECURITY.md
vendored
Normal file
38
.github/SECURITY.md
vendored
Normal file
@@ -0,0 +1,38 @@
|
||||
# Security Policy
|
||||
|
||||
This is a project of the [Apache Software Foundation](https://apache.org) and follows the
|
||||
ASF [vulnerability handling process](https://apache.org/security/#vulnerability-handling).
|
||||
|
||||
## Reporting Vulnerabilities
|
||||
|
||||
**⚠️ Please do not file GitHub issues for security vulnerabilities as they are public! ⚠️**
|
||||
|
||||
|
||||
Apache Software Foundation takes a rigorous standpoint in annihilating the security issues
|
||||
in its software projects. Apache Superset is highly sensitive and forthcoming to issues
|
||||
pertaining to its features and functionality.
|
||||
If you have any concern or believe you have found a vulnerability in Apache Superset,
|
||||
please get in touch with the Apache Superset Security Team privately at
|
||||
e-mail address [security@superset.apache.org](mailto:security@superset.apache.org).
|
||||
|
||||
More details can be found on the ASF website at
|
||||
[ASF vulnerability reporting process](https://apache.org/security/#reporting-a-vulnerability)
|
||||
|
||||
We kindly ask you to include the following information in your report:
|
||||
- Apache Superset version that you are using
|
||||
- A sanitized copy of your `superset_config.py` file or any config overrides
|
||||
- Detailed steps to reproduce the vulnerability
|
||||
|
||||
Note that Apache Superset is not responsible for any third-party dependencies that may
|
||||
have security issues. Any vulnerabilities found in third-party dependencies should be
|
||||
reported to the maintainers of those projects. Results from security scans of Apache
|
||||
Superset dependencies found on its official Docker image can be remediated at release time
|
||||
by extending the image itself.
|
||||
|
||||
**Your responsible disclosure and collaboration are invaluable.**
|
||||
|
||||
## Extra Information
|
||||
|
||||
- [Apache Superset documentation](https://superset.apache.org/docs/security)
|
||||
- [Common Vulnerabilities and Exposures by release](https://superset.apache.org/docs/security/cves)
|
||||
- [How Security Vulnerabilities are Reported & Handled in Apache Superset (Blog)](https://preset.io/blog/how-security-vulnerabilities-are-reported-and-handled-in-apache-superset/)
|
||||
34
.github/actions/change-detector/action.yml
vendored
34
.github/actions/change-detector/action.yml
vendored
@@ -1,34 +0,0 @@
|
||||
name: Change Detector
|
||||
description: Detects file changes for pull request and push events
|
||||
inputs:
|
||||
token:
|
||||
description: GitHub token for authentication
|
||||
required: true
|
||||
outputs:
|
||||
python:
|
||||
description: Whether Python-related files were changed
|
||||
value: ${{ steps.change-detector.outputs.python }}
|
||||
frontend:
|
||||
description: Whether frontend-related files were changed
|
||||
value: ${{ steps.change-detector.outputs.frontend }}
|
||||
docker:
|
||||
description: Whether docker-related files were changed
|
||||
value: ${{ steps.change-detector.outputs.docker }}
|
||||
docs:
|
||||
description: Whether docs-related files were changed
|
||||
value: ${{ steps.change-detector.outputs.docs }}
|
||||
superset-extensions-cli:
|
||||
description: Whether superset-extensions-cli package-related files were changed
|
||||
value: ${{ steps.change-detector.outputs.superset-extensions-cli }}
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Detect file changes
|
||||
id: change-detector
|
||||
run: |
|
||||
python --version
|
||||
python scripts/change_detector.py
|
||||
shell: bash
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ inputs.token }}
|
||||
GITHUB_OUTPUT: ${{ github.output }}
|
||||
@@ -1,23 +0,0 @@
|
||||
name: Label Draft PRs
|
||||
on:
|
||||
pull_request:
|
||||
types:
|
||||
- opened
|
||||
- converted_to_draft
|
||||
jobs:
|
||||
label-draft:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Check if the PR is a draft
|
||||
id: check-draft
|
||||
uses: actions/github-script@v8
|
||||
with:
|
||||
script: |
|
||||
const isDraft = context.payload.pull_request.draft;
|
||||
core.setOutput('isDraft', isDraft);
|
||||
- name: Add `review:draft` Label
|
||||
if: steps.check-draft.outputs.isDraft == 'true'
|
||||
uses: actions-ecosystem/action-add-labels@v1
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
labels: "review:draft"
|
||||
2
.github/actions/chart-releaser-action
vendored
2
.github/actions/chart-releaser-action
vendored
Submodule .github/actions/chart-releaser-action updated: a917fd15b2...120944e663
53
.github/actions/setup-backend/action.yml
vendored
53
.github/actions/setup-backend/action.yml
vendored
@@ -2,9 +2,9 @@ name: 'Setup Python Environment'
|
||||
description: 'Set up Python and install dependencies with optional configurations.'
|
||||
inputs:
|
||||
python-version:
|
||||
description: 'Python version to set up. Accepts a version number, "current", or "next".'
|
||||
description: 'Python version to set up.'
|
||||
required: true
|
||||
default: 'current'
|
||||
default: '3.9'
|
||||
cache:
|
||||
description: 'Cache dependencies. Options: pip'
|
||||
required: false
|
||||
@@ -13,53 +13,22 @@ inputs:
|
||||
description: 'Type of requirements to install. Options: base, development, default'
|
||||
required: false
|
||||
default: 'dev'
|
||||
install-superset:
|
||||
description: 'Whether to install Superset itself. If false, only python is installed'
|
||||
required: false
|
||||
default: 'true'
|
||||
|
||||
runs:
|
||||
using: 'composite'
|
||||
steps:
|
||||
- name: Interpret Python Version
|
||||
id: set-python-version
|
||||
shell: bash
|
||||
env:
|
||||
INPUT_PYTHON_VERSION: ${{ inputs.python-version }}
|
||||
run: |
|
||||
if [ "$INPUT_PYTHON_VERSION" = "current" ]; then
|
||||
RESOLVED_VERSION="3.11"
|
||||
elif [ "$INPUT_PYTHON_VERSION" = "next" ]; then
|
||||
# currently disabled in GHA matrixes because of library compatibility issues
|
||||
RESOLVED_VERSION="3.12"
|
||||
elif printf '%s' "$INPUT_PYTHON_VERSION" | grep -Eq '^[0-9]+\.[0-9]+(\.[0-9]+)?$'; then
|
||||
RESOLVED_VERSION="$INPUT_PYTHON_VERSION"
|
||||
else
|
||||
echo "Invalid python-version: '$INPUT_PYTHON_VERSION'" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "python-version=$RESOLVED_VERSION" >> "$GITHUB_OUTPUT"
|
||||
- name: Set up Python ${{ steps.set-python-version.outputs.python-version }}
|
||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
- name: Set up Python ${{ inputs.python-version }}
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ steps.set-python-version.outputs.python-version }}
|
||||
python-version: ${{ inputs.python-version }}
|
||||
cache: ${{ inputs.cache }}
|
||||
- name: Install dependencies
|
||||
env:
|
||||
INPUT_INSTALL_SUPERSET: ${{ inputs.install-superset }}
|
||||
INPUT_REQUIREMENTS_TYPE: ${{ inputs.requirements-type }}
|
||||
run: |
|
||||
if [ "$INPUT_INSTALL_SUPERSET" = "true" ]; then
|
||||
sudo apt-get update && sudo apt-get -y install libldap2-dev libsasl2-dev
|
||||
|
||||
pip install --upgrade pip setuptools wheel uv
|
||||
|
||||
if [ "$INPUT_REQUIREMENTS_TYPE" = "dev" ]; then
|
||||
uv pip install --system -r requirements/development.txt
|
||||
elif [ "$INPUT_REQUIREMENTS_TYPE" = "base" ]; then
|
||||
uv pip install --system -r requirements/base.txt
|
||||
fi
|
||||
|
||||
uv pip install --system -e .
|
||||
sudo apt-get update && sudo apt-get -y install libldap2-dev libsasl2-dev
|
||||
pip install --upgrade pip setuptools wheel
|
||||
if [ "${{ inputs.requirements-type }}" = "dev" ]; then
|
||||
pip install -r requirements/development.txt
|
||||
elif [ "${{ inputs.requirements-type }}" = "base" ]; then
|
||||
pip install -r requirements/base.txt
|
||||
fi
|
||||
shell: bash
|
||||
|
||||
78
.github/actions/setup-docker/action.yml
vendored
78
.github/actions/setup-docker/action.yml
vendored
@@ -1,78 +0,0 @@
|
||||
name: "Setup Docker Environment"
|
||||
description: "Reusable steps for setting up QEMU, Docker Buildx, DockerHub login, Supersetbot, and optionally Docker Compose"
|
||||
inputs:
|
||||
build:
|
||||
description: "Used for building?"
|
||||
required: false
|
||||
default: "false"
|
||||
dockerhub-user:
|
||||
description: "DockerHub username"
|
||||
required: false
|
||||
dockerhub-token:
|
||||
description: "DockerHub token"
|
||||
required: false
|
||||
install-docker-compose:
|
||||
description: "Flag to install Docker Compose"
|
||||
required: false
|
||||
default: "true"
|
||||
login-to-dockerhub:
|
||||
description: "Whether you want to log into dockerhub"
|
||||
required: false
|
||||
default: "true"
|
||||
outputs: {}
|
||||
runs:
|
||||
using: "composite"
|
||||
steps:
|
||||
|
||||
- name: Set up QEMU
|
||||
if: ${{ inputs.build == 'true' }}
|
||||
uses: docker/setup-qemu-action@06116385d9baf250c9f4dcb4858b16962ea869c3 # v4.1.0
|
||||
with:
|
||||
# Pin the binfmt image to a specific QEMU release. The default
|
||||
# (`tonistiigi/binfmt:latest`) is a moving target, and drift across
|
||||
# QEMU's x86_64→aarch64 translator has been the proximate cause of
|
||||
# intermittent `exit code: 132` (SIGILL) failures during the arm64
|
||||
# leg of the multi-platform docker build — newer Node native modules
|
||||
# emit instructions QEMU's user-mode emulation occasionally drops on
|
||||
# the floor. Pinning a known-good release stabilises that path.
|
||||
image: tonistiigi/binfmt:qemu-v8.1.5
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
if: ${{ inputs.build == 'true' }}
|
||||
uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0
|
||||
|
||||
- name: Try to login to DockerHub
|
||||
if: ${{ inputs.login-to-dockerhub == 'true' }}
|
||||
continue-on-error: true
|
||||
uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0
|
||||
with:
|
||||
username: ${{ inputs.dockerhub-user }}
|
||||
password: ${{ inputs.dockerhub-token }}
|
||||
|
||||
- name: Install Docker Compose
|
||||
if: ${{ inputs.install-docker-compose == 'true' }}
|
||||
shell: bash
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y ca-certificates curl
|
||||
sudo install -m 0755 -d /etc/apt/keyrings
|
||||
|
||||
# Download and save the Docker GPG key in the correct format
|
||||
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
|
||||
|
||||
# Ensure the key file is readable
|
||||
sudo chmod a+r /etc/apt/keyrings/docker.gpg
|
||||
|
||||
# Add the Docker repository using the correct key
|
||||
echo \
|
||||
"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \
|
||||
$(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \
|
||||
sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
|
||||
|
||||
# Update package lists and install Docker Compose plugin
|
||||
sudo apt update
|
||||
sudo apt install -y docker-compose-plugin
|
||||
|
||||
- name: Docker Version Info
|
||||
shell: bash
|
||||
run: docker info
|
||||
42
.github/actions/setup-supersetbot/action.yml
vendored
42
.github/actions/setup-supersetbot/action.yml
vendored
@@ -1,43 +1,11 @@
|
||||
name: 'Setup supersetbot'
|
||||
description: 'Sets up supersetbot npm lib from the repo or npm'
|
||||
inputs:
|
||||
from-npm:
|
||||
description: 'Install from npm instead of local setup'
|
||||
required: false
|
||||
default: 'true' # Defaults to using the local setup
|
||||
description: 'Sets up supersetbot npm lib from the repo'
|
||||
runs:
|
||||
using: 'composite'
|
||||
steps:
|
||||
|
||||
- name: Setup Node Env
|
||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
with:
|
||||
node-version: '20'
|
||||
|
||||
- name: Install supersetbot from npm
|
||||
if: ${{ inputs.from-npm == 'true' }}
|
||||
- name: Install dependencies
|
||||
shell: bash
|
||||
# zizmor: ignore[adhoc-packages] - supersetbot is a first-party Apache CLI (apache-superset/supersetbot) installed globally as a tool; a global CLI install has no application manifest/lockfile context
|
||||
run: npm install -g supersetbot
|
||||
|
||||
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
||||
if: ${{ inputs.from-npm == 'false' }}
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
repository: apache-superset/supersetbot
|
||||
path: supersetbot
|
||||
|
||||
- name: Setup supersetbot from repo
|
||||
if: ${{ inputs.from-npm == 'false' }}
|
||||
shell: bash
|
||||
working-directory: supersetbot
|
||||
# zizmor: ignore[adhoc-packages] - installs the locally packed supersetbot tarball built from the trusted apache-superset/supersetbot checkout; no lockfile applies to a global CLI install
|
||||
run: |
|
||||
# simple trick to install globally with dependencies
|
||||
npm pack
|
||||
npm install -g ./supersetbot*.tgz
|
||||
|
||||
- name: echo supersetbot version
|
||||
shell: bash
|
||||
run: supersetbot version
|
||||
cd .github/supersetbot
|
||||
npm install
|
||||
npm link
|
||||
|
||||
1
.github/copilot-instructions.md
vendored
1
.github/copilot-instructions.md
vendored
@@ -1 +0,0 @@
|
||||
../AGENTS.md
|
||||
358
.github/dependabot.yml
vendored
358
.github/dependabot.yml
vendored
@@ -1,104 +1,354 @@
|
||||
version: 2
|
||||
enable-beta-ecosystems: true
|
||||
updates:
|
||||
|
||||
- package-ecosystem: "github-actions"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "daily"
|
||||
cooldown:
|
||||
default-days: 7
|
||||
interval: "monthly"
|
||||
|
||||
- package-ecosystem: "npm"
|
||||
ignore:
|
||||
- dependency-name: "@rjsf/*"
|
||||
# TODO: remove below entries until React >= 19.0.0
|
||||
- dependency-name: "react-icons"
|
||||
# JSDOM v30 doesn't play well with Jest v30
|
||||
# Source: https://jestjs.io/blog#known-issues
|
||||
# GH thread: https://github.com/jsdom/jsdom/issues/3492
|
||||
- dependency-name: "jest-environment-jsdom"
|
||||
# `@swc/plugin-transform-imports` doesn't work with current Webpack-SWC hybrid setup
|
||||
# See https://github.com/apache/superset/pull/37384#issuecomment-3793991389
|
||||
# TODO: remove the plugin once Lodash usage has been migrated to a more readily tree-shakeable alternative
|
||||
- dependency-name: "@swc/plugin-transform-imports"
|
||||
# `just-handlerbars-helpers` library in plugin-chart-handlebars requires `currencyformatter`` to be < 2
|
||||
- dependency-name: "currencyformatter.js"
|
||||
update-types: ["version-update:semver-major"]
|
||||
# TODO: remove below clause once https://github.com/pmmmwh/react-refresh-webpack-plugin/pull/940 lands onto a future release
|
||||
# and confirm the issue https://github.com/apache/superset/issues/39600 is fixed
|
||||
- dependency-name: "react-checkbox-tree"
|
||||
update-types: ["version-update:semver-major"]
|
||||
# Babel 8 (7.x -> 8.x) is blocked on the surrounding ecosystem: @emotion/babel-plugin
|
||||
# (NodePath#hoist), babel-plugin-jsx-remove-data-test-id (t.jSXOpeningElement), and
|
||||
# ts-jest all rely on Babel APIs removed in v8 and have not shipped Babel 8 support.
|
||||
# Ignore the coordinated major bump until the ecosystem catches up; it must be done
|
||||
# as a single manual upgrade anyway. TODO: remove when Babel 8 support is viable.
|
||||
- dependency-name: "@babel/*"
|
||||
update-types: ["version-update:semver-major"]
|
||||
# not until node >= 18.12.0
|
||||
- dependency-name: "css-minimizer-webpack-plugin"
|
||||
directory: "/superset-frontend/"
|
||||
schedule:
|
||||
interval: "daily"
|
||||
interval: "monthly"
|
||||
labels:
|
||||
- npm
|
||||
- dependabot
|
||||
open-pull-requests-limit: 30
|
||||
versioning-strategy: increase
|
||||
cooldown:
|
||||
default-days: 7
|
||||
|
||||
- package-ecosystem: "pip"
|
||||
directory: "/"
|
||||
open-pull-requests-limit: 10
|
||||
# Bump the lower bound to the new version, not just widen the upper
|
||||
# bound. Without this, a `sqlglot>=28.10.0, <29` constraint upgraded
|
||||
# to `<30` would keep the stale lower bound forever, dragging
|
||||
# transitively-resolved versions with it. See #40186 (review thread).
|
||||
versioning-strategy: increase
|
||||
directory: "/requirements/"
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
interval: "monthly"
|
||||
labels:
|
||||
- pip
|
||||
- dependabot
|
||||
cooldown:
|
||||
default-days: 7
|
||||
open-pull-requests-limit: 30
|
||||
|
||||
- package-ecosystem: "npm"
|
||||
directory: ".github/actions"
|
||||
schedule:
|
||||
interval: "daily"
|
||||
interval: "monthly"
|
||||
open-pull-requests-limit: 10
|
||||
versioning-strategy: increase
|
||||
cooldown:
|
||||
default-days: 7
|
||||
|
||||
- package-ecosystem: "npm"
|
||||
directory: "/docs/"
|
||||
schedule:
|
||||
interval: "daily"
|
||||
interval: "monthly"
|
||||
open-pull-requests-limit: 10
|
||||
versioning-strategy: increase
|
||||
cooldown:
|
||||
default-days: 7
|
||||
|
||||
- package-ecosystem: "npm"
|
||||
directory: "/superset-websocket/"
|
||||
schedule:
|
||||
interval: "daily"
|
||||
interval: "monthly"
|
||||
labels:
|
||||
- npm
|
||||
- dependabot
|
||||
versioning-strategy: increase
|
||||
cooldown:
|
||||
default-days: 7
|
||||
|
||||
- package-ecosystem: "npm"
|
||||
directory: "/superset-websocket/utils/client-ws-app/"
|
||||
schedule:
|
||||
interval: "daily"
|
||||
interval: "monthly"
|
||||
labels:
|
||||
- npm
|
||||
- dependabot
|
||||
open-pull-requests-limit: 10
|
||||
versioning-strategy: increase
|
||||
cooldown:
|
||||
default-days: 7
|
||||
|
||||
# Now for all of our plugins and packages!
|
||||
|
||||
- package-ecosystem: "npm"
|
||||
directory: "/superset-frontend/plugins/legacy-plugin-chart-calendar/"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
labels:
|
||||
- npm
|
||||
- dependabot
|
||||
open-pull-requests-limit: 5
|
||||
versioning-strategy: increase
|
||||
|
||||
- package-ecosystem: "npm"
|
||||
directory: "/superset-frontend/plugins/legacy-plugin-chart-histogram/"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
labels:
|
||||
- npm
|
||||
- dependabot
|
||||
open-pull-requests-limit: 5
|
||||
versioning-strategy: increase
|
||||
|
||||
- package-ecosystem: "npm"
|
||||
directory: "/superset-frontend/plugins/legacy-plugin-chart-partition/"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
labels:
|
||||
- npm
|
||||
- dependabot
|
||||
open-pull-requests-limit: 5
|
||||
versioning-strategy: increase
|
||||
|
||||
- package-ecosystem: "npm"
|
||||
directory: "/superset-frontend/plugins/legacy-plugin-chart-world-map/"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
labels:
|
||||
- npm
|
||||
- dependabot
|
||||
open-pull-requests-limit: 5
|
||||
versioning-strategy: increase
|
||||
|
||||
- package-ecosystem: "npm"
|
||||
directory: "/superset-frontend/plugins/plugin-chart-pivot-table/"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
labels:
|
||||
- npm
|
||||
- dependabot
|
||||
open-pull-requests-limit: 5
|
||||
versioning-strategy: increase
|
||||
|
||||
- package-ecosystem: "npm"
|
||||
directory: "/superset-frontend/plugins/legacy-plugin-chart-chord/"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
labels:
|
||||
- npm
|
||||
- dependabot
|
||||
open-pull-requests-limit: 5
|
||||
versioning-strategy: increase
|
||||
|
||||
- package-ecosystem: "npm"
|
||||
directory: "/superset-frontend/plugins/legacy-plugin-chart-horizon/"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
labels:
|
||||
- npm
|
||||
- dependabot
|
||||
open-pull-requests-limit: 5
|
||||
versioning-strategy: increase
|
||||
|
||||
- package-ecosystem: "npm"
|
||||
directory: "/superset-frontend/plugins/legacy-plugin-chart-rose/"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
labels:
|
||||
- npm
|
||||
- dependabot
|
||||
open-pull-requests-limit: 5
|
||||
versioning-strategy: increase
|
||||
|
||||
- package-ecosystem: "npm"
|
||||
directory: "/superset-frontend/plugins/legacy-preset-chart-deckgl/"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
labels:
|
||||
- npm
|
||||
- dependabot
|
||||
open-pull-requests-limit: 5
|
||||
versioning-strategy: increase
|
||||
|
||||
- package-ecosystem: "npm"
|
||||
directory: "/superset-frontend/plugins/plugin-chart-table/"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
labels:
|
||||
- npm
|
||||
- dependabot
|
||||
open-pull-requests-limit: 5
|
||||
versioning-strategy: increase
|
||||
|
||||
- package-ecosystem: "npm"
|
||||
directory: "/superset-frontend/plugins/legacy-plugin-chart-country-map/"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
labels:
|
||||
- npm
|
||||
- dependabot
|
||||
open-pull-requests-limit: 5
|
||||
versioning-strategy: increase
|
||||
|
||||
- package-ecosystem: "npm"
|
||||
directory: "/superset-frontend/plugins/legacy-plugin-chart-map-box/"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
labels:
|
||||
- npm
|
||||
- dependabot
|
||||
open-pull-requests-limit: 5
|
||||
versioning-strategy: increase
|
||||
|
||||
- package-ecosystem: "npm"
|
||||
directory: "/superset-frontend/plugins/legacy-plugin-chart-sankey/"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
labels:
|
||||
- npm
|
||||
- dependabot
|
||||
open-pull-requests-limit: 5
|
||||
versioning-strategy: increase
|
||||
|
||||
- package-ecosystem: "npm"
|
||||
directory: "/superset-frontend/plugins/legacy-preset-chart-nvd3/"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
labels:
|
||||
- npm
|
||||
- dependabot
|
||||
open-pull-requests-limit: 5
|
||||
versioning-strategy: increase
|
||||
|
||||
- package-ecosystem: "npm"
|
||||
directory: "/superset-frontend/plugins/plugin-chart-word-cloud/"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
labels:
|
||||
- npm
|
||||
- dependabot
|
||||
open-pull-requests-limit: 5
|
||||
versioning-strategy: increase
|
||||
|
||||
- package-ecosystem: "npm"
|
||||
directory: "/superset-frontend/plugins/legacy-plugin-chart-event-flow/"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
labels:
|
||||
- npm
|
||||
- dependabot
|
||||
open-pull-requests-limit: 5
|
||||
versioning-strategy: increase
|
||||
|
||||
- package-ecosystem: "npm"
|
||||
directory: "/superset-frontend/plugins/legacy-plugin-chart-paired-t-test/"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
labels:
|
||||
- npm
|
||||
- dependabot
|
||||
open-pull-requests-limit: 5
|
||||
versioning-strategy: increase
|
||||
|
||||
- package-ecosystem: "npm"
|
||||
directory: "/superset-frontend/plugins/legacy-plugin-chart-sankey-loop/"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
labels:
|
||||
- npm
|
||||
- dependabot
|
||||
open-pull-requests-limit: 5
|
||||
versioning-strategy: increase
|
||||
|
||||
- package-ecosystem: "npm"
|
||||
directory: "/superset-frontend/plugins/plugin-chart-echarts/"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
labels:
|
||||
- npm
|
||||
- dependabot
|
||||
open-pull-requests-limit: 5
|
||||
versioning-strategy: increase
|
||||
|
||||
- package-ecosystem: "npm"
|
||||
directory: "/superset-frontend/plugins/preset-chart-xy/"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
labels:
|
||||
- npm
|
||||
- dependabot
|
||||
open-pull-requests-limit: 5
|
||||
versioning-strategy: increase
|
||||
|
||||
- package-ecosystem: "npm"
|
||||
directory: "/superset-frontend/plugins/legacy-plugin-chart-heatmap/"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
labels:
|
||||
- npm
|
||||
- dependabot
|
||||
open-pull-requests-limit: 5
|
||||
versioning-strategy: increase
|
||||
|
||||
- package-ecosystem: "npm"
|
||||
directory: "/superset-frontend/plugins/legacy-plugin-chart-parallel-coordinates/"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
labels:
|
||||
- npm
|
||||
- dependabot
|
||||
open-pull-requests-limit: 5
|
||||
versioning-strategy: increase
|
||||
|
||||
- package-ecosystem: "npm"
|
||||
directory: "/superset-frontend/plugins/legacy-plugin-chart-sunburst/"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
labels:
|
||||
- npm
|
||||
- dependabot
|
||||
open-pull-requests-limit: 5
|
||||
versioning-strategy: increase
|
||||
|
||||
- package-ecosystem: "npm"
|
||||
directory: "/superset-frontend/plugins/plugin-chart-handlebars/"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
labels:
|
||||
- npm
|
||||
- dependabot
|
||||
open-pull-requests-limit: 5
|
||||
versioning-strategy: increase
|
||||
|
||||
- package-ecosystem: "npm"
|
||||
directory: "/superset-frontend/packages/generator-superset/"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
labels:
|
||||
- npm
|
||||
- dependabot
|
||||
open-pull-requests-limit: 5
|
||||
versioning-strategy: increase
|
||||
|
||||
- package-ecosystem: "npm"
|
||||
directory: "/superset-frontend/packages/superset-ui-chart-controls/"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
labels:
|
||||
- npm
|
||||
- dependabot
|
||||
open-pull-requests-limit: 5
|
||||
versioning-strategy: increase
|
||||
|
||||
- package-ecosystem: "npm"
|
||||
directory: "/superset-frontend/packages/superset-ui-core/"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
labels:
|
||||
- npm
|
||||
- dependabot
|
||||
open-pull-requests-limit: 5
|
||||
versioning-strategy: increase
|
||||
|
||||
- package-ecosystem: "npm"
|
||||
directory: "/superset-frontend/packages/superset-ui-demo/"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
labels:
|
||||
- npm
|
||||
- dependabot
|
||||
open-pull-requests-limit: 5
|
||||
versioning-strategy: increase
|
||||
|
||||
- package-ecosystem: "npm"
|
||||
directory: "/superset-frontend/packages/superset-ui-switchboard/"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
labels:
|
||||
- npm
|
||||
- dependabot
|
||||
open-pull-requests-limit: 5
|
||||
versioning-strategy: increase
|
||||
|
||||
25
.github/labeler.yml
vendored
25
.github/labeler.yml
vendored
@@ -17,11 +17,6 @@
|
||||
- any-glob-to-any-file:
|
||||
- 'superset/migrations/**'
|
||||
|
||||
"risk:ci-script":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- 'scripts/**'
|
||||
|
||||
############################################
|
||||
# Dependencies
|
||||
############################################
|
||||
@@ -77,16 +72,6 @@
|
||||
- any-glob-to-any-file:
|
||||
- 'superset/translations/zh/**'
|
||||
|
||||
"i18n:czech":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- 'superset/translations/cs/**'
|
||||
|
||||
"i18n:traditional-chinese":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- 'superset/translations/zh_TW/**'
|
||||
|
||||
"i18n:dutch":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
@@ -127,11 +112,6 @@
|
||||
- any-glob-to-any-file:
|
||||
- 'superset/translations/sk/**'
|
||||
|
||||
"i18n:latvian":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- 'superset/translations/lv/**'
|
||||
|
||||
"i18n:ukrainian":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
@@ -142,11 +122,6 @@
|
||||
- any-glob-to-any-file:
|
||||
- 'superset/translations/es/**'
|
||||
|
||||
"i18n:persian":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- 'superset/translations/fa/**'
|
||||
|
||||
############################################
|
||||
# Sub-projects and monorepo packages
|
||||
############################################
|
||||
|
||||
22
.github/supersetbot/.eslintrc.json
vendored
Normal file
22
.github/supersetbot/.eslintrc.json
vendored
Normal file
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"extends": "airbnb-base",
|
||||
"rules": {
|
||||
"import/extensions": 0,
|
||||
"import/prefer-default-export": 0,
|
||||
"func-names": 0,
|
||||
"no-console": 0,
|
||||
"class-methods-use-this": 0
|
||||
},
|
||||
"parserOptions": {
|
||||
"ecmaVersion": 2020,
|
||||
"sourceType": "module"
|
||||
},
|
||||
"parserOptions": {
|
||||
"ecmaVersion": "latest",
|
||||
"sourceType": "module",
|
||||
"requireConfigFile": false
|
||||
},
|
||||
"env": {
|
||||
"jest": true
|
||||
}
|
||||
}
|
||||
37
.github/supersetbot/README.md
vendored
Normal file
37
.github/supersetbot/README.md
vendored
Normal file
@@ -0,0 +1,37 @@
|
||||
# supersetbot
|
||||
|
||||
supersetbot is a utility bot that can be used to help around GitHub, CI and beyond.
|
||||
|
||||
The bot can be used as a local CLI OR, for a subset of fitted use cases, can be invoked directly
|
||||
from GitHub comments.
|
||||
|
||||
Because it's its own npm app, it can be tested/deployed/used in isolation from the rest of
|
||||
Superset, and take on some of the complexity from GitHub actions and onto a nifty
|
||||
utility that can be used in different contexts.
|
||||
|
||||
## Features
|
||||
|
||||
```bash
|
||||
$ use nvm 20
|
||||
$ npm i -g supersetbot
|
||||
$ supersetbot
|
||||
Usage: supersetbot [options] [command]
|
||||
|
||||
Options:
|
||||
-v, --verbose Output extra debugging information
|
||||
-r, --repo <repo> The GitHub repo to use (ie: "apache/superset")
|
||||
-d, --dry-run Run the command in dry-run mode
|
||||
-a, --actor <actor> The actor
|
||||
-h, --help display help for command
|
||||
|
||||
Commands:
|
||||
label [options] <label> Add a label to an issue or PR
|
||||
unlabel [options] <label> Remove a label from an issue or PR
|
||||
release-label-pr [options] <prId> Figure out first release for PR and label it
|
||||
version Prints supersetbot's version number
|
||||
release-label-prs [options] Given a set of PRs, auto-release label them
|
||||
release-label [options] <release> Figure out first release for PR and label it
|
||||
orglabel [options] Add an org label based on the author
|
||||
docker [options] Generates/run docker build commands use in CI
|
||||
help [command] display help for command
|
||||
```
|
||||
8
.github/supersetbot/jest.config.js
vendored
Normal file
8
.github/supersetbot/jest.config.js
vendored
Normal file
@@ -0,0 +1,8 @@
|
||||
export default {
|
||||
transform: {
|
||||
},
|
||||
testEnvironment: 'node',
|
||||
moduleNameMapper: {
|
||||
'^(\\.{1,2}/.*)\\.js$': '$1',
|
||||
},
|
||||
};
|
||||
11020
.github/supersetbot/package-lock.json
generated
vendored
Normal file
11020
.github/supersetbot/package-lock.json
generated
vendored
Normal file
File diff suppressed because it is too large
Load Diff
36
.github/supersetbot/package.json
vendored
Normal file
36
.github/supersetbot/package.json
vendored
Normal file
@@ -0,0 +1,36 @@
|
||||
{
|
||||
"name": "supersetbot",
|
||||
"version": "0.4.2",
|
||||
"description": "A bot for the Superset GitHub repo",
|
||||
"type": "module",
|
||||
"main": "src/index.js",
|
||||
"scripts": {
|
||||
"test": "node --experimental-vm-modules node_modules/jest/bin/jest.js",
|
||||
"eslint": "eslint",
|
||||
"supersetbot": "supersetbot"
|
||||
},
|
||||
"keywords": [],
|
||||
"author": "",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"@octokit/plugin-throttling": "^8.1.3",
|
||||
"@octokit/rest": "^20.0.2",
|
||||
"commander": "^11.0.0",
|
||||
"semver": "^7.6.0",
|
||||
"simple-git": "^3.22.0",
|
||||
"string-argv": "^0.3.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@jest/globals": "^29.7.0",
|
||||
"eslint": "^8.56.0",
|
||||
"eslint-config-airbnb": "^19.0.4",
|
||||
"eslint-plugin-import": "^2.29.1",
|
||||
"eslint-plugin-jsx-a11y": "^6.8.0",
|
||||
"eslint-plugin-react": "^7.33.2",
|
||||
"eslint-plugin-react-hooks": "^4.6.0",
|
||||
"jest": "^29.7.0"
|
||||
},
|
||||
"bin": {
|
||||
"supersetbot": "./src/supersetbot"
|
||||
}
|
||||
}
|
||||
175
.github/supersetbot/src/cli.js
vendored
Executable file
175
.github/supersetbot/src/cli.js
vendored
Executable file
@@ -0,0 +1,175 @@
|
||||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
import { Command, Option } from 'commander';
|
||||
|
||||
import * as docker from './docker.js';
|
||||
import * as utils from './utils.js';
|
||||
import Github from './github.js';
|
||||
import Git from './git.js';
|
||||
|
||||
export default function getCLI(context) {
|
||||
const program = new Command();
|
||||
|
||||
// Some reusable options
|
||||
const issueOption = new Option('-i, --issue <issue>', 'The issue number', process.env.GITHUB_ISSUE_NUMBER);
|
||||
const excludeCherriesOption = new Option('-c, --exclude-cherries', 'Generate cherry labels point to each release where the PR has been cherried');
|
||||
|
||||
// Setting up top-level CLI options
|
||||
program
|
||||
.option('-v, --verbose', 'Output extra debugging information')
|
||||
.option('-r, --repo <repo>', 'The GitHub repo to use (ie: "apache/superset")', process.env.GITHUB_REPOSITORY)
|
||||
.option('-d, --dry-run', 'Run the command in dry-run mode')
|
||||
.option('-a, --actor <actor>', 'The actor', process.env.GITHUB_ACTOR);
|
||||
|
||||
program.command('label <label>')
|
||||
.description('Add a label to an issue or PR')
|
||||
.addOption(issueOption)
|
||||
.action(async function (label) {
|
||||
const opts = context.processOptions(this, ['issue', 'repo']);
|
||||
const github = new Github({ context, issue: opts.issue });
|
||||
await github.label(opts.issue, label, context, opts.actor, opts.verbose, opts.dryRun);
|
||||
});
|
||||
|
||||
program.command('unlabel <label>')
|
||||
.description('Remove a label from an issue or PR')
|
||||
.addOption(issueOption)
|
||||
.action(async function (label) {
|
||||
const opts = context.processOptions(this, ['issue', 'repo']);
|
||||
const github = new Github({ context, issueNumber: opts.issue });
|
||||
await github.unlabel(opts.issue, label, context, opts.actor, opts.verbose, opts.dryRun);
|
||||
});
|
||||
|
||||
program.command('release-label-pr <prId>')
|
||||
.description('Figure out first release for PR and label it')
|
||||
.addOption(excludeCherriesOption)
|
||||
.action(async function (prId) {
|
||||
const opts = context.processOptions(this, ['repo']);
|
||||
const git = new Git(context);
|
||||
await git.loadReleases();
|
||||
|
||||
let wrapped = context.commandWrapper({
|
||||
func: git.getReleaseLabels,
|
||||
verbose: opts.verbose,
|
||||
});
|
||||
const labels = await wrapped(parseInt(prId, 10), opts.verbose, opts.excludeCherries);
|
||||
const github = new Github({ context, issueNumber: opts.issue });
|
||||
wrapped = context.commandWrapper({
|
||||
func: github.syncLabels,
|
||||
verbose: opts.verbose,
|
||||
});
|
||||
await wrapped({labels, prId, actor: opts.actor, verbose: opts.verbose, dryRun: opts.dryRun});
|
||||
});
|
||||
|
||||
program.command('version')
|
||||
.description("Prints supersetbot's version number")
|
||||
.action(async () => {
|
||||
const version = await utils.currentPackageVersion();
|
||||
context.log(version);
|
||||
});
|
||||
|
||||
if (context.source === 'CLI') {
|
||||
program.command('release-label-prs')
|
||||
.description('Given a set of PRs, auto-release label them')
|
||||
.option('-s, --search <search>', 'extra search string to append using the GitHub mini-language')
|
||||
.option('-p, --pages <pages>', 'the number of pages (100 per page) to fetch and process', 10)
|
||||
.action(async function () {
|
||||
const opts = context.processOptions(this, ['repo']);
|
||||
|
||||
const github = new Github({ context, issueNumber: opts.issue });
|
||||
const prs = await github.searchMergedPRs({
|
||||
query: opts.search,
|
||||
onlyUnlabeled: true,
|
||||
verbose: opts.verbose,
|
||||
pages: opts.pages,
|
||||
});
|
||||
const prIdLabelMap = new Map(prs.map((pr) => [pr.number, pr.labels]));
|
||||
const git = new Git(context);
|
||||
await git.loadReleases();
|
||||
|
||||
const prsPromises = prs.map(async (pr) => {
|
||||
const labels = await git.getReleaseLabels(pr.number, opts.verbose);
|
||||
return { prId: pr.number, labels };
|
||||
});
|
||||
const prsTargetLabel = await Promise.all(prsPromises);
|
||||
// eslint-disable-next-line no-restricted-syntax
|
||||
for (const { prId, labels } of prsTargetLabel) {
|
||||
// Running sequentially to avoid rate limiting
|
||||
// eslint-disable-next-line no-await-in-loop
|
||||
await github.syncLabels({
|
||||
labels,
|
||||
existingLabels: prIdLabelMap.get(prId).map(l => l.name),
|
||||
prId,
|
||||
...opts,
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
program.command('release-label <release>')
|
||||
.description('Figure out first release for PR and label it')
|
||||
.addOption(excludeCherriesOption)
|
||||
.action(async function (release) {
|
||||
const opts = context.processOptions(this, ['repo']);
|
||||
const git = new Git(context);
|
||||
await git.loadReleases();
|
||||
const prs = await git.getPRsToSync(release, opts.verbose, opts.excludeCherries);
|
||||
|
||||
const github = new Github({ context });
|
||||
// eslint-disable-next-line no-restricted-syntax
|
||||
for (const { prId, labels } of prs) {
|
||||
// Running sequentially to avoid rate limiting
|
||||
// eslint-disable-next-line no-await-in-loop
|
||||
await github.syncLabels({
|
||||
prId,
|
||||
labels,
|
||||
...opts,
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
program.command('orglabel')
|
||||
.description('Add an org label based on the author')
|
||||
.addOption(issueOption)
|
||||
.action(async function () {
|
||||
const opts = context.processOptions(this, ['issue', 'repo']);
|
||||
const github = new Github({ context, issueNumber: opts.issue });
|
||||
await github.assignOrgLabel(opts.issue, opts.verbose, opts.dryRun);
|
||||
});
|
||||
|
||||
|
||||
program.command('docker')
|
||||
.description('Generates/run docker build commands use in CI')
|
||||
.option('-t, --preset <preset>', 'Build preset', /^(lean|dev|dockerize|websocket|py310|ci)$/i, 'lean')
|
||||
.option('-c, --context <context>', 'Build context', /^(push|pull_request|release)$/i, 'local')
|
||||
.option('-r, --context-ref <ref>', 'Reference to the PR, release, or branch')
|
||||
.option('-p, --platform <platform...>', 'Platforms (multiple values allowed)')
|
||||
.option('-f, --force-latest', 'Force the "latest" tag on the release')
|
||||
.option('-v, --verbose', 'Print more info')
|
||||
.action(function () {
|
||||
const opts = context.processOptions(this, ['preset']);
|
||||
opts.platform = opts.platform || ['linux/arm64'];
|
||||
const cmd = docker.getDockerCommand({ ...opts });
|
||||
context.log(cmd);
|
||||
if (!opts.dryRun) {
|
||||
utils.runShellCommand(cmd, false);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
return program;
|
||||
}
|
||||
12
.github/supersetbot/src/cli.test.js
vendored
Normal file
12
.github/supersetbot/src/cli.test.js
vendored
Normal file
@@ -0,0 +1,12 @@
|
||||
import { spawnSync } from 'child_process';
|
||||
|
||||
describe('CLI Test', () => {
|
||||
test.each([
|
||||
['./src/supersetbot', ['docker', '--preset', 'dev', '--dry-run'], '--target dev'],
|
||||
['./src/supersetbot', ['docker', '--dry-run'], '--target lean'],
|
||||
])('returns %s for release %s', (command, arg, contains) => {
|
||||
const result = spawnSync(command, arg);
|
||||
const output = result.stdout.toString();
|
||||
expect(result.stdout.toString()).toContain(contains);
|
||||
});
|
||||
});
|
||||
152
.github/supersetbot/src/context.js
vendored
Normal file
152
.github/supersetbot/src/context.js
vendored
Normal file
@@ -0,0 +1,152 @@
|
||||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
import { parseArgsStringToArgv } from 'string-argv';
|
||||
|
||||
class Context {
|
||||
constructor(source) {
|
||||
this.hasErrors = false;
|
||||
this.source = source;
|
||||
this.options = {};
|
||||
this.errorLogs = [];
|
||||
this.logs = [];
|
||||
this.repo = null;
|
||||
this.optToEnvMap = {
|
||||
issue: 'GITHUB_ISSUE_NUMBER',
|
||||
repo: 'GITHUB_REPOSITORY',
|
||||
};
|
||||
}
|
||||
|
||||
requireOption(optionName, options) {
|
||||
const optionValue = options[optionName];
|
||||
if (optionValue === undefined || optionValue === null) {
|
||||
this.logError(`option [${optionName}] is required`);
|
||||
this.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
parseArgs(s) {
|
||||
return parseArgsStringToArgv(s);
|
||||
}
|
||||
|
||||
requireOptions(optionNames, options) {
|
||||
optionNames.forEach((optionName) => {
|
||||
this.requireOption(optionName, options);
|
||||
});
|
||||
}
|
||||
|
||||
processOptions(command, requiredOptions = []) {
|
||||
const raw = command.parent?.rawArgs;
|
||||
this.command = '???';
|
||||
if (raw) {
|
||||
this.command = raw.map((s) => (s.includes(' ') ? `"${s}"` : s)).join(' ').replace('node ', '');
|
||||
}
|
||||
this.options = { ...command.opts(), ...command.parent.opts() };
|
||||
|
||||
// Runtime defaults for unit tests since commanders can't receive callables as default
|
||||
Object.entries(this.optToEnvMap).forEach(([k, v]) => {
|
||||
if (!this.options[k]) {
|
||||
this.options[k] = process.env[v];
|
||||
}
|
||||
});
|
||||
this.requireOptions(requiredOptions, this.options);
|
||||
this.issueNumber = this.options.issue;
|
||||
|
||||
if (this.source === 'GHA') {
|
||||
this.options.actor = process.env.GITHUB_ACTOR || 'UNKNOWN';
|
||||
this.options.repo = process.env.GITHUB_REPOSITORY;
|
||||
}
|
||||
this.repo = this.options.repo;
|
||||
|
||||
return this.options;
|
||||
}
|
||||
|
||||
log(msg) {
|
||||
console.log(msg);
|
||||
this.logs = [...this.logs, msg];
|
||||
}
|
||||
|
||||
logSuccess(msg) {
|
||||
const augMsg = `🟢 SUCCESS: ${msg}`;
|
||||
console.log(augMsg);
|
||||
this.logs.push(augMsg);
|
||||
}
|
||||
|
||||
logError(msg) {
|
||||
this.hasErrors = true;
|
||||
const augMsg = `🔴 ERROR: ${msg}`;
|
||||
console.error(augMsg);
|
||||
this.errorLogs.push(augMsg);
|
||||
}
|
||||
|
||||
exit(code = 0) {
|
||||
this.onDone();
|
||||
process.exit(code);
|
||||
}
|
||||
|
||||
commandWrapper({
|
||||
func, successMsg, errorMsg = null, verbose = false, dryRun = false,
|
||||
}) {
|
||||
return async (...args) => {
|
||||
let resp;
|
||||
let hasError = false;
|
||||
|
||||
try {
|
||||
if (!dryRun) {
|
||||
resp = await func(...args);
|
||||
}
|
||||
if (verbose && resp) {
|
||||
console.log(resp);
|
||||
}
|
||||
} catch (error) {
|
||||
hasError = true;
|
||||
if (errorMsg) {
|
||||
this.logError(errorMsg);
|
||||
} else {
|
||||
this.logError(error);
|
||||
}
|
||||
throw (error);
|
||||
}
|
||||
if (successMsg && !hasError) {
|
||||
this.logSuccess(successMsg);
|
||||
}
|
||||
return resp;
|
||||
};
|
||||
}
|
||||
|
||||
doneComment() {
|
||||
const msgs = [...this.logs, ...this.errorLogs];
|
||||
let comment = '';
|
||||
comment += `> \`${this.command}\`\n`;
|
||||
comment += '```\n';
|
||||
comment += msgs.join('\n');
|
||||
comment += '\n```';
|
||||
return comment;
|
||||
}
|
||||
|
||||
async onDone() {
|
||||
let msg;
|
||||
if (this.source === 'GHA') {
|
||||
msg = this.doneComment();
|
||||
}
|
||||
return msg;
|
||||
}
|
||||
}
|
||||
|
||||
export default Context;
|
||||
142
.github/supersetbot/src/docker.js
vendored
Normal file
142
.github/supersetbot/src/docker.js
vendored
Normal file
@@ -0,0 +1,142 @@
|
||||
import { spawnSync } from 'child_process';
|
||||
|
||||
const REPO = 'apache/superset';
|
||||
const CACHE_REPO = `${REPO}-cache`;
|
||||
const BASE_PY_IMAGE = '3.9-slim-bookworm';
|
||||
|
||||
export function runCmd(command, raiseOnFailure = true) {
|
||||
const { stdout, stderr } = spawnSync(command, { shell: true, encoding: 'utf-8', env: process.env });
|
||||
|
||||
if (stderr && raiseOnFailure) {
|
||||
throw new Error(stderr);
|
||||
}
|
||||
return stdout;
|
||||
}
|
||||
|
||||
function getGitSha() {
|
||||
return runCmd('git rev-parse HEAD').trim();
|
||||
}
|
||||
|
||||
function getBuildContextRef(buildContext) {
|
||||
const event = buildContext || process.env.GITHUB_EVENT_NAME;
|
||||
const githubRef = process.env.GITHUB_REF || '';
|
||||
|
||||
if (event === 'pull_request') {
|
||||
const githubHeadRef = process.env.GITHUB_HEAD_REF || '';
|
||||
return githubHeadRef.replace(/[^a-zA-Z0-9]/g, '-').slice(0, 40);
|
||||
} if (event === 'release') {
|
||||
return githubRef.replace('refs/tags/', '').slice(0, 40);
|
||||
} if (event === 'push') {
|
||||
return githubRef.replace('refs/heads/', '').replace(/[^a-zA-Z0-9]/g, '-').slice(0, 40);
|
||||
}
|
||||
return '';
|
||||
}
|
||||
|
||||
export function isLatestRelease(release) {
|
||||
const output = runCmd(`../../scripts/tag_latest_release.sh ${release} --dry-run`, false) || '';
|
||||
return output.includes('SKIP_TAG::false');
|
||||
}
|
||||
|
||||
function makeDockerTag(parts) {
|
||||
return `${REPO}:${parts.filter((part) => part).join('-')}`;
|
||||
}
|
||||
|
||||
export function getDockerTags({
|
||||
preset, platforms, sha, buildContext, buildContextRef, forceLatest = false,
|
||||
}) {
|
||||
const tags = new Set();
|
||||
const tagChunks = [];
|
||||
|
||||
const isLatest = isLatestRelease(buildContextRef);
|
||||
|
||||
if (preset !== 'lean') {
|
||||
tagChunks.push(preset);
|
||||
}
|
||||
|
||||
if (platforms.length === 1) {
|
||||
const platform = platforms[0];
|
||||
const shortBuildPlatform = platform.replace('linux/', '').replace('64', '');
|
||||
if (shortBuildPlatform !== 'amd') {
|
||||
tagChunks.push(shortBuildPlatform);
|
||||
}
|
||||
}
|
||||
|
||||
tags.add(makeDockerTag([sha, ...tagChunks]));
|
||||
tags.add(makeDockerTag([sha.slice(0, 7), ...tagChunks]));
|
||||
|
||||
if (buildContext === 'release') {
|
||||
tags.add(makeDockerTag([buildContextRef, ...tagChunks]));
|
||||
if (isLatest || forceLatest) {
|
||||
tags.add(makeDockerTag(['latest', ...tagChunks]));
|
||||
}
|
||||
} else if (buildContext === 'push' && buildContextRef === 'master') {
|
||||
tags.add(makeDockerTag(['master', ...tagChunks]));
|
||||
} else if (buildContext === 'pull_request') {
|
||||
tags.add(makeDockerTag([`pr-${buildContextRef}`, ...tagChunks]));
|
||||
}
|
||||
|
||||
return [...tags];
|
||||
}
|
||||
|
||||
export function getDockerCommand({
|
||||
preset, platform, buildContext, buildContextRef, forceLatest = false,
|
||||
}) {
|
||||
const platforms = platform;
|
||||
|
||||
let buildTarget = '';
|
||||
let pyVer = BASE_PY_IMAGE;
|
||||
let dockerContext = '.';
|
||||
|
||||
if (preset === 'dev') {
|
||||
buildTarget = 'dev';
|
||||
} else if (preset === 'lean') {
|
||||
buildTarget = 'lean';
|
||||
} else if (preset === 'py310') {
|
||||
buildTarget = 'lean';
|
||||
pyVer = '3.10-slim-bookworm';
|
||||
} else if (preset === 'websocket') {
|
||||
dockerContext = 'superset-websocket';
|
||||
} else if (preset === 'ci') {
|
||||
buildTarget = 'ci';
|
||||
} else if (preset === 'dockerize') {
|
||||
dockerContext = '-f dockerize.Dockerfile .';
|
||||
} else {
|
||||
console.error(`Invalid build preset: ${preset}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
let ref = buildContextRef;
|
||||
if (!ref) {
|
||||
ref = getBuildContextRef(buildContext);
|
||||
}
|
||||
const sha = getGitSha();
|
||||
const tags = getDockerTags({
|
||||
preset, platforms, sha, buildContext, buildContextRef: ref, forceLatest,
|
||||
}).map((tag) => `-t ${tag}`).join(' \\\n ');
|
||||
const isAuthenticated = !!(process.env.DOCKERHUB_TOKEN);
|
||||
|
||||
const dockerArgs = isAuthenticated ? '--push' : '--load';
|
||||
const targetArgument = buildTarget ? `--target ${buildTarget}` : '';
|
||||
const cacheRef = `${CACHE_REPO}:${pyVer}`;
|
||||
const platformArg = `--platform ${platforms.join(',')}`;
|
||||
const cacheFromArg = `--cache-from=type=registry,ref=${cacheRef}`;
|
||||
const cacheToArg = isAuthenticated ? `--cache-to=type=registry,mode=max,ref=${cacheRef}` : '';
|
||||
const buildArg = pyVer ? `--build-arg PY_VER=${pyVer}` : '';
|
||||
const actor = process.env.GITHUB_ACTOR;
|
||||
|
||||
return `docker buildx build \\
|
||||
${dockerArgs} \\
|
||||
${tags} \\
|
||||
${cacheFromArg} \\
|
||||
${cacheToArg} \\
|
||||
${targetArgument} \\
|
||||
${buildArg} \\
|
||||
${platformArg} \\
|
||||
--label sha=${sha} \\
|
||||
--label target=${buildTarget} \\
|
||||
--label build_trigger=${ref} \\
|
||||
--label base=${pyVer} \\
|
||||
--label build_actor=${actor} \\
|
||||
${dockerContext}
|
||||
`;
|
||||
}
|
||||
244
.github/supersetbot/src/docker.test.js
vendored
Normal file
244
.github/supersetbot/src/docker.test.js
vendored
Normal file
@@ -0,0 +1,244 @@
|
||||
import * as dockerUtils from './docker.js';
|
||||
|
||||
const SHA = '22e7c602b9aa321ec7e0df4bb0033048664dcdf0';
|
||||
const PR_ID = '666';
|
||||
const OLD_REL = '2.1.0';
|
||||
const NEW_REL = '2.1.1';
|
||||
const REPO = 'apache/superset';
|
||||
|
||||
beforeEach(() => {
|
||||
process.env.TEST_ENV = 'true';
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
delete process.env.TEST_ENV;
|
||||
});
|
||||
|
||||
describe('isLatestRelease', () => {
|
||||
test.each([
|
||||
['2.1.0', false],
|
||||
['2.1.1', true],
|
||||
['1.0.0', false],
|
||||
['3.0.0', true],
|
||||
])('returns %s for release %s', (release, expectedBool) => {
|
||||
expect(dockerUtils.isLatestRelease(release)).toBe(expectedBool);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getDockerTags', () => {
|
||||
test.each([
|
||||
// PRs
|
||||
[
|
||||
'lean',
|
||||
['linux/arm64'],
|
||||
SHA,
|
||||
'pull_request',
|
||||
PR_ID,
|
||||
[`${REPO}:22e7c60-arm`, `${REPO}:${SHA}-arm`, `${REPO}:pr-${PR_ID}-arm`],
|
||||
],
|
||||
[
|
||||
'ci',
|
||||
['linux/amd64'],
|
||||
SHA,
|
||||
'pull_request',
|
||||
PR_ID,
|
||||
[`${REPO}:22e7c60-ci`, `${REPO}:${SHA}-ci`, `${REPO}:pr-${PR_ID}-ci`],
|
||||
],
|
||||
[
|
||||
'lean',
|
||||
['linux/amd64'],
|
||||
SHA,
|
||||
'pull_request',
|
||||
PR_ID,
|
||||
[`${REPO}:22e7c60`, `${REPO}:${SHA}`, `${REPO}:pr-${PR_ID}`],
|
||||
],
|
||||
[
|
||||
'dev',
|
||||
['linux/arm64'],
|
||||
SHA,
|
||||
'pull_request',
|
||||
PR_ID,
|
||||
[
|
||||
`${REPO}:22e7c60-dev-arm`,
|
||||
`${REPO}:${SHA}-dev-arm`,
|
||||
`${REPO}:pr-${PR_ID}-dev-arm`,
|
||||
],
|
||||
],
|
||||
[
|
||||
'dev',
|
||||
['linux/amd64'],
|
||||
SHA,
|
||||
'pull_request',
|
||||
PR_ID,
|
||||
[`${REPO}:22e7c60-dev`, `${REPO}:${SHA}-dev`, `${REPO}:pr-${PR_ID}-dev`],
|
||||
],
|
||||
// old releases
|
||||
[
|
||||
'lean',
|
||||
['linux/arm64'],
|
||||
SHA,
|
||||
'release',
|
||||
OLD_REL,
|
||||
[`${REPO}:22e7c60-arm`, `${REPO}:${SHA}-arm`, `${REPO}:${OLD_REL}-arm`],
|
||||
],
|
||||
[
|
||||
'lean',
|
||||
['linux/amd64'],
|
||||
SHA,
|
||||
'release',
|
||||
OLD_REL,
|
||||
[`${REPO}:22e7c60`, `${REPO}:${SHA}`, `${REPO}:${OLD_REL}`],
|
||||
],
|
||||
[
|
||||
'dev',
|
||||
['linux/arm64'],
|
||||
SHA,
|
||||
'release',
|
||||
OLD_REL,
|
||||
[
|
||||
`${REPO}:22e7c60-dev-arm`,
|
||||
`${REPO}:${SHA}-dev-arm`,
|
||||
`${REPO}:${OLD_REL}-dev-arm`,
|
||||
],
|
||||
],
|
||||
[
|
||||
'dev',
|
||||
['linux/amd64'],
|
||||
SHA,
|
||||
'release',
|
||||
OLD_REL,
|
||||
[`${REPO}:22e7c60-dev`, `${REPO}:${SHA}-dev`, `${REPO}:${OLD_REL}-dev`],
|
||||
],
|
||||
// new releases
|
||||
[
|
||||
'lean',
|
||||
['linux/arm64'],
|
||||
SHA,
|
||||
'release',
|
||||
NEW_REL,
|
||||
[
|
||||
`${REPO}:22e7c60-arm`,
|
||||
`${REPO}:${SHA}-arm`,
|
||||
`${REPO}:${NEW_REL}-arm`,
|
||||
`${REPO}:latest-arm`,
|
||||
],
|
||||
],
|
||||
[
|
||||
'lean',
|
||||
['linux/amd64'],
|
||||
SHA,
|
||||
'release',
|
||||
NEW_REL,
|
||||
[`${REPO}:22e7c60`, `${REPO}:${SHA}`, `${REPO}:${NEW_REL}`, `${REPO}:latest`],
|
||||
],
|
||||
[
|
||||
'dev',
|
||||
['linux/arm64'],
|
||||
SHA,
|
||||
'release',
|
||||
NEW_REL,
|
||||
[
|
||||
`${REPO}:22e7c60-dev-arm`,
|
||||
`${REPO}:${SHA}-dev-arm`,
|
||||
`${REPO}:${NEW_REL}-dev-arm`,
|
||||
`${REPO}:latest-dev-arm`,
|
||||
],
|
||||
],
|
||||
[
|
||||
'dev',
|
||||
['linux/amd64'],
|
||||
SHA,
|
||||
'release',
|
||||
NEW_REL,
|
||||
[
|
||||
`${REPO}:22e7c60-dev`,
|
||||
`${REPO}:${SHA}-dev`,
|
||||
`${REPO}:${NEW_REL}-dev`,
|
||||
`${REPO}:latest-dev`,
|
||||
],
|
||||
],
|
||||
// merge on master
|
||||
[
|
||||
'lean',
|
||||
['linux/arm64'],
|
||||
SHA,
|
||||
'push',
|
||||
'master',
|
||||
[`${REPO}:22e7c60-arm`, `${REPO}:${SHA}-arm`, `${REPO}:master-arm`],
|
||||
],
|
||||
[
|
||||
'lean',
|
||||
['linux/amd64'],
|
||||
SHA,
|
||||
'push',
|
||||
'master',
|
||||
[`${REPO}:22e7c60`, `${REPO}:${SHA}`, `${REPO}:master`],
|
||||
],
|
||||
[
|
||||
'dev',
|
||||
['linux/arm64'],
|
||||
SHA,
|
||||
'push',
|
||||
'master',
|
||||
[
|
||||
`${REPO}:22e7c60-dev-arm`,
|
||||
`${REPO}:${SHA}-dev-arm`,
|
||||
`${REPO}:master-dev-arm`,
|
||||
],
|
||||
],
|
||||
[
|
||||
'dev',
|
||||
['linux/amd64'],
|
||||
SHA,
|
||||
'push',
|
||||
'master',
|
||||
[`${REPO}:22e7c60-dev`, `${REPO}:${SHA}-dev`, `${REPO}:master-dev`],
|
||||
],
|
||||
|
||||
])('returns expected tags', (preset, platforms, sha, buildContext, buildContextRef, expectedTags) => {
|
||||
const tags = dockerUtils.getDockerTags({
|
||||
preset, platforms, sha, buildContext, buildContextRef,
|
||||
});
|
||||
expect(tags).toEqual(expect.arrayContaining(expectedTags));
|
||||
});
|
||||
});
|
||||
|
||||
describe('getDockerCommand', () => {
|
||||
test.each([
|
||||
[
|
||||
'lean',
|
||||
['linux/amd64'],
|
||||
true,
|
||||
SHA,
|
||||
'push',
|
||||
'master',
|
||||
['--push', `-t ${REPO}:master `],
|
||||
],
|
||||
[
|
||||
'dev',
|
||||
['linux/amd64'],
|
||||
false,
|
||||
SHA,
|
||||
'push',
|
||||
'master',
|
||||
['--load', `-t ${REPO}:master-dev `],
|
||||
],
|
||||
// multi-platform
|
||||
[
|
||||
'lean',
|
||||
['linux/arm64', 'linux/amd64'],
|
||||
true,
|
||||
SHA,
|
||||
'push',
|
||||
'master',
|
||||
['--platform linux/arm64,linux/amd64'],
|
||||
],
|
||||
])('returns expected docker command', (preset, platform, isAuthenticated, sha, buildContext, buildContextRef, contains) => {
|
||||
const cmd = dockerUtils.getDockerCommand({
|
||||
preset, platform, isAuthenticated, sha, buildContext, buildContextRef,
|
||||
});
|
||||
contains.forEach((expectedSubstring) => {
|
||||
expect(cmd).toContain(expectedSubstring);
|
||||
});
|
||||
});
|
||||
});
|
||||
120
.github/supersetbot/src/git.js
vendored
Normal file
120
.github/supersetbot/src/git.js
vendored
Normal file
@@ -0,0 +1,120 @@
|
||||
import simpleGit from 'simple-git';
|
||||
import semver from 'semver';
|
||||
|
||||
import GitRelease from './git_release.js';
|
||||
|
||||
export default class Git {
|
||||
#releaseTags;
|
||||
|
||||
constructor(context, mainBranch = 'master') {
|
||||
this.context = context;
|
||||
this.mainBranch = mainBranch;
|
||||
this.releases = new Map();
|
||||
this.git = simpleGit();
|
||||
this.mainBranchGitRelease = this.mainBranchGitRelease.bind(this);
|
||||
this.getReleaseLabels = this.getReleaseLabels.bind(this);
|
||||
}
|
||||
|
||||
async mainBranchGitRelease() {
|
||||
let rel = this.releases.get(this.mainBranch);
|
||||
if (!rel) {
|
||||
rel = await this.loadRelease(this.mainBranch);
|
||||
}
|
||||
return rel;
|
||||
}
|
||||
|
||||
async releaseTags() {
|
||||
if (!this.#releaseTags) {
|
||||
const tags = await this.git.tags();
|
||||
// Filter tags to include only those that match semver and are official releases
|
||||
const semverTags = tags.all.filter((tag) => semver.valid(tag) && !tag.includes('-') && !tag.includes('v'));
|
||||
semverTags.sort((a, b) => semver.compare(a, b));
|
||||
this.#releaseTags = semverTags;
|
||||
}
|
||||
return this.#releaseTags;
|
||||
}
|
||||
|
||||
async loadMainBranch() {
|
||||
await this.loadRelease(this.mainBranch);
|
||||
}
|
||||
|
||||
async loadReleases(tags = null) {
|
||||
const tagsToFetch = tags || await this.releaseTags();
|
||||
if (!tags) {
|
||||
await this.loadMainBranch();
|
||||
}
|
||||
const promises = [];
|
||||
tagsToFetch.forEach((tag) => {
|
||||
promises.push(this.loadRelease(tag));
|
||||
});
|
||||
await Promise.all(promises);
|
||||
}
|
||||
|
||||
async loadRelease(tag) {
|
||||
const release = new GitRelease(tag, this.context);
|
||||
await release.load();
|
||||
this.releases.set(tag, release);
|
||||
return release;
|
||||
}
|
||||
|
||||
static shortenSHA(sha) {
|
||||
return sha.substring(0, 7);
|
||||
}
|
||||
|
||||
async getReleaseLabels(prNumber, verbose, excludeCherries = false) {
|
||||
const labels = [];
|
||||
const main = await this.mainBranchGitRelease();
|
||||
const commit = main.prIdCommitMap.get(prNumber);
|
||||
if (commit) {
|
||||
const { sha } = commit;
|
||||
const shortSHA = Git.shortenSHA(sha);
|
||||
if (verbose) {
|
||||
console.log(`PR ${prNumber} is ${shortSHA} on branch ${this.mainBranch}`);
|
||||
}
|
||||
|
||||
let firstGitReleased = null;
|
||||
const tags = await this.releaseTags();
|
||||
tags.forEach((tag) => {
|
||||
const release = this.releases.get(tag);
|
||||
if (release.shaCommitMap.get(sha) && !firstGitReleased && release.tag !== this.mainBranch) {
|
||||
firstGitReleased = release.tag;
|
||||
labels.push(`🚢 ${release.tag}`);
|
||||
}
|
||||
const commitInGitRelease = release.prIdCommitMap.get(prNumber);
|
||||
if (!excludeCherries && commitInGitRelease && commitInGitRelease.sha !== sha) {
|
||||
labels.push(`🍒 ${release.tag}`);
|
||||
}
|
||||
});
|
||||
if (labels.length >= 1) {
|
||||
// using this emoji to show it's been labeled by the bot
|
||||
labels.push('🏷️ bot');
|
||||
}
|
||||
}
|
||||
return labels;
|
||||
}
|
||||
|
||||
async previousRelease(release) {
|
||||
const tags = await this.releaseTags();
|
||||
return tags[tags.indexOf(release) - 1];
|
||||
}
|
||||
|
||||
async getPRsToSync(release, verbose = false, excludeCherries = false) {
|
||||
const prevRelease = await this.previousRelease(release);
|
||||
const releaseRange = new GitRelease(release, this.context, prevRelease);
|
||||
await releaseRange.load();
|
||||
const prIds = releaseRange.prIdCommitMap.keys();
|
||||
|
||||
const prs = [];
|
||||
const promises = [];
|
||||
[...prIds].forEach(prId => {
|
||||
promises.push(
|
||||
this.getReleaseLabels(prId, verbose, excludeCherries)
|
||||
.then((labels) => {
|
||||
prs.push({ prId, labels });
|
||||
}),
|
||||
);
|
||||
});
|
||||
await Promise.all(promises);
|
||||
return prs;
|
||||
}
|
||||
}
|
||||
50
.github/supersetbot/src/git_release.js
vendored
Normal file
50
.github/supersetbot/src/git_release.js
vendored
Normal file
@@ -0,0 +1,50 @@
|
||||
import simpleGit from 'simple-git';
|
||||
|
||||
export default class GitRelease {
|
||||
constructor(tag, context, from = null) {
|
||||
this.tag = tag;
|
||||
this.context = context;
|
||||
this.prNumberRegex = /\(#(\d+)\)/;
|
||||
this.shaCommitMap = null;
|
||||
this.prIdCommitMap = null;
|
||||
this.prCommitMap = null;
|
||||
this.git = simpleGit();
|
||||
this.from = from;
|
||||
}
|
||||
|
||||
extractPRNumber(commitMessage) {
|
||||
const match = (commitMessage || '').match(this.prNumberRegex);
|
||||
return match ? parseInt(match[1], 10) : null;
|
||||
}
|
||||
|
||||
async load() {
|
||||
let from = this.from || await this.git.firstCommit();
|
||||
if (from.includes('\n')) {
|
||||
[from] = from.split('\n');
|
||||
}
|
||||
const range = `${this.from || 'first'}..${this.tag}`;
|
||||
const commits = await this.git.log({ from, to: this.tag });
|
||||
this.context.log(`${range} - fetched ${commits.all.length} commits`);
|
||||
|
||||
this.shaCommitMap = new Map();
|
||||
commits.all.forEach((commit) => {
|
||||
const sha = commit.hash.substring(0, 7);
|
||||
this.shaCommitMap.set(
|
||||
sha,
|
||||
{
|
||||
prId: this.extractPRNumber(commit.message),
|
||||
message: commit.message,
|
||||
sha,
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
this.prIdCommitMap = new Map();
|
||||
// eslint-disable-next-line no-restricted-syntax
|
||||
for (const commit of this.shaCommitMap.values()) {
|
||||
if (commit.prId) {
|
||||
this.prIdCommitMap.set(commit.prId, commit);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
252
.github/supersetbot/src/github.js
vendored
Normal file
252
.github/supersetbot/src/github.js
vendored
Normal file
@@ -0,0 +1,252 @@
|
||||
import { Octokit } from '@octokit/rest';
|
||||
import { throttling } from '@octokit/plugin-throttling';
|
||||
|
||||
import { ORG_LIST, PROTECTED_LABEL_PATTERNS, COMMITTER_TEAM } from './metadata.js';
|
||||
|
||||
class Github {
|
||||
#userInTeamCache;
|
||||
|
||||
constructor({ context, issueNumber = null, token = null }) {
|
||||
this.context = context;
|
||||
this.issueNumber = issueNumber;
|
||||
const githubToken = token || process.env.GITHUB_TOKEN;
|
||||
if (!githubToken) {
|
||||
const msg = 'GITHUB_TOKEN is not set';
|
||||
this.context.logError(msg);
|
||||
}
|
||||
const throttledOctokit = Octokit.plugin(throttling);
|
||||
// eslint-disable-next-line new-cap
|
||||
this.octokit = new throttledOctokit({
|
||||
auth: githubToken,
|
||||
throttle: {
|
||||
id: 'supersetbot',
|
||||
onRateLimit: (retryAfter, options, octokit, retryCount) => {
|
||||
const howManyRetries = 10;
|
||||
octokit.log.warn(`Retry ${retryCount} out of ${howManyRetries} - retrying in ${retryAfter} seconds!`);
|
||||
if (retryCount < howManyRetries) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
},
|
||||
onSecondaryRateLimit: (retryAfter, options, octokit) => {
|
||||
octokit.log.warn(`SecondaryRateLimit detected for request ${options.method} ${options.url}`);
|
||||
},
|
||||
},
|
||||
});
|
||||
this.syncLabels = this.syncLabels.bind(this);
|
||||
this.#userInTeamCache = new Map();
|
||||
}
|
||||
|
||||
unPackRepo() {
|
||||
const [owner, repo] = this.context.repo.split('/');
|
||||
return { repo, owner };
|
||||
}
|
||||
|
||||
async label(issueNumber, label, actor = null, verbose = false, dryRun = false) {
|
||||
let hasPerm = true;
|
||||
if (actor && Github.isLabelProtected(label)) {
|
||||
hasPerm = await this.checkIfUserInTeam(actor, COMMITTER_TEAM, verbose);
|
||||
}
|
||||
if (hasPerm) {
|
||||
const addLabelWrapped = this.context.commandWrapper({
|
||||
func: this.octokit.rest.issues.addLabels,
|
||||
successMsg: `label "${label}" added to issue ${issueNumber}`,
|
||||
verbose,
|
||||
dryRun,
|
||||
});
|
||||
await addLabelWrapped({
|
||||
...this.unPackRepo(),
|
||||
issue_number: issueNumber,
|
||||
labels: [label],
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async createComment(body) {
|
||||
if (this.issueNumber) {
|
||||
await this.octokit.rest.issues.createComment({
|
||||
...this.unPackRepo(),
|
||||
body,
|
||||
issue_number: this.issueNumber,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async unlabel(issueNumber, label, actor = null, verbose = false, dryRun = false) {
|
||||
let hasPerm = true;
|
||||
if (actor && Github.isLabelProtected(label)) {
|
||||
hasPerm = await this.checkIfUserInTeam(actor, COMMITTER_TEAM, verbose);
|
||||
}
|
||||
if (hasPerm) {
|
||||
const removeLabelWrapped = this.context.commandWrapper({
|
||||
func: this.octokit.rest.issues.removeLabel,
|
||||
successMsg: `label "${label}" removed from issue ${issueNumber}`,
|
||||
verbose,
|
||||
dryRun,
|
||||
});
|
||||
await removeLabelWrapped({
|
||||
...this.unPackRepo(),
|
||||
issue_number: issueNumber,
|
||||
name: label,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async assignOrgLabel(issueNumber, verbose = false, dryRun = false) {
|
||||
const issue = await this.octokit.rest.issues.get({
|
||||
...this.unPackRepo(),
|
||||
issue_number: issueNumber,
|
||||
});
|
||||
const username = issue.data.user.login;
|
||||
const orgs = await this.octokit.orgs.listForUser({ username });
|
||||
const orgNames = orgs.data.map((v) => v.login);
|
||||
|
||||
// get list of matching github orgs
|
||||
const matchingOrgs = orgNames.filter((org) => ORG_LIST.includes(org));
|
||||
if (matchingOrgs.length) {
|
||||
const wrapped = this.context.commandWrapper({
|
||||
func: this.octokit.rest.issues.addLabels,
|
||||
successMsg: `added label(s) ${matchingOrgs} to issue ${issueNumber}`,
|
||||
errorMsg: "couldn't add labels to issue",
|
||||
verbose,
|
||||
dryRun,
|
||||
});
|
||||
wrapped({
|
||||
...this.unPackRepo(),
|
||||
issue_number: issueNumber,
|
||||
labels: matchingOrgs,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async searchMergedPRs({
|
||||
query = '',
|
||||
onlyUnlabeled = true,
|
||||
verbose = false,
|
||||
startPage = 0,
|
||||
pages = 5,
|
||||
}) {
|
||||
// look for PRs
|
||||
let q = `repo:${this.context.repo} is:merged ${query}`;
|
||||
if (onlyUnlabeled) {
|
||||
q = `${q} -label:"🏷️ bot"`;
|
||||
}
|
||||
if (verbose) {
|
||||
this.context.log(`Query: ${q}`);
|
||||
}
|
||||
let prs = [];
|
||||
for (let i = 0; i < pages; i += 1) {
|
||||
if (verbose) {
|
||||
this.context.log(`Fetching PRs to process page ${i + 1} out of ${pages}`);
|
||||
}
|
||||
// eslint-disable-next-line no-await-in-loop
|
||||
const data = await this.octokit.search.issuesAndPullRequests({
|
||||
q,
|
||||
per_page: 100,
|
||||
page: startPage + i,
|
||||
});
|
||||
prs = [...prs, ...data.data.items];
|
||||
}
|
||||
if (verbose) {
|
||||
this.context.log(`Fetched ${prs.length}`);
|
||||
}
|
||||
return prs;
|
||||
}
|
||||
|
||||
async syncLabels({
|
||||
labels,
|
||||
prId,
|
||||
actor = null,
|
||||
verbose = false,
|
||||
dryRun = false,
|
||||
existingLabels = null,
|
||||
}) {
|
||||
if (verbose) {
|
||||
this.context.log(`[PR: ${prId}] - sync labels ${labels}`);
|
||||
}
|
||||
let hasPerm = true;
|
||||
if (actor) {
|
||||
hasPerm = await this.checkIfUserInTeam(actor, COMMITTER_TEAM, verbose);
|
||||
}
|
||||
if (!hasPerm) {
|
||||
return;
|
||||
}
|
||||
let targetLabels = existingLabels;
|
||||
if (targetLabels === null) {
|
||||
// No labels have been passed as an array, so checking against GitHub
|
||||
const resp = await this.octokit.issues.listLabelsOnIssue({
|
||||
...this.unPackRepo(),
|
||||
issue_number: prId,
|
||||
});
|
||||
targetLabels = resp.data.map((l) => l.name);
|
||||
}
|
||||
|
||||
if (verbose) {
|
||||
this.context.log(`[PR: ${prId}] - target release labels: ${labels}`);
|
||||
this.context.log(`[PR: ${prId}] - existing labels on issue: ${existingLabels}`);
|
||||
}
|
||||
|
||||
// Extract existing labels with the given prefixes
|
||||
const prefixes = ['🚢', '🍒', '🎯', '🏷️'];
|
||||
const existingPrefixLabels = targetLabels
|
||||
.filter((label) => prefixes.some((s) => typeof(label) === 'string' && label.startsWith(s)));
|
||||
|
||||
// Labels to add
|
||||
const labelsToAdd = labels.filter((label) => !existingPrefixLabels.includes(label));
|
||||
if (verbose) {
|
||||
this.context.log(`[PR: ${prId}] - labels to add: ${labelsToAdd}`);
|
||||
}
|
||||
// Labels to remove
|
||||
const labelsToRemove = existingPrefixLabels.filter((label) => !labels.includes(label));
|
||||
if (verbose) {
|
||||
this.context.log(`[PR: ${prId}] - labels to remove: ${labelsToRemove}`);
|
||||
}
|
||||
|
||||
// Add labels
|
||||
if (labelsToAdd.length > 0 && !dryRun) {
|
||||
await this.octokit.issues.addLabels({
|
||||
...this.unPackRepo(),
|
||||
issue_number: prId,
|
||||
labels: labelsToAdd,
|
||||
});
|
||||
}
|
||||
|
||||
// Remove labels
|
||||
if (labelsToRemove.length > 0 && !dryRun) {
|
||||
await Promise.all(labelsToRemove.map((label) => this.octokit.issues.removeLabel({
|
||||
...this.unPackRepo(),
|
||||
issue_number: prId,
|
||||
name: label,
|
||||
})));
|
||||
}
|
||||
this.context.logSuccess(`synched labels for PR ${prId} with labels ${labels}`);
|
||||
}
|
||||
|
||||
async checkIfUserInTeam(username, team, verbose = false) {
|
||||
let isInTeam = this.#userInTeamCache.get([username, team]);
|
||||
if (isInTeam !== undefined) {
|
||||
return isInTeam;
|
||||
}
|
||||
|
||||
const [org, teamSlug] = team.split('/');
|
||||
const wrapped = this.context.commandWrapper({
|
||||
func: this.octokit.teams.getMembershipForUserInOrg,
|
||||
errorMsg: `User "${username}" is not authorized to alter protected labels.`,
|
||||
verbose,
|
||||
});
|
||||
const resp = await wrapped({
|
||||
org,
|
||||
team_slug: teamSlug,
|
||||
username,
|
||||
});
|
||||
isInTeam = resp?.data?.state === 'active';
|
||||
this.#userInTeamCache.set([username, team], isInTeam);
|
||||
return isInTeam;
|
||||
}
|
||||
|
||||
static isLabelProtected(label) {
|
||||
return PROTECTED_LABEL_PATTERNS.some((pattern) => new RegExp(pattern).test(label));
|
||||
}
|
||||
}
|
||||
|
||||
export default Github;
|
||||
39
.github/supersetbot/src/index.js
vendored
Normal file
39
.github/supersetbot/src/index.js
vendored
Normal file
@@ -0,0 +1,39 @@
|
||||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
import getCLI from './cli.js';
|
||||
import Context from './context.js';
|
||||
import Github from './github.js';
|
||||
|
||||
async function runCommandFromGithubAction(rawCommand) {
|
||||
const context = new Context('GHA');
|
||||
const cli = getCLI(context);
|
||||
const github = new Github(context);
|
||||
|
||||
// Make rawCommand look like argv
|
||||
const cmd = rawCommand.trim().replace('@supersetbot', 'supersetbot');
|
||||
const args = context.parseArgs(cmd);
|
||||
|
||||
await cli.parseAsync(['node', ...args]);
|
||||
const msg = await context.onDone();
|
||||
|
||||
github.createComment(msg);
|
||||
}
|
||||
|
||||
export { runCommandFromGithubAction };
|
||||
51
.github/supersetbot/src/index.test.js
vendored
Normal file
51
.github/supersetbot/src/index.test.js
vendored
Normal file
@@ -0,0 +1,51 @@
|
||||
// import * as stringArgv from 'string-argv';
|
||||
import { jest } from '@jest/globals';
|
||||
|
||||
import Context from './context.js';
|
||||
import Github from './github.js';
|
||||
import * as index from './index.js';
|
||||
|
||||
describe('runCommandFromGithubAction', () => {
|
||||
const labelSpy = jest.spyOn(Github.prototype, 'label').mockImplementation(jest.fn());
|
||||
// mocking some of the Context object
|
||||
const onDoneSpy = jest.spyOn(Context.prototype, 'onDone');
|
||||
const doneCommentSpy = jest.spyOn(Context.prototype, 'doneComment');
|
||||
const parseArgsSpy = jest.spyOn(Context.prototype, 'parseArgs');
|
||||
jest.spyOn(Github.prototype, 'createComment').mockImplementation(jest.fn());
|
||||
|
||||
let originalEnv;
|
||||
|
||||
afterEach(() => {
|
||||
process.env = originalEnv;
|
||||
});
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
originalEnv = process.env;
|
||||
process.env.GITHUB_ISSUE_NUMBER = '666';
|
||||
process.env.GITHUB_REPOSITORY = 'apache/superset';
|
||||
});
|
||||
|
||||
it('should strip the command', async () => {
|
||||
await index.runCommandFromGithubAction(' @supersetbot label test-label ');
|
||||
expect(parseArgsSpy).toHaveBeenCalledWith('supersetbot label test-label');
|
||||
|
||||
await index.runCommandFromGithubAction(' \n @supersetbot label test-label \n \n \n');
|
||||
expect(parseArgsSpy).toHaveBeenCalledWith('supersetbot label test-label');
|
||||
|
||||
await index.runCommandFromGithubAction(' \n \t@supersetbot label test-label \t \n \n\t \n');
|
||||
expect(parseArgsSpy).toHaveBeenCalledWith('supersetbot label test-label');
|
||||
});
|
||||
|
||||
it('should parse the raw command correctly and call commands.label and context.onDone', async () => {
|
||||
await index.runCommandFromGithubAction('@supersetbot label test-label');
|
||||
|
||||
expect(labelSpy).toHaveBeenCalled();
|
||||
expect(onDoneSpy).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should generate a good comment message', async () => {
|
||||
await index.runCommandFromGithubAction('@supersetbot label test-label');
|
||||
const comment = doneCommentSpy.mock.results[0].value;
|
||||
expect(comment).toContain('> `supersetbot label test-label`');
|
||||
});
|
||||
});
|
||||
35
.github/supersetbot/src/metadata.js
vendored
Normal file
35
.github/supersetbot/src/metadata.js
vendored
Normal file
@@ -0,0 +1,35 @@
|
||||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
export const ORG_LIST = [
|
||||
'preset-io',
|
||||
'airbnb',
|
||||
'dropbox',
|
||||
'lyft',
|
||||
'Turing',
|
||||
'Superset-Community-Partners',
|
||||
'CybercentreCanada',
|
||||
];
|
||||
export const PROTECTED_LABEL_PATTERNS = [
|
||||
'protected.*',
|
||||
'released.*',
|
||||
'hold.*',
|
||||
'^v\\d+(\\.\\d+)*$',
|
||||
'(🚢|🍒|🎯).*',
|
||||
];
|
||||
export const COMMITTER_TEAM = 'apache/superset-committers';
|
||||
27
.github/supersetbot/src/supersetbot
vendored
Executable file
27
.github/supersetbot/src/supersetbot
vendored
Executable file
@@ -0,0 +1,27 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
import getCLI from './cli.js';
|
||||
import Context from './context.js';
|
||||
|
||||
const envContext = new Context('CLI');
|
||||
const cli = getCLI(envContext);
|
||||
|
||||
cli.parse();
|
||||
78
.github/supersetbot/src/utils.js
vendored
Normal file
78
.github/supersetbot/src/utils.js
vendored
Normal file
@@ -0,0 +1,78 @@
|
||||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
import { spawn } from 'child_process';
|
||||
import { readFile } from 'fs/promises';
|
||||
import { fileURLToPath } from 'url';
|
||||
import path from 'path';
|
||||
|
||||
const dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
async function loadPackageJson() {
|
||||
try {
|
||||
const packageJsonPath = path.join(dirname, '../package.json');
|
||||
const data = await readFile(packageJsonPath, 'utf8');
|
||||
const packageJson = JSON.parse(data);
|
||||
return packageJson;
|
||||
} catch (error) {
|
||||
console.error('Error reading package.json:', error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
export async function currentPackageVersion() {
|
||||
const data = await loadPackageJson();
|
||||
return data.version;
|
||||
}
|
||||
|
||||
export function runShellCommand(command, raiseOnError = true) {
|
||||
return new Promise((resolve, reject) => {
|
||||
// Split the command string into an array of arguments
|
||||
const args = command.split(/\s+/).filter((s) => !!s && s !== '\\');
|
||||
const childProcess = spawn(args.shift(), args);
|
||||
let stdoutData = '';
|
||||
let stderrData = '';
|
||||
|
||||
// Capture stdout data
|
||||
childProcess.stdout.on('data', (data) => {
|
||||
stdoutData += data;
|
||||
console.log(`stdout: ${data}`);
|
||||
});
|
||||
|
||||
// Capture stderr data
|
||||
childProcess.stderr.on('data', (data) => {
|
||||
stderrData += data;
|
||||
console.error(`stderr: ${data}`);
|
||||
});
|
||||
|
||||
// Handle process exit
|
||||
childProcess.on('close', (code) => {
|
||||
if (code === 0) {
|
||||
resolve(stdoutData);
|
||||
} else {
|
||||
const msg = `Command failed with code ${code}: ${stderrData}`;
|
||||
if (raiseOnError) {
|
||||
reject(new Error(msg));
|
||||
} else {
|
||||
console.error(msg);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
335
.github/workflows/bashlib.sh
vendored
335
.github/workflows/bashlib.sh
vendored
@@ -20,6 +20,10 @@ set -e
|
||||
GITHUB_WORKSPACE=${GITHUB_WORKSPACE:-.}
|
||||
ASSETS_MANIFEST="$GITHUB_WORKSPACE/superset/static/assets/manifest.json"
|
||||
|
||||
# Rounded job start time, used to create a unique Cypress build id for
|
||||
# parallelization so we can manually rerun a job after 20 minutes
|
||||
NONCE=$(echo "$(date "+%Y%m%d%H%M") - ($(date +%M)%20)" | bc)
|
||||
|
||||
# Echo only when not in parallel mode
|
||||
say() {
|
||||
if [[ $(echo "$INPUT_PARALLEL" | tr '[:lower:]' '[:upper:]') != 'TRUE' ]]; then
|
||||
@@ -55,15 +59,6 @@ build-assets() {
|
||||
say "::endgroup::"
|
||||
}
|
||||
|
||||
build-embedded-sdk() {
|
||||
cd "$GITHUB_WORKSPACE/superset-embedded-sdk"
|
||||
|
||||
say "::group::Build embedded SDK bundle for E2E tests"
|
||||
npm ci
|
||||
npm run build
|
||||
say "::endgroup::"
|
||||
}
|
||||
|
||||
build-instrumented-assets() {
|
||||
cd "$GITHUB_WORKSPACE/superset-frontend"
|
||||
|
||||
@@ -94,8 +89,6 @@ EOF
|
||||
setup-mysql() {
|
||||
say "::group::Initialize database"
|
||||
mysql -h 127.0.0.1 -P 13306 -u root --password=root <<-EOF
|
||||
SET GLOBAL transaction_isolation='READ-COMMITTED';
|
||||
SET GLOBAL TRANSACTION ISOLATION LEVEL READ COMMITTED;
|
||||
DROP DATABASE IF EXISTS superset;
|
||||
CREATE DATABASE superset DEFAULT CHARACTER SET utf8 COLLATE utf8_unicode_ci;
|
||||
DROP DATABASE IF EXISTS sqllab_test_db;
|
||||
@@ -114,7 +107,7 @@ testdata() {
|
||||
say "::group::Load test data"
|
||||
# must specify PYTHONPATH to make `tests.superset_test_config` importable
|
||||
export PYTHONPATH="$GITHUB_WORKSPACE"
|
||||
uv pip install --system -e .
|
||||
pip install -e .
|
||||
superset db upgrade
|
||||
superset load_test_users
|
||||
superset load_examples --load-test-data
|
||||
@@ -122,44 +115,9 @@ testdata() {
|
||||
say "::endgroup::"
|
||||
}
|
||||
|
||||
playwright_testdata() {
|
||||
cd "$GITHUB_WORKSPACE"
|
||||
say "::group::Load all examples for Playwright tests"
|
||||
# must specify PYTHONPATH to make `tests.superset_test_config` importable
|
||||
export PYTHONPATH="$GITHUB_WORKSPACE"
|
||||
uv pip install --system -e .
|
||||
superset db upgrade
|
||||
superset load_test_users
|
||||
superset load_examples
|
||||
superset init
|
||||
# Enable DML on the examples database so Playwright tests can create/drop
|
||||
# temporary tables via SQL Lab without depending on external data sources.
|
||||
superset shell <<'PYEOF'
|
||||
import sys
|
||||
from superset.extensions import db
|
||||
from superset.models.core import Database
|
||||
examples_db = db.session.query(Database).filter_by(database_name='examples').first()
|
||||
if not examples_db:
|
||||
sys.exit('ERROR: examples database not found. load_examples may have failed.')
|
||||
|
||||
examples_db.allow_dml = True
|
||||
db.session.commit()
|
||||
print('Enabled allow_dml on examples database')
|
||||
PYEOF
|
||||
say "::endgroup::"
|
||||
}
|
||||
|
||||
celery-worker() {
|
||||
cd "$GITHUB_WORKSPACE"
|
||||
say "::group::Start Celery worker"
|
||||
# must specify PYTHONPATH to make `tests.superset_test_config` importable
|
||||
export PYTHONPATH="$GITHUB_WORKSPACE"
|
||||
celery \
|
||||
--app=superset.tasks.celery_app:app \
|
||||
worker \
|
||||
--concurrency=2 \
|
||||
--detach \
|
||||
--optimization=fair
|
||||
codecov() {
|
||||
say "::group::Upload code coverage"
|
||||
bash ".github/workflows/codecov.sh" "$@"
|
||||
say "::endgroup::"
|
||||
}
|
||||
|
||||
@@ -175,196 +133,73 @@ cypress-install() {
|
||||
cache-save cypress
|
||||
}
|
||||
|
||||
cypress-run-all() {
|
||||
local USE_DASHBOARD=$1
|
||||
local APP_ROOT=$2
|
||||
# Run Cypress and upload coverage reports
|
||||
cypress-run() {
|
||||
cd "$GITHUB_WORKSPACE/superset-frontend/cypress-base"
|
||||
|
||||
# Start the Superset backend via gunicorn (not `flask run`). The Flask
|
||||
# development server is single-threaded and has no crash-recovery, so
|
||||
# heavy tests (dashboard import/export, SQL Lab) can knock it offline
|
||||
# for the rest of the run — surfacing as `ECONNREFUSED` / `socket hang up`
|
||||
# / `Missing CSRF token` cascades. Gunicorn gives us multiple workers,
|
||||
# a request timeout, and worker-recycling under load.
|
||||
local serverlog="${HOME}/superset-cypress.log"
|
||||
local port=8081
|
||||
CYPRESS_BASE_URL="http://localhost:${port}"
|
||||
if [ -n "$APP_ROOT" ]; then
|
||||
export SUPERSET_APP_ROOT=$APP_ROOT
|
||||
CYPRESS_BASE_URL=${CYPRESS_BASE_URL}${APP_ROOT}
|
||||
fi
|
||||
export CYPRESS_BASE_URL
|
||||
local page=$1
|
||||
local group=${2:-Default}
|
||||
local cypress="./node_modules/.bin/cypress run"
|
||||
local browser=${CYPRESS_BROWSER:-chrome}
|
||||
|
||||
# Mirrors the args in docker/entrypoints/run-server.sh (1 worker × 20
|
||||
# gthread threads) to keep parity with production. Multi-worker
|
||||
# configurations expose timing-sensitive races in the SQL Lab → Explore
|
||||
# navigation flow under E2E. We diverge from the entrypoint on:
|
||||
# --timeout 120: heavy dashboard import/export specs exceed the 60s
|
||||
# default
|
||||
# --max-requests / --max-requests-jitter: recycle the worker under
|
||||
# test load to avoid leaks accumulating across the run
|
||||
# superset.app:create_app(): explicit factory so we don't depend on
|
||||
# FLASK_APP being exported
|
||||
nohup gunicorn \
|
||||
--bind "127.0.0.1:$port" \
|
||||
--workers 1 \
|
||||
--worker-class gthread \
|
||||
--threads 20 \
|
||||
--timeout 120 \
|
||||
--max-requests 500 \
|
||||
--max-requests-jitter 50 \
|
||||
--access-logfile - \
|
||||
--error-logfile - \
|
||||
"superset.app:create_app()" \
|
||||
>"$serverlog" 2>&1 </dev/null &
|
||||
local serverPid=$!
|
||||
export TERM="xterm"
|
||||
export ELECTRON_DISABLE_GPU=true # Attempt to disable GPU for Electron-based Cypress
|
||||
|
||||
# Ensure the backend is cleaned up and its log is emitted even when the
|
||||
# test runner fails under `set -e`.
|
||||
trap '
|
||||
echo "::group::gunicorn log for Cypress run"
|
||||
cat "'"$serverlog"'" || true
|
||||
echo "::endgroup::"
|
||||
kill '"$serverPid"' 2>/dev/null || true
|
||||
' EXIT
|
||||
|
||||
# Wait for the backend to be ready before launching Cypress; otherwise
|
||||
# the first spec can race the server bind and see connection errors.
|
||||
local timeout=60
|
||||
say "Waiting for gunicorn server to start on port $port..."
|
||||
while [ $timeout -gt 0 ]; do
|
||||
if curl -f "http://localhost:${port}${APP_ROOT}/health" >/dev/null 2>&1; then
|
||||
say "gunicorn server is ready"
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
timeout=$((timeout - 1))
|
||||
done
|
||||
if [ $timeout -eq 0 ]; then
|
||||
echo "::error::gunicorn server failed to start within 60 seconds"
|
||||
echo "::group::Server startup log"
|
||||
cat "$serverlog"
|
||||
echo "::endgroup::"
|
||||
return 1
|
||||
fi
|
||||
|
||||
USE_DASHBOARD_FLAG=''
|
||||
if [ "$USE_DASHBOARD" = "true" ]; then
|
||||
USE_DASHBOARD_FLAG='--use-dashboard'
|
||||
fi
|
||||
|
||||
# UNCOMMENT the next few commands to monitor memory usage
|
||||
# monitor_memory & # Start memory monitoring in the background
|
||||
# memoryMonitorPid=$!
|
||||
python ../../scripts/cypress_run.py --parallelism $PARALLELISM --parallelism-id $PARALLEL_ID --group $PARALLEL_ID --retries 5 $USE_DASHBOARD_FLAG
|
||||
# kill $memoryMonitorPid
|
||||
}
|
||||
|
||||
playwright-install() {
|
||||
cd "$GITHUB_WORKSPACE/superset-frontend"
|
||||
|
||||
say "::group::Install Playwright browsers"
|
||||
npx playwright install --with-deps chromium
|
||||
# Create output directories for test results and debugging
|
||||
mkdir -p playwright-results
|
||||
mkdir -p test-results
|
||||
say "::endgroup::"
|
||||
}
|
||||
|
||||
playwright-run() {
|
||||
local APP_ROOT=$1
|
||||
local TEST_PATH=$2
|
||||
|
||||
# Start the Superset backend via gunicorn from the project root.
|
||||
# See cypress-run-all() above for the rationale — the Flask dev server
|
||||
# cannot survive the dashboard import/export tests under load.
|
||||
cd "$GITHUB_WORKSPACE"
|
||||
local serverlog="${HOME}/superset-playwright.log"
|
||||
local port=8081
|
||||
# Use 127.0.0.1 explicitly: `flask run` binds IPv4 only, and Node's DNS
|
||||
# resolution for `localhost` can return `::1` first (IPv6), which then
|
||||
# refuses against the IPv4 listener and surfaces as
|
||||
# `connect ECONNREFUSED ::1:<port>` in API helpers driven from Node
|
||||
# (e.g., the embedded test app's exposed token fetcher).
|
||||
PLAYWRIGHT_BASE_URL="http://127.0.0.1:${port}"
|
||||
if [ -n "$APP_ROOT" ]; then
|
||||
export SUPERSET_APP_ROOT=$APP_ROOT
|
||||
PLAYWRIGHT_BASE_URL=${PLAYWRIGHT_BASE_URL}${APP_ROOT}/
|
||||
fi
|
||||
export PLAYWRIGHT_BASE_URL
|
||||
|
||||
# See cypress-run-all() above for the args rationale (1 worker × 20
|
||||
# gthread threads matching docker/entrypoints/run-server.sh, plus a
|
||||
# 120s timeout and request-recycling for heavy E2E load).
|
||||
nohup gunicorn \
|
||||
--bind "127.0.0.1:$port" \
|
||||
--workers 1 \
|
||||
--worker-class gthread \
|
||||
--threads 20 \
|
||||
--timeout 120 \
|
||||
--max-requests 500 \
|
||||
--max-requests-jitter 50 \
|
||||
--access-logfile - \
|
||||
--error-logfile - \
|
||||
"superset.app:create_app()" \
|
||||
>"$serverlog" 2>&1 </dev/null &
|
||||
local serverPid=$!
|
||||
|
||||
# Ensure cleanup on exit (and emit the server log on failure)
|
||||
trap '
|
||||
echo "::group::gunicorn log for Playwright run"
|
||||
cat "'"$serverlog"'" || true
|
||||
echo "::endgroup::"
|
||||
kill '"$serverPid"' 2>/dev/null || true
|
||||
' EXIT
|
||||
|
||||
# Wait for server to be ready with health check
|
||||
local timeout=60
|
||||
say "Waiting for gunicorn server to start on port $port..."
|
||||
while [ $timeout -gt 0 ]; do
|
||||
if curl -f ${PLAYWRIGHT_BASE_URL}/health >/dev/null 2>&1; then
|
||||
say "gunicorn server is ready"
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
timeout=$((timeout - 1))
|
||||
done
|
||||
|
||||
if [ $timeout -eq 0 ]; then
|
||||
echo "::error::gunicorn server failed to start within 60 seconds"
|
||||
echo "::group::Server startup log"
|
||||
cat "$serverlog"
|
||||
echo "::endgroup::"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Change to frontend directory for Playwright execution
|
||||
cd "$GITHUB_WORKSPACE/superset-frontend"
|
||||
|
||||
say "::group::Run Playwright tests"
|
||||
echo "Running Playwright with baseURL: ${PLAYWRIGHT_BASE_URL}"
|
||||
if [ -n "$TEST_PATH" ]; then
|
||||
# Check if there are any test files in the specified path
|
||||
if ! find "playwright/tests/${TEST_PATH}" -name "*.spec.ts" -type f 2>/dev/null | grep -q .; then
|
||||
echo "No test files found in ${TEST_PATH} - skipping test run"
|
||||
say "::endgroup::"
|
||||
return 0
|
||||
fi
|
||||
echo "Running tests: ${TEST_PATH}"
|
||||
# Set INCLUDE_EXPERIMENTAL=true to allow experimental tests to run
|
||||
export INCLUDE_EXPERIMENTAL=true
|
||||
npx playwright test "${TEST_PATH}" --output=playwright-results
|
||||
local status=$?
|
||||
# Unset to prevent leaking into subsequent commands
|
||||
unset INCLUDE_EXPERIMENTAL
|
||||
say "::group::Run Cypress for [$page]"
|
||||
if [[ -z $CYPRESS_KEY ]]; then
|
||||
xvfb-run --auto-servernum --server-args='-screen 0, 1024x768x24' $cypress --spec "cypress/e2e/$page" --browser "$browser"
|
||||
else
|
||||
echo "Running all required tests (experimental/ excluded via playwright.config.ts)"
|
||||
npx playwright test --output=playwright-results
|
||||
local status=$?
|
||||
export CYPRESS_RECORD_KEY=$(echo $CYPRESS_KEY | base64 --decode)
|
||||
# additional flags for Cypress dashboard recording
|
||||
xvfb-run --auto-servernum --server-args='-screen 0, 1024x768x24' $cypress --spec "cypress/e2e/$page" --browser "$browser" \
|
||||
--record --group "$group" --tag "${GITHUB_REPOSITORY},${GITHUB_EVENT_NAME}" \
|
||||
--parallel --ci-build-id "${GITHUB_SHA:0:8}-${NONCE}"
|
||||
|
||||
fi
|
||||
|
||||
# don't add quotes to $record because we do want word splitting
|
||||
say "::endgroup::"
|
||||
}
|
||||
|
||||
cypress-run-all() {
|
||||
# Start Flask and run it in background
|
||||
# --no-debugger means disable the interactive debugger on the 500 page
|
||||
# so errors can print to stderr.
|
||||
local flasklog="${HOME}/flask.log"
|
||||
local port=8081
|
||||
export CYPRESS_BASE_URL="http://localhost:${port}"
|
||||
|
||||
nohup flask run --no-debugger -p $port >"$flasklog" 2>&1 </dev/null &
|
||||
local flaskProcessId=$!
|
||||
|
||||
cypress-run "*/**/*"
|
||||
|
||||
# After job is done, print out Flask log for debugging
|
||||
say "::group::Flask log for default run"
|
||||
cat "$flasklog"
|
||||
say "::endgroup::"
|
||||
|
||||
return $status
|
||||
# Rerun SQL Lab tests with backend persist disabled
|
||||
export SUPERSET_CONFIG=tests.integration_tests.superset_test_config_sqllab_backend_persist_off
|
||||
|
||||
# Restart Flask with new configs
|
||||
kill $flaskProcessId
|
||||
nohup flask run --no-debugger -p $port >"$flasklog" 2>&1 </dev/null &
|
||||
local flaskProcessId=$!
|
||||
|
||||
cypress-run "sqllab/*" "Backend persist"
|
||||
|
||||
# Upload code coverage separately so each page can have separate flags
|
||||
# -c will clean existing coverage reports, -F means add flags
|
||||
# || true to prevent CI failure on codecov upload
|
||||
codecov -c -F "cypress" || true
|
||||
|
||||
say "::group::Flask log for backend persist"
|
||||
cat "$flasklog"
|
||||
say "::endgroup::"
|
||||
|
||||
# make sure the program exits
|
||||
kill $flaskProcessId
|
||||
}
|
||||
|
||||
eyes-storybook-dependencies() {
|
||||
@@ -373,17 +208,27 @@ eyes-storybook-dependencies() {
|
||||
say "::endgroup::"
|
||||
}
|
||||
|
||||
monitor_memory() {
|
||||
# This is a small utility to monitor memory usage. Useful for debugging memory in GHA.
|
||||
# To use wrap your command as follows
|
||||
#
|
||||
# monitor_memory & # Start memory monitoring in the background
|
||||
# memoryMonitorPid=$!
|
||||
# YOUR_COMMAND_HERE
|
||||
# kill $memoryMonitorPid
|
||||
while true; do
|
||||
echo "$(date) - Top 5 memory-consuming processes:"
|
||||
ps -eo pid,comm,%mem --sort=-%mem | head -n 6 # First line is the header, next 5 are top processes
|
||||
sleep 2
|
||||
done
|
||||
cypress-run-applitools() {
|
||||
cd "$GITHUB_WORKSPACE/superset-frontend/cypress-base"
|
||||
|
||||
local flasklog="${HOME}/flask.log"
|
||||
local port=8081
|
||||
local cypress="./node_modules/.bin/cypress run"
|
||||
local browser=${CYPRESS_BROWSER:-chrome}
|
||||
|
||||
export CYPRESS_BASE_URL="http://localhost:${port}"
|
||||
|
||||
nohup flask run --no-debugger -p $port >"$flasklog" 2>&1 </dev/null &
|
||||
local flaskProcessId=$!
|
||||
|
||||
$cypress --spec "cypress/e2e/*/**/*.applitools.test.ts" --browser "$browser" --headless --config ignoreTestFiles="[]"
|
||||
|
||||
codecov -c -F "cypress" || true
|
||||
|
||||
say "::group::Flask log for default run"
|
||||
cat "$flasklog"
|
||||
say "::endgroup::"
|
||||
|
||||
# make sure the program exits
|
||||
kill $flaskProcessId
|
||||
}
|
||||
|
||||
80
.github/workflows/bump-python-package.yml
vendored
80
.github/workflows/bump-python-package.yml
vendored
@@ -1,80 +0,0 @@
|
||||
name: Bump Python Package
|
||||
|
||||
on:
|
||||
# Can be triggered manually
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
package:
|
||||
required: false
|
||||
description: The python package to bump (all if empty)
|
||||
group:
|
||||
required: false
|
||||
description: The optional dependency group to bump (as defined in pyproject.toml)
|
||||
limit:
|
||||
required: true
|
||||
description: Max number of PRs to open (0 for no limit)
|
||||
default: 5
|
||||
extra-flags:
|
||||
required: false
|
||||
default: --only-base
|
||||
description: Additional flags to pass to the bump-python command
|
||||
#schedule:
|
||||
# - cron: '0 0 * * *' # Runs daily at midnight UTC
|
||||
|
||||
jobs:
|
||||
bump-python-package:
|
||||
runs-on: ubuntu-24.04
|
||||
permissions:
|
||||
actions: write
|
||||
contents: write
|
||||
pull-requests: write
|
||||
checks: write
|
||||
steps:
|
||||
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
persist-credentials: true
|
||||
ref: master
|
||||
|
||||
- name: Setup supersetbot
|
||||
uses: ./.github/actions/setup-supersetbot/
|
||||
|
||||
- name: Set up Python ${{ inputs.python-version }}
|
||||
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
|
||||
with:
|
||||
python-version: "3.11"
|
||||
|
||||
- name: Install uv
|
||||
run: pip install uv
|
||||
|
||||
- name: supersetbot bump-python -p "${{ github.event.inputs.package }}"
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
INPUT_PACKAGE: ${{ github.event.inputs.package }}
|
||||
INPUT_GROUP: ${{ github.event.inputs.group }}
|
||||
INPUT_EXTRA_FLAGS: ${{ github.event.inputs.extra-flags }}
|
||||
INPUT_LIMIT: ${{ github.event.inputs.limit }}
|
||||
run: |
|
||||
git config --global user.email "action@github.com"
|
||||
git config --global user.name "GitHub Action"
|
||||
|
||||
PACKAGE_OPT=""
|
||||
if [ -n "${INPUT_PACKAGE}" ]; then
|
||||
PACKAGE_OPT="-p ${INPUT_PACKAGE}"
|
||||
fi
|
||||
|
||||
GROUP_OPT=""
|
||||
if [ -n "${INPUT_GROUP}" ]; then
|
||||
GROUP_OPT="-g ${INPUT_GROUP}"
|
||||
fi
|
||||
|
||||
EXTRA_FLAGS="${INPUT_EXTRA_FLAGS}"
|
||||
|
||||
supersetbot bump-python \
|
||||
--verbose \
|
||||
--use-current-repo \
|
||||
--include-subpackages \
|
||||
--limit ${INPUT_LIMIT} \
|
||||
$PACKAGE_OPT \
|
||||
$GROUP_OPT \
|
||||
$EXTRA_FLAGS
|
||||
43
.github/workflows/cancel_duplicates.yml
vendored
Normal file
43
.github/workflows/cancel_duplicates.yml
vendored
Normal file
@@ -0,0 +1,43 @@
|
||||
name: Cancel Duplicates
|
||||
on:
|
||||
workflow_run:
|
||||
workflows:
|
||||
- "Miscellaneous"
|
||||
types:
|
||||
- requested
|
||||
|
||||
jobs:
|
||||
cancel-duplicate-runs:
|
||||
name: Cancel duplicate workflow runs
|
||||
runs-on: ubuntu-20.04
|
||||
permissions:
|
||||
actions: write
|
||||
contents: read
|
||||
steps:
|
||||
- name: Check number of queued tasks
|
||||
id: check_queued
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
GITHUB_REPO: ${{ github.repository }}
|
||||
run: |
|
||||
get_count() {
|
||||
echo $(curl -s -H "Authorization: token $GITHUB_TOKEN" \
|
||||
"https://api.github.com/repos/$GITHUB_REPO/actions/runs?status=$1" | \
|
||||
jq ".total_count")
|
||||
}
|
||||
count=$(( `get_count queued` + `get_count in_progress` ))
|
||||
echo "Found $count unfinished jobs."
|
||||
echo "count=$count" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
||||
if: steps.check_queued.outputs.count >= 20
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Cancel duplicate workflow runs
|
||||
if: steps.check_queued.outputs.count >= 20
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
GITHUB_REPOSITORY: ${{ github.repository }}
|
||||
run: |
|
||||
pip install click requests typing_extensions python-dateutil
|
||||
python ./scripts/cancel_github_workflows.py
|
||||
76
.github/workflows/check-python-deps.yml
vendored
76
.github/workflows/check-python-deps.yml
vendored
@@ -1,76 +0,0 @@
|
||||
name: Check python dependencies
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- "master"
|
||||
- "[0-9].[0-9]*"
|
||||
pull_request:
|
||||
types: [synchronize, opened, reopened, ready_for_review]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
|
||||
# cancel previous workflow jobs for PRs
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
check-python-deps:
|
||||
runs-on: ubuntu-22.04
|
||||
steps:
|
||||
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: recursive
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Check for file changes
|
||||
id: check
|
||||
uses: ./.github/actions/change-detector/
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Setup Python
|
||||
if: steps.check.outputs.python
|
||||
uses: ./.github/actions/setup-backend/
|
||||
|
||||
# Authenticate the Docker daemon so the python:slim pull in
|
||||
# uv-pip-compile.sh uses our (much higher) authenticated rate limit
|
||||
# instead of the shared-runner anonymous one. Best-effort: on fork PRs the
|
||||
# secrets are unavailable, so this no-ops and the pull falls back to
|
||||
# anonymous (covered by the retry loop in the script).
|
||||
- name: Login to Docker Hub
|
||||
if: steps.check.outputs.python
|
||||
continue-on-error: true
|
||||
uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USER }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Run uv
|
||||
if: steps.check.outputs.python
|
||||
run: ./scripts/uv-pip-compile.sh
|
||||
|
||||
- name: Check for uncommitted changes
|
||||
if: steps.check.outputs.python
|
||||
run: |
|
||||
echo "Full diff (for logging/debugging):"
|
||||
git diff
|
||||
|
||||
echo "Filtered diff (excluding comments and whitespace):"
|
||||
filtered_diff=$(git diff -U0 | grep '^[-+]' | grep -vE '^[-+]{3}' | grep -vE '^[-+][[:space:]]*#' | grep -vE '^[-+][[:space:]]*$' || true)
|
||||
echo "$filtered_diff"
|
||||
|
||||
if [[ -n "$filtered_diff" ]]; then
|
||||
echo
|
||||
echo "ERROR: The pinned dependencies are not up-to-date."
|
||||
echo "Please run './scripts/uv-pip-compile.sh' and commit the changes."
|
||||
echo "More info: https://github.com/apache/superset/tree/master/requirements"
|
||||
exit 1
|
||||
else
|
||||
echo "Pinned dependencies are up-to-date."
|
||||
fi
|
||||
12
.github/workflows/check_db_migration_confict.yml
vendored
12
.github/workflows/check_db_migration_confict.yml
vendored
@@ -5,7 +5,7 @@ on:
|
||||
- "superset/migrations/**"
|
||||
branches:
|
||||
- "master"
|
||||
- "[0-9].[0-9]*"
|
||||
- "[0-9].[0-9]"
|
||||
pull_request:
|
||||
paths:
|
||||
- "superset/migrations/**"
|
||||
@@ -19,17 +19,15 @@ concurrency:
|
||||
jobs:
|
||||
check_db_migration_conflict:
|
||||
name: Check DB migration conflict
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-20.04
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
uses: actions/checkout@v4
|
||||
- name: Check and notify
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
uses: actions/github-script@v7
|
||||
with:
|
||||
github-token: ${{ github.token }}
|
||||
script: |
|
||||
@@ -71,7 +69,7 @@ jobs:
|
||||
`❗ @${pull.user.login} Your base branch \`${currentBranch}\` has ` +
|
||||
'also updated `superset/migrations`.\n' +
|
||||
'\n' +
|
||||
'**Please consider rebasing your branch and [resolving potential db migration conflicts](https://superset.apache.org/docs/contributing/development#merging-db-migrations).**',
|
||||
'**Please consider rebasing your branch and [resolving potential db migration conflicts](https://github.com/apache/superset/blob/master/CONTRIBUTING.md#merging-db-migrations).**',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
1903
.github/workflows/codecov.sh
vendored
Executable file
1903
.github/workflows/codecov.sh
vendored
Executable file
File diff suppressed because it is too large
Load Diff
43
.github/workflows/codeql-analysis.yml
vendored
43
.github/workflows/codeql-analysis.yml
vendored
@@ -2,10 +2,14 @@ name: "CodeQL"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: ["master", "[0-9].[0-9]*"]
|
||||
branches: ["master", "[0-9].[0-9]"]
|
||||
paths:
|
||||
- "superset/**"
|
||||
pull_request:
|
||||
# The branches below must be a subset of the branches above
|
||||
branches: ["master"]
|
||||
paths:
|
||||
- "superset/**"
|
||||
schedule:
|
||||
- cron: "0 4 * * *"
|
||||
|
||||
@@ -15,39 +19,12 @@ concurrency:
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
changes:
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
outputs:
|
||||
python: ${{ steps.check.outputs.python }}
|
||||
frontend: ${{ steps.check.outputs.frontend }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Check for file changes
|
||||
id: check
|
||||
uses: ./.github/actions/change-detector/
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
analyze:
|
||||
name: Analyze
|
||||
needs: changes
|
||||
# Skip on PRs that touch neither code group (e.g. docs-only) so the
|
||||
# analysis runners don't spin up. push/schedule runs always proceed:
|
||||
# the change-detector returns "all changed" for non-PR events.
|
||||
if: needs.changes.outputs.python == 'true' || needs.changes.outputs.frontend == 'true'
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
runs-on: ubuntu-22.04
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
pull-requests: read
|
||||
security-events: write
|
||||
|
||||
strategy:
|
||||
@@ -58,13 +35,11 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
uses: actions/checkout@v4
|
||||
|
||||
# Initializes the CodeQL tools for scanning.
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@54f647b7e1bb85c95cddabcd46b0c578ec92bc1a # v4.36.3
|
||||
uses: github/codeql-action/init@v3
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
# If you wish to specify custom queries, you can do so here or in a config file.
|
||||
@@ -75,6 +50,6 @@ jobs:
|
||||
# queries: security-extended,security-and-quality
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@54f647b7e1bb85c95cddabcd46b0c578ec92bc1a # v4.36.3
|
||||
uses: github/codeql-action/analyze@v3
|
||||
with:
|
||||
category: "/language:${{matrix.language}}"
|
||||
|
||||
56
.github/workflows/dependency-review.yml
vendored
56
.github/workflows/dependency-review.yml
vendored
@@ -5,67 +5,23 @@
|
||||
# Source repository: https://github.com/actions/dependency-review-action
|
||||
# Public documentation: https://docs.github.com/en/code-security/supply-chain-security/understanding-your-software-supply-chain/about-dependency-review#dependency-review-enforcement
|
||||
name: "Dependency Review"
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- "master"
|
||||
- "[0-9].[0-9]*"
|
||||
pull_request:
|
||||
types: [synchronize, opened, reopened, ready_for_review]
|
||||
|
||||
# cancel previous workflow jobs for PRs
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
on: [pull_request]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
dependency-review:
|
||||
if: github.event_name == 'pull_request'
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: "Checkout Repository"
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
uses: actions/checkout@v4
|
||||
- name: "Dependency Review"
|
||||
uses: actions/dependency-review-action@a1d282b36b6f3519aa1f3fc636f609c47dddb294 # v5.0.0
|
||||
continue-on-error: true
|
||||
uses: actions/dependency-review-action@v4
|
||||
with:
|
||||
fail-on-severity: critical
|
||||
# compatible/incompatible licenses addressed here: https://www.apache.org/legal/resolved.html
|
||||
# find SPDX identifiers here: https://spdx.org/licenses/
|
||||
deny-licenses: MS-LPL, BUSL-1.1, QPL-1.0, Sleepycat, SSPL-1.0, CPOL-1.02, AGPL-3.0, GPL-1.0+, BSD-4-Clause-UC, NPL-1.0, NPL-1.1, JSON
|
||||
# pkg:npm/store2@2.14.2
|
||||
# adding an exception for an ambigious license on store2, which has been resolved in
|
||||
# the latest version. It's MIT: https://github.com/nbubna/store/blob/master/LICENSE-MIT
|
||||
# pkg:npm/node-forge@1.3.1
|
||||
# selecting BSD-3-Clause licensing terms for node-forge to ensure compatibility with Apache
|
||||
allow-dependencies-licenses: pkg:npm/rgbcolor, pkg:npm/jszip@3.10.1
|
||||
|
||||
python-dependency-liccheck:
|
||||
# NOTE: Configuration for liccheck lives in our pyproject.yml.
|
||||
# You cannot use a liccheck.ini file in this workflow.
|
||||
runs-on: ubuntu-22.04
|
||||
steps:
|
||||
- name: "Checkout Repository"
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Python
|
||||
uses: ./.github/actions/setup-backend/
|
||||
with:
|
||||
requirements-type: base
|
||||
|
||||
- name: "Set up liccheck"
|
||||
run: |
|
||||
uv pip install --system liccheck
|
||||
- name: "Run liccheck"
|
||||
run: |
|
||||
# run the checks
|
||||
liccheck -R output.txt
|
||||
# Print the report
|
||||
cat output.txt
|
||||
# adding an exception for an ambigious license on store2, which has been resolved in the latest version. It's MIT: https://github.com/nbubna/store/blob/master/LICENSE-MIT
|
||||
allow-dependencies-licenses: 'pkg:npm/store2@2.14.2'
|
||||
|
||||
91
.github/workflows/docker-release.yml
vendored
Normal file
91
.github/workflows/docker-release.yml
vendored
Normal file
@@ -0,0 +1,91 @@
|
||||
name: Docker Publish Release
|
||||
|
||||
on:
|
||||
release:
|
||||
types: [published, edited]
|
||||
|
||||
# Can be triggered manually
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
release:
|
||||
required: true
|
||||
description: The version to generate
|
||||
git-ref:
|
||||
required: true
|
||||
description: The git reference to checkout prior to running the docker build
|
||||
force-latest:
|
||||
required: true
|
||||
type: choice
|
||||
default: 'false'
|
||||
description: Whether to force a latest tag on the release
|
||||
options:
|
||||
- true
|
||||
- false
|
||||
jobs:
|
||||
config:
|
||||
runs-on: "ubuntu-latest"
|
||||
outputs:
|
||||
has-secrets: ${{ steps.check.outputs.has-secrets }}
|
||||
steps:
|
||||
- name: "Check for secrets"
|
||||
id: check
|
||||
shell: bash
|
||||
run: |
|
||||
if [ -n "${{ (secrets.DOCKERHUB_USER != '' && secrets.DOCKERHUB_TOKEN != '') || '' }}" ]; then
|
||||
echo "has-secrets=1" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
docker-release:
|
||||
needs: config
|
||||
if: needs.config.outputs.has-secrets
|
||||
name: docker-release
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
build_preset: ["dev", "lean", "py310", "websocket", "dockerize"]
|
||||
fail-fast: false
|
||||
steps:
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Setup Node Env
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '20'
|
||||
|
||||
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup supersetbot
|
||||
uses: ./.github/actions/setup-supersetbot/
|
||||
|
||||
- name: Execute custom Node.js script
|
||||
env:
|
||||
DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
|
||||
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
run: |
|
||||
RELEASE="${{ github.event.release.tag_name }}"
|
||||
FORCE_LATEST=""
|
||||
EVENT="${{github.event_name}}"
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
# in the case of a manually-triggered run, read release from input
|
||||
RELEASE="${{ github.event.inputs.release }}"
|
||||
if [ "${{ github.event.inputs.force-latest }}" = "true" ]; then
|
||||
FORCE_LATEST="--force-latest"
|
||||
fi
|
||||
# build_docker.py may not exist on that SHA, let's switcharoo to /tmp
|
||||
cp ./scripts/build_docker.py /tmp
|
||||
git checkout "${{ github.event.inputs.git-ref }}"
|
||||
cp /tmp/build_docker.py scripts/
|
||||
EVENT="release"
|
||||
fi
|
||||
|
||||
supersetbot docker \
|
||||
--preset ${{ matrix.build_preset }} \
|
||||
--context "$EVENT" \
|
||||
--context-ref "$RELEASE" $FORCE_LATEST \
|
||||
--platform "linux/arm64" \
|
||||
--platform "linux/amd64"
|
||||
197
.github/workflows/docker.yml
vendored
197
.github/workflows/docker.yml
vendored
@@ -4,62 +4,31 @@ on:
|
||||
push:
|
||||
branches:
|
||||
- "master"
|
||||
- "[0-9].[0-9]*"
|
||||
- "[0-9].[0-9]"
|
||||
pull_request:
|
||||
branches:
|
||||
- "master"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
changes:
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
outputs:
|
||||
python: ${{ steps.check.outputs.python }}
|
||||
frontend: ${{ steps.check.outputs.frontend }}
|
||||
docker: ${{ steps.check.outputs.docker }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Check for file changes
|
||||
id: check
|
||||
uses: ./.github/actions/change-detector/
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
setup_matrix:
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 5
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
matrix_config: ${{ steps.set_matrix.outputs.matrix_config }}
|
||||
steps:
|
||||
- id: set_matrix
|
||||
run: |
|
||||
MATRIX_CONFIG=$(if [ "${{ github.event_name }}" == "pull_request" ]; then echo '["dev", "lean"]'; else echo '["dev", "lean", "websocket", "dockerize", "py311", "py312"]'; fi)
|
||||
MATRIX_CONFIG=$(if [ "${{ github.event_name }}" == "pull_request" ]; then echo '["ci"]'; else echo '["dev", "lean", "py310", "websocket", "dockerize"]'; fi)
|
||||
echo "matrix_config=${MATRIX_CONFIG}" >> $GITHUB_OUTPUT
|
||||
echo $GITHUB_OUTPUT
|
||||
|
||||
docker-build:
|
||||
name: docker-build
|
||||
needs: [setup_matrix, changes]
|
||||
if: >-
|
||||
needs.changes.outputs.python == 'true' ||
|
||||
needs.changes.outputs.frontend == 'true' ||
|
||||
needs.changes.outputs.docker == 'true'
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 60
|
||||
needs: setup_matrix
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
build_preset: ${{fromJson(needs.setup_matrix.outputs.matrix_config)}}
|
||||
@@ -67,149 +36,47 @@ jobs:
|
||||
env:
|
||||
DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
|
||||
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
IMAGE_TAG: apache/superset:GHA-${{ matrix.build_preset }}-${{ github.run_id }}
|
||||
|
||||
steps:
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Try to login to DockerHub
|
||||
continue-on-error: true
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USER }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Setup Node Env
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '20'
|
||||
|
||||
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Free up disk space
|
||||
shell: bash
|
||||
run: |
|
||||
# Reclaim large preinstalled toolchains we don't use. The image
|
||||
# build, and especially the docker-compose sanity check (which
|
||||
# rebuilds from scratch whenever the registry cache image
|
||||
# apache/superset-cache is unavailable), can otherwise exhaust the
|
||||
# runner's root disk and fail with "no space left on device".
|
||||
echo "Disk before cleanup:"; df -h /
|
||||
sudo rm -rf \
|
||||
/usr/share/dotnet \
|
||||
/usr/local/lib/android \
|
||||
/opt/ghc \
|
||||
/usr/local/.ghcup \
|
||||
/opt/hostedtoolcache/CodeQL \
|
||||
/usr/local/share/boost || true
|
||||
echo "Disk after cleanup:"; df -h /
|
||||
|
||||
- name: Setup Docker Environment
|
||||
uses: ./.github/actions/setup-docker
|
||||
with:
|
||||
dockerhub-user: ${{ secrets.DOCKERHUB_USER }}
|
||||
dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
build: "true"
|
||||
|
||||
- name: Setup supersetbot
|
||||
uses: ./.github/actions/setup-supersetbot/
|
||||
|
||||
- name: Build Docker Image
|
||||
shell: bash
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
BUILD_PRESET: ${{ matrix.build_preset }}
|
||||
run: |
|
||||
# Single platform builds in pull_request context to speed things up
|
||||
if [ "$GITHUB_EVENT_NAME" = "push" ]; then
|
||||
if [ "${{ github.event_name }}" = "push" ]; then
|
||||
PLATFORM_ARG="--platform linux/arm64 --platform linux/amd64"
|
||||
# can only --load images in single-platform builds
|
||||
PUSH_OR_LOAD="--push"
|
||||
elif [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then
|
||||
elif [ "${{ github.event_name }}" = "pull_request" ]; then
|
||||
PLATFORM_ARG="--platform linux/amd64"
|
||||
PUSH_OR_LOAD="--load"
|
||||
fi
|
||||
|
||||
# Retry to absorb transient Docker Hub registry errors (base-image
|
||||
# pull timeouts, 504/401 on push, ECONNRESET) that otherwise fail
|
||||
# the whole job. buildx reuses the buildkit layer cache from the
|
||||
# failed attempt, so a retry mostly re-does just the failed push.
|
||||
#
|
||||
# supersetbot's "dev"/"lean" presets pin their own --build-arg
|
||||
# PY_VER, which lands ahead of --extra-flags on the assembled
|
||||
# buildx command line; docker/buildx keeps the last value for a
|
||||
# repeated --build-arg key, so appending PY_VER here overrides
|
||||
# supersetbot's pin and keeps the build on the Dockerfile's own
|
||||
# supported Python version.
|
||||
for attempt in 1 2 3; do
|
||||
if supersetbot docker \
|
||||
$PUSH_OR_LOAD \
|
||||
--preset "$BUILD_PRESET" \
|
||||
--context "$EVENT" \
|
||||
--context-ref "$RELEASE" $FORCE_LATEST \
|
||||
--extra-flags "--build-arg PY_VER=3.11.14-slim-trixie --build-arg INCLUDE_CHROMIUM=false --tag $IMAGE_TAG" \
|
||||
$PLATFORM_ARG; then
|
||||
break
|
||||
fi
|
||||
if [ "$attempt" -eq 3 ]; then
|
||||
echo "::error::supersetbot docker build failed after 3 attempts"
|
||||
exit 1
|
||||
fi
|
||||
echo "::warning::Build attempt ${attempt} failed; retrying in 30s..."
|
||||
sleep 30
|
||||
done
|
||||
|
||||
# in the context of push (using multi-platform build), we need to pull the image locally
|
||||
- name: Docker pull
|
||||
if: github.event_name == 'push'
|
||||
run: |
|
||||
for i in 1 2 3; do
|
||||
docker pull $IMAGE_TAG && break
|
||||
[ $i -lt 3 ] && sleep 30
|
||||
done
|
||||
|
||||
- name: Print docker stats
|
||||
run: |
|
||||
echo "SHA: ${{ github.sha }}"
|
||||
echo "IMAGE: $IMAGE_TAG"
|
||||
docker images $IMAGE_TAG
|
||||
docker history $IMAGE_TAG
|
||||
|
||||
- name: docker-compose sanity check
|
||||
if: matrix.build_preset == 'dev'
|
||||
shell: bash
|
||||
env:
|
||||
BUILD_PRESET: ${{ matrix.build_preset }}
|
||||
run: |
|
||||
export SUPERSET_BUILD_TARGET=$BUILD_PRESET
|
||||
# This should reuse the CACHED image built in the previous steps
|
||||
docker compose build superset-init --build-arg DEV_MODE=false --build-arg INCLUDE_CHROMIUM=false
|
||||
docker compose up superset-init --exit-code-from superset-init
|
||||
|
||||
docker-compose-image-tag:
|
||||
# Run this job only on pushes to master (not for PRs)
|
||||
# goal is to check that building the latest image works, not required for all PR pushes
|
||||
needs: changes
|
||||
if: github.event_name == 'push' && github.ref == 'refs/heads/master' && needs.changes.outputs.docker == 'true'
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Free up disk space
|
||||
shell: bash
|
||||
run: |
|
||||
# The sanity check rebuilds the image from scratch whenever the
|
||||
# registry cache image apache/superset-cache is unavailable, which
|
||||
# can exhaust the runner's root disk ("no space left on device").
|
||||
echo "Disk before cleanup:"; df -h /
|
||||
sudo rm -rf \
|
||||
/usr/share/dotnet \
|
||||
/usr/local/lib/android \
|
||||
/opt/ghc \
|
||||
/usr/local/.ghcup \
|
||||
/opt/hostedtoolcache/CodeQL \
|
||||
/usr/local/share/boost || true
|
||||
echo "Disk after cleanup:"; df -h /
|
||||
- name: Setup Docker Environment
|
||||
uses: ./.github/actions/setup-docker
|
||||
with:
|
||||
dockerhub-user: ${{ secrets.DOCKERHUB_USER }}
|
||||
dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
build: "false"
|
||||
install-docker-compose: "true"
|
||||
- name: docker-compose sanity check
|
||||
shell: bash
|
||||
run: |
|
||||
docker compose -f docker-compose-image-tag.yml up superset-init --exit-code-from superset-init
|
||||
supersetbot docker \
|
||||
--preset ${{ matrix.build_preset }} \
|
||||
--context "$EVENT" \
|
||||
--context-ref "$RELEASE" $FORCE_LATEST \
|
||||
$PLATFORM_ARG
|
||||
|
||||
45
.github/workflows/embedded-sdk-release.yml
vendored
45
.github/workflows/embedded-sdk-release.yml
vendored
@@ -4,35 +4,36 @@ on:
|
||||
push:
|
||||
branches:
|
||||
- "master"
|
||||
- "[0-9].[0-9]*"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
- "[0-9].[0-9]"
|
||||
|
||||
jobs:
|
||||
config:
|
||||
runs-on: "ubuntu-latest"
|
||||
outputs:
|
||||
has-secrets: ${{ steps.check.outputs.has-secrets }}
|
||||
steps:
|
||||
- name: "Check for secrets"
|
||||
id: check
|
||||
shell: bash
|
||||
run: |
|
||||
if [ -n "${{ (secrets.NPM_TOKEN != '') || '' }}" ]; then
|
||||
echo "has-secrets=1" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
build:
|
||||
# Publishing uses npm trusted publishing (OIDC), so there is no NPM_TOKEN to
|
||||
# gate on. Restrict to the canonical repo: forks cannot mint a valid OIDC
|
||||
# token for this package and must not publish.
|
||||
if: github.repository == 'apache/superset'
|
||||
runs-on: ubuntu-24.04
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write # required for npm trusted publishing (OIDC)
|
||||
needs: config
|
||||
if: needs.config.outputs.has-secrets
|
||||
runs-on: ubuntu-20.04
|
||||
defaults:
|
||||
run:
|
||||
working-directory: superset-embedded-sdk
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
# Note: registry-url is intentionally omitted. When set, actions/setup-node
|
||||
# writes an .npmrc with `_authToken=${NODE_AUTH_TOKEN}` and a placeholder
|
||||
# token, which makes npm attempt token auth and skip the OIDC
|
||||
# trusted-publishing exchange. With no .npmrc auth line, npm authenticates
|
||||
# via OIDC against the default registry (registry.npmjs.org).
|
||||
- uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
with:
|
||||
node-version-file: "./superset-embedded-sdk/.nvmrc"
|
||||
node-version: "16"
|
||||
registry-url: "https://registry.npmjs.org"
|
||||
- run: npm ci
|
||||
- run: npm run ci:release
|
||||
env:
|
||||
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
|
||||
|
||||
15
.github/workflows/embedded-sdk-test.yml
vendored
15
.github/workflows/embedded-sdk-test.yml
vendored
@@ -6,9 +6,6 @@ on:
|
||||
- "superset-embedded-sdk/**"
|
||||
types: [synchronize, opened, reopened, ready_for_review]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
# cancel previous workflow jobs for PRs
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
|
||||
@@ -16,18 +13,16 @@ concurrency:
|
||||
|
||||
jobs:
|
||||
embedded-sdk-test:
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-20.04
|
||||
defaults:
|
||||
run:
|
||||
working-directory: superset-embedded-sdk
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
with:
|
||||
node-version-file: "./superset-embedded-sdk/.nvmrc"
|
||||
registry-url: "https://registry.npmjs.org"
|
||||
node-version: "16"
|
||||
registry-url: 'https://registry.npmjs.org'
|
||||
- run: npm ci
|
||||
- run: npm test
|
||||
- run: npm run build
|
||||
|
||||
75
.github/workflows/ephemeral-env-pr-close.yml
vendored
Normal file
75
.github/workflows/ephemeral-env-pr-close.yml
vendored
Normal file
@@ -0,0 +1,75 @@
|
||||
name: Cleanup ephemeral envs (PR close)
|
||||
|
||||
on:
|
||||
pull_request_target:
|
||||
types: [closed]
|
||||
|
||||
jobs:
|
||||
config:
|
||||
runs-on: "ubuntu-latest"
|
||||
outputs:
|
||||
has-secrets: ${{ steps.check.outputs.has-secrets }}
|
||||
steps:
|
||||
- name: "Check for secrets"
|
||||
id: check
|
||||
shell: bash
|
||||
run: |
|
||||
if [ -n "${{ (secrets.AWS_ACCESS_KEY_ID != '' && secrets.AWS_SECRET_ACCESS_KEY != '') || '' }}" ]; then
|
||||
echo "has-secrets=1" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
ephemeral-env-cleanup:
|
||||
needs: config
|
||||
if: needs.config.outputs.has-secrets
|
||||
name: Cleanup ephemeral envs
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Configure AWS credentials
|
||||
uses: aws-actions/configure-aws-credentials@v4
|
||||
with:
|
||||
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
aws-region: us-west-2
|
||||
|
||||
- name: Describe ECS service
|
||||
id: describe-services
|
||||
run: |
|
||||
echo "active=$(aws ecs describe-services --cluster superset-ci --services pr-${{ github.event.number }}-service | jq '.services[] | select(.status == "ACTIVE") | any')" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Delete ECS service
|
||||
if: steps.describe-services.outputs.active == 'true'
|
||||
id: delete-service
|
||||
run: |
|
||||
aws ecs delete-service \
|
||||
--cluster superset-ci \
|
||||
--service pr-${{ github.event.number }}-service \
|
||||
--force
|
||||
|
||||
- name: Login to Amazon ECR
|
||||
if: steps.describe-services.outputs.active == 'true'
|
||||
id: login-ecr
|
||||
uses: aws-actions/amazon-ecr-login@v2
|
||||
|
||||
- name: Delete ECR image tag
|
||||
if: steps.describe-services.outputs.active == 'true'
|
||||
id: delete-image-tag
|
||||
run: |
|
||||
aws ecr batch-delete-image \
|
||||
--registry-id $(echo "${{ steps.login-ecr.outputs.registry }}" | grep -Eo "^[0-9]+") \
|
||||
--repository-name superset-ci \
|
||||
--image-ids imageTag=pr-${{ github.event.number }}
|
||||
|
||||
- name: Comment (success)
|
||||
if: steps.describe-services.outputs.active == 'true'
|
||||
uses: actions/github-script@v7
|
||||
with:
|
||||
github-token: ${{github.token}}
|
||||
script: |
|
||||
github.rest.issues.createComment({
|
||||
issue_number: ${{ github.event.number }},
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
body: 'Ephemeral environment shutdown and build artifacts deleted.'
|
||||
})
|
||||
283
.github/workflows/ephemeral-env.yml
vendored
Normal file
283
.github/workflows/ephemeral-env.yml
vendored
Normal file
@@ -0,0 +1,283 @@
|
||||
name: Ephemeral env workflow
|
||||
|
||||
on:
|
||||
issue_comment:
|
||||
types: [created]
|
||||
|
||||
jobs:
|
||||
config:
|
||||
runs-on: "ubuntu-latest"
|
||||
if: github.event.issue.pull_request
|
||||
outputs:
|
||||
has-secrets: ${{ steps.check.outputs.has-secrets }}
|
||||
steps:
|
||||
- name: "Check for secrets"
|
||||
id: check
|
||||
shell: bash
|
||||
run: |
|
||||
if [ -n "${{ (secrets.AWS_ACCESS_KEY_ID != '' && secrets.AWS_SECRET_ACCESS_KEY != '') || '' }}" ]; then
|
||||
echo "has-secrets=1" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
ephemeral-env-comment:
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.issue.number || github.run_id }}-comment
|
||||
cancel-in-progress: true
|
||||
needs: config
|
||||
if: needs.config.outputs.has-secrets
|
||||
name: Evaluate ephemeral env comment trigger (/testenv)
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
pull-requests: write
|
||||
outputs:
|
||||
slash-command: ${{ steps.eval-body.outputs.result }}
|
||||
feature-flags: ${{ steps.eval-feature-flags.outputs.result }}
|
||||
|
||||
steps:
|
||||
- name: Debug
|
||||
run: |
|
||||
echo "Comment on PR #${{ github.event.issue.number }} by ${{ github.event.issue.user.login }}, ${{ github.event.comment.author_association }}"
|
||||
|
||||
- name: Eval comment body for /testenv slash command
|
||||
uses: actions/github-script@v7
|
||||
id: eval-body
|
||||
with:
|
||||
result-encoding: string
|
||||
script: |
|
||||
const pattern = /^\/testenv (up|down)/
|
||||
const result = pattern.exec(context.payload.comment.body)
|
||||
return result === null ? 'noop' : result[1]
|
||||
|
||||
- name: Eval comment body for feature flags
|
||||
uses: actions/github-script@v7
|
||||
id: eval-feature-flags
|
||||
with:
|
||||
script: |
|
||||
const pattern = /FEATURE_(\w+)=(\w+)/g;
|
||||
let results = [];
|
||||
[...context.payload.comment.body.matchAll(pattern)].forEach(match => {
|
||||
const config = {
|
||||
name: `SUPERSET_FEATURE_${match[1]}`,
|
||||
value: match[2],
|
||||
};
|
||||
results.push(config);
|
||||
});
|
||||
return results;
|
||||
|
||||
- name: Limit to committers
|
||||
if: >
|
||||
steps.eval-body.outputs.result != 'noop' &&
|
||||
github.event.comment.author_association != 'MEMBER' &&
|
||||
github.event.comment.author_association != 'OWNER'
|
||||
uses: actions/github-script@v7
|
||||
with:
|
||||
github-token: ${{github.token}}
|
||||
script: |
|
||||
const errMsg = '@${{ github.event.comment.user.login }} Ephemeral environment creation is currently limited to committers.'
|
||||
github.rest.issues.createComment({
|
||||
issue_number: ${{ github.event.issue.number }},
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
body: errMsg
|
||||
})
|
||||
core.setFailed(errMsg)
|
||||
|
||||
ephemeral-docker-build:
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.issue.number || github.run_id }}-build
|
||||
cancel-in-progress: true
|
||||
needs: ephemeral-env-comment
|
||||
name: ephemeral-docker-build
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Get Info from comment
|
||||
uses: actions/github-script@v7
|
||||
id: get-pr-info
|
||||
with:
|
||||
script: |
|
||||
const request = {
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
pull_number: ${{ github.event.issue.number }},
|
||||
}
|
||||
core.info(`Getting PR #${request.pull_number} from ${request.owner}/${request.repo}`)
|
||||
const pr = await github.rest.pulls.get(request);
|
||||
return pr.data;
|
||||
|
||||
- name: Debug
|
||||
id: get-sha
|
||||
run: |
|
||||
echo "sha=${{ fromJSON(steps.get-pr-info.outputs.result).head.sha }}" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} : ${{steps.get-sha.outputs.sha}} )"
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ steps.get-sha.outputs.sha }}
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Build ephemeral env image
|
||||
run: |
|
||||
./scripts/build_docker.py \
|
||||
"ci" \
|
||||
"pull_request" \
|
||||
--build_context_ref ${{ github.event.issue.number }}
|
||||
|
||||
- name: Configure AWS credentials
|
||||
uses: aws-actions/configure-aws-credentials@v4
|
||||
with:
|
||||
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
aws-region: us-west-2
|
||||
|
||||
- name: Login to Amazon ECR
|
||||
id: login-ecr
|
||||
uses: aws-actions/amazon-ecr-login@v2
|
||||
|
||||
- name: Load, tag and push image to ECR
|
||||
id: push-image
|
||||
env:
|
||||
ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
|
||||
ECR_REPOSITORY: superset-ci
|
||||
IMAGE_TAG: apache/superset:${{ steps.get-sha.outputs.sha }}-ci
|
||||
run: |
|
||||
docker tag $IMAGE_TAG $ECR_REGISTRY/$ECR_REPOSITORY:pr-${{ github.event.issue.number }}-ci
|
||||
docker push -a $ECR_REGISTRY/$ECR_REPOSITORY
|
||||
|
||||
ephemeral-env-up:
|
||||
needs: [ephemeral-env-comment, ephemeral-docker-build]
|
||||
if: needs.ephemeral-env-comment.outputs.slash-command == 'up'
|
||||
name: Spin up an ephemeral environment
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Configure AWS credentials
|
||||
uses: aws-actions/configure-aws-credentials@v4
|
||||
with:
|
||||
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
aws-region: us-west-2
|
||||
|
||||
- name: Login to Amazon ECR
|
||||
id: login-ecr
|
||||
uses: aws-actions/amazon-ecr-login@v2
|
||||
|
||||
- name: Check target image exists in ECR
|
||||
id: check-image
|
||||
continue-on-error: true
|
||||
run: |
|
||||
aws ecr describe-images \
|
||||
--registry-id $(echo "${{ steps.login-ecr.outputs.registry }}" | grep -Eo "^[0-9]+") \
|
||||
--repository-name superset-ci \
|
||||
--image-ids imageTag=pr-${{ github.event.issue.number }}-ci
|
||||
|
||||
- name: Fail on missing container image
|
||||
if: steps.check-image.outcome == 'failure'
|
||||
uses: actions/github-script@v7
|
||||
with:
|
||||
github-token: ${{github.token}}
|
||||
script: |
|
||||
const errMsg = '@${{ github.event.comment.user.login }} Container image not yet published for this PR. Please try again when build is complete.'
|
||||
github.rest.issues.createComment({
|
||||
issue_number: ${{ github.event.issue.number }},
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
body: errMsg
|
||||
})
|
||||
core.setFailed(errMsg)
|
||||
|
||||
- name: Fill in the new image ID in the Amazon ECS task definition
|
||||
id: task-def
|
||||
uses: aws-actions/amazon-ecs-render-task-definition@v1
|
||||
with:
|
||||
task-definition: .github/workflows/ecs-task-definition.json
|
||||
container-name: superset-ci
|
||||
image: ${{ steps.login-ecr.outputs.registry }}/superset-ci:pr-${{ github.event.issue.number }}-ci
|
||||
|
||||
- name: Update env vars in the Amazon ECS task definition
|
||||
run: |
|
||||
cat <<< "$(jq '.containerDefinitions[0].environment += ${{ needs.ephemeral-env-comment.outputs.feature-flags }}' < ${{ steps.task-def.outputs.task-definition }})" > ${{ steps.task-def.outputs.task-definition }}
|
||||
|
||||
- name: Describe ECS service
|
||||
id: describe-services
|
||||
run: |
|
||||
echo "active=$(aws ecs describe-services --cluster superset-ci --services pr-${{ github.event.issue.number }}-service | jq '.services[] | select(.status == "ACTIVE") | any')" >> $GITHUB_OUTPUT
|
||||
- name: Create ECS service
|
||||
if: steps.describe-services.outputs.active != 'true'
|
||||
id: create-service
|
||||
env:
|
||||
ECR_SUBNETS: subnet-0e15a5034b4121710,subnet-0e8efef4a72224974
|
||||
ECR_SECURITY_GROUP: sg-092ff3a6ae0574d91
|
||||
run: |
|
||||
aws ecs create-service \
|
||||
--cluster superset-ci \
|
||||
--service-name pr-${{ github.event.issue.number }}-service \
|
||||
--task-definition superset-ci \
|
||||
--launch-type FARGATE \
|
||||
--desired-count 1 \
|
||||
--platform-version LATEST \
|
||||
--network-configuration "awsvpcConfiguration={subnets=[$ECR_SUBNETS],securityGroups=[$ECR_SECURITY_GROUP],assignPublicIp=ENABLED}" \
|
||||
--tags key=pr,value=${{ github.event.issue.number }} key=github_user,value=${{ github.actor }}
|
||||
|
||||
- name: Deploy Amazon ECS task definition
|
||||
id: deploy-task
|
||||
uses: aws-actions/amazon-ecs-deploy-task-definition@v1
|
||||
with:
|
||||
task-definition: ${{ steps.task-def.outputs.task-definition }}
|
||||
service: pr-${{ github.event.issue.number }}-service
|
||||
cluster: superset-ci
|
||||
wait-for-service-stability: true
|
||||
wait-for-minutes: 10
|
||||
|
||||
- name: List tasks
|
||||
id: list-tasks
|
||||
run: |
|
||||
echo "task=$(aws ecs list-tasks --cluster superset-ci --service-name pr-${{ github.event.issue.number }}-service | jq '.taskArns | first')" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Get network interface
|
||||
id: get-eni
|
||||
run: |
|
||||
echo "eni=$(aws ecs describe-tasks --cluster superset-ci --tasks ${{ steps.list-tasks.outputs.task }} | jq '.tasks | .[0] | .attachments | .[0] | .details | map(select(.name=="networkInterfaceId")) | .[0] | .value')" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Get public IP
|
||||
id: get-ip
|
||||
run: |
|
||||
echo "ip=$(aws ec2 describe-network-interfaces --network-interface-ids ${{ steps.get-eni.outputs.eni }} | jq -r '.NetworkInterfaces | first | .Association.PublicIp')" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Comment (success)
|
||||
if: ${{ success() }}
|
||||
uses: actions/github-script@v7
|
||||
with:
|
||||
github-token: ${{github.token}}
|
||||
script: |
|
||||
github.rest.issues.createComment({
|
||||
issue_number: ${{ github.event.issue.number }},
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
body: '@${{ github.event.comment.user.login }} Ephemeral environment spinning up at http://${{ steps.get-ip.outputs.ip }}:8080. Credentials are `admin`/`admin`. Please allow several minutes for bootstrapping and startup.'
|
||||
})
|
||||
|
||||
- name: Comment (failure)
|
||||
if: ${{ failure() }}
|
||||
uses: actions/github-script@v7
|
||||
with:
|
||||
github-token: ${{github.token}}
|
||||
script: |
|
||||
github.rest.issues.createComment({
|
||||
issue_number: ${{ github.event.issue.number }},
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
body: '@${{ github.event.comment.user.login }} Ephemeral environment creation failed. Please check the Actions logs for details.'
|
||||
})
|
||||
17
.github/workflows/generate-FOSSA-report.yml
vendored
17
.github/workflows/generate-FOSSA-report.yml
vendored
@@ -4,14 +4,11 @@ on:
|
||||
push:
|
||||
branches:
|
||||
- "master"
|
||||
- "[0-9].[0-9]*"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
- "[0-9].[0-9]"
|
||||
|
||||
jobs:
|
||||
config:
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: "ubuntu-latest"
|
||||
outputs:
|
||||
has-secrets: ${{ steps.check.outputs.has-secrets }}
|
||||
steps:
|
||||
@@ -19,25 +16,23 @@ jobs:
|
||||
id: check
|
||||
shell: bash
|
||||
run: |
|
||||
if [ -n "${FOSSA_API_KEY}" ]; then
|
||||
if [ -n "${{ (secrets.FOSSA_API_KEY != '' ) || '' }}" ]; then
|
||||
echo "has-secrets=1" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
env:
|
||||
FOSSA_API_KEY: ${{ (secrets.FOSSA_API_KEY != '' ) || '' }}
|
||||
license_check:
|
||||
needs: config
|
||||
if: needs.config.outputs.has-secrets
|
||||
name: Generate Report
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-20.04
|
||||
steps:
|
||||
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: recursive
|
||||
- name: Setup Java
|
||||
uses: actions/setup-java@0f481fcb613427c0f801b606911222b5b6f3083a # v5.5.0
|
||||
uses: actions/setup-java@v4
|
||||
with:
|
||||
distribution: "temurin"
|
||||
java-version: "11"
|
||||
|
||||
28
.github/workflows/github-action-validator.sh
vendored
28
.github/workflows/github-action-validator.sh
vendored
@@ -1,28 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# Inspired from https://github.com/mpalmer/action-validator?tab=readme-ov-file#pre-commit-hook-example
|
||||
echo "Running pre-commit hook for GitHub Actions: https://github.com/mpalmer/action-validator"
|
||||
for action in $(git ls-files .github/ | grep -E '^\.github/(workflows|actions)/.*\.ya?ml$'); do
|
||||
if action-validator "$action"; then
|
||||
echo "✅ $action"
|
||||
else
|
||||
echo "❌ $action"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
49
.github/workflows/github-action-validator.yml
vendored
49
.github/workflows/github-action-validator.yml
vendored
@@ -1,49 +0,0 @@
|
||||
name: Validate All GitHub Actions
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- "master"
|
||||
- "[0-9].[0-9]*"
|
||||
pull_request:
|
||||
branches:
|
||||
- "**"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
# cancel previous workflow jobs for PRs
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
validate-all-ghas:
|
||||
runs-on: ubuntu-24.04
|
||||
permissions:
|
||||
contents: read
|
||||
# Required for the zizmor action to upload its SARIF results to
|
||||
# GitHub code scanning (advanced-security is enabled by default).
|
||||
security-events: write
|
||||
steps:
|
||||
- name: Checkout Repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
with:
|
||||
node-version: "20"
|
||||
|
||||
- name: Install Dependencies
|
||||
# Versions are pinned to avoid ad-hoc, unpinned package installs.
|
||||
# Bump deliberately when upgrading.
|
||||
# zizmor: ignore[adhoc-packages] - @action-validator is a global CLI tool installed to validate the repo's workflows; a global CLI install has no application manifest/lockfile context, and the versions are pinned above
|
||||
run: npm install -g @action-validator/core@0.6.0 @action-validator/cli@0.6.0
|
||||
|
||||
- name: Run Script
|
||||
run: bash .github/workflows/github-action-validator.sh
|
||||
|
||||
- name: Check for security issues on GHA workflows
|
||||
uses: zizmorcore/zizmor-action@192e21d79ab29983730a13d1382995c2307fbcaa # v0.5.7
|
||||
19
.github/workflows/issue_creation.yml
vendored
19
.github/workflows/issue_creation.yml
vendored
@@ -9,25 +9,24 @@ on:
|
||||
|
||||
jobs:
|
||||
superbot-orglabel:
|
||||
runs-on: ubuntu-24.04
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
issues: write
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
- name: Execute SupersetBot Command
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
node-version: '20'
|
||||
|
||||
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup supersetbot
|
||||
uses: ./.github/actions/setup-supersetbot/
|
||||
|
||||
- name: Execute supersetbot orglabel command
|
||||
- name: Execute custom Node.js script
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
# Label the issue with the appropriate org using supersetbot
|
||||
# - this requires for the author to be publicly associated with their org
|
||||
# - and for the org to be listed in `supersetbot/src/metadata.js`
|
||||
supersetbot orglabel --issue ${{ github.event.number }} --repo ${{ github.repository }} || true
|
||||
supersetbot orglabel --issue ${{ github.event.number }} --repo ${{ github.repository }}
|
||||
|
||||
9
.github/workflows/labeler.yml
vendored
9
.github/workflows/labeler.yml
vendored
@@ -2,19 +2,14 @@ name: "Pull Request Labeler"
|
||||
on:
|
||||
- pull_request_target
|
||||
|
||||
# cancel previous workflow jobs for PRs
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
labeler:
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/labeler@b8dd2d9be0f68b860e7dae5dae7d772984eacd6d # v6.2.0
|
||||
- uses: actions/labeler@v5
|
||||
with:
|
||||
sync-labels: true
|
||||
|
||||
|
||||
46
.github/workflows/latest-release-tag.yml
vendored
46
.github/workflows/latest-release-tag.yml
vendored
@@ -6,34 +6,32 @@ on:
|
||||
jobs:
|
||||
latest-release:
|
||||
name: Add/update tag to new release
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
steps:
|
||||
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: recursive
|
||||
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: recursive
|
||||
|
||||
- name: Check for latest tag
|
||||
id: latest-tag
|
||||
env:
|
||||
RELEASE_TAG_NAME: ${{ github.event.release.tag_name }}
|
||||
run: |
|
||||
source ./scripts/tag_latest_release.sh "$RELEASE_TAG_NAME" --dry-run
|
||||
- name: Check for latest tag
|
||||
id: latest-tag
|
||||
run: |
|
||||
source ./scripts/tag_latest_release.sh $(echo ${{ github.event.release.tag_name }}) --dry-run
|
||||
|
||||
- name: Configure Git
|
||||
run: |
|
||||
git config user.name "$GITHUB_ACTOR"
|
||||
git config user.email "$GITHUB_ACTOR@users.noreply.github.com"
|
||||
- name: Configure Git
|
||||
run: |
|
||||
git config user.name "$GITHUB_ACTOR"
|
||||
git config user.email "$GITHUB_ACTOR@users.noreply.github.com"
|
||||
|
||||
- name: Run latest-tag
|
||||
uses: ./.github/actions/latest-tag
|
||||
if: steps.latest-tag.outputs.SKIP_TAG != 'true'
|
||||
with:
|
||||
description: Superset latest release
|
||||
tag-name: latest
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
- name: Run latest-tag
|
||||
uses: ./.github/actions/latest-tag
|
||||
if: (! ${{ steps.latest-tag.outputs.SKIP_TAG }} )
|
||||
with:
|
||||
description: Superset latest release
|
||||
tag-name: latest
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
|
||||
13
.github/workflows/license-check.yml
vendored
13
.github/workflows/license-check.yml
vendored
@@ -4,9 +4,6 @@ on:
|
||||
pull_request:
|
||||
types: [synchronize, opened, reopened, ready_for_review]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
# cancel previous workflow jobs for PRs
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
|
||||
@@ -15,17 +12,17 @@ concurrency:
|
||||
jobs:
|
||||
license_check:
|
||||
name: License Check
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-20.04
|
||||
steps:
|
||||
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: recursive
|
||||
- name: Setup Java
|
||||
uses: actions/setup-java@0f481fcb613427c0f801b606911222b5b6f3083a # v5.5.0
|
||||
uses: actions/setup-java@v4
|
||||
with:
|
||||
distribution: "temurin"
|
||||
java-version: "11"
|
||||
distribution: 'temurin'
|
||||
java-version: '11'
|
||||
- name: Run license check
|
||||
run: ./scripts/check_license.sh
|
||||
|
||||
14
.github/workflows/no-hold-label.yml
vendored
14
.github/workflows/no-hold-label.yml
vendored
@@ -4,23 +4,17 @@ on:
|
||||
pull_request:
|
||||
types: [labeled, unlabeled, opened, reopened, synchronize]
|
||||
|
||||
permissions:
|
||||
pull-requests: read
|
||||
|
||||
# Let each label event run to completion. Cancelling in-progress runs leaves
|
||||
# CANCELLED entries in the PR's check-suite rollup, which poisons GitHub's
|
||||
# `status:success` search filter even though all real CI passed. The job is
|
||||
# a tiny no-op github-script call, so the wasted compute is negligible.
|
||||
# cancel previous workflow jobs for PRs
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
|
||||
cancel-in-progress: false
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
check-hold-label:
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Check for 'hold' label
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
uses: actions/github-script@v7
|
||||
with:
|
||||
github-token: ${{secrets.GITHUB_TOKEN}}
|
||||
script: |
|
||||
|
||||
84
.github/workflows/no-op.yml
vendored
Normal file
84
.github/workflows/no-op.yml
vendored
Normal file
@@ -0,0 +1,84 @@
|
||||
# no-op.yml
|
||||
#
|
||||
# Purpose:
|
||||
# This workflow provides a workaround for the "required status checks" feature in GitHub Actions
|
||||
# when using path-specific conditions in other workflows. Required checks might remain in a "Pending"
|
||||
# state if the conditions are not met, thus blocking pull requests from being merged.
|
||||
# This no-op (no operation) workflow provides dummy success statuses for these required jobs when
|
||||
# the real jobs do not run due to path-specific conditions.
|
||||
#
|
||||
# How it works:
|
||||
# - It defines jobs with the same names as the required jobs in the main workflows.
|
||||
# - These jobs simply execute a command (`exit 0`) to succeed immediately.
|
||||
# - When a pull request is created or updated, both this no-op workflow and the main workflows are triggered.
|
||||
# - If the main workflows' jobs don't run (due to path conditions), these no-op jobs provide successful statuses.
|
||||
# - If the main workflows' jobs do run and fail, their failure statuses take precedence,
|
||||
# ensuring that pull requests are not merged with failing checks.
|
||||
#
|
||||
# Usage:
|
||||
# - Ensure that the job names in this workflow match exactly the names of the corresponding jobs in the main workflows.
|
||||
# - This workflow should be kept as-is, without path-specific conditions.
|
||||
|
||||
name: no-op Checks
|
||||
on: pull_request
|
||||
|
||||
jobs:
|
||||
frontend-build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: No-op for frontend-build
|
||||
run: |
|
||||
echo "This is a no-op step for frontend-build to ensure a successful status."
|
||||
exit 0
|
||||
|
||||
pre-commit:
|
||||
strategy:
|
||||
matrix:
|
||||
python-version: ["3.9"]
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: No-op for pre-commit
|
||||
run: |
|
||||
echo "This is a no-op step for pre-commit to ensure a successful status."
|
||||
exit 0
|
||||
|
||||
python-lint:
|
||||
strategy:
|
||||
matrix:
|
||||
python-version: ["3.9"]
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: No-op for python-lint
|
||||
run: |
|
||||
echo "This is a no-op step for python-lint to ensure a successful status."
|
||||
exit 0
|
||||
test-postgres-hive:
|
||||
strategy:
|
||||
matrix:
|
||||
python-version: ["3.9"]
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: No-op for frontend-build
|
||||
run: |
|
||||
echo "This is a no-op step for test-postgres-postgres to ensure a successful status when skipped."
|
||||
exit 0
|
||||
test-postgres-presto:
|
||||
strategy:
|
||||
matrix:
|
||||
python-version: ["3.9"]
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: No-op for frontend-build
|
||||
run: |
|
||||
echo "This is a no-op step for test-postgres-postgres to ensure a successful status when skipped."
|
||||
exit 0
|
||||
unit-tests:
|
||||
strategy:
|
||||
matrix:
|
||||
python-version: ["3.9"]
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: No-op for frontend-build
|
||||
run: |
|
||||
echo "This is a no-op step for unit-tests to ensure a successful status when skipped."
|
||||
exit 0
|
||||
12
.github/workflows/pr-lint.yml
vendored
12
.github/workflows/pr-lint.yml
vendored
@@ -8,20 +8,15 @@ on:
|
||||
# Possible values: https://help.github.com/en/actions/reference/events-that-trigger-workflows#pull-request-event-pull_request
|
||||
types: [opened, edited, reopened, synchronize]
|
||||
|
||||
# cancel previous workflow jobs for PRs
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
lint-check:
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: recursive
|
||||
@@ -31,5 +26,6 @@ jobs:
|
||||
on-failed-regex-fail-action: true
|
||||
on-failed-regex-request-changes: false
|
||||
on-failed-regex-create-review: false
|
||||
on-failed-regex-comment: "Please format your PR title to match: `%regex%`!"
|
||||
on-failed-regex-comment:
|
||||
"Please format your PR title to match: `%regex%`!"
|
||||
repo-token: "${{ github.token }}"
|
||||
|
||||
88
.github/workflows/pre-commit.yml
vendored
88
.github/workflows/pre-commit.yml
vendored
@@ -4,13 +4,10 @@ on:
|
||||
push:
|
||||
branches:
|
||||
- "master"
|
||||
- "[0-9].[0-9]*"
|
||||
- "[0-9].[0-9]"
|
||||
pull_request:
|
||||
types: [synchronize, opened, reopened, ready_for_review]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
# cancel previous workflow jobs for PRs
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
|
||||
@@ -18,86 +15,31 @@ concurrency:
|
||||
|
||||
jobs:
|
||||
pre-commit:
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 20
|
||||
runs-on: ubuntu-20.04
|
||||
strategy:
|
||||
matrix:
|
||||
# Run the full version spread on push (master/release) and nightly,
|
||||
# but only the current version on PRs — lint/format/type results
|
||||
# rarely differ across patch versions, so 3x per PR is wasteful.
|
||||
python-version: ${{ github.event_name == 'pull_request' && fromJSON('["current"]') || fromJSON('["current", "next"]') }}
|
||||
python-version: ["3.9"]
|
||||
steps:
|
||||
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: recursive
|
||||
|
||||
- name: Setup Python
|
||||
uses: ./.github/actions/setup-backend/
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
- name: Setup Go
|
||||
uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0
|
||||
|
||||
- name: Install helm-docs
|
||||
run: go install github.com/norwoodj/helm-docs/cmd/helm-docs@v1.14.2
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
with:
|
||||
node-version-file: "superset-frontend/.nvmrc"
|
||||
cache: "npm"
|
||||
cache-dependency-path: "superset-frontend/package-lock.json"
|
||||
|
||||
- name: Install Frontend Dependencies
|
||||
- name: Enable brew and helm-docs
|
||||
# Add brew to the path - see https://github.com/actions/runner-images/issues/6283
|
||||
run: |
|
||||
cd superset-frontend
|
||||
npm ci
|
||||
|
||||
- name: Install Docs Dependencies
|
||||
run: |
|
||||
cd docs
|
||||
yarn install --immutable
|
||||
|
||||
- name: Cache pre-commit environments
|
||||
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||
with:
|
||||
path: ~/.cache/pre-commit
|
||||
key: pre-commit-v2-${{ runner.os }}-py${{ matrix.python-version }}-${{ hashFiles('.pre-commit-config.yaml') }}
|
||||
restore-keys: |
|
||||
pre-commit-v2-${{ runner.os }}-py${{ matrix.python-version }}-
|
||||
|
||||
- name: Get changed files
|
||||
id: changed_files
|
||||
uses: ./.github/actions/file-changes-action
|
||||
with:
|
||||
output: " "
|
||||
|
||||
echo "/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/sbin" >> $GITHUB_PATH
|
||||
eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"
|
||||
echo "HOMEBREW_PREFIX=$HOMEBREW_PREFIX" >>"${GITHUB_ENV}"
|
||||
echo "HOMEBREW_CELLAR=$HOMEBREW_CELLAR" >>"${GITHUB_ENV}"
|
||||
echo "HOMEBREW_REPOSITORY=$HOMEBREW_REPOSITORY" >>"${GITHUB_ENV}"
|
||||
brew install norwoodj/tap/helm-docs
|
||||
- name: pre-commit
|
||||
env:
|
||||
CHANGED_FILES: ${{ steps.changed_files.outputs.files }}
|
||||
run: |
|
||||
set +e # Don't exit immediately on failure
|
||||
export SKIP=type-checking-frontend
|
||||
pre-commit run --files $CHANGED_FILES
|
||||
PRE_COMMIT_EXIT_CODE=$?
|
||||
git diff --quiet --exit-code
|
||||
GIT_DIFF_EXIT_CODE=$?
|
||||
if [ "${PRE_COMMIT_EXIT_CODE}" -ne 0 ] || [ "${GIT_DIFF_EXIT_CODE}" -ne 0 ]; then
|
||||
if [ "${PRE_COMMIT_EXIT_CODE}" -ne 0 ]; then
|
||||
echo "❌ Pre-commit check failed (exit code: ${PRE_COMMIT_EXIT_CODE})."
|
||||
echo "🔍 Modified files:"
|
||||
git diff --name-only
|
||||
else
|
||||
echo "❌ Git working directory is dirty."
|
||||
echo "📌 This likely means that pre-commit made changes that were not committed."
|
||||
echo "🔍 Modified files:"
|
||||
git diff --name-only
|
||||
fi
|
||||
|
||||
echo "🚒 To prevent/address this CI issue, please install/use pre-commit locally."
|
||||
echo "📖 More details here: https://superset.apache.org/docs/contributing/development#git-hooks"
|
||||
if ! pre-commit run --all-files; then
|
||||
git status
|
||||
git diff
|
||||
exit 1
|
||||
fi
|
||||
|
||||
70
.github/workflows/prefer-typescript.yml
vendored
Normal file
70
.github/workflows/prefer-typescript.yml
vendored
Normal file
@@ -0,0 +1,70 @@
|
||||
name: Prefer TypeScript
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- "master"
|
||||
- "[0-9].[0-9]"
|
||||
paths:
|
||||
- "superset-frontend/src/**"
|
||||
pull_request:
|
||||
types: [synchronize, opened, reopened, ready_for_review]
|
||||
paths:
|
||||
- "superset-frontend/src/**"
|
||||
|
||||
# cancel previous workflow jobs for PRs
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
prefer_typescript:
|
||||
if: github.ref == 'ref/heads/master' && github.event_name == 'pull_request'
|
||||
name: Prefer TypeScript
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: recursive
|
||||
- name: Get changed files
|
||||
id: changed
|
||||
uses: ./.github/actions/file-changes-action
|
||||
with:
|
||||
githubToken: ${{ github.token }}
|
||||
|
||||
- name: Determine if a .js or .jsx file was added
|
||||
id: check
|
||||
run: |
|
||||
js_files_added() {
|
||||
jq -r '
|
||||
map(
|
||||
select(
|
||||
endswith(".js") or endswith(".jsx")
|
||||
)
|
||||
) | join("\n")
|
||||
' ${HOME}/files_added.json
|
||||
}
|
||||
echo "js_files_added=$(js_files_added)" >> $GITHUB_OUTPUT
|
||||
|
||||
- if: steps.check.outputs.js_files_added
|
||||
name: Add Comment to PR
|
||||
uses: ./.github/actions/comment-on-pr
|
||||
continue-on-error: true
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
with:
|
||||
msg: |
|
||||
### WARNING: Prefer TypeScript
|
||||
|
||||
Looks like your PR contains new `.js` or `.jsx` files:
|
||||
|
||||
```
|
||||
${{steps.check.outputs.js_files_added}}
|
||||
```
|
||||
|
||||
As decided in [SIP-36](https://github.com/apache/superset/issues/9101), all new frontend code should be written in TypeScript. Please convert above files to TypeScript then re-request review.
|
||||
34
.github/workflows/release.yml
vendored
34
.github/workflows/release.yml
vendored
@@ -4,14 +4,11 @@ on:
|
||||
push:
|
||||
branches:
|
||||
- "master"
|
||||
- "[0-9].[0-9]*"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
- "[0-9].[0-9]"
|
||||
|
||||
jobs:
|
||||
config:
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: "ubuntu-latest"
|
||||
outputs:
|
||||
has-secrets: ${{ steps.check.outputs.has-secrets }}
|
||||
steps:
|
||||
@@ -19,23 +16,24 @@ jobs:
|
||||
id: check
|
||||
shell: bash
|
||||
run: |
|
||||
if [ -n "${NPM_TOKEN}" ]; then
|
||||
if [ -n "${{ (secrets.NPM_TOKEN != '' && secrets.GH_PERSONAL_ACCESS_TOKEN != '') || '' }}" ]; then
|
||||
echo "has-secrets=1" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
env:
|
||||
NPM_TOKEN: ${{ (secrets.NPM_TOKEN != '' && secrets.GH_PERSONAL_ACCESS_TOKEN != '') || '' }}
|
||||
build:
|
||||
needs: config
|
||||
if: needs.config.outputs.has-secrets
|
||||
name: Bump version and publish package(s)
|
||||
runs-on: ubuntu-24.04
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
runs-on: ubuntu-20.04
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
node-version: [16]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
# pulls all commits (needed for lerna / semantic release to correctly version)
|
||||
fetch-depth: 0
|
||||
- name: Get tags and filter trigger tags
|
||||
@@ -48,15 +46,15 @@ jobs:
|
||||
git fetch --prune --unshallow
|
||||
git tag -d `git tag | grep -E '^trigger-'`
|
||||
|
||||
- name: Install Node.js
|
||||
- name: Use Node.js ${{ matrix.node-version }}
|
||||
if: env.HAS_TAGS
|
||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version-file: "./superset-frontend/.nvmrc"
|
||||
node-version: ${{ matrix.node-version }}
|
||||
|
||||
- name: Cache npm
|
||||
if: env.HAS_TAGS
|
||||
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.npm # npm cache files are stored in `~/.npm` on Linux/macOS
|
||||
key: ${{ runner.OS }}-node-${{ hashFiles('**/package-lock.json') }}
|
||||
@@ -70,7 +68,7 @@ jobs:
|
||||
run: echo "dir=$(npm config get cache)" >> $GITHUB_OUTPUT
|
||||
- name: Cache npm
|
||||
if: env.HAS_TAGS
|
||||
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||
uses: actions/cache@v4
|
||||
id: npm-cache # use this to check for `cache-hit` (`steps.npm-cache.outputs.cache-hit != 'true'`)
|
||||
with:
|
||||
path: ${{ steps.npm-cache-dir-path.outputs.dir }}
|
||||
|
||||
177
.github/workflows/scheduled-docker-image-refresh.yml
vendored
177
.github/workflows/scheduled-docker-image-refresh.yml
vendored
@@ -1,177 +0,0 @@
|
||||
name: Scheduled Docker image refresh
|
||||
|
||||
# Re-runs the Docker image build against the latest published release on a
|
||||
# weekly cadence. The code being built doesn't change — but the base image
|
||||
# layers (python:*-slim-trixie and its OS packages) DO get upstream
|
||||
# security patches between Superset releases, and those patches don't
|
||||
# reach our published images unless we rebuild.
|
||||
#
|
||||
# Without this workflow, `apache/superset:<latest>` lags behind upstream
|
||||
# Debian/Python base patches by whatever interval falls between Superset
|
||||
# releases (typically 3–6 weeks). With it, the lag drops to at most one
|
||||
# week regardless of release cadence.
|
||||
#
|
||||
# This is a security-hygiene cron, not a release. It overwrites the
|
||||
# existing tags for the most recent release (e.g. `apache/superset:5.0.0`
|
||||
# and `apache/superset:latest`) with bit-for-bit-equivalent contents
|
||||
# layered on a refreshed base. Image digests change; everything users
|
||||
# actually pin against (image content, code, deps) does not.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# Mondays at 06:00 UTC — gives the weekend for upstream patches to
|
||||
# settle and surfaces failures at the start of the work week so a
|
||||
# human can react.
|
||||
- cron: "0 6 * * 1"
|
||||
|
||||
# Manual trigger so operators can force a refresh on demand (e.g.
|
||||
# immediately after a high-severity base-image CVE drops).
|
||||
workflow_dispatch: {}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
# Serialize with itself and with the release publisher (tag-release.yml) —
|
||||
# both push to the same Docker Hub tags, so a race could end with stale
|
||||
# layers winning. Both workflows must declare this group for the lock to work.
|
||||
concurrency:
|
||||
group: docker-publish-latest-release
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
config:
|
||||
runs-on: ubuntu-24.04
|
||||
outputs:
|
||||
has-secrets: ${{ steps.check.outputs.has-secrets }}
|
||||
latest-release: ${{ steps.latest.outputs.tag }}
|
||||
force-latest: ${{ steps.latest.outputs.force-latest }}
|
||||
steps:
|
||||
- name: Check for Docker Hub secrets
|
||||
id: check
|
||||
shell: bash
|
||||
run: |
|
||||
if [ -n "${DOCKERHUB_USER}" ]; then
|
||||
echo "has-secrets=1" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
env:
|
||||
DOCKERHUB_USER: ${{ (secrets.DOCKERHUB_USER != '' && secrets.DOCKERHUB_TOKEN != '') || '' }}
|
||||
|
||||
- name: Look up latest published release
|
||||
id: latest
|
||||
shell: bash
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
REPOSITORY: ${{ github.repository }}
|
||||
run: |
|
||||
# `releases/latest` returns the latest non-prerelease, non-draft
|
||||
# release — which is exactly what `apache/superset:latest`
|
||||
# should reflect.
|
||||
TAG=$(gh api "repos/${REPOSITORY}/releases/latest" --jq .tag_name)
|
||||
if [ -z "$TAG" ] || [ "$TAG" = "null" ]; then
|
||||
echo "::error::Could not determine latest release tag"
|
||||
exit 1
|
||||
fi
|
||||
echo "Latest release: $TAG"
|
||||
echo "tag=$TAG" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# Only move `:latest` when the release flagged "latest" is also the
|
||||
# highest semver release. This guards against a mis-click leaving an
|
||||
# older maintenance release (e.g. a 5.x patch shipped after 6.0 GA)
|
||||
# marked latest, which would otherwise roll `:latest` back a major
|
||||
# version on the next cron run. If it isn't the newest, we still
|
||||
# refresh that release's own version tag but leave `:latest` alone.
|
||||
HIGHEST=$(gh api --paginate "repos/${REPOSITORY}/releases" \
|
||||
--jq '.[] | select(.draft|not) | select(.prerelease|not) | .tag_name' \
|
||||
| sed 's/^v//' | sort -V | tail -n1)
|
||||
if [ "${TAG#v}" = "$HIGHEST" ]; then
|
||||
echo "force-latest=1" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "::warning::Latest-flagged release $TAG is not the highest semver ($HIGHEST); refreshing its version tag but leaving :latest untouched"
|
||||
fi
|
||||
|
||||
docker-rebuild:
|
||||
needs: config
|
||||
if: needs.config.outputs.has-secrets == '1'
|
||||
name: docker-rebuild
|
||||
runs-on: ubuntu-24.04
|
||||
strategy:
|
||||
# Mirror the same matrix the release publisher uses so every variant
|
||||
# operators consume from Docker Hub gets the refreshed base.
|
||||
matrix:
|
||||
build_preset: ["dev", "lean", "websocket", "dockerize", "py311", "py312"]
|
||||
fail-fast: false
|
||||
steps:
|
||||
- name: "Checkout release tag: ${{ needs.config.outputs.latest-release }}"
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
ref: ${{ needs.config.outputs.latest-release }}
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Docker Environment
|
||||
uses: ./.github/actions/setup-docker
|
||||
with:
|
||||
dockerhub-user: ${{ secrets.DOCKERHUB_USER }}
|
||||
dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
install-docker-compose: "false"
|
||||
build: "true"
|
||||
|
||||
- name: Use Node.js 20
|
||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
with:
|
||||
node-version: 20
|
||||
|
||||
- name: Setup supersetbot
|
||||
uses: ./.github/actions/setup-supersetbot/
|
||||
|
||||
- name: Rebuild and push
|
||||
env:
|
||||
DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
|
||||
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
BUILD_PRESET: ${{ matrix.build_preset }}
|
||||
LATEST_RELEASE: ${{ needs.config.outputs.latest-release }}
|
||||
FORCE_LATEST_FLAG: ${{ needs.config.outputs.force-latest == '1' && '--force-latest' || '' }}
|
||||
run: |
|
||||
# Reuses the same supersetbot invocation as the release
|
||||
# publisher (`tag-release.yml`), so the resulting tags are
|
||||
# identical to what a manual release dispatch would produce —
|
||||
# just with a freshly-pulled base image layer underneath.
|
||||
# `--force-latest` is only passed when the config job confirmed the
|
||||
# fetched release is the newest one (see FORCE_LATEST_FLAG above).
|
||||
supersetbot docker \
|
||||
--push \
|
||||
--preset "$BUILD_PRESET" \
|
||||
--context release \
|
||||
--context-ref "$LATEST_RELEASE" \
|
||||
$FORCE_LATEST_FLAG \
|
||||
--platform "linux/arm64" \
|
||||
--platform "linux/amd64"
|
||||
|
||||
# The whole point of this cron is catching base-image CVEs, so a silent
|
||||
# failure is the expensive case — a red X in the Actions tab nobody is
|
||||
# watching on a Monday. File a tracked issue when any rebuild leg fails so
|
||||
# a missed security refresh surfaces instead of sitting unnoticed.
|
||||
notify-on-failure:
|
||||
needs: [config, docker-rebuild]
|
||||
if: failure() && needs.config.outputs.has-secrets == '1'
|
||||
runs-on: ubuntu-24.04
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
steps:
|
||||
- name: Open a tracking issue
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
REPOSITORY: ${{ github.repository }}
|
||||
LATEST_RELEASE: ${{ needs.config.outputs.latest-release }}
|
||||
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
run: |
|
||||
gh issue create \
|
||||
--repo "$REPOSITORY" \
|
||||
--title "Scheduled Docker image refresh failed for ${LATEST_RELEASE}" \
|
||||
--label "infra:container" \
|
||||
--label "bug" \
|
||||
--body "The weekly Docker base-image refresh failed for release \`${LATEST_RELEASE}\`. Published images may be missing upstream base-layer security patches until this is resolved.
|
||||
|
||||
Failed run: ${RUN_URL}"
|
||||
37
.github/workflows/showtime-cleanup.yml
vendored
37
.github/workflows/showtime-cleanup.yml
vendored
@@ -1,37 +0,0 @@
|
||||
name: 🎪 Showtime Cleanup
|
||||
|
||||
# Scheduled cleanup of expired environments
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 */6 * * *' # Every 6 hours
|
||||
|
||||
# Manual trigger for testing
|
||||
workflow_dispatch:
|
||||
|
||||
# Common environment variables
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
AWS_REGION: ${{ vars.AWS_REGION || 'us-west-2' }}
|
||||
GITHUB_ORG: ${{ github.repository_owner }}
|
||||
GITHUB_REPO: ${{ github.event.repository.name }}
|
||||
|
||||
jobs:
|
||||
cleanup-expired:
|
||||
name: Clean up expired showtime environments
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write # delete orphaned showtime label definitions (label CRUD is the issues API)
|
||||
pull-requests: write
|
||||
|
||||
steps:
|
||||
- name: Install Superset Showtime
|
||||
run: pip install superset-showtime
|
||||
|
||||
- name: Cleanup expired environments and orphaned labels
|
||||
run: |
|
||||
echo "Cleaning up environments respecting TTL labels, and pruning orphaned 🎪 labels"
|
||||
python -m showtime cleanup --respect-ttl --force
|
||||
191
.github/workflows/showtime-trigger.yml
vendored
191
.github/workflows/showtime-trigger.yml
vendored
@@ -1,191 +0,0 @@
|
||||
name: 🎪 Superset Showtime
|
||||
|
||||
# Ultra-simple: just sync on any PR state change
|
||||
on:
|
||||
# zizmor: ignore[dangerous-triggers] - required to react to PR label changes; PR code is
|
||||
# only checked out and built after the maintainer-authorization gate (write/admin actors)
|
||||
pull_request_target:
|
||||
types: [labeled, unlabeled, synchronize, closed]
|
||||
|
||||
# Manual testing
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
pr_number:
|
||||
description: "PR number to sync"
|
||||
required: true
|
||||
type: number
|
||||
sha:
|
||||
description: "Specific SHA to deploy (optional, defaults to latest)"
|
||||
required: false
|
||||
type: string
|
||||
|
||||
# Common environment variables for all jobs (non-sensitive only)
|
||||
env:
|
||||
AWS_REGION: us-west-2
|
||||
GITHUB_ORG: ${{ github.repository_owner }}
|
||||
GITHUB_REPO: ${{ github.event.repository.name }}
|
||||
GITHUB_ACTOR: ${{ github.actor }}
|
||||
|
||||
jobs:
|
||||
sync:
|
||||
name: 🎪 Sync PR to desired state
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 90
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
|
||||
steps:
|
||||
- name: Security Check - Authorize Maintainers Only
|
||||
id: auth
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
with:
|
||||
script: |
|
||||
const actor = context.actor;
|
||||
console.log(`🔍 Checking authorization for ${actor}`);
|
||||
|
||||
// Early exit for workflow_dispatch - assume authorized since it's manually triggered
|
||||
if (context.eventName === 'workflow_dispatch') {
|
||||
console.log(`✅ Workflow dispatch event - assuming authorized for ${actor}`);
|
||||
core.setOutput('authorized', 'true');
|
||||
return;
|
||||
}
|
||||
|
||||
const { data: permission } = await github.rest.repos.getCollaboratorPermissionLevel({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
username: actor
|
||||
});
|
||||
|
||||
console.log(`📊 Permission level for ${actor}: ${permission.permission}`);
|
||||
const authorized = ['write', 'admin'].includes(permission.permission);
|
||||
|
||||
// If this is a synchronize event from unauthorized user, check if Showtime is active and set blocked label
|
||||
if (!authorized && context.eventName === 'pull_request_target' && context.payload.action === 'synchronize') {
|
||||
console.log(`🔒 Synchronize event detected - checking if Showtime is active`);
|
||||
|
||||
// Check if PR has any circus tent labels (Showtime is in use)
|
||||
const { data: issue } = await github.rest.issues.get({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: context.payload.pull_request.number
|
||||
});
|
||||
|
||||
const hasCircusLabels = issue.labels.some(label => label.name.startsWith('🎪 '));
|
||||
|
||||
if (hasCircusLabels) {
|
||||
console.log(`🎪 Circus labels found - setting blocked label to prevent auto-deployment`);
|
||||
|
||||
await github.rest.issues.addLabels({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: context.payload.pull_request.number,
|
||||
labels: ['🎪 🔒 showtime-blocked']
|
||||
});
|
||||
|
||||
console.log(`✅ Blocked label set - Showtime will detect and skip operations`);
|
||||
} else {
|
||||
console.log(`ℹ️ No circus labels found - Showtime not in use, skipping block`);
|
||||
}
|
||||
}
|
||||
|
||||
if (!authorized) {
|
||||
console.log(`🚨 Unauthorized user ${actor} - skipping all operations`);
|
||||
core.setOutput('authorized', 'false');
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`✅ Authorized maintainer: ${actor}`);
|
||||
core.setOutput('authorized', 'true');
|
||||
|
||||
- name: Install Superset Showtime
|
||||
if: steps.auth.outputs.authorized == 'true'
|
||||
run: |
|
||||
echo "::notice::Maintainer $GITHUB_ACTOR triggered deploy for PR ${PULL_REQUEST_NUMBER}"
|
||||
pip install --upgrade superset-showtime
|
||||
showtime version
|
||||
|
||||
env:
|
||||
PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number || github.event.inputs.pr_number }}
|
||||
- name: Check what actions are needed
|
||||
if: steps.auth.outputs.authorized == 'true'
|
||||
id: check
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
INPUT_PR_NUMBER: ${{ github.event.inputs.pr_number }}
|
||||
INPUT_SHA: ${{ github.event.inputs.sha }}
|
||||
run: |
|
||||
# Bulletproof PR number extraction
|
||||
if [[ -n "${{ github.event.pull_request.number }}" ]]; then
|
||||
PR_NUM="${{ github.event.pull_request.number }}"
|
||||
elif [[ -n "${INPUT_PR_NUMBER}" ]]; then
|
||||
PR_NUM="${INPUT_PR_NUMBER}"
|
||||
else
|
||||
echo "❌ No PR number found in event or inputs"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Using PR number: $PR_NUM"
|
||||
|
||||
# Run sync check-only with optional SHA override
|
||||
if [[ -n "${INPUT_SHA}" ]]; then
|
||||
OUTPUT=$(python -m showtime sync $PR_NUM --check-only --sha "${INPUT_SHA}")
|
||||
else
|
||||
OUTPUT=$(python -m showtime sync $PR_NUM --check-only)
|
||||
fi
|
||||
echo "$OUTPUT"
|
||||
|
||||
# Extract the outputs we need for conditional steps
|
||||
BUILD=$(echo "$OUTPUT" | grep "build_needed=" | cut -d'=' -f2)
|
||||
SYNC=$(echo "$OUTPUT" | grep "sync_needed=" | cut -d'=' -f2)
|
||||
PR_NUM_OUT=$(echo "$OUTPUT" | grep "pr_number=" | cut -d'=' -f2)
|
||||
TARGET_SHA=$(echo "$OUTPUT" | grep "target_sha=" | cut -d'=' -f2)
|
||||
|
||||
echo "build_needed=$BUILD" >> $GITHUB_OUTPUT
|
||||
echo "sync_needed=$SYNC" >> $GITHUB_OUTPUT
|
||||
echo "pr_number=$PR_NUM_OUT" >> $GITHUB_OUTPUT
|
||||
echo "target_sha=$TARGET_SHA" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Checkout PR code (only if build needed)
|
||||
if: steps.auth.outputs.authorized == 'true' && steps.check.outputs.build_needed == 'true'
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
ref: ${{ steps.check.outputs.target_sha }}
|
||||
persist-credentials: false
|
||||
# Building fork PR code is Showtime's purpose: deploys are gated on the
|
||||
# maintainer-authorization step above (write/admin actors only), so this
|
||||
# checkout is an explicit, authorized opt-in rather than an automatic one.
|
||||
allow-unsafe-pr-checkout: true
|
||||
|
||||
- name: Setup Docker Environment (only if build needed)
|
||||
if: steps.auth.outputs.authorized == 'true' && steps.check.outputs.build_needed == 'true'
|
||||
uses: ./.github/actions/setup-docker
|
||||
with:
|
||||
dockerhub-user: ${{ secrets.DOCKERHUB_USER }}
|
||||
dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
build: "true"
|
||||
install-docker-compose: "false"
|
||||
|
||||
- name: Execute sync (handles everything)
|
||||
if: steps.auth.outputs.authorized == 'true' && steps.check.outputs.sync_needed == 'true'
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
|
||||
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
CHECK_PR_NUMBER: ${{ steps.check.outputs.pr_number }}
|
||||
CHECK_TARGET_SHA: ${{ steps.check.outputs.target_sha }}
|
||||
run: |
|
||||
PR_NUM="$CHECK_PR_NUMBER"
|
||||
TARGET_SHA="$CHECK_TARGET_SHA"
|
||||
if [[ -n "$TARGET_SHA" ]]; then
|
||||
python -m showtime sync $PR_NUM --sha "$TARGET_SHA"
|
||||
else
|
||||
python -m showtime sync $PR_NUM
|
||||
fi
|
||||
71
.github/workflows/superset-app-cli.yml
vendored
71
.github/workflows/superset-app-cli.yml
vendored
@@ -1,71 +0,0 @@
|
||||
name: Superset App CLI tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- "master"
|
||||
- "[0-9].[0-9]*"
|
||||
pull_request:
|
||||
types: [synchronize, opened, reopened, ready_for_review]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
|
||||
# cancel previous workflow jobs for PRs
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
test-load-examples:
|
||||
runs-on: ubuntu-24.04
|
||||
env:
|
||||
PYTHONPATH: ${{ github.workspace }}
|
||||
SUPERSET_CONFIG: tests.integration_tests.superset_test_config
|
||||
REDIS_PORT: 16379
|
||||
SUPERSET__SQLALCHEMY_DATABASE_URI: postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:17-alpine
|
||||
env:
|
||||
POSTGRES_USER: superset
|
||||
POSTGRES_PASSWORD: superset
|
||||
ports:
|
||||
# Use custom ports for services to avoid accidentally connecting to
|
||||
# GitHub action runner's default installations
|
||||
- 15432:5432
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
ports:
|
||||
- 16379:6379
|
||||
steps:
|
||||
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: recursive
|
||||
- name: Check for file changes
|
||||
id: check
|
||||
uses: ./.github/actions/change-detector/
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
- name: Setup Python
|
||||
if: steps.check.outputs.python
|
||||
uses: ./.github/actions/setup-backend/
|
||||
- name: Setup Postgres
|
||||
if: steps.check.outputs.python
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: setup-postgres
|
||||
- name: superset init
|
||||
if: steps.check.outputs.python
|
||||
run: |
|
||||
uv pip install --system -e .
|
||||
superset db upgrade
|
||||
superset load_test_users
|
||||
- name: superset load_examples
|
||||
if: steps.check.outputs.python
|
||||
run: |
|
||||
# load examples without test data
|
||||
superset load_examples --load-big-data
|
||||
92
.github/workflows/superset-applitool-cypress.yml
vendored
Normal file
92
.github/workflows/superset-applitool-cypress.yml
vendored
Normal file
@@ -0,0 +1,92 @@
|
||||
name: Applitools Cypress
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "0 1 * * *"
|
||||
|
||||
jobs:
|
||||
config:
|
||||
runs-on: "ubuntu-latest"
|
||||
outputs:
|
||||
has-secrets: ${{ steps.check.outputs.has-secrets }}
|
||||
steps:
|
||||
- name: "Check for secrets"
|
||||
id: check
|
||||
shell: bash
|
||||
run: |
|
||||
if [ -n "${{ (secrets.APPLITOOLS_API_KEY != '' && secrets.APPLITOOLS_API_KEY != '') || '' }}" ]; then
|
||||
echo "has-secrets=1" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
cypress-applitools:
|
||||
needs: config
|
||||
if: needs.config.outputs.has-secrets
|
||||
runs-on: ubuntu-20.04
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
browser: ["chrome"]
|
||||
node: [16]
|
||||
env:
|
||||
SUPERSET_ENV: development
|
||||
SUPERSET_CONFIG: tests.integration_tests.superset_test_config
|
||||
SUPERSET__SQLALCHEMY_DATABASE_URI: postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset
|
||||
PYTHONPATH: ${{ github.workspace }}
|
||||
REDIS_PORT: 16379
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
APPLITOOLS_APP_NAME: Superset
|
||||
APPLITOOLS_API_KEY: ${{ secrets.APPLITOOLS_API_KEY }}
|
||||
APPLITOOLS_BATCH_ID: ${{ github.sha }}
|
||||
APPLITOOLS_BATCH_NAME: Superset Cypress
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:15-alpine
|
||||
env:
|
||||
POSTGRES_USER: superset
|
||||
POSTGRES_PASSWORD: superset
|
||||
ports:
|
||||
- 15432:5432
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
ports:
|
||||
- 16379:6379
|
||||
steps:
|
||||
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: recursive
|
||||
ref: master
|
||||
- name: Setup Python
|
||||
uses: ./.github/actions/setup-backend/
|
||||
- name: Import test data
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: testdata
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: ${{ matrix.node }}
|
||||
- name: Install npm dependencies
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: npm-install
|
||||
- name: Build javascript packages
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: build-instrumented-assets
|
||||
- name: Setup Postgres
|
||||
if: steps.check.outcome == 'failure'
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: setup-postgres
|
||||
- name: Install cypress
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: cypress-install
|
||||
- name: Run Cypress
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
env:
|
||||
CYPRESS_BROWSER: ${{ matrix.browser }}
|
||||
with:
|
||||
run: cypress-run-applitools
|
||||
55
.github/workflows/superset-applitools-storybook.yml
vendored
Normal file
55
.github/workflows/superset-applitools-storybook.yml
vendored
Normal file
@@ -0,0 +1,55 @@
|
||||
name: Applitools Storybook
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "0 0 * * *"
|
||||
|
||||
env:
|
||||
APPLITOOLS_APP_NAME: Superset
|
||||
APPLITOOLS_API_KEY: ${{ secrets.APPLITOOLS_API_KEY }}
|
||||
APPLITOOLS_BATCH_ID: ${{ github.sha }}
|
||||
APPLITOOLS_BATCH_NAME: Superset Storybook
|
||||
|
||||
jobs:
|
||||
config:
|
||||
runs-on: "ubuntu-latest"
|
||||
outputs:
|
||||
has-secrets: ${{ steps.check.outputs.has-secrets }}
|
||||
steps:
|
||||
- name: "Check for secrets"
|
||||
id: check
|
||||
shell: bash
|
||||
run: |
|
||||
if [ -n "${{ (secrets.APPLITOOLS_API_KEY != '' && secrets.APPLITOOLS_API_KEY != '') || '' }}" ]; then
|
||||
echo "has-secrets=1" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
cron:
|
||||
needs: config
|
||||
if: needs.config.outputs.has-secrets
|
||||
runs-on: ubuntu-20.04
|
||||
strategy:
|
||||
matrix:
|
||||
node: [16]
|
||||
steps:
|
||||
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: recursive
|
||||
ref: master
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: ${{ matrix.node }}
|
||||
- name: Install eyes-storybook dependencies
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: eyes-storybook-dependencies
|
||||
- name: Install NPM dependencies
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: npm-install
|
||||
- name: Run Applitools Eyes-Storybook
|
||||
working-directory: ./superset-frontend
|
||||
run: npx eyes-storybook -u https://superset-storybook.netlify.app/
|
||||
72
.github/workflows/superset-cli.yml
vendored
Normal file
72
.github/workflows/superset-cli.yml
vendored
Normal file
@@ -0,0 +1,72 @@
|
||||
name: Superset CLI tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- "master"
|
||||
- "[0-9].[0-9]"
|
||||
pull_request:
|
||||
types: [synchronize, opened, reopened, ready_for_review]
|
||||
|
||||
# cancel previous workflow jobs for PRs
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
test-load-examples:
|
||||
runs-on: ubuntu-20.04
|
||||
strategy:
|
||||
matrix:
|
||||
python-version: ["3.9"]
|
||||
env:
|
||||
PYTHONPATH: ${{ github.workspace }}
|
||||
SUPERSET_CONFIG: tests.integration_tests.superset_test_config
|
||||
REDIS_PORT: 16379
|
||||
SUPERSET__SQLALCHEMY_DATABASE_URI: postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:15-alpine
|
||||
env:
|
||||
POSTGRES_USER: superset
|
||||
POSTGRES_PASSWORD: superset
|
||||
ports:
|
||||
# Use custom ports for services to avoid accidentally connecting to
|
||||
# GitHub action runner's default installations
|
||||
- 15432:5432
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
ports:
|
||||
- 16379:6379
|
||||
steps:
|
||||
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: recursive
|
||||
- name: Check if python changes are present
|
||||
id: check
|
||||
env:
|
||||
GITHUB_REPO: ${{ github.repository }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
continue-on-error: true
|
||||
run: ./scripts/ci_check_no_file_changes.sh python
|
||||
- name: Setup Python
|
||||
uses: ./.github/actions/setup-backend/
|
||||
if: steps.check.outcome == 'failure'
|
||||
- name: Setup Postgres
|
||||
if: steps.check.outcome == 'failure'
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: setup-postgres
|
||||
- name: superset init
|
||||
if: steps.check.outcome == 'failure'
|
||||
run: |
|
||||
pip install -e .
|
||||
superset db upgrade
|
||||
superset load_test_users
|
||||
- name: superset load_examples
|
||||
if: steps.check.outcome == 'failure'
|
||||
run: |
|
||||
# load examples without test data
|
||||
superset load_examples --load-big-data
|
||||
104
.github/workflows/superset-docs-deploy.yml
vendored
104
.github/workflows/superset-docs-deploy.yml
vendored
@@ -1,39 +1,16 @@
|
||||
name: Docs Deployment
|
||||
|
||||
on:
|
||||
# Deploy after integration tests complete on master
|
||||
# zizmor: ignore[dangerous-triggers] - runs in base-branch context after a trusted upstream workflow; scoped to master
|
||||
workflow_run:
|
||||
workflows: ["Python-Integration"]
|
||||
types: [completed]
|
||||
branches: [master]
|
||||
|
||||
# Also allow manual trigger and direct pushes to docs
|
||||
push:
|
||||
paths:
|
||||
- "docs/**"
|
||||
- "README.md"
|
||||
branches:
|
||||
- "master"
|
||||
|
||||
workflow_dispatch: {}
|
||||
|
||||
# Serialize deploys: the action pushes to apache/superset-site without
|
||||
# rebasing, so concurrent runs race on the final push and the loser fails
|
||||
# with `! [rejected] asf-site -> asf-site (fetch first)`. Cancel any
|
||||
# in-progress run as soon as a newer one starts — the destination repo
|
||||
# isn't touched until the final push step, so canceling mid-build is safe,
|
||||
# and the freshest content always wins.
|
||||
concurrency:
|
||||
group: docs-deploy-asf-site
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
- "[0-9].[0-9]"
|
||||
|
||||
jobs:
|
||||
config:
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: "ubuntu-latest"
|
||||
outputs:
|
||||
has-secrets: ${{ steps.check.outputs.has-secrets }}
|
||||
steps:
|
||||
@@ -41,87 +18,36 @@ jobs:
|
||||
id: check
|
||||
shell: bash
|
||||
run: |
|
||||
if [ -n "${SUPERSET_SITE_BUILD}" ]; then
|
||||
if [ -n "${{ (secrets.SUPERSET_SITE_BUILD != '' && secrets.SUPERSET_SITE_BUILD != '') || '' }}" ]; then
|
||||
echo "has-secrets=1" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
env:
|
||||
SUPERSET_SITE_BUILD: ${{ (secrets.SUPERSET_SITE_BUILD != '' && secrets.SUPERSET_SITE_BUILD != '') || '' }}
|
||||
build-deploy:
|
||||
needs: config
|
||||
# For workflow_run triggers, only deploy when the triggering run originated
|
||||
# from this repository (not a fork), ensuring the checked-out code and any
|
||||
# local actions executed with deploy credentials are trusted.
|
||||
if: >-
|
||||
needs.config.outputs.has-secrets &&
|
||||
(github.event_name != 'workflow_run' ||
|
||||
github.event.workflow_run.head_repository.full_name == github.repository)
|
||||
if: needs.config.outputs.has-secrets
|
||||
name: Build & Deploy
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: "ubuntu-latest"
|
||||
defaults:
|
||||
run:
|
||||
working-directory: docs
|
||||
steps:
|
||||
- name: "Checkout ${{ github.event.workflow_run.head_sha || github.sha }}"
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.event.workflow_run.head_sha || github.sha }}
|
||||
persist-credentials: false
|
||||
submodules: recursive
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
- name: Set up Node.js 16
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version-file: "./docs/.nvmrc"
|
||||
- name: Setup Python
|
||||
uses: ./.github/actions/setup-backend/
|
||||
- uses: actions/setup-java@0f481fcb613427c0f801b606911222b5b6f3083a # v5.5.0
|
||||
with:
|
||||
distribution: "zulu"
|
||||
java-version: "21"
|
||||
- name: Install Graphviz
|
||||
run: sudo apt-get install -y graphviz
|
||||
- name: Compute Entity Relationship diagram (ERD)
|
||||
env:
|
||||
SUPERSET_SECRET_KEY: not-a-secret
|
||||
run: |
|
||||
python scripts/erd/erd.py
|
||||
curl -L http://sourceforge.net/projects/plantuml/files/1.2023.7/plantuml.1.2023.7.jar/download > ~/plantuml.jar
|
||||
java -jar ~/plantuml.jar -v -tsvg -r -o "${{ github.workspace }}/docs/static/img/" "${{ github.workspace }}/scripts/erd/erd.puml"
|
||||
node-version: "16"
|
||||
- name: yarn install
|
||||
working-directory: docs
|
||||
run: |
|
||||
yarn install --check-cache
|
||||
- name: Download database diagnostics (if triggered by integration tests)
|
||||
if: github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success'
|
||||
uses: dawidd6/action-download-artifact@b6e2e70617bc3265edd6dab6c906732b2f1ae151 # v21
|
||||
continue-on-error: true
|
||||
with:
|
||||
workflow: superset-python-integrationtest.yml
|
||||
run_id: ${{ github.event.workflow_run.id }}
|
||||
name: database-diagnostics
|
||||
path: docs/src/data/
|
||||
- name: Try to download latest diagnostics (for push/dispatch triggers)
|
||||
if: github.event_name != 'workflow_run'
|
||||
uses: dawidd6/action-download-artifact@b6e2e70617bc3265edd6dab6c906732b2f1ae151 # v21
|
||||
continue-on-error: true
|
||||
with:
|
||||
workflow: superset-python-integrationtest.yml
|
||||
name: database-diagnostics
|
||||
path: docs/src/data/
|
||||
branch: master
|
||||
search_artifacts: true
|
||||
if_no_artifact_found: warn
|
||||
- name: Use diagnostics artifact if available
|
||||
working-directory: docs
|
||||
run: |
|
||||
if [ -f "src/data/databases-diagnostics.json" ]; then
|
||||
echo "Using fresh diagnostics from integration tests"
|
||||
mv src/data/databases-diagnostics.json src/data/databases.json
|
||||
else
|
||||
echo "Using committed databases.json (no artifact found)"
|
||||
fi
|
||||
- name: yarn build
|
||||
working-directory: docs
|
||||
run: |
|
||||
yarn build
|
||||
- name: deploy docs
|
||||
if: github.ref == 'refs/heads/master'
|
||||
uses: ./.github/actions/github-action-push-to-another-repository
|
||||
env:
|
||||
API_TOKEN_GITHUB: ${{ secrets.SUPERSET_SITE_BUILD }}
|
||||
@@ -130,5 +56,5 @@ jobs:
|
||||
destination-github-username: "apache"
|
||||
destination-repository-name: "superset-site"
|
||||
target-branch: "asf-site"
|
||||
commit-message: "deploying docs: ${{ github.event.head_commit.message || 'triggered by integration tests' }} (apache/superset@${{ github.event.workflow_run.head_sha || github.sha }})"
|
||||
commit-message: "deploying docs: ${{ github.event.head_commit.message }} (apache/superset@${{ github.sha }})"
|
||||
user-email: dev@superset.apache.org
|
||||
|
||||
125
.github/workflows/superset-docs-verify.yml
vendored
125
.github/workflows/superset-docs-verify.yml
vendored
@@ -4,142 +4,33 @@ on:
|
||||
pull_request:
|
||||
paths:
|
||||
- "docs/**"
|
||||
- "superset/db_engine_specs/**"
|
||||
- ".github/workflows/superset-docs-verify.yml"
|
||||
types: [synchronize, opened, reopened, ready_for_review]
|
||||
# zizmor: ignore[dangerous-triggers] - runs in base-branch context and only consumes artifacts from the trusted upstream workflow
|
||||
workflow_run:
|
||||
workflows: ["Python-Integration"]
|
||||
types: [completed]
|
||||
|
||||
# cancel previous workflow jobs for PRs
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.event.workflow_run.head_sha || github.run_id }}
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
linkinator:
|
||||
# See docs here: https://github.com/marketplace/actions/linkinator
|
||||
# Only run on pull_request, not workflow_run
|
||||
if: github.event_name == 'pull_request'
|
||||
name: Link Checking
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
# Do not bump this linkinator-action version without opening
|
||||
# an ASF Infra ticket to allow the new version first!
|
||||
- uses: JustinBeckwith/linkinator-action@af984b9f30f63e796ae2ea5be5e07cb587f1bbd9 # v2.3
|
||||
continue-on-error: true # This will make the job advisory (non-blocking, no red X)
|
||||
with:
|
||||
paths: "**/*.md, **/*.mdx"
|
||||
linksToSkip: >-
|
||||
^https://github.com/apache/(superset|incubator-superset)/(pull|issues)/\d+,
|
||||
^https://github.com/apache/(superset|incubator-superset)/commit/[a-f0-9]+,
|
||||
superset-frontend/.*CHANGELOG\.md,
|
||||
http://localhost:8088/,
|
||||
http://127.0.0.1:3000/,
|
||||
http://localhost:9001/,
|
||||
https://charts.bitnami.com/bitnami,
|
||||
https://www.li.me/,
|
||||
https://www.fanatics.com/,
|
||||
https://tails.com/gb/,
|
||||
https://www.techaudit.info/,
|
||||
https://avetilearning.com/,
|
||||
https://www.udemy.com/,
|
||||
https://trustmedis.com/,
|
||||
http://theiconic.com.au/,
|
||||
https://dev.mysql.com/doc/refman/5.7/en/innodb-limits.html,
|
||||
^https://img\.shields\.io/.*,
|
||||
https://vkusvill.ru/,
|
||||
https://www.linkedin.com/in/mark-thomas-b16751158/,
|
||||
https://theiconic.com.au/,
|
||||
https://wattbewerb.de/,
|
||||
https://timbr.ai/,
|
||||
https://opensource.org/license/apache-2-0,
|
||||
https://www.plaidcloud.com/
|
||||
|
||||
build-on-pr:
|
||||
# Build docs when PR changes docs/** (uses committed databases.json)
|
||||
if: github.event_name == 'pull_request'
|
||||
name: Build (PR trigger)
|
||||
runs-on: ubuntu-24.04
|
||||
build-deploy:
|
||||
name: Build & Deploy
|
||||
runs-on: ubuntu-20.04
|
||||
defaults:
|
||||
run:
|
||||
working-directory: docs
|
||||
steps:
|
||||
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: recursive
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
- name: Set up Node.js 16
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version-file: "./docs/.nvmrc"
|
||||
node-version: '16'
|
||||
- name: yarn install
|
||||
run: |
|
||||
yarn install --check-cache
|
||||
- name: Lint docs links
|
||||
# Fast source-level check for bare relative internal links
|
||||
# like `[Foo](../foo)` that Docusaurus's onBrokenLinks
|
||||
# setting can't catch. Runs in seconds; fails fast before
|
||||
# the expensive build step.
|
||||
run: |
|
||||
yarn lint:docs-links
|
||||
- name: yarn typecheck
|
||||
run: |
|
||||
yarn typecheck
|
||||
- name: yarn build
|
||||
run: |
|
||||
yarn build
|
||||
|
||||
build-after-tests:
|
||||
# Build docs after integration tests complete (uses fresh diagnostics)
|
||||
# Only runs if integration tests succeeded
|
||||
if: >
|
||||
github.event_name == 'workflow_run' &&
|
||||
github.event.workflow_run.conclusion == 'success' &&
|
||||
github.event.workflow_run.head_repository.full_name == github.repository
|
||||
name: Build (after integration tests)
|
||||
runs-on: ubuntu-24.04
|
||||
defaults:
|
||||
run:
|
||||
working-directory: docs
|
||||
steps:
|
||||
- name: "Checkout PR head: ${{ github.event.workflow_run.head_sha }}"
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
ref: ${{ github.event.workflow_run.head_sha }}
|
||||
persist-credentials: false
|
||||
submodules: recursive
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
with:
|
||||
node-version-file: "./docs/.nvmrc"
|
||||
- name: yarn install
|
||||
run: |
|
||||
yarn install --check-cache
|
||||
- name: Download database diagnostics from integration tests
|
||||
uses: dawidd6/action-download-artifact@b6e2e70617bc3265edd6dab6c906732b2f1ae151 # v21
|
||||
with:
|
||||
workflow: superset-python-integrationtest.yml
|
||||
run_id: ${{ github.event.workflow_run.id }}
|
||||
name: database-diagnostics
|
||||
path: docs/src/data/
|
||||
if_no_artifact_found: "warning"
|
||||
- name: Use fresh diagnostics
|
||||
run: |
|
||||
if [ -f "src/data/databases-diagnostics.json" ]; then
|
||||
echo "Using fresh diagnostics from integration tests"
|
||||
mv src/data/databases-diagnostics.json src/data/databases.json
|
||||
else
|
||||
echo "Warning: No diagnostics artifact found, using committed data"
|
||||
fi
|
||||
- name: yarn typecheck
|
||||
run: |
|
||||
yarn typecheck
|
||||
|
||||
292
.github/workflows/superset-e2e.yml
vendored
292
.github/workflows/superset-e2e.yml
vendored
@@ -4,73 +4,29 @@ on:
|
||||
push:
|
||||
branches:
|
||||
- "master"
|
||||
- "[0-9].[0-9]*"
|
||||
- "[0-9].[0-9]"
|
||||
pull_request:
|
||||
types: [synchronize, opened, reopened, ready_for_review]
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
use_dashboard:
|
||||
description: "Use Cypress Dashboard (true/false) [paid service - trigger manually when needed]. You MUST provide a branch and/or PR number below for this to work."
|
||||
required: false
|
||||
default: "false"
|
||||
ref:
|
||||
description: "The branch or tag to checkout"
|
||||
required: false
|
||||
default: ""
|
||||
pr_id:
|
||||
description: "The pull request ID to checkout"
|
||||
required: false
|
||||
default: ""
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
changes:
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
outputs:
|
||||
python: ${{ steps.check.outputs.python }}
|
||||
frontend: ${{ steps.check.outputs.frontend }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Check for file changes
|
||||
id: check
|
||||
uses: ./.github/actions/change-detector/
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
cypress-matrix:
|
||||
needs: changes
|
||||
if: (needs.changes.outputs.python == 'true' || needs.changes.outputs.frontend == 'true') && github.event.pull_request.draft == false
|
||||
# Somehow one test flakes on 24.04 for unknown reasons, this is the only GHA left on 22.04
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
runs-on: ubuntu-20.04
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
strategy:
|
||||
# when one test fails, DO NOT cancel the other
|
||||
# parallel_id, because this will kill Cypress processes
|
||||
# containers, because this will kill Cypress processes
|
||||
# leaving the Dashboard hanging ...
|
||||
# https://github.com/cypress-io/github-action/issues/48
|
||||
fail-fast: false
|
||||
matrix:
|
||||
parallel_id: [0, 1]
|
||||
containers: [1, 2, 3]
|
||||
browser: ["chrome"]
|
||||
app_root: ${{ github.event_name == 'push' && fromJSON('["", "/app/prefix"]') || fromJSON('[""]') }}
|
||||
# The /app/prefix variant (push events only) is smoke-tested on a single
|
||||
# shard rather than the full matrix, so exclude it from the other shards.
|
||||
exclude:
|
||||
- parallel_id: 1
|
||||
app_root: "/app/prefix"
|
||||
env:
|
||||
SUPERSET_ENV: development
|
||||
SUPERSET_CONFIG: tests.integration_tests.superset_test_config
|
||||
@@ -78,11 +34,9 @@ jobs:
|
||||
PYTHONPATH: ${{ github.workspace }}
|
||||
REDIS_PORT: 16379
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
# Only use dashboard when explicitly requested via workflow_dispatch
|
||||
USE_DASHBOARD: ${{ github.event.inputs.use_dashboard == 'true' || 'false' }}
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:17-alpine
|
||||
image: postgres:15-alpine
|
||||
env:
|
||||
POSTGRES_USER: superset
|
||||
POSTGRES_PASSWORD: superset
|
||||
@@ -93,251 +47,69 @@ jobs:
|
||||
ports:
|
||||
- 16379:6379
|
||||
steps:
|
||||
# -------------------------------------------------------
|
||||
# Conditional checkout based on context
|
||||
- name: Checkout for push or pull_request event
|
||||
if: github.event_name == 'push' || github.event_name == 'pull_request'
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
- name: "Checkout (pull) ${{ github.ref }} ( ${{ github.sha }} )"
|
||||
uses: actions/checkout@v4
|
||||
if: github.event_name == 'push'
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: recursive
|
||||
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
|
||||
- name: Checkout using ref (workflow_dispatch)
|
||||
if: github.event_name == 'workflow_dispatch' && github.event.inputs.ref != ''
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
- name: "Checkout (pull_request) ${{ github.ref }} ( ${{ github.sha }} )"
|
||||
uses: actions/checkout@v4
|
||||
if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target'
|
||||
with:
|
||||
ref: "refs/pull/${{ github.event.number }}/merge"
|
||||
persist-credentials: false
|
||||
ref: ${{ github.event.inputs.ref }}
|
||||
submodules: recursive
|
||||
- name: Checkout using PR ID (workflow_dispatch)
|
||||
if: github.event_name == 'workflow_dispatch' && github.event.inputs.pr_id != ''
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: refs/pull/${{ github.event.inputs.pr_id }}/merge
|
||||
submodules: recursive
|
||||
# -------------------------------------------------------
|
||||
- name: Check if python or frontend changes are present
|
||||
id: check
|
||||
env:
|
||||
GITHUB_REPO: ${{ github.repository }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
continue-on-error: true
|
||||
run: ./scripts/ci_check_no_file_changes.sh python frontend
|
||||
- name: Setup Python
|
||||
uses: ./.github/actions/setup-backend/
|
||||
if: steps.check.outcome == 'failure'
|
||||
- name: Setup postgres
|
||||
if: steps.check.outcome == 'failure'
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: setup-postgres
|
||||
- name: Import test data
|
||||
if: steps.check.outcome == 'failure'
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: testdata
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version-file: "./superset-frontend/.nvmrc"
|
||||
cache: "npm"
|
||||
cache-dependency-path: "superset-frontend/package-lock.json"
|
||||
node-version: "16"
|
||||
- name: Install npm dependencies
|
||||
if: steps.check.outcome == 'failure'
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: npm-install
|
||||
- name: Build javascript packages
|
||||
if: steps.check.outcome == 'failure'
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: build-instrumented-assets
|
||||
- name: Install cypress
|
||||
if: steps.check.outcome == 'failure'
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: cypress-install
|
||||
- name: Run Cypress
|
||||
if: steps.check.outcome == 'failure'
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
env:
|
||||
CYPRESS_BROWSER: ${{ matrix.browser }}
|
||||
PARALLEL_ID: ${{ matrix.parallel_id }}
|
||||
PARALLELISM: 2
|
||||
CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
|
||||
NODE_OPTIONS: "--max-old-space-size=4096"
|
||||
CYPRESS_KEY: YjljODE2MzAtODcwOC00NTA3LWE4NmMtMTU3YmFmMjIzOTRhCg==
|
||||
with:
|
||||
run: cypress-run-all ${{ env.USE_DASHBOARD }} ${{ matrix.app_root }}
|
||||
- name: Set safe app root
|
||||
if: failure()
|
||||
id: set-safe-app-root
|
||||
env:
|
||||
APP_ROOT: ${{ matrix.app_root }}
|
||||
run: |
|
||||
SAFE_APP_ROOT=${APP_ROOT//\//_}
|
||||
echo "safe_app_root=$SAFE_APP_ROOT" >> $GITHUB_OUTPUT
|
||||
run: cypress-run-all
|
||||
- name: Upload Artifacts
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
uses: actions/upload-artifact@v4
|
||||
if: failure()
|
||||
with:
|
||||
name: screenshots
|
||||
path: ${{ github.workspace }}/superset-frontend/cypress-base/cypress/screenshots
|
||||
name: cypress-artifact-${{ github.run_id }}-${{ github.job }}-${{ matrix.browser }}-${{ matrix.parallel_id }}--${{ steps.set-safe-app-root.outputs.safe_app_root }}
|
||||
|
||||
playwright-tests:
|
||||
needs: changes
|
||||
if: needs.changes.outputs.python == 'true' || needs.changes.outputs.frontend == 'true'
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
browser: ["chromium"]
|
||||
app_root: ${{ github.event_name == 'push' && fromJSON('["", "/app/prefix"]') || fromJSON('[""]') }}
|
||||
env:
|
||||
SUPERSET_ENV: development
|
||||
SUPERSET_CONFIG: tests.integration_tests.superset_test_config
|
||||
SUPERSET__SQLALCHEMY_DATABASE_URI: postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset
|
||||
PYTHONPATH: ${{ github.workspace }}
|
||||
REDIS_PORT: 16379
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:17-alpine
|
||||
env:
|
||||
POSTGRES_USER: superset
|
||||
POSTGRES_PASSWORD: superset
|
||||
ports:
|
||||
- 15432:5432
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
ports:
|
||||
- 16379:6379
|
||||
steps:
|
||||
# -------------------------------------------------------
|
||||
# Conditional checkout based on context (same as Cypress workflow)
|
||||
- name: Checkout for push or pull_request event
|
||||
if: github.event_name == 'push' || github.event_name == 'pull_request'
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: recursive
|
||||
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
|
||||
- name: Checkout using ref (workflow_dispatch)
|
||||
if: github.event_name == 'workflow_dispatch' && github.event.inputs.ref != ''
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: ${{ github.event.inputs.ref }}
|
||||
submodules: recursive
|
||||
- name: Checkout using PR ID (workflow_dispatch)
|
||||
if: github.event_name == 'workflow_dispatch' && github.event.inputs.pr_id != ''
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: refs/pull/${{ github.event.inputs.pr_id }}/merge
|
||||
submodules: recursive
|
||||
# -------------------------------------------------------
|
||||
- name: Setup Python
|
||||
uses: ./.github/actions/setup-backend/
|
||||
- name: Setup postgres
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: setup-postgres
|
||||
- name: Import test data
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: playwright_testdata
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
with:
|
||||
node-version-file: "./superset-frontend/.nvmrc"
|
||||
cache: "npm"
|
||||
cache-dependency-path: "superset-frontend/package-lock.json"
|
||||
- name: Install npm dependencies
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: npm-install
|
||||
- name: Build javascript packages
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: build-instrumented-assets
|
||||
- name: Build embedded SDK
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: build-embedded-sdk
|
||||
- name: Install Playwright
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: playwright-install
|
||||
- name: Run Playwright (Required Tests)
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
env:
|
||||
NODE_OPTIONS: "--max-old-space-size=4096"
|
||||
with:
|
||||
run: playwright-run "${{ matrix.app_root }}"
|
||||
- name: Set safe app root
|
||||
if: failure()
|
||||
id: set-safe-app-root
|
||||
env:
|
||||
APP_ROOT: ${{ matrix.app_root }}
|
||||
run: |
|
||||
SAFE_APP_ROOT=${APP_ROOT//\//_}
|
||||
echo "safe_app_root=$SAFE_APP_ROOT" >> $GITHUB_OUTPUT
|
||||
- name: Upload Playwright Artifacts
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
if: failure()
|
||||
with:
|
||||
path: |
|
||||
${{ github.workspace }}/superset-frontend/playwright-results/
|
||||
${{ github.workspace }}/superset-frontend/test-results/
|
||||
name: playwright-artifact-${{ github.run_id }}-${{ github.job }}-${{ matrix.browser }}--${{ steps.set-safe-app-root.outputs.safe_app_root }}
|
||||
|
||||
# Stable required-status-check anchors. cypress-matrix and playwright-tests
|
||||
# are matrix jobs gated on change detection (python || frontend). On a PR
|
||||
# that touches neither — e.g. a docs-only PR — they are skipped at the job
|
||||
# level, which happens before matrix expansion, so the per-combination
|
||||
# contexts (`cypress-matrix (0, chrome)`, `playwright-tests (chromium)`) are
|
||||
# never produced and branch protection waits on them forever. These
|
||||
# always-running jobs report a single stable context that passes when the
|
||||
# underlying matrix job succeeded or was skipped, and fails only on a real
|
||||
# failure. Require these in .asf.yaml instead of the matrix-expanded names.
|
||||
#
|
||||
# A matrix job reads as "skipped" in two distinct cases, and only the first
|
||||
# is a legitimate pass: (a) change detection succeeded and gated the job off
|
||||
# (docs-only PR); (b) the `changes` job itself failed or was cancelled, in
|
||||
# which case GHA skips its dependents too. Accepting (b) would let a broken
|
||||
# change-detector report a false green, so each anchor first requires
|
||||
# `changes` to have succeeded before honouring a skip.
|
||||
cypress-matrix-required:
|
||||
needs: [changes, cypress-matrix]
|
||||
if: always()
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 5
|
||||
permissions: {}
|
||||
steps:
|
||||
- name: Check cypress-matrix result
|
||||
env:
|
||||
CHANGES: ${{ needs.changes.result }}
|
||||
RESULT: ${{ needs.cypress-matrix.result }}
|
||||
run: |
|
||||
if [ "$CHANGES" != "success" ]; then
|
||||
echo "change detection did not succeed (result: $CHANGES); refusing to pass on a skipped matrix"
|
||||
exit 1
|
||||
fi
|
||||
if [ "$RESULT" != "success" ] && [ "$RESULT" != "skipped" ]; then
|
||||
echo "cypress-matrix did not pass (result: $RESULT)"
|
||||
exit 1
|
||||
fi
|
||||
echo "cypress-matrix result: $RESULT (changes: $CHANGES)"
|
||||
|
||||
playwright-tests-required:
|
||||
needs: [changes, playwright-tests]
|
||||
if: always()
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 5
|
||||
permissions: {}
|
||||
steps:
|
||||
- name: Check playwright-tests result
|
||||
env:
|
||||
CHANGES: ${{ needs.changes.result }}
|
||||
RESULT: ${{ needs.playwright-tests.result }}
|
||||
run: |
|
||||
if [ "$CHANGES" != "success" ]; then
|
||||
echo "change detection did not succeed (result: $CHANGES); refusing to pass on a skipped matrix"
|
||||
exit 1
|
||||
fi
|
||||
if [ "$RESULT" != "success" ] && [ "$RESULT" != "skipped" ]; then
|
||||
echo "playwright-tests did not pass (result: $RESULT)"
|
||||
exit 1
|
||||
fi
|
||||
echo "playwright-tests result: $RESULT (changes: $CHANGES)"
|
||||
|
||||
71
.github/workflows/superset-extensions-cli.yml
vendored
71
.github/workflows/superset-extensions-cli.yml
vendored
@@ -1,71 +0,0 @@
|
||||
name: Superset Extensions CLI Package Tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- "master"
|
||||
- "[0-9].[0-9]*"
|
||||
pull_request:
|
||||
types: [synchronize, opened, reopened, ready_for_review]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
|
||||
# cancel previous workflow jobs for PRs
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
test-superset-extensions-cli-package:
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
matrix:
|
||||
# Full version spread on push (master/release) + nightly; current only
|
||||
# on PRs to cut runner cost (cross-version breaks are caught at merge).
|
||||
python-version: ${{ github.event_name == 'pull_request' && fromJSON('["current"]') || fromJSON('["current", "next"]') }}
|
||||
defaults:
|
||||
run:
|
||||
working-directory: superset-extensions-cli
|
||||
steps:
|
||||
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: recursive
|
||||
|
||||
- name: Check for file changes
|
||||
id: check
|
||||
uses: ./.github/actions/change-detector/
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Setup Python
|
||||
if: steps.check.outputs.superset-extensions-cli
|
||||
uses: ./.github/actions/setup-backend/
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
requirements-type: dev
|
||||
|
||||
- name: Run pytest with coverage
|
||||
if: steps.check.outputs.superset-extensions-cli
|
||||
run: |
|
||||
pytest --cov=superset_extensions_cli --cov-report=xml --cov-report=term-missing --cov-report=html -v --tb=short
|
||||
|
||||
- name: Upload coverage reports to Codecov
|
||||
if: steps.check.outputs.superset-extensions-cli
|
||||
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
|
||||
with:
|
||||
file: ./coverage.xml
|
||||
flags: superset-extensions-cli
|
||||
name: superset-extensions-cli-coverage
|
||||
fail_ci_if_error: false
|
||||
|
||||
- name: Upload HTML coverage report
|
||||
if: steps.check.outputs.superset-extensions-cli
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
with:
|
||||
name: superset-extensions-cli-coverage-html
|
||||
path: htmlcov/
|
||||
242
.github/workflows/superset-frontend.yml
vendored
242
.github/workflows/superset-frontend.yml
vendored
@@ -1,209 +1,89 @@
|
||||
name: "Frontend Build CI (unit tests, linting & sanity checks)"
|
||||
name: Frontend
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- "master"
|
||||
- "[0-9].[0-9]*"
|
||||
- "[0-9].[0-9]"
|
||||
paths:
|
||||
- "superset-frontend/**"
|
||||
pull_request:
|
||||
types: [synchronize, opened, reopened, ready_for_review]
|
||||
paths:
|
||||
- "superset-frontend/**"
|
||||
|
||||
# cancel previous workflow jobs for PRs
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
TAG: apache/superset:GHA-${{ github.run_id }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
frontend-build:
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
outputs:
|
||||
should-run: ${{ steps.check.outputs.frontend }}
|
||||
runs-on: ubuntu-20.04
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
fetch-depth: 0
|
||||
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
|
||||
|
||||
- name: Check for File Changes
|
||||
submodules: recursive
|
||||
- name: Check npm lock file version
|
||||
run: ./scripts/ci_check_npm_lock_version.sh ./superset-frontend/package-lock.json
|
||||
- name: Check if frontend changes are present
|
||||
id: check
|
||||
uses: ./.github/actions/change-detector/
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Build Docker Image
|
||||
if: steps.check.outputs.frontend
|
||||
shell: bash
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
echo "git rev-parse --short HEAD"
|
||||
git rev-parse --short HEAD
|
||||
echo "git show -s --format=raw HEAD"
|
||||
git show -s --format=raw HEAD
|
||||
docker buildx build \
|
||||
-t $TAG \
|
||||
--cache-from=type=registry,ref=apache/superset-cache:3.11-slim-trixie \
|
||||
--target superset-node-ci \
|
||||
.
|
||||
|
||||
- name: Save Docker Image as Artifact
|
||||
if: steps.check.outputs.frontend
|
||||
run: |
|
||||
docker save $TAG | zstd -3 --threads=0 > docker-image.tar.zst
|
||||
|
||||
- name: Upload Docker Image Artifact
|
||||
if: steps.check.outputs.frontend
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
GITHUB_REPO: ${{ github.repository }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
continue-on-error: true
|
||||
run: ./scripts/ci_check_no_file_changes.sh frontend
|
||||
- name: Setup Node.js
|
||||
if: steps.check.outcome == 'failure'
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
name: docker-image
|
||||
path: docker-image.tar.zst
|
||||
|
||||
sharded-jest-tests:
|
||||
needs: frontend-build
|
||||
if: needs.frontend-build.outputs.should-run == 'true'
|
||||
strategy:
|
||||
matrix:
|
||||
shard: [1, 2, 3, 4, 5, 6, 7, 8]
|
||||
fail-fast: false
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- name: Download Docker Image Artifact
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
|
||||
node-version: "16"
|
||||
- name: Install dependencies
|
||||
if: steps.check.outcome == 'failure'
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
name: docker-image
|
||||
|
||||
- name: Load Docker Image
|
||||
run: npm-install
|
||||
- name: eslint
|
||||
if: steps.check.outcome == 'failure'
|
||||
working-directory: ./superset-frontend
|
||||
run: |
|
||||
zstd -d < docker-image.tar.zst | docker load
|
||||
|
||||
- name: npm run test with coverage
|
||||
run: |
|
||||
mkdir -p ${{ github.workspace }}/superset-frontend/coverage
|
||||
docker run \
|
||||
-v ${{ github.workspace }}/superset-frontend/coverage:/app/superset-frontend/coverage \
|
||||
--rm $TAG \
|
||||
bash -c \
|
||||
"npm run test -- --coverage --shard=${{ matrix.shard }}/8 --coverageReporters=json"
|
||||
|
||||
- name: Upload Coverage Artifact
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
with:
|
||||
name: coverage-artifacts-${{ matrix.shard }}
|
||||
path: superset-frontend/coverage
|
||||
|
||||
report-coverage:
|
||||
needs: [sharded-jest-tests]
|
||||
if: needs.frontend-build.outputs.should-run == 'true'
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 15
|
||||
permissions:
|
||||
id-token: write
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
fetch-depth: 0
|
||||
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
|
||||
|
||||
- name: Download Coverage Artifacts
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
|
||||
with:
|
||||
pattern: coverage-artifacts-*
|
||||
path: coverage/
|
||||
|
||||
- name: Reorganize test result reports
|
||||
run: |
|
||||
find coverage/
|
||||
for i in {1..8}; do
|
||||
mv coverage/coverage-artifacts-${i}/coverage-final.json coverage/coverage-shard-${i}.json
|
||||
done
|
||||
shell: bash
|
||||
|
||||
- name: Merge Code Coverage
|
||||
run: npx nyc merge coverage/ merged-output/coverage-summary.json
|
||||
|
||||
- name: Upload Code Coverage
|
||||
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
|
||||
with:
|
||||
flags: javascript
|
||||
use_oidc: true
|
||||
verbose: true
|
||||
disable_search: true
|
||||
files: merged-output/coverage-summary.json
|
||||
slug: apache/superset
|
||||
|
||||
lint-frontend:
|
||||
needs: frontend-build
|
||||
if: needs.frontend-build.outputs.should-run == 'true'
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- name: Download Docker Image Artifact
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
|
||||
with:
|
||||
name: docker-image
|
||||
|
||||
- name: Load Docker Image
|
||||
run: |
|
||||
zstd -d < docker-image.tar.zst | docker load
|
||||
|
||||
- name: lint
|
||||
run: |
|
||||
docker run --rm $TAG bash -c \
|
||||
"npm i && npm run lint"
|
||||
|
||||
npm run eslint -- . --quiet
|
||||
- name: tsc
|
||||
if: steps.check.outcome == 'failure'
|
||||
working-directory: ./superset-frontend
|
||||
run: |
|
||||
docker run --rm $TAG bash -c \
|
||||
"npm i && npm run plugins:build && npm run type"
|
||||
|
||||
validate-frontend:
|
||||
needs: frontend-build
|
||||
if: needs.frontend-build.outputs.should-run == 'true'
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- name: Download Docker Image Artifact
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
|
||||
with:
|
||||
name: docker-image
|
||||
|
||||
- name: Load Docker Image
|
||||
npm run type
|
||||
- name: prettier
|
||||
if: steps.check.outcome == 'failure'
|
||||
working-directory: ./superset-frontend
|
||||
run: |
|
||||
zstd -d < docker-image.tar.zst | docker load
|
||||
|
||||
- name: Build Plugins Packages
|
||||
npm run prettier-check
|
||||
- name: Build plugins packages
|
||||
if: steps.check.outcome == 'failure'
|
||||
working-directory: ./superset-frontend
|
||||
run: npm run plugins:build
|
||||
- name: Build plugins Storybook
|
||||
if: steps.check.outcome == 'failure'
|
||||
working-directory: ./superset-frontend
|
||||
run: npm run plugins:build-storybook
|
||||
- name: superset-ui/core coverage
|
||||
if: steps.check.outcome == 'failure'
|
||||
working-directory: ./superset-frontend
|
||||
run: |
|
||||
docker run --rm $TAG bash -c \
|
||||
"npm run plugins:build"
|
||||
|
||||
test-storybook:
|
||||
needs: frontend-build
|
||||
if: needs.frontend-build.outputs.should-run == 'true'
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 25
|
||||
steps:
|
||||
- name: Download Docker Image Artifact
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
|
||||
with:
|
||||
name: docker-image
|
||||
|
||||
- name: Load Docker Image
|
||||
npm run core:cover
|
||||
- name: unit tests
|
||||
if: steps.check.outcome == 'failure'
|
||||
working-directory: ./superset-frontend
|
||||
run: |
|
||||
zstd -d < docker-image.tar.zst | docker load
|
||||
|
||||
- name: Build Storybook and Run Tests
|
||||
run: |
|
||||
docker run --rm $TAG bash -c \
|
||||
"npm run build-storybook && npx playwright install-deps && npx playwright install chromium && npm run test-storybook:ci"
|
||||
npm run test -- --coverage --silent
|
||||
# todo: remove this step when fix generator as a project in root jest.config.js
|
||||
- name: generator-superset unit tests
|
||||
if: steps.check.outcome == 'failure'
|
||||
working-directory: ./superset-frontend/packages/generator-superset
|
||||
run: npx jest
|
||||
- name: Upload code coverage
|
||||
if: steps.check.outcome == 'failure'
|
||||
working-directory: ./superset-frontend
|
||||
run: ../.github/workflows/codecov.sh -c -F javascript
|
||||
|
||||
18
.github/workflows/superset-helm-lint.yml
vendored
18
.github/workflows/superset-helm-lint.yml
vendored
@@ -1,4 +1,4 @@
|
||||
name: "Helm: lint and test charts"
|
||||
name: Lint and Test Charts
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
@@ -6,9 +6,6 @@ on:
|
||||
paths:
|
||||
- "helm/**"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
# cancel previous workflow jobs for PRs
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
|
||||
@@ -16,24 +13,23 @@ concurrency:
|
||||
|
||||
jobs:
|
||||
lint-test:
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: recursive
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Set up Helm
|
||||
uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1
|
||||
uses: azure/setup-helm@v3
|
||||
with:
|
||||
version: v3.16.4
|
||||
version: v3.5.4
|
||||
|
||||
- name: Setup Python
|
||||
uses: ./.github/actions/setup-backend/
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
install-superset: "false"
|
||||
python-version: "3.9"
|
||||
|
||||
- name: Set up chart-testing
|
||||
uses: ./.github/actions/chart-testing-action
|
||||
|
||||
96
.github/workflows/superset-helm-release.yml
vendored
96
.github/workflows/superset-helm-release.yml
vendored
@@ -1,38 +1,24 @@
|
||||
# This workflow automates the release process for Helm charts.
|
||||
# The workflow creates a new branch for the release and opens a pull request against the 'gh-pages' branch,
|
||||
# allowing the changes to be reviewed and merged manually.
|
||||
|
||||
name: "Helm: release charts"
|
||||
name: Release Charts
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- "master"
|
||||
- "[0-9].[0-9]*"
|
||||
- "[0-9].[0-9]"
|
||||
paths:
|
||||
- "helm/**"
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
ref:
|
||||
description: "The branch, tag, or commit SHA to check out"
|
||||
required: false
|
||||
default: "master"
|
||||
|
||||
jobs:
|
||||
release:
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref || github.ref_name }}
|
||||
persist-credentials: true
|
||||
persist-credentials: false
|
||||
submodules: recursive
|
||||
fetch-depth: 0
|
||||
|
||||
@@ -42,86 +28,18 @@ jobs:
|
||||
git config user.email "$GITHUB_ACTOR@users.noreply.github.com"
|
||||
|
||||
- name: Install Helm
|
||||
uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1
|
||||
uses: azure/setup-helm@v3
|
||||
with:
|
||||
version: v3.5.4
|
||||
|
||||
- name: Add bitnami repo dependency
|
||||
run: helm repo add bitnami https://charts.bitnami.com/bitnami
|
||||
|
||||
- name: Fetch/list all tags
|
||||
run: |
|
||||
# Debugging tags
|
||||
git fetch --tags --force
|
||||
git tag -d superset-helm-chart-0.13.4 || true
|
||||
echo "DEBUG TAGS"
|
||||
git show-ref --tags
|
||||
|
||||
- name: Create unique pages branch name
|
||||
id: vars
|
||||
run: echo "branch_name=helm-publish-${GITHUB_SHA:0:7}" >> $GITHUB_ENV
|
||||
|
||||
- name: Force recreate branch from gh-pages
|
||||
env:
|
||||
BRANCH_NAME: ${{ env.branch_name }}
|
||||
run: |
|
||||
# Ensure a clean working directory
|
||||
git reset --hard
|
||||
git clean -fdx
|
||||
git checkout -b local_gha_temp
|
||||
git submodule update
|
||||
|
||||
# Fetch the latest gh-pages branch
|
||||
git fetch origin gh-pages
|
||||
|
||||
# Check out and reset the target branch based on gh-pages
|
||||
git checkout -B "$BRANCH_NAME" origin/gh-pages
|
||||
|
||||
# Remove submodules from the branch
|
||||
git submodule deinit -f --all
|
||||
|
||||
# Force push to the remote branch
|
||||
git push origin "$BRANCH_NAME" --force
|
||||
|
||||
# Return to the original branch
|
||||
git checkout local_gha_temp
|
||||
|
||||
- name: Fetch/list all tags
|
||||
run: |
|
||||
git submodule update
|
||||
cat .github/actions/chart-releaser-action/action.yml
|
||||
|
||||
- name: Run chart-releaser
|
||||
uses: ./.github/actions/chart-releaser-action
|
||||
with:
|
||||
version: v1.6.0
|
||||
charts_dir: helm
|
||||
mark_as_latest: false
|
||||
pages_branch: ${{ env.branch_name }}
|
||||
env:
|
||||
CR_TOKEN: "${{ github.token }}"
|
||||
CR_RELEASE_NAME_TEMPLATE: "superset-helm-chart-{{ .Version }}"
|
||||
|
||||
- name: Open Pull Request
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
script: |
|
||||
const branchName = process.env.BRANCH_NAME;
|
||||
const [owner, repo] = process.env.GITHUB_REPOSITORY.split('/');
|
||||
|
||||
if (!branchName) {
|
||||
throw new Error("Branch name is not defined.");
|
||||
}
|
||||
|
||||
const pr = await github.rest.pulls.create({
|
||||
owner,
|
||||
repo,
|
||||
title: `Helm chart release for ${branchName}`,
|
||||
head: branchName,
|
||||
base: "gh-pages", // Adjust if the target branch is different
|
||||
body: `This PR releases Helm charts to the gh-pages branch.`,
|
||||
});
|
||||
|
||||
core.info(`Pull request created: ${pr.data.html_url}`);
|
||||
env:
|
||||
BRANCH_NAME: ${{ env.branch_name }}
|
||||
|
||||
172
.github/workflows/superset-playwright.yml
vendored
172
.github/workflows/superset-playwright.yml
vendored
@@ -1,172 +0,0 @@
|
||||
name: Playwright Experimental Tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- "master"
|
||||
- "[0-9].[0-9]*"
|
||||
pull_request:
|
||||
types: [synchronize, opened, reopened, ready_for_review]
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
ref:
|
||||
description: "The branch or tag to checkout"
|
||||
required: false
|
||||
default: ""
|
||||
pr_id:
|
||||
description: "The pull request ID to checkout"
|
||||
required: false
|
||||
default: ""
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
changes:
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
outputs:
|
||||
python: ${{ steps.check.outputs.python }}
|
||||
frontend: ${{ steps.check.outputs.frontend }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Check for file changes
|
||||
id: check
|
||||
uses: ./.github/actions/change-detector/
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
# NOTE: Required Playwright tests are in superset-e2e.yml (E2E / playwright-tests)
|
||||
# This workflow contains only experimental tests that run in shadow mode
|
||||
playwright-tests-experimental:
|
||||
needs: changes
|
||||
if: needs.changes.outputs.python == 'true' || needs.changes.outputs.frontend == 'true'
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
continue-on-error: true
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
browser: ["chromium"]
|
||||
app_root: ["", "/app/prefix"]
|
||||
env:
|
||||
SUPERSET_ENV: development
|
||||
SUPERSET_CONFIG: tests.integration_tests.superset_test_config
|
||||
SUPERSET__SQLALCHEMY_DATABASE_URI: postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset
|
||||
PYTHONPATH: ${{ github.workspace }}
|
||||
REDIS_PORT: 16379
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:17-alpine
|
||||
env:
|
||||
POSTGRES_USER: superset
|
||||
POSTGRES_PASSWORD: superset
|
||||
ports:
|
||||
- 15432:5432
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
ports:
|
||||
- 16379:6379
|
||||
steps:
|
||||
# -------------------------------------------------------
|
||||
# Conditional checkout based on context (same as Cypress workflow)
|
||||
- name: Checkout for push or pull_request event
|
||||
if: github.event_name == 'push' || github.event_name == 'pull_request'
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: recursive
|
||||
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
|
||||
- name: Checkout using ref (workflow_dispatch)
|
||||
if: github.event_name == 'workflow_dispatch' && github.event.inputs.ref != ''
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: ${{ github.event.inputs.ref }}
|
||||
submodules: recursive
|
||||
- name: Checkout using PR ID (workflow_dispatch)
|
||||
if: github.event_name == 'workflow_dispatch' && github.event.inputs.pr_id != ''
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: refs/pull/${{ github.event.inputs.pr_id }}/merge
|
||||
submodules: recursive
|
||||
# -------------------------------------------------------
|
||||
- name: Setup Python
|
||||
uses: ./.github/actions/setup-backend/
|
||||
- name: Setup postgres
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: setup-postgres
|
||||
- name: Import test data
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: playwright_testdata
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
with:
|
||||
node-version-file: "./superset-frontend/.nvmrc"
|
||||
cache: "npm"
|
||||
cache-dependency-path: "superset-frontend/package-lock.json"
|
||||
- name: Install npm dependencies
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: npm-install
|
||||
- name: Build javascript packages
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: build-instrumented-assets
|
||||
- name: Build embedded SDK
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: build-embedded-sdk
|
||||
- name: Install Playwright
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: playwright-install
|
||||
- name: Run Playwright (Experimental Tests)
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
env:
|
||||
NODE_OPTIONS: "--max-old-space-size=4096"
|
||||
with:
|
||||
run: playwright-run "${{ matrix.app_root }}" experimental/
|
||||
- name: Run Playwright (Embedded Tests)
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
env:
|
||||
NODE_OPTIONS: "--max-old-space-size=4096"
|
||||
# Scope embedded-only env vars to this step. Setting them at the job
|
||||
# level enabled the EMBEDDED_SUPERSET feature flag inside Flask for
|
||||
# the preceding "Required Tests" and "Experimental Tests" steps too,
|
||||
# which loads extra handlers and destabilizes the werkzeug dev
|
||||
# server under the 2-worker Playwright load. Required Tests should
|
||||
# match master's Flask configuration.
|
||||
SUPERSET_FEATURE_EMBEDDED_SUPERSET: "true"
|
||||
INCLUDE_EMBEDDED: "true"
|
||||
with:
|
||||
run: playwright-run "${{ matrix.app_root }}" embedded
|
||||
- name: Set safe app root
|
||||
if: failure()
|
||||
id: set-safe-app-root
|
||||
run: |
|
||||
APP_ROOT="${{ matrix.app_root }}"
|
||||
SAFE_APP_ROOT=${APP_ROOT//\//_}
|
||||
echo "safe_app_root=$SAFE_APP_ROOT" >> $GITHUB_OUTPUT
|
||||
- name: Upload Playwright Artifacts
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
if: failure()
|
||||
with:
|
||||
path: |
|
||||
${{ github.workspace }}/superset-frontend/playwright-results/
|
||||
${{ github.workspace }}/superset-frontend/test-results/
|
||||
name: playwright-experimental-artifact-${{ github.run_id }}-${{ github.job }}-${{ matrix.browser }}--${{ steps.set-safe-app-root.outputs.safe_app_root }}
|
||||
@@ -1,16 +1,11 @@
|
||||
# Python integration tests
|
||||
name: Python-Integration
|
||||
|
||||
# Least-privilege default for GITHUB_TOKEN. Jobs that need more (e.g. OIDC for
|
||||
# codecov uploads) opt in via their own job-level `permissions:` block.
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- "master"
|
||||
- "[0-9].[0-9]*"
|
||||
- "[0-9].[0-9]"
|
||||
pull_request:
|
||||
types: [synchronize, opened, reopened, ready_for_review]
|
||||
|
||||
@@ -19,32 +14,11 @@ concurrency:
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
changes:
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
outputs:
|
||||
python: ${{ steps.check.outputs.python }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Check for file changes
|
||||
id: check
|
||||
uses: ./.github/actions/change-detector/
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
test-mysql:
|
||||
needs: changes
|
||||
if: needs.changes.outputs.python == 'true'
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 45
|
||||
permissions:
|
||||
id-token: write
|
||||
runs-on: ubuntu-20.04
|
||||
strategy:
|
||||
matrix:
|
||||
python-version: ["3.9"]
|
||||
env:
|
||||
PYTHONPATH: ${{ github.workspace }}
|
||||
SUPERSET_CONFIG: tests.integration_tests.superset_test_config
|
||||
@@ -53,18 +27,11 @@ jobs:
|
||||
mysql+mysqldb://superset:superset@127.0.0.1:13306/superset?charset=utf8mb4&binary_prefix=true
|
||||
services:
|
||||
mysql:
|
||||
image: mysql:8.0
|
||||
# Authenticated pulls use our higher Docker Hub rate limit. Empty on
|
||||
# fork PRs (secrets unavailable) -> runner falls back to anonymous.
|
||||
image: mysql:5.7
|
||||
env:
|
||||
MYSQL_ROOT_PASSWORD: root
|
||||
ports:
|
||||
- 13306:3306
|
||||
options: >-
|
||||
--health-cmd="mysqladmin ping --silent"
|
||||
--health-interval=10s
|
||||
--health-timeout=5s
|
||||
--health-retries=5
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
options: --entrypoint redis-server
|
||||
@@ -72,70 +39,44 @@ jobs:
|
||||
- 16379:6379
|
||||
steps:
|
||||
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: recursive
|
||||
- name: Check if python changes are present
|
||||
id: check
|
||||
env:
|
||||
GITHUB_REPO: ${{ github.repository }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
continue-on-error: true
|
||||
run: ./scripts/ci_check_no_file_changes.sh python
|
||||
- name: Setup Python
|
||||
uses: ./.github/actions/setup-backend/
|
||||
if: steps.check.outcome == 'failure'
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
- name: Setup MySQL
|
||||
if: steps.check.outcome == 'failure'
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: setup-mysql
|
||||
- name: Start Celery worker
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: celery-worker
|
||||
run: |
|
||||
setup-mysql
|
||||
- name: Run celery
|
||||
if: steps.check.outcome == 'failure'
|
||||
run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 &
|
||||
- name: Python integration tests (MySQL)
|
||||
if: steps.check.outcome == 'failure'
|
||||
run: |
|
||||
./scripts/python_tests.sh
|
||||
- name: Upload code coverage
|
||||
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
|
||||
with:
|
||||
flags: python,mysql
|
||||
verbose: true
|
||||
use_oidc: true
|
||||
slug: apache/superset
|
||||
- name: Generate database diagnostics for docs
|
||||
env:
|
||||
SUPERSET_CONFIG: tests.integration_tests.superset_test_config
|
||||
SUPERSET__SQLALCHEMY_DATABASE_URI: |
|
||||
mysql+mysqldb://superset:superset@127.0.0.1:13306/superset?charset=utf8mb4&binary_prefix=true
|
||||
if: steps.check.outcome == 'failure'
|
||||
run: |
|
||||
python -c "
|
||||
import json
|
||||
from superset.app import create_app
|
||||
from superset.db_engine_specs.lib import generate_yaml_docs
|
||||
app = create_app()
|
||||
with app.app_context():
|
||||
docs = generate_yaml_docs()
|
||||
# Wrap in the expected format
|
||||
output = {
|
||||
'generated': '$(date -Iseconds)',
|
||||
'databases': docs
|
||||
}
|
||||
with open('databases-diagnostics.json', 'w') as f:
|
||||
json.dump(output, f, indent=2, default=str)
|
||||
print(f'Generated diagnostics for {len(docs)} databases')
|
||||
"
|
||||
- name: Upload database diagnostics artifact
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
with:
|
||||
name: database-diagnostics
|
||||
path: databases-diagnostics.json
|
||||
retention-days: 7
|
||||
bash .github/workflows/codecov.sh -c -F python -F mysql
|
||||
test-postgres:
|
||||
needs: changes
|
||||
if: needs.changes.outputs.python == 'true'
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 45
|
||||
permissions:
|
||||
id-token: write
|
||||
runs-on: ubuntu-20.04
|
||||
strategy:
|
||||
matrix:
|
||||
# Full version spread on push (master/release) + nightly; current only
|
||||
# on PRs to cut runner cost (cross-version breaks are caught at merge).
|
||||
python-version: ${{ github.event_name == 'pull_request' && fromJSON('["current"]') || fromJSON('["current", "next"]') }}
|
||||
python-version: ["3.9", "3.10"]
|
||||
env:
|
||||
PYTHONPATH: ${{ github.workspace }}
|
||||
SUPERSET_CONFIG: tests.integration_tests.superset_test_config
|
||||
@@ -143,7 +84,7 @@ jobs:
|
||||
SUPERSET__SQLALCHEMY_DATABASE_URI: postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:17-alpine
|
||||
image: postgres:15-alpine
|
||||
env:
|
||||
POSTGRES_USER: superset
|
||||
POSTGRES_PASSWORD: superset
|
||||
@@ -157,41 +98,45 @@ jobs:
|
||||
- 16379:6379
|
||||
steps:
|
||||
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: recursive
|
||||
- name: Check if python changes are present
|
||||
id: check
|
||||
env:
|
||||
GITHUB_REPO: ${{ github.repository }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
continue-on-error: true
|
||||
run: ./scripts/ci_check_no_file_changes.sh python
|
||||
- name: Setup Python
|
||||
uses: ./.github/actions/setup-backend/
|
||||
if: steps.check.outcome == 'failure'
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
- name: Setup Postgres
|
||||
if: steps.check.outcome == 'failure'
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: |
|
||||
setup-postgres
|
||||
- name: Start Celery worker
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: celery-worker
|
||||
- name: Run celery
|
||||
if: steps.check.outcome == 'failure'
|
||||
run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 &
|
||||
- name: Python integration tests (PostgreSQL)
|
||||
if: steps.check.outcome == 'failure'
|
||||
run: |
|
||||
./scripts/python_tests.sh
|
||||
- name: Upload code coverage
|
||||
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
|
||||
with:
|
||||
flags: python,postgres
|
||||
verbose: true
|
||||
use_oidc: true
|
||||
slug: apache/superset
|
||||
if: steps.check.outcome == 'failure'
|
||||
run: |
|
||||
bash .github/workflows/codecov.sh -c -F python -F postgres
|
||||
|
||||
test-sqlite:
|
||||
needs: changes
|
||||
if: needs.changes.outputs.python == 'true'
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 45
|
||||
permissions:
|
||||
id-token: write
|
||||
runs-on: ubuntu-20.04
|
||||
strategy:
|
||||
matrix:
|
||||
python-version: ["3.9"]
|
||||
env:
|
||||
PYTHONPATH: ${{ github.workspace }}
|
||||
SUPERSET_CONFIG: tests.integration_tests.superset_test_config
|
||||
@@ -207,51 +152,37 @@ jobs:
|
||||
- 16379:6379
|
||||
steps:
|
||||
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: recursive
|
||||
- name: Check if python changes are present
|
||||
id: check
|
||||
env:
|
||||
GITHUB_REPO: ${{ github.repository }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
continue-on-error: true
|
||||
run: ./scripts/ci_check_no_file_changes.sh python
|
||||
- name: Setup Python
|
||||
uses: ./.github/actions/setup-backend/
|
||||
if: steps.check.outcome == 'failure'
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
- name: Install dependencies
|
||||
if: steps.check.outcome == 'failure'
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: |
|
||||
# sqlite needs this working directory
|
||||
mkdir ${{ github.workspace }}/.temp
|
||||
- name: Start Celery worker
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: celery-worker
|
||||
- name: Run celery
|
||||
if: steps.check.outcome == 'failure'
|
||||
run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 &
|
||||
- name: Python integration tests (SQLite)
|
||||
if: steps.check.outcome == 'failure'
|
||||
run: |
|
||||
./scripts/python_tests.sh
|
||||
- name: Upload code coverage
|
||||
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
|
||||
with:
|
||||
flags: python,sqlite
|
||||
verbose: true
|
||||
use_oidc: true
|
||||
slug: apache/superset
|
||||
|
||||
# Stable required-status-check anchor for the matrix-based test-postgres job.
|
||||
# It is gated on change detection, so on non-Python PRs it is skipped and
|
||||
# never produces its `test-postgres (current)` context (a job-level skip
|
||||
# happens before matrix expansion). This always-running job reports a single
|
||||
# context branch protection can require: it passes when test-postgres
|
||||
# succeeded or was skipped, and fails only on a real failure.
|
||||
test-postgres-required:
|
||||
needs: [changes, test-postgres]
|
||||
if: always()
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- name: Check test-postgres result
|
||||
env:
|
||||
RESULT: ${{ needs.test-postgres.result }}
|
||||
if: steps.check.outcome == 'failure'
|
||||
run: |
|
||||
if [ "$RESULT" != "success" ] && [ "$RESULT" != "skipped" ]; then
|
||||
echo "test-postgres did not pass (result: $RESULT)"
|
||||
exit 1
|
||||
fi
|
||||
echo "test-postgres result: $RESULT"
|
||||
bash .github/workflows/codecov.sh -c -F python -F sqlite
|
||||
|
||||
66
.github/workflows/superset-python-misc.yml
vendored
Normal file
66
.github/workflows/superset-python-misc.yml
vendored
Normal file
@@ -0,0 +1,66 @@
|
||||
# Python Misc unit tests
|
||||
name: Python Misc
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- "master"
|
||||
- "[0-9].[0-9]"
|
||||
paths:
|
||||
- "superset/**"
|
||||
pull_request:
|
||||
types: [synchronize, opened, reopened, ready_for_review]
|
||||
paths:
|
||||
- "superset/**"
|
||||
|
||||
# cancel previous workflow jobs for PRs
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
python-lint:
|
||||
runs-on: ubuntu-20.04
|
||||
strategy:
|
||||
matrix:
|
||||
python-version: ["3.9"]
|
||||
steps:
|
||||
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: recursive
|
||||
- name: Check if python changes are present
|
||||
id: check
|
||||
env:
|
||||
GITHUB_REPO: ${{ github.repository }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
continue-on-error: true
|
||||
run: ./scripts/ci_check_no_file_changes.sh python
|
||||
- name: Setup Python
|
||||
uses: ./.github/actions/setup-backend/
|
||||
if: steps.check.outcome == 'failure'
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
- name: pylint
|
||||
if: steps.check.outcome == 'failure'
|
||||
# `-j 0` run Pylint in parallel
|
||||
run: pylint -j 0 superset
|
||||
|
||||
babel-extract:
|
||||
runs-on: ubuntu-20.04
|
||||
strategy:
|
||||
matrix:
|
||||
python-version: ["3.9"]
|
||||
steps:
|
||||
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: recursive
|
||||
- name: Setup Python
|
||||
uses: ./.github/actions/setup-backend/
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
- name: Test babel extraction
|
||||
run: flask fab babel-extract --target superset/translations --output superset/translations/messages.pot --config superset/translations/babel.cfg -k _,__,t,tn,tct
|
||||
109
.github/workflows/superset-python-presto-hive.yml
vendored
109
.github/workflows/superset-python-presto-hive.yml
vendored
@@ -5,9 +5,13 @@ on:
|
||||
push:
|
||||
branches:
|
||||
- "master"
|
||||
- "[0-9].[0-9]*"
|
||||
- "[0-9].[0-9]"
|
||||
paths:
|
||||
- "superset/**"
|
||||
pull_request:
|
||||
types: [synchronize, opened, reopened, ready_for_review]
|
||||
paths:
|
||||
- "superset/**"
|
||||
|
||||
# cancel previous workflow jobs for PRs
|
||||
concurrency:
|
||||
@@ -15,32 +19,11 @@ concurrency:
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
changes:
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
outputs:
|
||||
python: ${{ steps.check.outputs.python }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Check for file changes
|
||||
id: check
|
||||
uses: ./.github/actions/change-detector/
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
test-postgres-presto:
|
||||
needs: changes
|
||||
if: needs.changes.outputs.python == 'true'
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 45
|
||||
permissions:
|
||||
id-token: write
|
||||
runs-on: ubuntu-20.04
|
||||
strategy:
|
||||
matrix:
|
||||
python-version: ["3.9"]
|
||||
env:
|
||||
PYTHONPATH: ${{ github.workspace }}
|
||||
SUPERSET_CONFIG: tests.integration_tests.superset_test_config
|
||||
@@ -49,7 +32,7 @@ jobs:
|
||||
SUPERSET__SQLALCHEMY_EXAMPLES_URI: presto://localhost:15433/memory/default
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:17-alpine
|
||||
image: postgres:15-alpine
|
||||
env:
|
||||
POSTGRES_USER: superset
|
||||
POSTGRES_PASSWORD: superset
|
||||
@@ -72,38 +55,42 @@ jobs:
|
||||
- 16379:6379
|
||||
steps:
|
||||
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: recursive
|
||||
- name: Check if python changes are present
|
||||
id: check
|
||||
env:
|
||||
GITHUB_REPO: ${{ github.repository }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
continue-on-error: true
|
||||
run: ./scripts/ci_check_no_file_changes.sh python
|
||||
- name: Setup Python
|
||||
uses: ./.github/actions/setup-backend/
|
||||
if: steps.check.outcome == 'failure'
|
||||
- name: Setup Postgres
|
||||
if: steps.check.outcome == 'failure'
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: setup-postgres
|
||||
- name: Start Celery worker
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: celery-worker
|
||||
- name: Run celery
|
||||
if: steps.check.outcome == 'failure'
|
||||
run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 &
|
||||
- name: Python unit tests (PostgreSQL)
|
||||
if: steps.check.outcome == 'failure'
|
||||
run: |
|
||||
./scripts/python_tests.sh -m 'chart_data_flow or sql_json_flow'
|
||||
- name: Upload code coverage
|
||||
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
|
||||
with:
|
||||
flags: python,presto
|
||||
verbose: true
|
||||
use_oidc: true
|
||||
slug: apache/superset
|
||||
if: steps.check.outcome == 'failure'
|
||||
run: |
|
||||
bash .github/workflows/codecov.sh -c -F python -F presto
|
||||
|
||||
test-postgres-hive:
|
||||
needs: changes
|
||||
if: needs.changes.outputs.python == 'true'
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 45
|
||||
permissions:
|
||||
id-token: write
|
||||
runs-on: ubuntu-20.04
|
||||
strategy:
|
||||
matrix:
|
||||
python-version: ["3.9"]
|
||||
env:
|
||||
PYTHONPATH: ${{ github.workspace }}
|
||||
SUPERSET_CONFIG: tests.integration_tests.superset_test_config
|
||||
@@ -113,7 +100,7 @@ jobs:
|
||||
UPLOAD_FOLDER: /tmp/.superset/uploads/
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:17-alpine
|
||||
image: postgres:15-alpine
|
||||
env:
|
||||
POSTGRES_USER: superset
|
||||
POSTGRES_PASSWORD: superset
|
||||
@@ -127,34 +114,42 @@ jobs:
|
||||
- 16379:6379
|
||||
steps:
|
||||
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: recursive
|
||||
- name: Check if python changes are present
|
||||
id: check
|
||||
env:
|
||||
GITHUB_REPO: ${{ github.repository }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
continue-on-error: true
|
||||
run: ./scripts/ci_check_no_file_changes.sh python
|
||||
- name: Create csv upload directory
|
||||
if: steps.check.outcome == 'failure'
|
||||
run: sudo mkdir -p /tmp/.superset/uploads
|
||||
- name: Give write access to the csv upload directory
|
||||
if: steps.check.outcome == 'failure'
|
||||
run: sudo chown -R $USER:$USER /tmp/.superset
|
||||
- name: Start hadoop and hive
|
||||
if: steps.check.outcome == 'failure'
|
||||
run: docker compose -f scripts/databases/hive/docker-compose.yml up -d
|
||||
- name: Setup Python
|
||||
uses: ./.github/actions/setup-backend/
|
||||
if: steps.check.outcome == 'failure'
|
||||
- name: Setup Postgres
|
||||
if: steps.check.outcome == 'failure'
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: setup-postgres
|
||||
- name: Start Celery worker
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: celery-worker
|
||||
- name: Run celery
|
||||
if: steps.check.outcome == 'failure'
|
||||
run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 &
|
||||
- name: Python unit tests (PostgreSQL)
|
||||
if: steps.check.outcome == 'failure'
|
||||
run: |
|
||||
uv pip install --system -e .[hive]
|
||||
./scripts/python_tests.sh -m 'chart_data_flow or sql_json_flow'
|
||||
- name: Upload code coverage
|
||||
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
|
||||
with:
|
||||
flags: python,hive
|
||||
verbose: true
|
||||
use_oidc: true
|
||||
slug: apache/superset
|
||||
if: steps.check.outcome == 'failure'
|
||||
run: |
|
||||
bash .github/workflows/codecov.sh -c -F python -F hive
|
||||
|
||||
102
.github/workflows/superset-python-unittest.yml
vendored
102
.github/workflows/superset-python-unittest.yml
vendored
@@ -1,18 +1,23 @@
|
||||
# Python unit tests
|
||||
name: Python-Unit
|
||||
|
||||
# Least-privilege default for GITHUB_TOKEN. Jobs that need more (e.g. OIDC for
|
||||
# codecov uploads) opt in via their own job-level `permissions:` block.
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- "master"
|
||||
- "[0-9].[0-9]*"
|
||||
- "[0-9].[0-9]"
|
||||
paths:
|
||||
- "superset/**"
|
||||
- "requirements/**"
|
||||
- "tests/unit_tests/**"
|
||||
- "scripts/**"
|
||||
pull_request:
|
||||
types: [synchronize, opened, reopened, ready_for_review]
|
||||
paths:
|
||||
- "superset/**"
|
||||
- "requirements/**"
|
||||
- "tests/unit_tests/**"
|
||||
- "scripts/**"
|
||||
|
||||
# cancel previous workflow jobs for PRs
|
||||
concurrency:
|
||||
@@ -20,94 +25,39 @@ concurrency:
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
changes:
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
outputs:
|
||||
python: ${{ steps.check.outputs.python }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Check for file changes
|
||||
id: check
|
||||
uses: ./.github/actions/change-detector/
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
unit-tests:
|
||||
needs: changes
|
||||
if: needs.changes.outputs.python == 'true'
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
id-token: write
|
||||
runs-on: ubuntu-20.04
|
||||
strategy:
|
||||
matrix:
|
||||
# Full version spread on push (master/release) + nightly; current only
|
||||
# on PRs to cut runner cost (cross-version breaks are caught at merge).
|
||||
python-version: ${{ github.event_name == 'pull_request' && fromJSON('["current"]') || fromJSON('["current", "next"]') }}
|
||||
python-version: ["3.9", "3.10"]
|
||||
env:
|
||||
PYTHONPATH: ${{ github.workspace }}
|
||||
# Promotes the SQLAlchemy 2.0 deprecation warnings already locked in as
|
||||
# errors via pytest.ini's `filterwarnings` to actually run in CI, so a
|
||||
# regression on those fails the build instead of relying on a
|
||||
# contributor remembering to set this locally. See the migration
|
||||
# battleplan: https://github.com/apache/superset/discussions/40273
|
||||
SQLALCHEMY_WARN_20: "1"
|
||||
steps:
|
||||
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: recursive
|
||||
- name: Check if python changes are present
|
||||
id: check
|
||||
env:
|
||||
GITHUB_REPO: ${{ github.repository }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
continue-on-error: true
|
||||
run: ./scripts/ci_check_no_file_changes.sh python
|
||||
- name: Setup Python
|
||||
uses: ./.github/actions/setup-backend/
|
||||
if: steps.check.outcome == 'failure'
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
- name: Python unit tests
|
||||
if: steps.check.outcome == 'failure'
|
||||
env:
|
||||
SUPERSET_TESTENV: true
|
||||
SUPERSET_SECRET_KEY: not-a-secret
|
||||
run: |
|
||||
pytest --durations-min=0.5 --cov-report= --cov=superset ./tests/common ./tests/unit_tests --cache-clear --maxfail=50
|
||||
- name: Python 100% coverage unit tests
|
||||
env:
|
||||
SUPERSET_TESTENV: true
|
||||
SUPERSET_SECRET_KEY: not-a-secret
|
||||
run: |
|
||||
pytest --durations-min=0.5 --cov=superset/sql/ ./tests/unit_tests/sql/ --cache-clear --cov-fail-under=100
|
||||
pytest --durations-min=0.5 --cov=superset/semantic_layers/ ./tests/unit_tests/semantic_layers/ --cache-clear --cov-fail-under=100
|
||||
pytest --durations-min=0.5 --cov-report= --cov=superset ./tests/common ./tests/unit_tests --cache-clear
|
||||
- name: Upload code coverage
|
||||
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
|
||||
with:
|
||||
flags: python,unit
|
||||
verbose: true
|
||||
use_oidc: true
|
||||
slug: apache/superset
|
||||
|
||||
# Stable required-status-check anchor. `unit-tests` is a matrix job gated on
|
||||
# change detection, so on non-Python PRs it is skipped and never produces its
|
||||
# `unit-tests (current)` context (a job-level skip happens before matrix
|
||||
# expansion). This always-running job reports a single context that branch
|
||||
# protection can require: it passes when unit-tests succeeded or was skipped,
|
||||
# and fails only on a real failure.
|
||||
unit-tests-required:
|
||||
needs: [changes, unit-tests]
|
||||
if: always()
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- name: Check unit-tests result
|
||||
env:
|
||||
RESULT: ${{ needs.unit-tests.result }}
|
||||
if: steps.check.outcome == 'failure'
|
||||
run: |
|
||||
if [ "$RESULT" != "success" ] && [ "$RESULT" != "skipped" ]; then
|
||||
echo "unit-tests did not pass (result: $RESULT)"
|
||||
exit 1
|
||||
fi
|
||||
echo "unit-tests result: $RESULT"
|
||||
bash .github/workflows/codecov.sh -c -F python -F unit
|
||||
|
||||
@@ -1,88 +0,0 @@
|
||||
name: Translation Regression Comment
|
||||
|
||||
on:
|
||||
# zizmor: ignore[dangerous-triggers] - runs in base-branch context and only consumes the uploaded artifact; never checks out PR code (see note below)
|
||||
workflow_run:
|
||||
workflows: ["Translations"]
|
||||
types: [completed]
|
||||
|
||||
# This workflow posts a PR comment when the Translations workflow detects a
|
||||
# regression. It uses the workflow_run trigger so that it always runs in the
|
||||
# base-branch context and can safely be granted write permissions, even for
|
||||
# PRs from forks.
|
||||
#
|
||||
# IMPORTANT: This workflow must NEVER check out code from the PR branch.
|
||||
# All data comes from the artifact uploaded by the Translations workflow.
|
||||
|
||||
permissions:
|
||||
pull-requests: write
|
||||
actions: read
|
||||
|
||||
jobs:
|
||||
post-comment:
|
||||
runs-on: ubuntu-24.04
|
||||
# Only act when the Translations workflow failed (which means a regression
|
||||
# was detected — the workflow exits 1 on regression).
|
||||
if: github.event.workflow_run.conclusion == 'failure'
|
||||
|
||||
steps:
|
||||
- name: Download regression artifact
|
||||
id: download
|
||||
continue-on-error: true
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
|
||||
with:
|
||||
name: translation-regression
|
||||
run-id: ${{ github.event.workflow_run.id }}
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
path: /tmp/translation-regression
|
||||
|
||||
- name: Post or update PR comment
|
||||
if: steps.download.outcome == 'success'
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
script: |
|
||||
const fs = require('fs');
|
||||
|
||||
const prNumberFile = '/tmp/translation-regression/pr-number.txt';
|
||||
const reportFile = '/tmp/translation-regression/regression-report.md';
|
||||
|
||||
if (!fs.existsSync(prNumberFile) || !fs.existsSync(reportFile)) {
|
||||
console.log('Artifact files not found, skipping comment.');
|
||||
return;
|
||||
}
|
||||
|
||||
const prNumber = parseInt(fs.readFileSync(prNumberFile, 'utf8').trim(), 10);
|
||||
if (!prNumber) {
|
||||
console.log('Could not parse PR number, skipping comment.');
|
||||
return;
|
||||
}
|
||||
|
||||
const report = fs.readFileSync(reportFile, 'utf8');
|
||||
const marker = '<!-- translation-regression-bot -->';
|
||||
const body = `${marker}\n${report}`;
|
||||
|
||||
const { data: comments } = await github.rest.issues.listComments({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: prNumber,
|
||||
});
|
||||
|
||||
const existing = comments.find(c => c.body && c.body.includes(marker));
|
||||
|
||||
if (existing) {
|
||||
await github.rest.issues.updateComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
comment_id: existing.id,
|
||||
body,
|
||||
});
|
||||
console.log(`Updated existing comment ${existing.id} on PR #${prNumber}`);
|
||||
} else {
|
||||
await github.rest.issues.createComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: prNumber,
|
||||
body,
|
||||
});
|
||||
console.log(`Created new comment on PR #${prNumber}`);
|
||||
}
|
||||
131
.github/workflows/superset-translations.yml
vendored
131
.github/workflows/superset-translations.yml
vendored
@@ -4,152 +4,51 @@ on:
|
||||
push:
|
||||
branches:
|
||||
- "master"
|
||||
- "[0-9].[0-9]*"
|
||||
- "[0-9].[0-9]"
|
||||
pull_request:
|
||||
types: [synchronize, opened, reopened, ready_for_review]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
|
||||
# cancel previous workflow jobs for PRs
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
frontend-check-translations:
|
||||
runs-on: ubuntu-24.04
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
frontend-check:
|
||||
runs-on: ubuntu-20.04
|
||||
steps:
|
||||
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: recursive
|
||||
|
||||
- name: Check for file changes
|
||||
id: check
|
||||
uses: ./.github/actions/change-detector/
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Setup Node.js
|
||||
if: steps.check.outputs.frontend
|
||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version-file: "./superset-frontend/.nvmrc"
|
||||
cache: "npm"
|
||||
cache-dependency-path: "superset-frontend/package-lock.json"
|
||||
node-version: "16"
|
||||
- name: Install dependencies
|
||||
if: steps.check.outputs.frontend
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: npm-install
|
||||
- name: lint
|
||||
if: steps.check.outputs.frontend
|
||||
working-directory: ./superset-frontend
|
||||
run: |
|
||||
npm run build-translation
|
||||
npm run check-translation
|
||||
|
||||
babel-extract:
|
||||
runs-on: ubuntu-24.04
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
runs-on: ubuntu-20.04
|
||||
strategy:
|
||||
matrix:
|
||||
python-version: ["3.9"]
|
||||
steps:
|
||||
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: recursive
|
||||
|
||||
- name: Check for file changes
|
||||
id: check
|
||||
uses: ./.github/actions/change-detector/
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Setup Python
|
||||
if: steps.check.outputs.python == 'true' || steps.check.outputs.frontend == 'true'
|
||||
uses: ./.github/actions/setup-backend/
|
||||
|
||||
- name: Install gettext tools
|
||||
if: steps.check.outputs.python == 'true' || steps.check.outputs.frontend == 'true'
|
||||
run: sudo apt-get update && sudo apt-get install -y gettext
|
||||
|
||||
# Fetch the base ref so we can compare PR-introduced regressions
|
||||
# against a fair baseline (also runs babel_update against the base
|
||||
# source) — this isolates the PR's contribution from any pre-existing
|
||||
# drift on the base branch.
|
||||
- name: Fetch base ref and create comparison worktree
|
||||
if: steps.check.outputs.python == 'true' || steps.check.outputs.frontend == 'true'
|
||||
env:
|
||||
PR_BASE_REF: ${{ github.event.pull_request.base.ref }}
|
||||
run: |
|
||||
# For PRs use the base branch; for direct pushes compare against the previous commit.
|
||||
BASE_REF="$PR_BASE_REF"
|
||||
if [ -n "$BASE_REF" ]; then
|
||||
git fetch --depth=1 origin "$BASE_REF"
|
||||
else
|
||||
git fetch --depth=2 origin "$GITHUB_REF"
|
||||
fi
|
||||
git worktree add /tmp/base-worktree FETCH_HEAD
|
||||
|
||||
# Run babel_update against BASE source + BASE translations. Any drift
|
||||
# already present on the base branch (source strings that have changed
|
||||
# without .po updates) shows up here as fuzzies — and will also show
|
||||
# up in the PR run, so it cancels out in the comparison.
|
||||
- name: Baseline — run babel_update against BASE source
|
||||
if: steps.check.outputs.python == 'true' || steps.check.outputs.frontend == 'true'
|
||||
working-directory: /tmp/base-worktree
|
||||
run: ./scripts/translations/babel_update.sh
|
||||
|
||||
- name: Record baseline translation counts
|
||||
if: steps.check.outputs.python == 'true' || steps.check.outputs.frontend == 'true'
|
||||
run: |
|
||||
python scripts/translations/check_translation_regression.py \
|
||||
--count \
|
||||
--translations-dir /tmp/base-worktree/superset/translations \
|
||||
> /tmp/before.json
|
||||
|
||||
# Run babel_update against the PR source and PR translations. This keeps
|
||||
# committed .po fixes in play while the base babel_update above still
|
||||
# cancels out translation drift already present on the base branch.
|
||||
- name: Run babel_update against PR source
|
||||
if: steps.check.outputs.python == 'true' || steps.check.outputs.frontend == 'true'
|
||||
run: ./scripts/translations/babel_update.sh
|
||||
|
||||
- name: Check for translation regression
|
||||
id: regression
|
||||
if: steps.check.outputs.python == 'true' || steps.check.outputs.frontend == 'true'
|
||||
continue-on-error: true
|
||||
run: |
|
||||
python scripts/translations/check_translation_regression.py \
|
||||
--compare /tmp/before.json \
|
||||
--report /tmp/regression-report.md
|
||||
|
||||
# Save the PR number so the comment workflow can post the report without
|
||||
# needing write permissions on this pull_request-triggered job.
|
||||
- name: Save PR number for comment workflow
|
||||
if: >-
|
||||
github.event_name == 'pull_request' &&
|
||||
steps.regression.outcome == 'failure'
|
||||
run: echo "${{ github.event.pull_request.number }}" > /tmp/pr-number.txt
|
||||
|
||||
- name: Upload regression artifact
|
||||
if: >-
|
||||
github.event_name == 'pull_request' &&
|
||||
steps.regression.outcome == 'failure'
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
with:
|
||||
name: translation-regression
|
||||
path: |
|
||||
/tmp/regression-report.md
|
||||
/tmp/pr-number.txt
|
||||
|
||||
- name: Fail if regression detected
|
||||
if: steps.regression.outcome == 'failure'
|
||||
run: exit 1
|
||||
python-version: ${{ matrix.python-version }}
|
||||
- name: Test babel extraction
|
||||
run: ./scripts/babel_update.sh
|
||||
|
||||
14
.github/workflows/superset-websocket.yml
vendored
14
.github/workflows/superset-websocket.yml
vendored
@@ -3,7 +3,7 @@ on:
|
||||
push:
|
||||
branches:
|
||||
- "master"
|
||||
- "[0-9].[0-9]*"
|
||||
- "[0-9].[0-9]"
|
||||
paths:
|
||||
- "superset-websocket/**"
|
||||
pull_request:
|
||||
@@ -11,9 +11,6 @@ on:
|
||||
- "superset-websocket/**"
|
||||
types: [synchronize, opened, reopened, ready_for_review]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
# cancel previous workflow jobs for PRs
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
|
||||
@@ -21,17 +18,12 @@ concurrency:
|
||||
|
||||
jobs:
|
||||
app-checks:
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 20
|
||||
runs-on: ubuntu-20.04
|
||||
steps:
|
||||
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
with:
|
||||
node-version-file: './superset-websocket/.nvmrc'
|
||||
- name: Install dependencies
|
||||
working-directory: ./superset-websocket
|
||||
run: npm ci
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user