mirror of
https://github.com/apache/superset.git
synced 2026-08-04 04:52:32 +00:00
feat(build): migrate from Prettier to Oxfmt for performant code formatting (#42434)
This commit is contained in:
Đỗ Trọng Hải
@@ -28,8 +28,20 @@ import path from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
const sidebarTsPath = path.join(__dirname, '..', 'developer_docs', 'api', 'sidebar.ts');
|
||||
const sidebarJsPath = path.join(__dirname, '..', 'developer_docs', 'api', 'sidebar.js');
|
||||
const sidebarTsPath = path.join(
|
||||
__dirname,
|
||||
'..',
|
||||
'developer_docs',
|
||||
'api',
|
||||
'sidebar.ts',
|
||||
);
|
||||
const sidebarJsPath = path.join(
|
||||
__dirname,
|
||||
'..',
|
||||
'developer_docs',
|
||||
'api',
|
||||
'sidebar.js',
|
||||
);
|
||||
|
||||
if (!fs.existsSync(sidebarTsPath)) {
|
||||
console.log('No sidebar.ts found, skipping conversion');
|
||||
@@ -47,7 +59,7 @@ content = content.replace(/: SidebarsConfig/g, '');
|
||||
// Change export default to module.exports
|
||||
content = content.replace(
|
||||
/export default sidebar\.apisidebar;/,
|
||||
'module.exports = sidebar.apisidebar;'
|
||||
'module.exports = sidebar.apisidebar;',
|
||||
);
|
||||
|
||||
// Parse the sidebar to add unique keys for duplicate labels
|
||||
@@ -60,9 +72,9 @@ try {
|
||||
const sidebarObj = new Function(`return ${sidebarMatch[1]}`)();
|
||||
|
||||
// First pass: count labels
|
||||
const countLabels = (items) => {
|
||||
const countLabels = items => {
|
||||
const counts = {};
|
||||
const count = (item) => {
|
||||
const count = item => {
|
||||
if (item.type === 'doc' && item.label) {
|
||||
counts[item.label] = (counts[item.label] || 0) + 1;
|
||||
}
|
||||
@@ -105,7 +117,7 @@ module.exports = sidebar.apisidebar;
|
||||
// Fall back to simple conversion
|
||||
content = content.replace(
|
||||
/export default sidebar\.apisidebar;/,
|
||||
'module.exports = sidebar.apisidebar;'
|
||||
'module.exports = sidebar.apisidebar;',
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -34,28 +34,68 @@ import { fileURLToPath } from 'url';
|
||||
const require = createRequire(import.meta.url);
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
const SPEC_PATH = path.join(__dirname, '..', 'static', 'resources', 'openapi.json');
|
||||
const SIDEBAR_PATH = path.join(__dirname, '..', 'developer_docs', 'api', 'sidebar.js');
|
||||
const SPEC_PATH = path.join(
|
||||
__dirname,
|
||||
'..',
|
||||
'static',
|
||||
'resources',
|
||||
'openapi.json',
|
||||
);
|
||||
const SIDEBAR_PATH = path.join(
|
||||
__dirname,
|
||||
'..',
|
||||
'developer_docs',
|
||||
'api',
|
||||
'sidebar.js',
|
||||
);
|
||||
const OUTPUT_PATH = path.join(__dirname, '..', 'developer_docs', 'api.mdx');
|
||||
|
||||
// Category groupings for better organization
|
||||
const CATEGORY_GROUPS = {
|
||||
'Authentication': ['Security'],
|
||||
Authentication: ['Security'],
|
||||
'Core Resources': ['Dashboards', 'Charts', 'Datasets', 'Database'],
|
||||
'Data Exploration': ['Explore', 'SQL Lab', 'Queries', 'Datasources', 'Advanced Data Type'],
|
||||
'Organization & Customization': ['Tags', 'Annotation Layers', 'CSS Templates'],
|
||||
'Data Exploration': [
|
||||
'Explore',
|
||||
'SQL Lab',
|
||||
'Queries',
|
||||
'Datasources',
|
||||
'Advanced Data Type',
|
||||
],
|
||||
'Organization & Customization': [
|
||||
'Tags',
|
||||
'Annotation Layers',
|
||||
'CSS Templates',
|
||||
],
|
||||
'Sharing & Embedding': [
|
||||
'Dashboard Permanent Link', 'Explore Permanent Link', 'SQL Lab Permanent Link',
|
||||
'Embedded Dashboard', 'Dashboard Filter State', 'Explore Form Data'
|
||||
'Dashboard Permanent Link',
|
||||
'Explore Permanent Link',
|
||||
'SQL Lab Permanent Link',
|
||||
'Embedded Dashboard',
|
||||
'Dashboard Filter State',
|
||||
'Explore Form Data',
|
||||
],
|
||||
'Scheduling & Alerts': ['Report Schedules'],
|
||||
'Security & Access Control': [
|
||||
'Security Roles', 'Security Users', 'Security Permissions',
|
||||
'Security Resources (View Menus)', 'Security Permissions on Resources (View Menus)',
|
||||
'Row Level Security'
|
||||
'Security Roles',
|
||||
'Security Users',
|
||||
'Security Permissions',
|
||||
'Security Resources (View Menus)',
|
||||
'Security Permissions on Resources (View Menus)',
|
||||
'Row Level Security',
|
||||
],
|
||||
'Import/Export & Administration': [
|
||||
'Import/export',
|
||||
'CacheRestApi',
|
||||
'LogRestApi',
|
||||
],
|
||||
'User & System': [
|
||||
'Current User',
|
||||
'User',
|
||||
'Menu',
|
||||
'Available Domains',
|
||||
'AsyncEventsRestApi',
|
||||
'OpenApi',
|
||||
],
|
||||
'Import/Export & Administration': ['Import/export', 'CacheRestApi', 'LogRestApi'],
|
||||
'User & System': ['Current User', 'User', 'Menu', 'Available Domains', 'AsyncEventsRestApi', 'OpenApi'],
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -68,7 +108,7 @@ function buildSlugMap() {
|
||||
try {
|
||||
const sidebar = require(SIDEBAR_PATH);
|
||||
|
||||
const extractDocs = (items) => {
|
||||
const extractDocs = items => {
|
||||
for (const item of items) {
|
||||
if (item.type === 'doc' && item.label && item.id) {
|
||||
// id is like "api/create-security-login" → slug "create-security-login"
|
||||
@@ -80,7 +120,9 @@ function buildSlugMap() {
|
||||
};
|
||||
|
||||
extractDocs(sidebar);
|
||||
console.log(`Loaded ${Object.keys(labelToSlug).length} slug mappings from sidebar`);
|
||||
console.log(
|
||||
`Loaded ${Object.keys(labelToSlug).length} slug mappings from sidebar`,
|
||||
);
|
||||
} catch {
|
||||
console.warn('Could not read sidebar, will use computed slugs');
|
||||
}
|
||||
@@ -214,7 +256,9 @@ curl -H "Authorization: Bearer YOUR_ACCESS_TOKEN" \\
|
||||
for (const [groupName, groupTags] of Object.entries(CATEGORY_GROUPS)) {
|
||||
if (groupName === 'Authentication') continue; // Already rendered
|
||||
|
||||
const tagsInGroup = groupTags.filter(tag => tagEndpoints[tag] && !renderedTags.has(tag));
|
||||
const tagsInGroup = groupTags.filter(
|
||||
tag => tagEndpoints[tag] && !renderedTags.has(tag),
|
||||
);
|
||||
if (tagsInGroup.length === 0) continue;
|
||||
|
||||
mdx += `#### ${groupName}\n\n`;
|
||||
@@ -238,7 +282,9 @@ curl -H "Authorization: Bearer YOUR_ACCESS_TOKEN" \\
|
||||
}
|
||||
|
||||
// Render any remaining tags not in a group
|
||||
const remainingTags = Object.keys(tagEndpoints).filter(tag => !renderedTags.has(tag));
|
||||
const remainingTags = Object.keys(tagEndpoints).filter(
|
||||
tag => !renderedTags.has(tag),
|
||||
);
|
||||
if (remainingTags.length > 0) {
|
||||
mdx += `#### Other\n\n`;
|
||||
|
||||
|
||||
@@ -33,7 +33,13 @@ import { fileURLToPath } from 'url';
|
||||
const require = createRequire(import.meta.url);
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
const SPEC_PATH = path.join(__dirname, '..', 'static', 'resources', 'openapi.json');
|
||||
const SPEC_PATH = path.join(
|
||||
__dirname,
|
||||
'..',
|
||||
'static',
|
||||
'resources',
|
||||
'openapi.json',
|
||||
);
|
||||
const API_DOCS_DIR = path.join(__dirname, '..', 'developer_docs', 'api');
|
||||
const SIDEBAR_PATH = path.join(API_DOCS_DIR, 'sidebar.js');
|
||||
|
||||
@@ -53,7 +59,7 @@ function buildSlugMap() {
|
||||
try {
|
||||
const sidebar = require(SIDEBAR_PATH);
|
||||
|
||||
const extractDocs = (items) => {
|
||||
const extractDocs = items => {
|
||||
for (const item of items) {
|
||||
if (item.type === 'doc' && item.label && item.id) {
|
||||
const slug = item.id.replace(/^api\//, '');
|
||||
@@ -109,13 +115,15 @@ function main() {
|
||||
|
||||
// Sort endpoints within each tag by path then method
|
||||
for (const tag of Object.keys(tagEndpoints)) {
|
||||
tagEndpoints[tag].sort((a, b) =>
|
||||
a.path.localeCompare(b.path) || a.method.localeCompare(b.method)
|
||||
tagEndpoints[tag].sort(
|
||||
(a, b) =>
|
||||
a.path.localeCompare(b.path) || a.method.localeCompare(b.method),
|
||||
);
|
||||
}
|
||||
|
||||
// Scan existing .tag.mdx files and match by frontmatter title
|
||||
const tagFiles = fs.readdirSync(API_DOCS_DIR)
|
||||
const tagFiles = fs
|
||||
.readdirSync(API_DOCS_DIR)
|
||||
.filter(f => f.endsWith('.tag.mdx'));
|
||||
|
||||
let updated = 0;
|
||||
|
||||
@@ -564,7 +564,9 @@ print(json.dumps(databases, default=str))
|
||||
throw new Error('No metadata found in engine specs');
|
||||
}
|
||||
|
||||
console.log(`Extracted metadata from ${Object.keys(databases).length} engine specs`);
|
||||
console.log(
|
||||
`Extracted metadata from ${Object.keys(databases).length} engine specs`,
|
||||
);
|
||||
return databases;
|
||||
} catch (err) {
|
||||
console.log('Engine spec metadata extraction failed:', err.message);
|
||||
@@ -615,20 +617,20 @@ function buildStatistics(databases) {
|
||||
for (const cat of categories) {
|
||||
// Map category constant names to display names
|
||||
const categoryDisplayNames = {
|
||||
'CLOUD_AWS': 'Cloud - AWS',
|
||||
'CLOUD_GCP': 'Cloud - Google',
|
||||
'CLOUD_AZURE': 'Cloud - Azure',
|
||||
'CLOUD_DATA_WAREHOUSES': 'Cloud Data Warehouses',
|
||||
'APACHE_PROJECTS': 'Apache Projects',
|
||||
'TRADITIONAL_RDBMS': 'Traditional RDBMS',
|
||||
'ANALYTICAL_DATABASES': 'Analytical Databases',
|
||||
'SEARCH_NOSQL': 'Search & NoSQL',
|
||||
'QUERY_ENGINES': 'Query Engines',
|
||||
'TIME_SERIES': 'Time Series Databases',
|
||||
'OTHER': 'Other Databases',
|
||||
'OPEN_SOURCE': 'Open Source',
|
||||
'HOSTED_OPEN_SOURCE': 'Hosted Open Source',
|
||||
'PROPRIETARY': 'Proprietary',
|
||||
CLOUD_AWS: 'Cloud - AWS',
|
||||
CLOUD_GCP: 'Cloud - Google',
|
||||
CLOUD_AZURE: 'Cloud - Azure',
|
||||
CLOUD_DATA_WAREHOUSES: 'Cloud Data Warehouses',
|
||||
APACHE_PROJECTS: 'Apache Projects',
|
||||
TRADITIONAL_RDBMS: 'Traditional RDBMS',
|
||||
ANALYTICAL_DATABASES: 'Analytical Databases',
|
||||
SEARCH_NOSQL: 'Search & NoSQL',
|
||||
QUERY_ENGINES: 'Query Engines',
|
||||
TIME_SERIES: 'Time Series Databases',
|
||||
OTHER: 'Other Databases',
|
||||
OPEN_SOURCE: 'Open Source',
|
||||
HOSTED_OPEN_SOURCE: 'Hosted Open Source',
|
||||
PROPRIETARY: 'Proprietary',
|
||||
};
|
||||
const displayName = categoryDisplayNames[cat] || cat;
|
||||
if (!stats.byCategory[displayName]) {
|
||||
@@ -657,7 +659,9 @@ function toSlug(name) {
|
||||
* Generate MDX content for a single database page
|
||||
*/
|
||||
function generateDatabaseMDX(name, db) {
|
||||
const description = db.documentation?.description || `Documentation for ${name} database connection.`;
|
||||
const description =
|
||||
db.documentation?.description ||
|
||||
`Documentation for ${name} database connection.`;
|
||||
const shortDesc = description
|
||||
.slice(0, 160)
|
||||
.replace(/\\/g, '\\\\')
|
||||
@@ -704,7 +708,9 @@ export const databaseInfo = ${inlineData};
|
||||
* Generate the index MDX for the databases overview
|
||||
*/
|
||||
function generateIndexMDX(statistics, usedFlaskContext = true) {
|
||||
const fallbackNotice = usedFlaskContext ? '' : `
|
||||
const fallbackNotice = usedFlaskContext
|
||||
? ''
|
||||
: `
|
||||
:::info Developer Note
|
||||
This documentation was built without Flask context, so feature diagnostics (scores, time grain support, etc.)
|
||||
may not reflect actual database capabilities. For full diagnostics, build docs locally with:
|
||||
@@ -832,7 +838,10 @@ function getImageDimensions(imgPath) {
|
||||
const content = fs.readFileSync(imgPath, 'utf-8');
|
||||
const vbMatch = content.match(/viewBox=["']([^"']+)["']/);
|
||||
if (vbMatch) {
|
||||
const parts = vbMatch[1].trim().split(/[\s,]+/).map(Number);
|
||||
const parts = vbMatch[1]
|
||||
.trim()
|
||||
.split(/[\s,]+/)
|
||||
.map(Number);
|
||||
if (parts.length >= 4 && parts[2] > 0 && parts[3] > 0) {
|
||||
return { width: parts[2], height: parts[3] };
|
||||
}
|
||||
@@ -841,7 +850,9 @@ function getImageDimensions(imgPath) {
|
||||
if (dims.width > 0 && dims.height > 0) {
|
||||
return { width: dims.width, height: dims.height };
|
||||
}
|
||||
} catch { /* fall through */ }
|
||||
} catch {
|
||||
/* fall through */
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -879,7 +890,9 @@ function generateReadmeLogos(databases) {
|
||||
// deduplicated by logo filename (matches docs homepage logic in index.tsx)
|
||||
const seenLogos = new Set();
|
||||
const dbsWithLogos = Object.entries(databases)
|
||||
.filter(([, db]) => db.documentation?.logo && db.documentation?.homepage_url)
|
||||
.filter(
|
||||
([, db]) => db.documentation?.logo && db.documentation?.homepage_url,
|
||||
)
|
||||
.sort(([a], [b]) => a.localeCompare(b))
|
||||
.filter(([, db]) => {
|
||||
const logo = db.documentation.logo;
|
||||
@@ -901,16 +914,27 @@ function generateReadmeLogos(databases) {
|
||||
// Generate linked logo tags with aspect-ratio-preserving dimensions
|
||||
const logoTags = dbsWithLogos.map(([name, db]) => {
|
||||
const logo = db.documentation.logo;
|
||||
const slug = name.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '');
|
||||
const slug = name
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, '-')
|
||||
.replace(/^-|-$/g, '');
|
||||
const imgPath = path.join(IMAGES_DIR, logo);
|
||||
|
||||
const dims = getImageDimensions(imgPath);
|
||||
let sizeAttrs;
|
||||
if (dims) {
|
||||
const { width, height } = fitToBoundingBox(dims.width, dims.height, MAX_WIDTH, MAX_HEIGHT, MIN_HEIGHT);
|
||||
const { width, height } = fitToBoundingBox(
|
||||
dims.width,
|
||||
dims.height,
|
||||
MAX_WIDTH,
|
||||
MAX_HEIGHT,
|
||||
MIN_HEIGHT,
|
||||
);
|
||||
sizeAttrs = `width="${width}" height="${height}"`;
|
||||
} else {
|
||||
console.warn(` Could not read dimensions for ${logo}, using height-only fallback`);
|
||||
console.warn(
|
||||
` Could not read dimensions for ${logo}, using height-only fallback`,
|
||||
);
|
||||
sizeAttrs = `height="${MAX_HEIGHT}"`;
|
||||
}
|
||||
|
||||
@@ -936,9 +960,14 @@ function updateReadme(databases) {
|
||||
const content = fs.readFileSync(README_PATH, 'utf-8');
|
||||
|
||||
// Check if markers exist
|
||||
if (!content.includes(README_START_MARKER) || !content.includes(README_END_MARKER)) {
|
||||
if (
|
||||
!content.includes(README_START_MARKER) ||
|
||||
!content.includes(README_END_MARKER)
|
||||
) {
|
||||
console.log('README.md missing database markers, skipping update');
|
||||
console.log(` Add ${README_START_MARKER} and ${README_END_MARKER} to enable auto-generation`);
|
||||
console.log(
|
||||
` Add ${README_START_MARKER} and ${README_END_MARKER} to enable auto-generation`,
|
||||
);
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -948,11 +977,11 @@ function updateReadme(databases) {
|
||||
// Replace content between markers
|
||||
const pattern = new RegExp(
|
||||
`${README_START_MARKER}[\\s\\S]*?${README_END_MARKER}`,
|
||||
'g'
|
||||
'g',
|
||||
);
|
||||
const newContent = content.replace(
|
||||
pattern,
|
||||
`${README_START_MARKER}\n${logosHtml}\n${README_END_MARKER}`
|
||||
`${README_START_MARKER}\n${logosHtml}\n${README_END_MARKER}`,
|
||||
);
|
||||
|
||||
if (newContent !== content) {
|
||||
@@ -990,9 +1019,14 @@ function extractCustomErrors() {
|
||||
|
||||
const customErrors = JSON.parse(result.stdout);
|
||||
const moduleCount = Object.keys(customErrors).length;
|
||||
const errorCount = Object.values(customErrors).reduce((sum, classes) =>
|
||||
sum + Object.values(classes).reduce((s, errs) => s + errs.length, 0), 0);
|
||||
console.log(` Found ${errorCount} custom errors across ${moduleCount} modules`);
|
||||
const errorCount = Object.values(customErrors).reduce(
|
||||
(sum, classes) =>
|
||||
sum + Object.values(classes).reduce((s, errs) => s + errs.length, 0),
|
||||
0,
|
||||
);
|
||||
console.log(
|
||||
` Found ${errorCount} custom errors across ${moduleCount} modules`,
|
||||
);
|
||||
return customErrors;
|
||||
} catch (err) {
|
||||
console.log(' Could not extract custom_errors:', err.message);
|
||||
@@ -1110,7 +1144,9 @@ function mergeWithExistingDiagnostics(newDatabases, existingData) {
|
||||
|
||||
const preserved = Object.values(newDatabases).filter(d => d.score > 0).length;
|
||||
if (preserved > 0) {
|
||||
console.log(`Preserved score/time_grains for ${preserved} databases from existing data`);
|
||||
console.log(
|
||||
`Preserved score/time_grains for ${preserved} databases from existing data`,
|
||||
);
|
||||
}
|
||||
|
||||
return newDatabases;
|
||||
@@ -1153,7 +1189,7 @@ async function main() {
|
||||
console.log(`Processed ${Object.keys(databases).length} databases\n`);
|
||||
|
||||
// Check if new data has scores; if not, preserve existing diagnostics
|
||||
const hasNewScores = Object.values(databases).some((db) => db.score > 0);
|
||||
const hasNewScores = Object.values(databases).some(db => db.score > 0);
|
||||
if (!hasNewScores && existingData) {
|
||||
databases = mergeWithExistingDiagnostics(databases, existingData);
|
||||
}
|
||||
@@ -1182,16 +1218,21 @@ async function main() {
|
||||
fs.writeFileSync(DATA_OUTPUT_FILE, JSON.stringify(output, null, 2) + '\n');
|
||||
console.log(`Generated: ${path.relative(DOCS_DIR, DATA_OUTPUT_FILE)}`);
|
||||
|
||||
|
||||
// Ensure supported directory exists
|
||||
if (!fs.existsSync(MDX_SUPPORTED_DIR)) {
|
||||
fs.mkdirSync(MDX_SUPPORTED_DIR, { recursive: true });
|
||||
}
|
||||
|
||||
// Clean up old MDX files that are no longer in the database list
|
||||
console.log(`\nCleaning up old MDX files in ${path.relative(DOCS_DIR, MDX_SUPPORTED_DIR)}/`);
|
||||
const existingMdxFiles = fs.readdirSync(MDX_SUPPORTED_DIR).filter(f => f.endsWith('.mdx'));
|
||||
const validSlugs = new Set(Object.keys(databases).map(name => `${toSlug(name)}.mdx`));
|
||||
console.log(
|
||||
`\nCleaning up old MDX files in ${path.relative(DOCS_DIR, MDX_SUPPORTED_DIR)}/`,
|
||||
);
|
||||
const existingMdxFiles = fs
|
||||
.readdirSync(MDX_SUPPORTED_DIR)
|
||||
.filter(f => f.endsWith('.mdx'));
|
||||
const validSlugs = new Set(
|
||||
Object.keys(databases).map(name => `${toSlug(name)}.mdx`),
|
||||
);
|
||||
let removedCount = 0;
|
||||
for (const file of existingMdxFiles) {
|
||||
if (!validSlugs.has(file)) {
|
||||
@@ -1204,7 +1245,9 @@ async function main() {
|
||||
}
|
||||
|
||||
// Generate individual MDX files for each database in supported/ subdirectory
|
||||
console.log(`\nGenerating MDX files in ${path.relative(DOCS_DIR, MDX_SUPPORTED_DIR)}/`);
|
||||
console.log(
|
||||
`\nGenerating MDX files in ${path.relative(DOCS_DIR, MDX_SUPPORTED_DIR)}/`,
|
||||
);
|
||||
|
||||
let mdxCount = 0;
|
||||
for (const [name, db] of Object.entries(databases)) {
|
||||
@@ -1233,7 +1276,7 @@ async function main() {
|
||||
};
|
||||
fs.writeFileSync(
|
||||
path.join(MDX_OUTPUT_DIR, '_category_.json'),
|
||||
JSON.stringify(categoryJson, null, 2) + '\n'
|
||||
JSON.stringify(categoryJson, null, 2) + '\n',
|
||||
);
|
||||
|
||||
// Generate _category_.json for supported/ subdirectory (collapsible)
|
||||
@@ -1245,12 +1288,15 @@ async function main() {
|
||||
};
|
||||
fs.writeFileSync(
|
||||
path.join(MDX_SUPPORTED_DIR, '_category_.json'),
|
||||
JSON.stringify(supportedCategoryJson, null, 2) + '\n'
|
||||
JSON.stringify(supportedCategoryJson, null, 2) + '\n',
|
||||
);
|
||||
console.log(` Generated _category_.json files`);
|
||||
|
||||
// Update README.md database logos (only when explicitly requested)
|
||||
if (process.env.UPDATE_README === 'true' || process.argv.includes('--update-readme')) {
|
||||
if (
|
||||
process.env.UPDATE_README === 'true' ||
|
||||
process.argv.includes('--update-readme')
|
||||
) {
|
||||
console.log('');
|
||||
updateReadme(databases);
|
||||
}
|
||||
|
||||
@@ -58,16 +58,28 @@ const GENERATORS = [
|
||||
inputs: [
|
||||
{
|
||||
type: 'glob',
|
||||
base: path.join(ROOT_DIR, 'superset-frontend/packages/superset-ui-core/src/components'),
|
||||
base: path.join(
|
||||
ROOT_DIR,
|
||||
'superset-frontend/packages/superset-ui-core/src/components',
|
||||
),
|
||||
pattern: '**/*.stories.tsx',
|
||||
},
|
||||
{
|
||||
type: 'glob',
|
||||
base: path.join(ROOT_DIR, 'superset-frontend/packages/superset-core/src'),
|
||||
base: path.join(
|
||||
ROOT_DIR,
|
||||
'superset-frontend/packages/superset-core/src',
|
||||
),
|
||||
pattern: '**/*.stories.tsx',
|
||||
},
|
||||
{ type: 'file', path: path.join(DOCS_DIR, 'scripts/generate-superset-components.mjs') },
|
||||
{ type: 'file', path: path.join(DOCS_DIR, 'src/components/StorybookWrapper.jsx') },
|
||||
{
|
||||
type: 'file',
|
||||
path: path.join(DOCS_DIR, 'scripts/generate-superset-components.mjs'),
|
||||
},
|
||||
{
|
||||
type: 'file',
|
||||
path: path.join(DOCS_DIR, 'src/components/StorybookWrapper.jsx'),
|
||||
},
|
||||
],
|
||||
outputs: [
|
||||
path.join(DOCS_DIR, 'developer_docs/components/index.mdx'),
|
||||
@@ -84,7 +96,10 @@ const GENERATORS = [
|
||||
base: path.join(ROOT_DIR, 'superset/db_engine_specs'),
|
||||
pattern: '**/*.py',
|
||||
},
|
||||
{ type: 'file', path: path.join(DOCS_DIR, 'scripts/generate-database-docs.mjs') },
|
||||
{
|
||||
type: 'file',
|
||||
path: path.join(DOCS_DIR, 'scripts/generate-database-docs.mjs'),
|
||||
},
|
||||
],
|
||||
outputs: [
|
||||
path.join(DOCS_DIR, 'src/data/databases.json'),
|
||||
@@ -96,15 +111,28 @@ const GENERATORS = [
|
||||
command:
|
||||
'python3 scripts/fix-openapi-spec.py && docusaurus gen-api-docs superset && node scripts/convert-api-sidebar.mjs && node scripts/generate-api-index.mjs && node scripts/generate-api-tag-pages.mjs',
|
||||
inputs: [
|
||||
{ type: 'file', path: path.join(DOCS_DIR, 'static/resources/openapi.json') },
|
||||
{ type: 'file', path: path.join(DOCS_DIR, 'scripts/fix-openapi-spec.py') },
|
||||
{ type: 'file', path: path.join(DOCS_DIR, 'scripts/convert-api-sidebar.mjs') },
|
||||
{ type: 'file', path: path.join(DOCS_DIR, 'scripts/generate-api-index.mjs') },
|
||||
{ type: 'file', path: path.join(DOCS_DIR, 'scripts/generate-api-tag-pages.mjs') },
|
||||
],
|
||||
outputs: [
|
||||
path.join(DOCS_DIR, 'docs/api.mdx'),
|
||||
{
|
||||
type: 'file',
|
||||
path: path.join(DOCS_DIR, 'static/resources/openapi.json'),
|
||||
},
|
||||
{
|
||||
type: 'file',
|
||||
path: path.join(DOCS_DIR, 'scripts/fix-openapi-spec.py'),
|
||||
},
|
||||
{
|
||||
type: 'file',
|
||||
path: path.join(DOCS_DIR, 'scripts/convert-api-sidebar.mjs'),
|
||||
},
|
||||
{
|
||||
type: 'file',
|
||||
path: path.join(DOCS_DIR, 'scripts/generate-api-index.mjs'),
|
||||
},
|
||||
{
|
||||
type: 'file',
|
||||
path: path.join(DOCS_DIR, 'scripts/generate-api-tag-pages.mjs'),
|
||||
},
|
||||
],
|
||||
outputs: [path.join(DOCS_DIR, 'docs/api.mdx')],
|
||||
},
|
||||
];
|
||||
|
||||
@@ -121,11 +149,15 @@ function walkDir(dir, pattern) {
|
||||
for (const entry of fs.readdirSync(currentDir, { withFileTypes: true })) {
|
||||
const fullPath = path.join(currentDir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
if (entry.name === 'node_modules' || entry.name === '__pycache__') continue;
|
||||
if (entry.name === 'node_modules' || entry.name === '__pycache__')
|
||||
continue;
|
||||
walk(fullPath);
|
||||
} else {
|
||||
// Normalize to forward slashes so glob patterns work on all platforms
|
||||
const relativePath = path.relative(dir, fullPath).split(path.sep).join('/');
|
||||
const relativePath = path
|
||||
.relative(dir, fullPath)
|
||||
.split(path.sep)
|
||||
.join('/');
|
||||
if (regex.test(relativePath)) {
|
||||
results.push(fullPath);
|
||||
}
|
||||
@@ -160,7 +192,9 @@ function computeInputHash(inputs) {
|
||||
hash.update(`file:${input.path}:${hashFile(input.path)}\n`);
|
||||
} else if (input.type === 'glob') {
|
||||
const files = walkDir(input.base, input.pattern);
|
||||
hash.update(`glob:${input.base}:${input.pattern}:count=${files.length}\n`);
|
||||
hash.update(
|
||||
`glob:${input.base}:${input.pattern}:count=${files.length}\n`,
|
||||
);
|
||||
for (const file of files) {
|
||||
hash.update(` ${path.relative(input.base, file)}:${hashFile(file)}\n`);
|
||||
}
|
||||
@@ -170,7 +204,7 @@ function computeInputHash(inputs) {
|
||||
}
|
||||
|
||||
function outputsExist(outputs) {
|
||||
return outputs.every((p) => fs.existsSync(p));
|
||||
return outputs.every(p => fs.existsSync(p));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -211,11 +245,11 @@ async function main() {
|
||||
// parallel, then api-docs sequentially (it depends on docusaurus CLI
|
||||
// being available, not on other generators).
|
||||
|
||||
const independent = GENERATORS.filter((g) => g.name !== 'api-docs');
|
||||
const sequential = GENERATORS.filter((g) => g.name === 'api-docs');
|
||||
const independent = GENERATORS.filter(g => g.name !== 'api-docs');
|
||||
const sequential = GENERATORS.filter(g => g.name === 'api-docs');
|
||||
|
||||
// Check and run independent generators in parallel
|
||||
const parallelPromises = independent.map((gen) => {
|
||||
const parallelPromises = independent.map(gen => {
|
||||
const currentHash = computeInputHash(gen.inputs);
|
||||
const cachedHash = cache[gen.name];
|
||||
const hasOutputs = outputsExist(gen.outputs);
|
||||
@@ -240,7 +274,7 @@ async function main() {
|
||||
stdio: 'inherit',
|
||||
env: process.env,
|
||||
});
|
||||
child.on('close', (code) => {
|
||||
child.on('close', code => {
|
||||
if (code === 0) {
|
||||
updatedCache[gen.name] = currentHash;
|
||||
resolve();
|
||||
@@ -249,7 +283,7 @@ async function main() {
|
||||
reject(new Error(`${gen.name} failed with exit code ${code}`));
|
||||
}
|
||||
});
|
||||
child.on('error', (err) => {
|
||||
child.on('error', err => {
|
||||
console.error(` ✗ ${gen.name} failed to start`);
|
||||
reject(err);
|
||||
});
|
||||
@@ -301,7 +335,7 @@ async function main() {
|
||||
}
|
||||
|
||||
console.log('Checking generators for changes...\n');
|
||||
main().catch((err) => {
|
||||
main().catch(err => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -65,11 +65,25 @@ const docsRoot = path.join(__dirname, '..');
|
||||
const ROOTS = ['docs', 'admin_docs', 'developer_docs', 'components'];
|
||||
|
||||
const NON_DOC_EXTENSIONS = new Set([
|
||||
'.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg', '.ico',
|
||||
'.json', '.yaml', '.yml', '.txt', '.csv',
|
||||
'.zip', '.tar', '.gz',
|
||||
'.png',
|
||||
'.jpg',
|
||||
'.jpeg',
|
||||
'.gif',
|
||||
'.webp',
|
||||
'.svg',
|
||||
'.ico',
|
||||
'.json',
|
||||
'.yaml',
|
||||
'.yml',
|
||||
'.txt',
|
||||
'.csv',
|
||||
'.zip',
|
||||
'.tar',
|
||||
'.gz',
|
||||
'.pdf',
|
||||
'.mp4', '.webm', '.mov',
|
||||
'.mp4',
|
||||
'.webm',
|
||||
'.mov',
|
||||
]);
|
||||
|
||||
const LINK_RE = /\[[^\]\n]+?\]\((?<url>\.{1,2}\/[^)\s]+?)\)/g;
|
||||
@@ -180,19 +194,19 @@ for (const f of findings) {
|
||||
}
|
||||
|
||||
console.error(
|
||||
`✗ lint-docs-links: found ${findings.length} broken internal link(s)`
|
||||
`✗ lint-docs-links: found ${findings.length} broken internal link(s)`,
|
||||
);
|
||||
console.error('');
|
||||
|
||||
if (groups.bare.length) {
|
||||
console.error(
|
||||
` ${groups.bare.length} bare relative link(s) (no .md/.mdx extension)`
|
||||
` ${groups.bare.length} bare relative link(s) (no .md/.mdx extension)`,
|
||||
);
|
||||
console.error(
|
||||
" Docusaurus's file resolver skips these; the browser resolves them"
|
||||
" Docusaurus's file resolver skips these; the browser resolves them",
|
||||
);
|
||||
console.error(
|
||||
' against the current page URL — wrong directory for trailing-slash routes.'
|
||||
' against the current page URL — wrong directory for trailing-slash routes.',
|
||||
);
|
||||
console.error(' Add the extension so the file resolver picks them up.');
|
||||
console.error('');
|
||||
@@ -204,21 +218,19 @@ if (groups.bare.length) {
|
||||
|
||||
if (groups['wrong-extension'].length) {
|
||||
console.error(
|
||||
` ${groups['wrong-extension'].length} wrong-extension link(s) (.md vs .mdx mismatch)`
|
||||
` ${groups['wrong-extension'].length} wrong-extension link(s) (.md vs .mdx mismatch)`,
|
||||
);
|
||||
console.error(' The target file exists with the other extension on disk.');
|
||||
console.error('');
|
||||
for (const f of groups['wrong-extension']) {
|
||||
console.error(
|
||||
` ${f.file}:${f.line} ${f.url} → use ${f.actualExt}`
|
||||
);
|
||||
console.error(` ${f.file}:${f.line} ${f.url} → use ${f.actualExt}`);
|
||||
}
|
||||
console.error('');
|
||||
}
|
||||
|
||||
if (groups['missing-target'].length) {
|
||||
console.error(
|
||||
` ${groups['missing-target'].length} missing-target link(s) (file doesn't exist)`
|
||||
` ${groups['missing-target'].length} missing-target link(s) (file doesn't exist)`,
|
||||
);
|
||||
console.error('');
|
||||
for (const f of groups['missing-target']) {
|
||||
|
||||
@@ -32,7 +32,7 @@ const CONFIG_FILE = path.join(__dirname, '..', 'versions-config.json');
|
||||
// Parse command line arguments
|
||||
const rawArgs = process.argv.slice(2);
|
||||
const skipGenerate = rawArgs.includes('--skip-generate');
|
||||
const args = rawArgs.filter((a) => a !== '--skip-generate');
|
||||
const args = rawArgs.filter(a => a !== '--skip-generate');
|
||||
const command = args[0]; // 'add' or 'remove'
|
||||
const section = args[1]; // 'user_docs', 'admin_docs', 'developer_docs', or 'components'
|
||||
const version = args[2]; // version string like '1.2.0'
|
||||
@@ -73,7 +73,8 @@ function freezeDataImports(section, version) {
|
||||
// Matches data file imports in two flavors:
|
||||
// `from '../../foo/bar.json'` (relative, must escape one or more dirs)
|
||||
// `from '@site/static/foo.json'` (Docusaurus site-root alias)
|
||||
const dataImportRe = /(from\s+['"])((?:\.\.\/)+|@site\/)([^'"\s]+\.(?:json|ya?ml))(['"])/g;
|
||||
const dataImportRe =
|
||||
/(from\s+['"])((?:\.\.\/)+|@site\/)([^'"\s]+\.(?:json|ya?ml))(['"])/g;
|
||||
|
||||
function freezeOne(fullPath, depth, prefix, pathSpec, importPath, suffix) {
|
||||
let resolvedSource;
|
||||
@@ -87,7 +88,9 @@ function freezeDataImports(section, version) {
|
||||
const upCount = pathSpec.match(/\.\.\//g).length;
|
||||
if (upCount <= depth) return null;
|
||||
const relativeFromVersioned = path.relative(versionedDocsPath, fullPath);
|
||||
const originalDir = path.dirname(path.join(sectionRoot, relativeFromVersioned));
|
||||
const originalDir = path.dirname(
|
||||
path.join(sectionRoot, relativeFromVersioned),
|
||||
);
|
||||
resolvedSource = path.resolve(originalDir, pathSpec + importPath);
|
||||
}
|
||||
// Skip imports that land inside the section root — those get copied
|
||||
@@ -105,7 +108,9 @@ function freezeDataImports(section, version) {
|
||||
.relative(path.dirname(fullPath), destPath)
|
||||
.split(path.sep)
|
||||
.join('/');
|
||||
const finalImport = rewritten.startsWith('.') ? rewritten : `./${rewritten}`;
|
||||
const finalImport = rewritten.startsWith('.')
|
||||
? rewritten
|
||||
: `./${rewritten}`;
|
||||
return `${prefix}${finalImport}${suffix}`;
|
||||
}
|
||||
|
||||
@@ -119,19 +124,32 @@ function freezeDataImports(section, version) {
|
||||
const original = fs.readFileSync(fullPath, 'utf8');
|
||||
let inFence = false;
|
||||
let mutated = false;
|
||||
const updated = original.split('\n').map(line => {
|
||||
if (/^\s*(```|~~~)/.test(line)) {
|
||||
inFence = !inFence;
|
||||
return line;
|
||||
}
|
||||
if (inFence) return line;
|
||||
return line.replace(dataImportRe, (match, prefix, pathSpec, importPath, suffix) => {
|
||||
const rewritten = freezeOne(fullPath, depth, prefix, pathSpec, importPath, suffix);
|
||||
if (rewritten === null) return match;
|
||||
mutated = true;
|
||||
return rewritten;
|
||||
});
|
||||
}).join('\n');
|
||||
const updated = original
|
||||
.split('\n')
|
||||
.map(line => {
|
||||
if (/^\s*(```|~~~)/.test(line)) {
|
||||
inFence = !inFence;
|
||||
return line;
|
||||
}
|
||||
if (inFence) return line;
|
||||
return line.replace(
|
||||
dataImportRe,
|
||||
(match, prefix, pathSpec, importPath, suffix) => {
|
||||
const rewritten = freezeOne(
|
||||
fullPath,
|
||||
depth,
|
||||
prefix,
|
||||
pathSpec,
|
||||
importPath,
|
||||
suffix,
|
||||
);
|
||||
if (rewritten === null) return match;
|
||||
mutated = true;
|
||||
return rewritten;
|
||||
},
|
||||
);
|
||||
})
|
||||
.join('\n');
|
||||
if (mutated) {
|
||||
fs.writeFileSync(fullPath, updated);
|
||||
const rel = path.relative(versionedDocsPath, fullPath);
|
||||
@@ -171,20 +189,23 @@ function fixVersionedImports(section, version) {
|
||||
// Track fenced code blocks so we don't rewrite import samples inside
|
||||
// ```ts / ```js (etc.) blocks that are documentation, not real imports.
|
||||
let inFence = false;
|
||||
const updated = original.split('\n').map(line => {
|
||||
if (/^\s*(```|~~~)/.test(line)) {
|
||||
inFence = !inFence;
|
||||
return line;
|
||||
}
|
||||
if (inFence) return line;
|
||||
return line.replace(
|
||||
/(from\s+['"])((?:\.\.\/)+)/g,
|
||||
(match, prefix, dots) => {
|
||||
const upCount = dots.match(/\.\.\//g).length;
|
||||
return upCount > depth ? `${prefix}../${dots}` : match;
|
||||
},
|
||||
);
|
||||
}).join('\n');
|
||||
const updated = original
|
||||
.split('\n')
|
||||
.map(line => {
|
||||
if (/^\s*(```|~~~)/.test(line)) {
|
||||
inFence = !inFence;
|
||||
return line;
|
||||
}
|
||||
if (inFence) return line;
|
||||
return line.replace(
|
||||
/(from\s+['"])((?:\.\.\/)+)/g,
|
||||
(match, prefix, dots) => {
|
||||
const upCount = dots.match(/\.\.\//g).length;
|
||||
return upCount > depth ? `${prefix}../${dots}` : match;
|
||||
},
|
||||
);
|
||||
})
|
||||
.join('\n');
|
||||
if (updated !== original) {
|
||||
fs.writeFileSync(fullPath, updated);
|
||||
const rel = path.relative(versionedDocsPath, fullPath);
|
||||
@@ -254,14 +275,15 @@ function addVersion(section, version) {
|
||||
|
||||
// Update config
|
||||
// Add to onlyIncludeVersions array (after 'current')
|
||||
const versionIndex = config[section].onlyIncludeVersions.indexOf('current') + 1;
|
||||
const versionIndex =
|
||||
config[section].onlyIncludeVersions.indexOf('current') + 1;
|
||||
config[section].onlyIncludeVersions.splice(versionIndex, 0, version);
|
||||
|
||||
// Add version metadata
|
||||
config[section].versions[version] = {
|
||||
label: version,
|
||||
path: version,
|
||||
banner: 'none'
|
||||
banner: 'none',
|
||||
};
|
||||
|
||||
// Note: we deliberately do NOT auto-bump `lastVersion` to the new
|
||||
@@ -334,7 +356,10 @@ function removeVersion(section, version) {
|
||||
fs.unlinkSync(versionsJsonPath);
|
||||
console.log(` Removed empty ${versionsJsonFile}`);
|
||||
} else {
|
||||
fs.writeFileSync(versionsJsonPath, JSON.stringify(versions, null, 2) + '\n');
|
||||
fs.writeFileSync(
|
||||
versionsJsonPath,
|
||||
JSON.stringify(versions, null, 2) + '\n',
|
||||
);
|
||||
console.log(` Updated ${versionsJsonFile}`);
|
||||
}
|
||||
}
|
||||
@@ -348,8 +373,11 @@ function removeVersion(section, version) {
|
||||
// Update lastVersion if needed
|
||||
if (config[section].lastVersion === version) {
|
||||
// Set to the next available version or 'current'
|
||||
const remainingVersions = config[section].onlyIncludeVersions.filter(v => v !== 'current');
|
||||
config[section].lastVersion = remainingVersions.length > 0 ? remainingVersions[0] : 'current';
|
||||
const remainingVersions = config[section].onlyIncludeVersions.filter(
|
||||
v => v !== 'current',
|
||||
);
|
||||
config[section].lastVersion =
|
||||
remainingVersions.length > 0 ? remainingVersions[0] : 'current';
|
||||
console.log(` Updated lastVersion to ${config[section].lastVersion}`);
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user