mirror of
https://github.com/apache/superset.git
synced 2026-04-23 18:14:56 +00:00
feat(docs): add filterable UI Components table and improve build performance (#38253)
Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Evan Rusackas
307
docs/scripts/generate-if-changed.mjs
Normal file
307
docs/scripts/generate-if-changed.mjs
Normal file
@@ -0,0 +1,307 @@
|
||||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Smart generator wrapper: only runs generators whose input files have changed.
|
||||
*
|
||||
* Computes a hash of each generator's input files (stories, engine specs,
|
||||
* openapi.json, and the generator scripts themselves). Compares against a
|
||||
* stored cache. Skips generators whose inputs and outputs are unchanged.
|
||||
*
|
||||
* Usage:
|
||||
* node scripts/generate-if-changed.mjs # smart mode (default)
|
||||
* node scripts/generate-if-changed.mjs --force # force regenerate all
|
||||
*/
|
||||
|
||||
import { createHash } from 'crypto';
|
||||
import { execSync, spawn } from 'child_process';
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
const DOCS_DIR = path.resolve(__dirname, '..');
|
||||
const ROOT_DIR = path.resolve(DOCS_DIR, '..');
|
||||
const CACHE_FILE = path.join(DOCS_DIR, '.docusaurus', 'generator-hashes.json');
|
||||
|
||||
const FORCE = process.argv.includes('--force');
|
||||
|
||||
// Ensure local node_modules/.bin is on PATH (needed for docusaurus CLI)
|
||||
const localBin = path.join(DOCS_DIR, 'node_modules', '.bin');
|
||||
process.env.PATH = `${localBin}${path.delimiter}${process.env.PATH}`;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Generator definitions
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const GENERATORS = [
|
||||
{
|
||||
name: 'superset-components',
|
||||
command: 'node scripts/generate-superset-components.mjs',
|
||||
inputs: [
|
||||
{
|
||||
type: 'glob',
|
||||
base: path.join(ROOT_DIR, 'superset-frontend/packages/superset-ui-core/src/components'),
|
||||
pattern: '**/*.stories.tsx',
|
||||
},
|
||||
{
|
||||
type: 'glob',
|
||||
base: path.join(ROOT_DIR, 'superset-frontend/packages/superset-core/src'),
|
||||
pattern: '**/*.stories.tsx',
|
||||
},
|
||||
{ type: 'file', path: path.join(DOCS_DIR, 'scripts/generate-superset-components.mjs') },
|
||||
{ type: 'file', path: path.join(DOCS_DIR, 'src/components/StorybookWrapper.jsx') },
|
||||
],
|
||||
outputs: [
|
||||
path.join(DOCS_DIR, 'developer_docs/components/index.mdx'),
|
||||
path.join(DOCS_DIR, 'static/data/components.json'),
|
||||
path.join(DOCS_DIR, 'src/types/apache-superset-core/index.d.ts'),
|
||||
],
|
||||
},
|
||||
{
|
||||
name: 'database-docs',
|
||||
command: 'node scripts/generate-database-docs.mjs',
|
||||
inputs: [
|
||||
{
|
||||
type: 'glob',
|
||||
base: path.join(ROOT_DIR, 'superset/db_engine_specs'),
|
||||
pattern: '**/*.py',
|
||||
},
|
||||
{ type: 'file', path: path.join(DOCS_DIR, 'scripts/generate-database-docs.mjs') },
|
||||
],
|
||||
outputs: [
|
||||
path.join(DOCS_DIR, 'src/data/databases.json'),
|
||||
path.join(DOCS_DIR, 'docs/databases/supported'),
|
||||
],
|
||||
},
|
||||
{
|
||||
name: 'api-docs',
|
||||
command:
|
||||
'python3 scripts/fix-openapi-spec.py && docusaurus gen-api-docs superset && node scripts/convert-api-sidebar.mjs && node scripts/generate-api-index.mjs && node scripts/generate-api-tag-pages.mjs',
|
||||
inputs: [
|
||||
{ type: 'file', path: path.join(DOCS_DIR, 'static/resources/openapi.json') },
|
||||
{ type: 'file', path: path.join(DOCS_DIR, 'scripts/fix-openapi-spec.py') },
|
||||
{ type: 'file', path: path.join(DOCS_DIR, 'scripts/convert-api-sidebar.mjs') },
|
||||
{ type: 'file', path: path.join(DOCS_DIR, 'scripts/generate-api-index.mjs') },
|
||||
{ type: 'file', path: path.join(DOCS_DIR, 'scripts/generate-api-tag-pages.mjs') },
|
||||
],
|
||||
outputs: [
|
||||
path.join(DOCS_DIR, 'docs/api.mdx'),
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Hashing utilities
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function walkDir(dir, pattern) {
|
||||
const results = [];
|
||||
if (!fs.existsSync(dir)) return results;
|
||||
|
||||
const regex = globToRegex(pattern);
|
||||
function walk(currentDir) {
|
||||
for (const entry of fs.readdirSync(currentDir, { withFileTypes: true })) {
|
||||
const fullPath = path.join(currentDir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
if (entry.name === 'node_modules' || entry.name === '__pycache__') continue;
|
||||
walk(fullPath);
|
||||
} else {
|
||||
// Normalize to forward slashes so glob patterns work on all platforms
|
||||
const relativePath = path.relative(dir, fullPath).split(path.sep).join('/');
|
||||
if (regex.test(relativePath)) {
|
||||
results.push(fullPath);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
walk(dir);
|
||||
return results.sort();
|
||||
}
|
||||
|
||||
function globToRegex(pattern) {
|
||||
// Simple glob-to-regex: ** matches any path, * matches anything except /
|
||||
const escaped = pattern
|
||||
.replace(/[.+^${}()|[\]\\]/g, '\\$&')
|
||||
.replace(/\*\*/g, '<<<GLOBSTAR>>>')
|
||||
.replace(/\*/g, '[^/]*')
|
||||
.replace(/<<<GLOBSTAR>>>/g, '.*');
|
||||
return new RegExp(`^${escaped}$`);
|
||||
}
|
||||
|
||||
function hashFile(filePath) {
|
||||
if (!fs.existsSync(filePath)) return 'missing';
|
||||
const stat = fs.statSync(filePath);
|
||||
// Use mtime + size for speed (avoids reading file contents)
|
||||
return `${stat.mtimeMs}:${stat.size}`;
|
||||
}
|
||||
|
||||
function computeInputHash(inputs) {
|
||||
const hash = createHash('md5');
|
||||
for (const input of inputs) {
|
||||
if (input.type === 'file') {
|
||||
hash.update(`file:${input.path}:${hashFile(input.path)}\n`);
|
||||
} else if (input.type === 'glob') {
|
||||
const files = walkDir(input.base, input.pattern);
|
||||
hash.update(`glob:${input.base}:${input.pattern}:count=${files.length}\n`);
|
||||
for (const file of files) {
|
||||
hash.update(` ${path.relative(input.base, file)}:${hashFile(file)}\n`);
|
||||
}
|
||||
}
|
||||
}
|
||||
return hash.digest('hex');
|
||||
}
|
||||
|
||||
function outputsExist(outputs) {
|
||||
return outputs.every((p) => fs.existsSync(p));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Cache management
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function loadCache() {
|
||||
try {
|
||||
if (fs.existsSync(CACHE_FILE)) {
|
||||
return JSON.parse(fs.readFileSync(CACHE_FILE, 'utf8'));
|
||||
}
|
||||
} catch {
|
||||
// Corrupted cache — start fresh
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
function saveCache(cache) {
|
||||
const dir = path.dirname(CACHE_FILE);
|
||||
if (!fs.existsSync(dir)) {
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
}
|
||||
fs.writeFileSync(CACHE_FILE, JSON.stringify(cache, null, 2));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Main
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function main() {
|
||||
const cache = loadCache();
|
||||
const updatedCache = { ...cache };
|
||||
let skipped = 0;
|
||||
let ran = 0;
|
||||
|
||||
// First pass: determine which generators need to run
|
||||
// Run independent generators (superset-components, database-docs) in
|
||||
// parallel, then api-docs sequentially (it depends on docusaurus CLI
|
||||
// being available, not on other generators).
|
||||
|
||||
const independent = GENERATORS.filter((g) => g.name !== 'api-docs');
|
||||
const sequential = GENERATORS.filter((g) => g.name === 'api-docs');
|
||||
|
||||
// Check and run independent generators in parallel
|
||||
const parallelPromises = independent.map((gen) => {
|
||||
const currentHash = computeInputHash(gen.inputs);
|
||||
const cachedHash = cache[gen.name];
|
||||
const hasOutputs = outputsExist(gen.outputs);
|
||||
|
||||
if (!FORCE && currentHash === cachedHash && hasOutputs) {
|
||||
console.log(` ✓ ${gen.name} — no changes, skipping`);
|
||||
skipped++;
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
const reason = FORCE
|
||||
? 'forced'
|
||||
: !hasOutputs
|
||||
? 'output missing'
|
||||
: 'inputs changed';
|
||||
console.log(` ↻ ${gen.name} — ${reason}, regenerating...`);
|
||||
ran++;
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const child = spawn('sh', ['-c', gen.command], {
|
||||
cwd: DOCS_DIR,
|
||||
stdio: 'inherit',
|
||||
env: process.env,
|
||||
});
|
||||
child.on('close', (code) => {
|
||||
if (code === 0) {
|
||||
updatedCache[gen.name] = currentHash;
|
||||
resolve();
|
||||
} else {
|
||||
console.error(` ✗ ${gen.name} failed (exit ${code})`);
|
||||
reject(new Error(`${gen.name} failed with exit code ${code}`));
|
||||
}
|
||||
});
|
||||
child.on('error', (err) => {
|
||||
console.error(` ✗ ${gen.name} failed to start`);
|
||||
reject(err);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
await Promise.all(parallelPromises);
|
||||
|
||||
// Run sequential generators (api-docs)
|
||||
for (const gen of sequential) {
|
||||
const currentHash = computeInputHash(gen.inputs);
|
||||
const cachedHash = cache[gen.name];
|
||||
const hasOutputs = outputsExist(gen.outputs);
|
||||
|
||||
if (!FORCE && currentHash === cachedHash && hasOutputs) {
|
||||
console.log(` ✓ ${gen.name} — no changes, skipping`);
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
const reason = FORCE
|
||||
? 'forced'
|
||||
: !hasOutputs
|
||||
? 'output missing'
|
||||
: 'inputs changed';
|
||||
console.log(` ↻ ${gen.name} — ${reason}, regenerating...`);
|
||||
ran++;
|
||||
|
||||
try {
|
||||
execSync(gen.command, {
|
||||
cwd: DOCS_DIR,
|
||||
stdio: 'inherit',
|
||||
timeout: 300_000,
|
||||
});
|
||||
updatedCache[gen.name] = currentHash;
|
||||
} catch (err) {
|
||||
console.error(` ✗ ${gen.name} failed`);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
saveCache(updatedCache);
|
||||
|
||||
if (ran === 0) {
|
||||
console.log(`\nAll ${skipped} generators up-to-date — nothing to do!\n`);
|
||||
} else {
|
||||
console.log(`\nDone: ${ran} regenerated, ${skipped} skipped.\n`);
|
||||
}
|
||||
}
|
||||
|
||||
console.log('Checking generators for changes...\n');
|
||||
main().catch((err) => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user