/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ /** * Smart generator wrapper: only runs generators whose input files have changed. * * Computes a hash of each generator's input files (stories, engine specs, * openapi.json, and the generator scripts themselves). Compares against a * stored cache. Skips generators whose inputs and outputs are unchanged. * * Usage: * node scripts/generate-if-changed.mjs # smart mode (default) * node scripts/generate-if-changed.mjs --force # force regenerate all */ import { createHash } from 'crypto'; import { execSync, spawn } from 'child_process'; import fs from 'fs'; import path from 'path'; import { fileURLToPath } from 'url'; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); const DOCS_DIR = path.resolve(__dirname, '..'); const ROOT_DIR = path.resolve(DOCS_DIR, '..'); const CACHE_FILE = path.join(DOCS_DIR, '.docusaurus', 'generator-hashes.json'); const FORCE = process.argv.includes('--force'); // Ensure local node_modules/.bin is on PATH (needed for docusaurus CLI) const localBin = path.join(DOCS_DIR, 'node_modules', '.bin'); process.env.PATH = `${localBin}${path.delimiter}${process.env.PATH}`; // --------------------------------------------------------------------------- // Generator definitions // --------------------------------------------------------------------------- const GENERATORS = [ { name: 'superset-components', command: 'node scripts/generate-superset-components.mjs', inputs: [ { type: 'glob', base: path.join(ROOT_DIR, 'superset-frontend/packages/superset-ui-core/src/components'), pattern: '**/*.stories.tsx', }, { type: 'glob', base: path.join(ROOT_DIR, 'superset-frontend/packages/superset-core/src'), pattern: '**/*.stories.tsx', }, { type: 'file', path: path.join(DOCS_DIR, 'scripts/generate-superset-components.mjs') }, { type: 'file', path: path.join(DOCS_DIR, 'src/components/StorybookWrapper.jsx') }, ], outputs: [ path.join(DOCS_DIR, 'developer_docs/components/index.mdx'), path.join(DOCS_DIR, 'static/data/components.json'), path.join(DOCS_DIR, 'src/types/apache-superset-core/index.d.ts'), ], }, { name: 'database-docs', command: 'node scripts/generate-database-docs.mjs', inputs: [ { type: 'glob', base: path.join(ROOT_DIR, 'superset/db_engine_specs'), pattern: '**/*.py', }, { type: 'file', path: path.join(DOCS_DIR, 'scripts/generate-database-docs.mjs') }, ], outputs: [ path.join(DOCS_DIR, 'src/data/databases.json'), path.join(DOCS_DIR, 'docs/databases/supported'), ], }, { name: 'api-docs', command: 'python3 scripts/fix-openapi-spec.py && docusaurus gen-api-docs superset && node scripts/convert-api-sidebar.mjs && node scripts/generate-api-index.mjs && node scripts/generate-api-tag-pages.mjs', inputs: [ { type: 'file', path: path.join(DOCS_DIR, 'static/resources/openapi.json') }, { type: 'file', path: path.join(DOCS_DIR, 'scripts/fix-openapi-spec.py') }, { type: 'file', path: path.join(DOCS_DIR, 'scripts/convert-api-sidebar.mjs') }, { type: 'file', path: path.join(DOCS_DIR, 'scripts/generate-api-index.mjs') }, { type: 'file', path: path.join(DOCS_DIR, 'scripts/generate-api-tag-pages.mjs') }, ], outputs: [ path.join(DOCS_DIR, 'docs/api.mdx'), ], }, ]; // --------------------------------------------------------------------------- // Hashing utilities // --------------------------------------------------------------------------- function walkDir(dir, pattern) { const results = []; if (!fs.existsSync(dir)) return results; const regex = globToRegex(pattern); function walk(currentDir) { for (const entry of fs.readdirSync(currentDir, { withFileTypes: true })) { const fullPath = path.join(currentDir, entry.name); if (entry.isDirectory()) { if (entry.name === 'node_modules' || entry.name === '__pycache__') continue; walk(fullPath); } else { // Normalize to forward slashes so glob patterns work on all platforms const relativePath = path.relative(dir, fullPath).split(path.sep).join('/'); if (regex.test(relativePath)) { results.push(fullPath); } } } } walk(dir); return results.sort(); } function globToRegex(pattern) { // Simple glob-to-regex: ** matches any path, * matches anything except / const escaped = pattern .replace(/[.+^${}()|[\]\\]/g, '\\$&') .replace(/\*\*/g, '<<>>') .replace(/\*/g, '[^/]*') .replace(/<<>>/g, '.*'); return new RegExp(`^${escaped}$`); } function hashFile(filePath) { if (!fs.existsSync(filePath)) return 'missing'; const stat = fs.statSync(filePath); // Use mtime + size for speed (avoids reading file contents) return `${stat.mtimeMs}:${stat.size}`; } function computeInputHash(inputs) { const hash = createHash('md5'); for (const input of inputs) { if (input.type === 'file') { hash.update(`file:${input.path}:${hashFile(input.path)}\n`); } else if (input.type === 'glob') { const files = walkDir(input.base, input.pattern); hash.update(`glob:${input.base}:${input.pattern}:count=${files.length}\n`); for (const file of files) { hash.update(` ${path.relative(input.base, file)}:${hashFile(file)}\n`); } } } return hash.digest('hex'); } function outputsExist(outputs) { return outputs.every((p) => fs.existsSync(p)); } // --------------------------------------------------------------------------- // Cache management // --------------------------------------------------------------------------- function loadCache() { try { if (fs.existsSync(CACHE_FILE)) { return JSON.parse(fs.readFileSync(CACHE_FILE, 'utf8')); } } catch { // Corrupted cache — start fresh } return {}; } function saveCache(cache) { const dir = path.dirname(CACHE_FILE); if (!fs.existsSync(dir)) { fs.mkdirSync(dir, { recursive: true }); } fs.writeFileSync(CACHE_FILE, JSON.stringify(cache, null, 2)); } // --------------------------------------------------------------------------- // Main // --------------------------------------------------------------------------- async function main() { const cache = loadCache(); const updatedCache = { ...cache }; let skipped = 0; let ran = 0; // First pass: determine which generators need to run // Run independent generators (superset-components, database-docs) in // parallel, then api-docs sequentially (it depends on docusaurus CLI // being available, not on other generators). const independent = GENERATORS.filter((g) => g.name !== 'api-docs'); const sequential = GENERATORS.filter((g) => g.name === 'api-docs'); // Check and run independent generators in parallel const parallelPromises = independent.map((gen) => { const currentHash = computeInputHash(gen.inputs); const cachedHash = cache[gen.name]; const hasOutputs = outputsExist(gen.outputs); if (!FORCE && currentHash === cachedHash && hasOutputs) { console.log(` ✓ ${gen.name} — no changes, skipping`); skipped++; return Promise.resolve(); } const reason = FORCE ? 'forced' : !hasOutputs ? 'output missing' : 'inputs changed'; console.log(` ↻ ${gen.name} — ${reason}, regenerating...`); ran++; return new Promise((resolve, reject) => { const child = spawn('sh', ['-c', gen.command], { cwd: DOCS_DIR, stdio: 'inherit', env: process.env, }); child.on('close', (code) => { if (code === 0) { updatedCache[gen.name] = currentHash; resolve(); } else { console.error(` ✗ ${gen.name} failed (exit ${code})`); reject(new Error(`${gen.name} failed with exit code ${code}`)); } }); child.on('error', (err) => { console.error(` ✗ ${gen.name} failed to start`); reject(err); }); }); }); await Promise.all(parallelPromises); // Run sequential generators (api-docs) for (const gen of sequential) { const currentHash = computeInputHash(gen.inputs); const cachedHash = cache[gen.name]; const hasOutputs = outputsExist(gen.outputs); if (!FORCE && currentHash === cachedHash && hasOutputs) { console.log(` ✓ ${gen.name} — no changes, skipping`); skipped++; continue; } const reason = FORCE ? 'forced' : !hasOutputs ? 'output missing' : 'inputs changed'; console.log(` ↻ ${gen.name} — ${reason}, regenerating...`); ran++; try { execSync(gen.command, { cwd: DOCS_DIR, stdio: 'inherit', timeout: 300_000, }); updatedCache[gen.name] = currentHash; } catch (err) { console.error(` ✗ ${gen.name} failed`); throw err; } } saveCache(updatedCache); if (ran === 0) { console.log(`\nAll ${skipped} generators up-to-date — nothing to do!\n`); } else { console.log(`\nDone: ${ran} regenerated, ${skipped} skipped.\n`); } } console.log('Checking generators for changes...\n'); main().catch((err) => { console.error(err); process.exit(1); });