mirror of
https://github.com/apache/superset.git
synced 2026-07-20 05:36:00 +00:00
docs: remove screenshots from repo — should not be committed to codebase
This commit is contained in:
kasiazjc
@@ -1,123 +0,0 @@
|
||||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Convert the generated TypeScript API sidebar to CommonJS format.
|
||||
* This allows the sidebar to be imported by sidebars.js.
|
||||
* Also adds unique keys to duplicate labels to avoid translation conflicts.
|
||||
*/
|
||||
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
const sidebarTsPath = path.join(__dirname, '..', 'developer_docs', 'api', 'sidebar.ts');
|
||||
const sidebarJsPath = path.join(__dirname, '..', 'developer_docs', 'api', 'sidebar.js');
|
||||
|
||||
if (!fs.existsSync(sidebarTsPath)) {
|
||||
console.log('No sidebar.ts found, skipping conversion');
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
let content = fs.readFileSync(sidebarTsPath, 'utf8');
|
||||
|
||||
// Remove TypeScript import
|
||||
content = content.replace(/import type.*\n/g, '');
|
||||
|
||||
// Remove type annotation
|
||||
content = content.replace(/: SidebarsConfig/g, '');
|
||||
|
||||
// Change export default to module.exports
|
||||
content = content.replace(
|
||||
/export default sidebar\.apisidebar;/,
|
||||
'module.exports = sidebar.apisidebar;'
|
||||
);
|
||||
|
||||
// Parse the sidebar to add unique keys for duplicate labels
|
||||
// This avoids translation key conflicts when the same label appears multiple times
|
||||
try {
|
||||
// Extract the sidebar object
|
||||
const sidebarMatch = content.match(/const sidebar = (\{[\s\S]*\});/);
|
||||
if (sidebarMatch) {
|
||||
// Use Function constructor instead of eval for safer evaluation
|
||||
const sidebarObj = new Function(`return ${sidebarMatch[1]}`)();
|
||||
|
||||
// First pass: count labels
|
||||
const countLabels = (items) => {
|
||||
const counts = {};
|
||||
const count = (item) => {
|
||||
if (item.type === 'doc' && item.label) {
|
||||
counts[item.label] = (counts[item.label] || 0) + 1;
|
||||
}
|
||||
if (item.items) {
|
||||
item.items.forEach(count);
|
||||
}
|
||||
};
|
||||
items.forEach(count);
|
||||
return counts;
|
||||
};
|
||||
|
||||
const counts = countLabels(sidebarObj.apisidebar);
|
||||
|
||||
// Second pass: add keys to items with duplicate labels
|
||||
const addKeys = (items, prefix = 'api') => {
|
||||
for (const item of items) {
|
||||
if (item.type === 'doc' && item.label && counts[item.label] > 1) {
|
||||
item.key = item.id;
|
||||
}
|
||||
// Also add keys to categories to avoid conflicts with main sidebar categories
|
||||
if (item.type === 'category' && item.label) {
|
||||
item.key = `${prefix}-category-${item.label.toLowerCase().replace(/\s+/g, '-')}`;
|
||||
}
|
||||
if (item.items) {
|
||||
addKeys(item.items, prefix);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
addKeys(sidebarObj.apisidebar);
|
||||
|
||||
// Regenerate the content with the updated sidebar
|
||||
content = `const sidebar = ${JSON.stringify(sidebarObj, null, 2)};
|
||||
|
||||
module.exports = sidebar.apisidebar;
|
||||
`;
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn('Could not add unique keys to sidebar:', e.message);
|
||||
// Fall back to simple conversion
|
||||
content = content.replace(
|
||||
/export default sidebar\.apisidebar;/,
|
||||
'module.exports = sidebar.apisidebar;'
|
||||
);
|
||||
}
|
||||
|
||||
// Add header with eslint-disable to allow @ts-nocheck
|
||||
const header = `/* eslint-disable @typescript-eslint/ban-ts-comment */
|
||||
// @ts-nocheck
|
||||
/**
|
||||
* Auto-generated CommonJS sidebar from sidebar.ts
|
||||
* Do not edit directly - run 'yarn generate:api-docs' to regenerate
|
||||
*/
|
||||
|
||||
`;
|
||||
|
||||
fs.writeFileSync(sidebarJsPath, header + content);
|
||||
console.log('Converted sidebar.ts to sidebar.js');
|
||||
@@ -1,296 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""
|
||||
Extract custom_errors from database engine specs for documentation.
|
||||
|
||||
This script parses engine spec files to extract error handling information
|
||||
that can be displayed on database documentation pages.
|
||||
|
||||
Usage: python scripts/extract_custom_errors.py
|
||||
Output: JSON mapping of engine spec module names to their custom errors
|
||||
"""
|
||||
|
||||
import ast
|
||||
import json # noqa: TID251 - standalone docs script, not part of superset
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
# Map SupersetErrorType values to human-readable categories and issue codes
|
||||
ERROR_TYPE_INFO = {
|
||||
"CONNECTION_INVALID_USERNAME_ERROR": {
|
||||
"category": "Authentication",
|
||||
"description": "Invalid username",
|
||||
"issue_codes": [1012],
|
||||
},
|
||||
"CONNECTION_INVALID_PASSWORD_ERROR": {
|
||||
"category": "Authentication",
|
||||
"description": "Invalid password",
|
||||
"issue_codes": [1013],
|
||||
},
|
||||
"CONNECTION_ACCESS_DENIED_ERROR": {
|
||||
"category": "Authentication",
|
||||
"description": "Access denied",
|
||||
"issue_codes": [1014, 1015],
|
||||
},
|
||||
"CONNECTION_INVALID_HOSTNAME_ERROR": {
|
||||
"category": "Connection",
|
||||
"description": "Invalid hostname",
|
||||
"issue_codes": [1007],
|
||||
},
|
||||
"CONNECTION_PORT_CLOSED_ERROR": {
|
||||
"category": "Connection",
|
||||
"description": "Port closed or refused",
|
||||
"issue_codes": [1008],
|
||||
},
|
||||
"CONNECTION_HOST_DOWN_ERROR": {
|
||||
"category": "Connection",
|
||||
"description": "Host unreachable",
|
||||
"issue_codes": [1009],
|
||||
},
|
||||
"CONNECTION_UNKNOWN_DATABASE_ERROR": {
|
||||
"category": "Connection",
|
||||
"description": "Unknown database",
|
||||
"issue_codes": [1015],
|
||||
},
|
||||
"CONNECTION_DATABASE_PERMISSIONS_ERROR": {
|
||||
"category": "Permissions",
|
||||
"description": "Insufficient permissions",
|
||||
"issue_codes": [1017],
|
||||
},
|
||||
"CONNECTION_MISSING_PARAMETERS_ERROR": {
|
||||
"category": "Configuration",
|
||||
"description": "Missing parameters",
|
||||
"issue_codes": [1018],
|
||||
},
|
||||
"CONNECTION_DATABASE_TIMEOUT": {
|
||||
"category": "Connection",
|
||||
"description": "Connection timeout",
|
||||
"issue_codes": [1001, 1009],
|
||||
},
|
||||
"COLUMN_DOES_NOT_EXIST_ERROR": {
|
||||
"category": "Query",
|
||||
"description": "Column not found",
|
||||
"issue_codes": [1003, 1004],
|
||||
},
|
||||
"TABLE_DOES_NOT_EXIST_ERROR": {
|
||||
"category": "Query",
|
||||
"description": "Table not found",
|
||||
"issue_codes": [1003, 1005],
|
||||
},
|
||||
"SCHEMA_DOES_NOT_EXIST_ERROR": {
|
||||
"category": "Query",
|
||||
"description": "Schema not found",
|
||||
"issue_codes": [1003, 1016],
|
||||
},
|
||||
"SYNTAX_ERROR": {
|
||||
"category": "Query",
|
||||
"description": "SQL syntax error",
|
||||
"issue_codes": [1030],
|
||||
},
|
||||
"OBJECT_DOES_NOT_EXIST_ERROR": {
|
||||
"category": "Query",
|
||||
"description": "Object not found",
|
||||
"issue_codes": [1029],
|
||||
},
|
||||
"GENERIC_DB_ENGINE_ERROR": {
|
||||
"category": "General",
|
||||
"description": "Database engine error",
|
||||
"issue_codes": [1002],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def extract_string_from_call(node: ast.Call) -> str | None:
|
||||
"""Extract string from __() or _() translation calls."""
|
||||
if not node.args:
|
||||
return None
|
||||
arg = node.args[0]
|
||||
if isinstance(arg, ast.Constant) and isinstance(arg.value, str):
|
||||
return arg.value
|
||||
elif isinstance(arg, ast.JoinedStr):
|
||||
# f-string - try to reconstruct
|
||||
parts = []
|
||||
for value in arg.values:
|
||||
if isinstance(value, ast.Constant):
|
||||
parts.append(str(value.value))
|
||||
elif isinstance(value, ast.FormattedValue):
|
||||
# Just use a placeholder
|
||||
parts.append("{...}")
|
||||
return "".join(parts)
|
||||
return None
|
||||
|
||||
|
||||
def extract_custom_errors_from_file(filepath: Path) -> dict[str, list[dict[str, Any]]]:
|
||||
"""
|
||||
Extract custom_errors definitions from a Python engine spec file.
|
||||
|
||||
Returns a dict mapping class names to their custom errors list.
|
||||
"""
|
||||
results = {}
|
||||
|
||||
try:
|
||||
with open(filepath, "r", encoding="utf-8") as f:
|
||||
source = f.read()
|
||||
|
||||
tree = ast.parse(source)
|
||||
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.ClassDef):
|
||||
class_name = node.name
|
||||
|
||||
for item in node.body:
|
||||
# Look for custom_errors = { ... }
|
||||
if (
|
||||
isinstance(item, ast.AnnAssign)
|
||||
and isinstance(item.target, ast.Name)
|
||||
and item.target.id == "custom_errors"
|
||||
and isinstance(item.value, ast.Dict)
|
||||
):
|
||||
errors = extract_errors_from_dict(item.value, source)
|
||||
if errors:
|
||||
results[class_name] = errors
|
||||
|
||||
# Also handle simple assignment: custom_errors = { ... }
|
||||
elif (
|
||||
isinstance(item, ast.Assign)
|
||||
and len(item.targets) == 1
|
||||
and isinstance(item.targets[0], ast.Name)
|
||||
and item.targets[0].id == "custom_errors"
|
||||
and isinstance(item.value, ast.Dict)
|
||||
):
|
||||
errors = extract_errors_from_dict(item.value, source)
|
||||
if errors:
|
||||
results[class_name] = errors
|
||||
|
||||
except (OSError, SyntaxError, ValueError) as e:
|
||||
print(f"Error parsing {filepath}: {e}", file=sys.stderr)
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def extract_regex_info(key: ast.expr) -> dict[str, Any]:
|
||||
"""Extract regex pattern info from the dict key."""
|
||||
if isinstance(key, ast.Name):
|
||||
return {"regex_name": key.id}
|
||||
if isinstance(key, ast.Call):
|
||||
if (
|
||||
isinstance(key.func, ast.Attribute)
|
||||
and key.func.attr == "compile"
|
||||
and key.args
|
||||
and isinstance(key.args[0], ast.Constant)
|
||||
):
|
||||
return {"regex_pattern": key.args[0].value}
|
||||
return {}
|
||||
|
||||
|
||||
def extract_invalid_fields(extra_node: ast.Dict) -> list[str]:
|
||||
"""Extract invalid fields from the extra dict."""
|
||||
for k, v in zip(extra_node.keys, extra_node.values, strict=False):
|
||||
if (
|
||||
isinstance(k, ast.Constant)
|
||||
and k.value == "invalid"
|
||||
and isinstance(v, ast.List)
|
||||
):
|
||||
return [elem.value for elem in v.elts if isinstance(elem, ast.Constant)]
|
||||
return []
|
||||
|
||||
|
||||
def extract_error_tuple_info(value: ast.Tuple) -> dict[str, Any]:
|
||||
"""Extract error info from the (message, error_type, extra) tuple."""
|
||||
result: dict[str, Any] = {}
|
||||
|
||||
# First element: message template
|
||||
msg_node = value.elts[0]
|
||||
if isinstance(msg_node, ast.Call):
|
||||
message = extract_string_from_call(msg_node)
|
||||
if message:
|
||||
result["message_template"] = message
|
||||
elif isinstance(msg_node, ast.Constant):
|
||||
result["message_template"] = msg_node.value
|
||||
|
||||
# Second element: SupersetErrorType.SOMETHING
|
||||
type_node = value.elts[1]
|
||||
if isinstance(type_node, ast.Attribute):
|
||||
error_type = type_node.attr
|
||||
result["error_type"] = error_type
|
||||
if error_type in ERROR_TYPE_INFO:
|
||||
type_info = ERROR_TYPE_INFO[error_type]
|
||||
result["category"] = type_info["category"]
|
||||
result["description"] = type_info["description"]
|
||||
result["issue_codes"] = type_info["issue_codes"]
|
||||
|
||||
# Third element: extra dict with invalid fields
|
||||
if len(value.elts) >= 3 and isinstance(value.elts[2], ast.Dict):
|
||||
invalid_fields = extract_invalid_fields(value.elts[2])
|
||||
if invalid_fields:
|
||||
result["invalid_fields"] = invalid_fields
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def extract_errors_from_dict(dict_node: ast.Dict, source: str) -> list[dict[str, Any]]:
|
||||
"""Extract error information from a custom_errors dict AST node."""
|
||||
errors = []
|
||||
|
||||
for key, value in zip(dict_node.keys, dict_node.values, strict=False):
|
||||
if key is None or value is None:
|
||||
continue
|
||||
|
||||
error_info = extract_regex_info(key)
|
||||
|
||||
if isinstance(value, ast.Tuple) and len(value.elts) >= 2:
|
||||
error_info.update(extract_error_tuple_info(value))
|
||||
|
||||
if error_info.get("error_type") and error_info.get("message_template"):
|
||||
errors.append(error_info)
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Main function to extract custom_errors from all engine specs."""
|
||||
# Find the superset root directory
|
||||
script_dir = Path(__file__).parent
|
||||
root_dir = script_dir.parent.parent
|
||||
specs_dir = root_dir / "superset" / "db_engine_specs"
|
||||
|
||||
if not specs_dir.exists():
|
||||
print(f"Error: Engine specs directory not found: {specs_dir}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
all_errors = {}
|
||||
|
||||
# Process each Python file in the specs directory
|
||||
for filepath in sorted(specs_dir.glob("*.py")):
|
||||
if filepath.name.startswith("_"):
|
||||
continue
|
||||
|
||||
module_name = filepath.stem
|
||||
class_errors = extract_custom_errors_from_file(filepath)
|
||||
|
||||
if class_errors:
|
||||
# Store errors by module and class
|
||||
all_errors[module_name] = class_errors
|
||||
|
||||
# Output as JSON
|
||||
print(json.dumps(all_errors, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,853 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""
|
||||
Fix missing schema references in the OpenAPI spec.
|
||||
|
||||
This script patches the openapi.json file to add any missing schemas
|
||||
that are referenced but not defined.
|
||||
"""
|
||||
|
||||
import json # noqa: TID251 - standalone docs script
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
def add_missing_schemas(spec: dict[str, Any]) -> tuple[dict[str, Any], list[str]]:
|
||||
"""Add missing schema definitions to the OpenAPI spec."""
|
||||
schemas = spec.get("components", {}).get("schemas", {})
|
||||
fixed = []
|
||||
|
||||
# DashboardScreenshotPostSchema - based on superset/dashboards/schemas.py
|
||||
if "DashboardScreenshotPostSchema" not in schemas:
|
||||
schemas["DashboardScreenshotPostSchema"] = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"dataMask": {
|
||||
"type": "object",
|
||||
"description": "An object representing the data mask.",
|
||||
"additionalProperties": True,
|
||||
},
|
||||
"activeTabs": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"description": "A list representing active tabs.",
|
||||
},
|
||||
"anchor": {
|
||||
"type": "string",
|
||||
"description": "A string representing the anchor.",
|
||||
},
|
||||
"urlParams": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"minItems": 2,
|
||||
"maxItems": 2,
|
||||
},
|
||||
"description": "A list of tuples, each containing two strings.",
|
||||
},
|
||||
},
|
||||
}
|
||||
fixed.append("DashboardScreenshotPostSchema")
|
||||
|
||||
# DashboardNativeFiltersConfigUpdateSchema - based on superset/dashboards/schemas.py
|
||||
if "DashboardNativeFiltersConfigUpdateSchema" not in schemas:
|
||||
schemas["DashboardNativeFiltersConfigUpdateSchema"] = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"deleted": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"description": "List of deleted filter IDs.",
|
||||
},
|
||||
"modified": {
|
||||
"type": "array",
|
||||
"items": {"type": "object"},
|
||||
"description": "List of modified filter configurations.",
|
||||
},
|
||||
"reordered": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"description": "List of filter IDs in new order.",
|
||||
},
|
||||
},
|
||||
}
|
||||
fixed.append("DashboardNativeFiltersConfigUpdateSchema")
|
||||
|
||||
# DashboardColorsConfigUpdateSchema - based on superset/dashboards/schemas.py
|
||||
if "DashboardColorsConfigUpdateSchema" not in schemas:
|
||||
schemas["DashboardColorsConfigUpdateSchema"] = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"color_namespace": {
|
||||
"type": "string",
|
||||
"nullable": True,
|
||||
"description": "The color namespace.",
|
||||
},
|
||||
"color_scheme": {
|
||||
"type": "string",
|
||||
"nullable": True,
|
||||
"description": "The color scheme name.",
|
||||
},
|
||||
"map_label_colors": {
|
||||
"type": "object",
|
||||
"additionalProperties": {"type": "string"},
|
||||
"description": "Mapping of labels to colors.",
|
||||
},
|
||||
"shared_label_colors": {
|
||||
"type": "object",
|
||||
"additionalProperties": {"type": "string"},
|
||||
"description": "Shared label colors across charts.",
|
||||
},
|
||||
"label_colors": {
|
||||
"type": "object",
|
||||
"additionalProperties": {"type": "string"},
|
||||
"description": "Label to color mapping.",
|
||||
},
|
||||
"color_scheme_domain": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"description": "Color scheme domain values.",
|
||||
},
|
||||
},
|
||||
}
|
||||
fixed.append("DashboardColorsConfigUpdateSchema")
|
||||
|
||||
# DashboardChartCustomizationsConfigUpdateSchema (dashboards/schemas.py)
|
||||
if "DashboardChartCustomizationsConfigUpdateSchema" not in schemas:
|
||||
schemas["DashboardChartCustomizationsConfigUpdateSchema"] = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"deleted": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"description": "List of deleted chart customization IDs.",
|
||||
},
|
||||
"modified": {
|
||||
"type": "array",
|
||||
"items": {"type": "object"},
|
||||
"description": "List of modified chart customizations.",
|
||||
},
|
||||
"reordered": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"description": "List of chart customization IDs in new order.",
|
||||
},
|
||||
},
|
||||
}
|
||||
fixed.append("DashboardChartCustomizationsConfigUpdateSchema")
|
||||
|
||||
# FormatQueryPayloadSchema - based on superset/sqllab/schemas.py
|
||||
if "FormatQueryPayloadSchema" not in schemas:
|
||||
schemas["FormatQueryPayloadSchema"] = {
|
||||
"type": "object",
|
||||
"required": ["sql"],
|
||||
"properties": {
|
||||
"sql": {
|
||||
"type": "string",
|
||||
"description": "The SQL query to format.",
|
||||
},
|
||||
"engine": {
|
||||
"type": "string",
|
||||
"nullable": True,
|
||||
"description": "The database engine.",
|
||||
},
|
||||
"database_id": {
|
||||
"type": "integer",
|
||||
"nullable": True,
|
||||
"description": "The database id.",
|
||||
},
|
||||
"template_params": {
|
||||
"type": "string",
|
||||
"nullable": True,
|
||||
"description": "The SQL query template params as JSON string.",
|
||||
},
|
||||
},
|
||||
}
|
||||
fixed.append("FormatQueryPayloadSchema")
|
||||
|
||||
# get_slack_channels_schema - based on superset/reports/schemas.py
|
||||
if "get_slack_channels_schema" not in schemas:
|
||||
schemas["get_slack_channels_schema"] = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"search_string": {
|
||||
"type": "string",
|
||||
"description": "String to search for in channel names.",
|
||||
},
|
||||
"types": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string",
|
||||
"enum": ["public_channel", "private_channel"],
|
||||
},
|
||||
"description": "Types of channels to search.",
|
||||
},
|
||||
"exact_match": {
|
||||
"type": "boolean",
|
||||
"description": "Whether to match channel names exactly.",
|
||||
},
|
||||
},
|
||||
}
|
||||
fixed.append("get_slack_channels_schema")
|
||||
|
||||
if "components" not in spec:
|
||||
spec["components"] = {}
|
||||
spec["components"]["schemas"] = schemas
|
||||
|
||||
return spec, fixed
|
||||
|
||||
|
||||
def path_to_operation_id(path: str, method: str) -> str:
|
||||
"""Convert a path and method to an operationId."""
|
||||
# Remove /api/v1/ prefix
|
||||
clean_path = path.replace("/api/v1/", "").strip("/")
|
||||
|
||||
# Replace path parameters
|
||||
clean_path = clean_path.replace("{", "by_").replace("}", "")
|
||||
|
||||
# Create operation name
|
||||
method_prefix = {
|
||||
"get": "get",
|
||||
"post": "create",
|
||||
"put": "update",
|
||||
"delete": "delete",
|
||||
"patch": "patch",
|
||||
}.get(method.lower(), method.lower())
|
||||
|
||||
return f"{method_prefix}_{clean_path}".replace("/", "_").replace("-", "_")
|
||||
|
||||
|
||||
def path_to_summary(path: str, method: str) -> str:
|
||||
"""Generate a human-readable summary from path and method."""
|
||||
# Remove /api/v1/ prefix
|
||||
clean_path = path.replace("/api/v1/", "").strip("/")
|
||||
|
||||
# Handle path parameters
|
||||
parts = []
|
||||
for part in clean_path.split("/"):
|
||||
if part.startswith("{") and part.endswith("}"):
|
||||
param = part[1:-1]
|
||||
parts.append(f"by {param}")
|
||||
else:
|
||||
parts.append(part.replace("_", " ").replace("-", " "))
|
||||
|
||||
resource = " ".join(parts)
|
||||
|
||||
method_verb = {
|
||||
"get": "Get",
|
||||
"post": "Create",
|
||||
"put": "Update",
|
||||
"delete": "Delete",
|
||||
"patch": "Update",
|
||||
}.get(method.lower(), method.capitalize())
|
||||
|
||||
return f"{method_verb} {resource}"
|
||||
|
||||
|
||||
def add_missing_operation_ids(spec: dict[str, Any]) -> int:
|
||||
"""Add operationId and summary to operations that are missing them."""
|
||||
fixed_count = 0
|
||||
|
||||
for path, methods in spec.get("paths", {}).items():
|
||||
for method, details in methods.items():
|
||||
if method not in ["get", "post", "put", "delete", "patch"]:
|
||||
continue
|
||||
|
||||
if not isinstance(details, dict):
|
||||
continue
|
||||
|
||||
summary = details.get("summary")
|
||||
operation_id = details.get("operationId")
|
||||
|
||||
if not summary and not operation_id:
|
||||
details["operationId"] = path_to_operation_id(path, method)
|
||||
details["summary"] = path_to_summary(path, method)
|
||||
fixed_count += 1
|
||||
|
||||
return fixed_count
|
||||
|
||||
|
||||
TAG_DESCRIPTIONS = {
|
||||
"Advanced Data Type": "Advanced data type operations and conversions.",
|
||||
"Annotation Layers": "Manage annotation layers and annotations for charts.",
|
||||
"AsyncEventsRestApi": "Real-time event streaming via Server-Sent Events (SSE).",
|
||||
"Available Domains": "Get available domains for the Superset instance.",
|
||||
"CSS Templates": "Manage CSS templates for custom dashboard styling.",
|
||||
"CacheRestApi": "Cache management and invalidation operations.",
|
||||
"Charts": "Create, read, update, and delete charts (slices).",
|
||||
"Current User": "Get information about the authenticated user.",
|
||||
"Dashboard Filter State": "Manage temporary filter state for dashboards.",
|
||||
"Dashboard Permanent Link": "Permanent links to dashboard states.",
|
||||
"Dashboards": "Create, read, update, and delete dashboards.",
|
||||
"Database": "Manage database connections and metadata.",
|
||||
"Datasets": "Manage datasets (tables) used for building charts.",
|
||||
"Datasources": "Query datasource metadata and column values.",
|
||||
"Embedded Dashboard": "Configure embedded dashboard settings.",
|
||||
"Explore": "Chart exploration and data querying endpoints.",
|
||||
"Explore Form Data": "Manage temporary form data for chart exploration.",
|
||||
"Explore Permanent Link": "Permanent links to chart explore states.",
|
||||
"Import/export": "Import and export Superset assets.",
|
||||
"LogRestApi": "Access audit logs and activity history.",
|
||||
"Menu": "Get the Superset menu structure.",
|
||||
"OpenApi": "Access the OpenAPI specification.",
|
||||
"Queries": "View and manage SQL Lab query history.",
|
||||
"Report Schedules": "Configure scheduled reports and alerts.",
|
||||
"Row Level Security": "Manage row-level security rules for data access.",
|
||||
"SQL Lab": "Execute SQL queries and manage SQL Lab sessions.",
|
||||
"SQL Lab Permanent Link": "Permanent links to SQL Lab states.",
|
||||
"Security": "Authentication and token management.",
|
||||
"Security Permissions": "View available permissions.",
|
||||
"Security Permissions on Resources (View Menus)": "Permission-resource mappings.",
|
||||
"Security Resources (View Menus)": "Manage security resources (view menus).",
|
||||
"Security Roles": "Manage security roles and their permissions.",
|
||||
"Security Users": "Manage user accounts.",
|
||||
"Tags": "Organize assets with tags.",
|
||||
"Themes": "Manage UI themes for customizing Superset's appearance.",
|
||||
"User": "User profile and preferences.",
|
||||
}
|
||||
|
||||
|
||||
def generate_code_sample(
|
||||
method: str, path: str, has_body: bool = False
|
||||
) -> list[dict[str, str]]:
|
||||
"""Generate code samples for an endpoint in multiple languages."""
|
||||
# Clean up path for display
|
||||
example_path = path.replace("{pk}", "1").replace("{id_or_slug}", "1")
|
||||
|
||||
samples = []
|
||||
|
||||
# cURL sample
|
||||
curl_cmd = f'curl -X {method.upper()} "http://localhost:8088{example_path}"'
|
||||
curl_cmd += ' \\\n -H "Authorization: Bearer $ACCESS_TOKEN"'
|
||||
if has_body:
|
||||
curl_cmd += ' \\\n -H "Content-Type: application/json"'
|
||||
curl_cmd += ' \\\n -d \'{"key": "value"}\''
|
||||
|
||||
samples.append(
|
||||
{
|
||||
"lang": "cURL",
|
||||
"label": "cURL",
|
||||
"source": curl_cmd,
|
||||
}
|
||||
)
|
||||
|
||||
# Python sample
|
||||
if method.lower() == "get":
|
||||
python_code = f"""import requests
|
||||
|
||||
response = requests.get(
|
||||
"http://localhost:8088{example_path}",
|
||||
headers={{"Authorization": "Bearer " + access_token}}
|
||||
)
|
||||
print(response.json())"""
|
||||
elif method.lower() == "post":
|
||||
python_code = f"""import requests
|
||||
|
||||
response = requests.post(
|
||||
"http://localhost:8088{example_path}",
|
||||
headers={{"Authorization": "Bearer " + access_token}},
|
||||
json={{"key": "value"}}
|
||||
)
|
||||
print(response.json())"""
|
||||
elif method.lower() == "put":
|
||||
python_code = f"""import requests
|
||||
|
||||
response = requests.put(
|
||||
"http://localhost:8088{example_path}",
|
||||
headers={{"Authorization": "Bearer " + access_token}},
|
||||
json={{"key": "value"}}
|
||||
)
|
||||
print(response.json())"""
|
||||
elif method.lower() == "delete":
|
||||
python_code = f"""import requests
|
||||
|
||||
response = requests.delete(
|
||||
"http://localhost:8088{example_path}",
|
||||
headers={{"Authorization": "Bearer " + access_token}}
|
||||
)
|
||||
print(response.status_code)"""
|
||||
else:
|
||||
python_code = f"""import requests
|
||||
|
||||
response = requests.{method.lower()}(
|
||||
"http://localhost:8088{example_path}",
|
||||
headers={{"Authorization": "Bearer " + access_token}}
|
||||
)
|
||||
print(response.json())"""
|
||||
|
||||
samples.append(
|
||||
{
|
||||
"lang": "Python",
|
||||
"label": "Python",
|
||||
"source": python_code,
|
||||
}
|
||||
)
|
||||
|
||||
# JavaScript sample
|
||||
if method.lower() == "get":
|
||||
js_code = f"""const response = await fetch(
|
||||
"http://localhost:8088{example_path}",
|
||||
{{
|
||||
headers: {{
|
||||
"Authorization": `Bearer ${{accessToken}}`
|
||||
}}
|
||||
}}
|
||||
);
|
||||
const data = await response.json();
|
||||
console.log(data);"""
|
||||
elif method.lower() in ["post", "put", "patch"]:
|
||||
js_code = f"""const response = await fetch(
|
||||
"http://localhost:8088{example_path}",
|
||||
{{
|
||||
method: "{method.upper()}",
|
||||
headers: {{
|
||||
"Authorization": `Bearer ${{accessToken}}`,
|
||||
"Content-Type": "application/json"
|
||||
}},
|
||||
body: JSON.stringify({{ key: "value" }})
|
||||
}}
|
||||
);
|
||||
const data = await response.json();
|
||||
console.log(data);"""
|
||||
else:
|
||||
js_code = f"""const response = await fetch(
|
||||
"http://localhost:8088{example_path}",
|
||||
{{
|
||||
method: "{method.upper()}",
|
||||
headers: {{
|
||||
"Authorization": `Bearer ${{accessToken}}`
|
||||
}}
|
||||
}}
|
||||
);
|
||||
console.log(response.status);"""
|
||||
|
||||
samples.append(
|
||||
{
|
||||
"lang": "JavaScript",
|
||||
"label": "JavaScript",
|
||||
"source": js_code,
|
||||
}
|
||||
)
|
||||
|
||||
return samples
|
||||
|
||||
|
||||
def add_code_samples(spec: dict[str, Any]) -> int:
|
||||
"""Add code samples to all endpoints."""
|
||||
count = 0
|
||||
|
||||
for path, methods in spec.get("paths", {}).items():
|
||||
for method, details in methods.items():
|
||||
if method not in ["get", "post", "put", "delete", "patch"]:
|
||||
continue
|
||||
if not isinstance(details, dict):
|
||||
continue
|
||||
|
||||
# Skip if already has code samples
|
||||
if "x-codeSamples" in details:
|
||||
continue
|
||||
|
||||
# Check if endpoint has a request body
|
||||
has_body = "requestBody" in details
|
||||
|
||||
details["x-codeSamples"] = generate_code_sample(method, path, has_body)
|
||||
count += 1
|
||||
|
||||
return count
|
||||
|
||||
|
||||
def configure_servers(spec: dict[str, Any]) -> bool:
|
||||
"""Configure server URLs with variables for flexible API testing."""
|
||||
new_servers = [
|
||||
{
|
||||
"url": "http://localhost:8088",
|
||||
"description": "Local development server",
|
||||
},
|
||||
{
|
||||
"url": "{protocol}://{host}:{port}",
|
||||
"description": "Custom server",
|
||||
"variables": {
|
||||
"protocol": {
|
||||
"default": "http",
|
||||
"enum": ["http", "https"],
|
||||
"description": "HTTP protocol",
|
||||
},
|
||||
"host": {
|
||||
"default": "localhost",
|
||||
"description": "Server hostname or IP",
|
||||
},
|
||||
"port": {
|
||||
"default": "8088",
|
||||
"description": "Server port",
|
||||
},
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
# Check if already configured
|
||||
existing = spec.get("servers", [])
|
||||
if len(existing) >= 2 and any("variables" in s for s in existing):
|
||||
return False
|
||||
|
||||
spec["servers"] = new_servers
|
||||
return True
|
||||
|
||||
|
||||
def add_tag_definitions(spec: dict[str, Any]) -> int:
|
||||
"""Add tag definitions with descriptions to the OpenAPI spec."""
|
||||
# Collect all unique tags used in operations
|
||||
used_tags: set[str] = set()
|
||||
for _path, methods in spec.get("paths", {}).items():
|
||||
for method, details in methods.items():
|
||||
if method not in ["get", "post", "put", "delete", "patch"]:
|
||||
continue
|
||||
if not isinstance(details, dict):
|
||||
continue
|
||||
tags = details.get("tags", [])
|
||||
used_tags.update(tags)
|
||||
|
||||
# Create tag definitions
|
||||
tag_definitions = []
|
||||
for tag in sorted(used_tags):
|
||||
tag_def = {"name": tag}
|
||||
if tag in TAG_DESCRIPTIONS:
|
||||
tag_def["description"] = TAG_DESCRIPTIONS[tag]
|
||||
else:
|
||||
# Generate a generic description
|
||||
tag_def["description"] = f"Endpoints related to {tag}."
|
||||
tag_definitions.append(tag_def)
|
||||
|
||||
# Only update if we have new tags
|
||||
existing_tags = {t.get("name") for t in spec.get("tags", [])}
|
||||
new_tags = [t for t in tag_definitions if t["name"] not in existing_tags]
|
||||
|
||||
if new_tags or not spec.get("tags"):
|
||||
spec["tags"] = tag_definitions
|
||||
return len(tag_definitions)
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def generate_example_from_schema( # noqa: C901
|
||||
schema: dict[str, Any],
|
||||
spec: dict[str, Any],
|
||||
depth: int = 0,
|
||||
max_depth: int = 5,
|
||||
) -> dict[str, Any] | list[Any] | str | int | float | bool | None:
|
||||
"""Generate an example value from an OpenAPI schema definition."""
|
||||
if depth > max_depth:
|
||||
return None
|
||||
|
||||
# Handle $ref
|
||||
if "$ref" in schema:
|
||||
ref_path = schema["$ref"]
|
||||
if ref_path.startswith("#/components/schemas/"):
|
||||
schema_name = ref_path.split("/")[-1]
|
||||
ref_schema = (
|
||||
spec.get("components", {}).get("schemas", {}).get(schema_name, {})
|
||||
)
|
||||
return generate_example_from_schema(ref_schema, spec, depth + 1, max_depth)
|
||||
return None
|
||||
|
||||
# If schema already has an example, use it
|
||||
if "example" in schema:
|
||||
return schema["example"]
|
||||
|
||||
schema_type = schema.get("type", "object")
|
||||
|
||||
if schema_type == "object":
|
||||
properties = schema.get("properties", {})
|
||||
if not properties:
|
||||
# Check for additionalProperties
|
||||
if schema.get("additionalProperties"):
|
||||
return {"key": "value"}
|
||||
return {}
|
||||
|
||||
result = {}
|
||||
for prop_name, prop_schema in properties.items():
|
||||
# Limit object depth and skip large nested objects
|
||||
if depth < max_depth:
|
||||
example_val = generate_example_from_schema(
|
||||
prop_schema, spec, depth + 1, max_depth
|
||||
)
|
||||
if example_val is not None:
|
||||
result[prop_name] = example_val
|
||||
return result
|
||||
|
||||
elif schema_type == "array":
|
||||
items_schema = schema.get("items", {})
|
||||
if items_schema:
|
||||
item_example = generate_example_from_schema(
|
||||
items_schema, spec, depth + 1, max_depth
|
||||
)
|
||||
if item_example is not None:
|
||||
return [item_example]
|
||||
return []
|
||||
|
||||
elif schema_type == "string":
|
||||
# Check for enum
|
||||
if "enum" in schema:
|
||||
return schema["enum"][0]
|
||||
# Check for format
|
||||
fmt = schema.get("format", "")
|
||||
if fmt == "date-time":
|
||||
return "2024-01-15T10:30:00Z"
|
||||
elif fmt == "date":
|
||||
return "2024-01-15"
|
||||
elif fmt == "email":
|
||||
return "user@example.com"
|
||||
elif fmt == "uri" or fmt == "url":
|
||||
return "https://example.com"
|
||||
elif fmt == "uuid":
|
||||
return "550e8400-e29b-41d4-a716-446655440000"
|
||||
# Use description hints or prop name
|
||||
return "string"
|
||||
|
||||
elif schema_type == "integer":
|
||||
if "minimum" in schema:
|
||||
return schema["minimum"]
|
||||
return 1
|
||||
|
||||
elif schema_type == "number":
|
||||
if "minimum" in schema:
|
||||
return schema["minimum"]
|
||||
return 1.0
|
||||
|
||||
elif schema_type == "boolean":
|
||||
return True
|
||||
|
||||
elif schema_type == "null":
|
||||
return None
|
||||
|
||||
# Handle oneOf, anyOf
|
||||
if "oneOf" in schema and schema["oneOf"]:
|
||||
return generate_example_from_schema(
|
||||
schema["oneOf"][0], spec, depth + 1, max_depth
|
||||
)
|
||||
if "anyOf" in schema and schema["anyOf"]:
|
||||
return generate_example_from_schema(
|
||||
schema["anyOf"][0], spec, depth + 1, max_depth
|
||||
)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def add_response_examples(spec: dict[str, Any]) -> int: # noqa: C901
|
||||
"""Add example values to API responses for better documentation."""
|
||||
count = 0
|
||||
|
||||
# First, add examples to standard error responses in components
|
||||
standard_errors = {
|
||||
"400": {"message": "Bad request: Invalid parameters provided"},
|
||||
"401": {"message": "Unauthorized: Authentication required"},
|
||||
"403": {
|
||||
"message": "Forbidden: You don't have permission to access this resource"
|
||||
},
|
||||
"404": {"message": "Not found: The requested resource does not exist"},
|
||||
"422": {"message": "Unprocessable entity: Validation error"},
|
||||
"500": {"message": "Internal server error: An unexpected error occurred"},
|
||||
}
|
||||
|
||||
responses = spec.get("components", {}).get("responses", {})
|
||||
for code, example_value in standard_errors.items():
|
||||
if code in responses:
|
||||
response = responses[code]
|
||||
content = response.get("content", {}).get("application/json", {})
|
||||
if content and "example" not in content:
|
||||
content["example"] = example_value
|
||||
count += 1
|
||||
|
||||
# Now add examples to inline response schemas in operations
|
||||
for _path, methods in spec.get("paths", {}).items():
|
||||
for method, details in methods.items():
|
||||
if method not in ["get", "post", "put", "delete", "patch"]:
|
||||
continue
|
||||
if not isinstance(details, dict):
|
||||
continue
|
||||
|
||||
responses_dict = details.get("responses", {})
|
||||
for _status_code, response in responses_dict.items():
|
||||
# Skip $ref responses (already handled above)
|
||||
if "$ref" in response:
|
||||
continue
|
||||
|
||||
content = response.get("content", {}).get("application/json", {})
|
||||
if not content:
|
||||
continue
|
||||
|
||||
# Skip if already has an example
|
||||
if "example" in content:
|
||||
continue
|
||||
|
||||
schema = content.get("schema", {})
|
||||
if schema:
|
||||
example = generate_example_from_schema(
|
||||
schema, spec, depth=0, max_depth=3
|
||||
)
|
||||
if example is not None and example != {}:
|
||||
content["example"] = example
|
||||
count += 1
|
||||
|
||||
return count
|
||||
|
||||
|
||||
def add_request_body_examples(spec: dict[str, Any]) -> int:
|
||||
"""Add example values to API request bodies for better documentation."""
|
||||
count = 0
|
||||
|
||||
for _path, methods in spec.get("paths", {}).items():
|
||||
for method, details in methods.items():
|
||||
if method not in ["post", "put", "patch"]:
|
||||
continue
|
||||
if not isinstance(details, dict):
|
||||
continue
|
||||
|
||||
request_body = details.get("requestBody", {})
|
||||
if not request_body or "$ref" in request_body:
|
||||
continue
|
||||
|
||||
content = request_body.get("content", {}).get("application/json", {})
|
||||
if not content:
|
||||
continue
|
||||
|
||||
# Skip if already has an example
|
||||
if "example" in content:
|
||||
continue
|
||||
|
||||
schema = content.get("schema", {})
|
||||
if schema:
|
||||
example = generate_example_from_schema(
|
||||
schema, spec, depth=0, max_depth=4
|
||||
)
|
||||
if example is not None and example != {}:
|
||||
content["example"] = example
|
||||
count += 1
|
||||
|
||||
return count
|
||||
|
||||
|
||||
def make_summaries_unique(spec: dict[str, Any]) -> int: # noqa: C901
|
||||
"""Make duplicate summaries unique by adding context from the path."""
|
||||
summary_info: dict[str, list[tuple[str, str]]] = {}
|
||||
fixed_count = 0
|
||||
|
||||
# First pass: collect all summaries and their paths (regardless of method)
|
||||
for path, methods in spec.get("paths", {}).items():
|
||||
for method, details in methods.items():
|
||||
if method not in ["get", "post", "put", "delete", "patch"]:
|
||||
continue
|
||||
if not isinstance(details, dict):
|
||||
continue
|
||||
summary = details.get("summary")
|
||||
if summary:
|
||||
if summary not in summary_info:
|
||||
summary_info[summary] = []
|
||||
summary_info[summary].append((path, method))
|
||||
|
||||
# Second pass: make duplicate summaries unique
|
||||
for path, methods in spec.get("paths", {}).items():
|
||||
for method, details in methods.items():
|
||||
if method not in ["get", "post", "put", "delete", "patch"]:
|
||||
continue
|
||||
if not isinstance(details, dict):
|
||||
continue
|
||||
summary = details.get("summary")
|
||||
if summary and len(summary_info.get(summary, [])) > 1:
|
||||
# Create a unique suffix from the full path
|
||||
# e.g., /api/v1/chart/{pk}/cache_screenshot/ -> "chart-cache-screenshot"
|
||||
clean_path = path.replace("/api/v1/", "").strip("/")
|
||||
# Remove parameter placeholders and convert to slug
|
||||
clean_path = clean_path.replace("{", "").replace("}", "")
|
||||
path_slug = clean_path.replace("/", "-").replace("_", "-")
|
||||
|
||||
# Check if this suffix is already in the summary
|
||||
if path_slug not in summary.lower():
|
||||
new_summary = f"{summary} ({path_slug})"
|
||||
details["summary"] = new_summary
|
||||
fixed_count += 1
|
||||
|
||||
return fixed_count
|
||||
|
||||
|
||||
def main() -> None: # noqa: C901
|
||||
"""Main function to fix the OpenAPI spec."""
|
||||
script_dir = Path(__file__).parent
|
||||
spec_path = script_dir.parent / "static" / "resources" / "openapi.json"
|
||||
|
||||
if not spec_path.exists():
|
||||
print(f"Error: OpenAPI spec not found at {spec_path}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
print(f"Reading OpenAPI spec from {spec_path}")
|
||||
|
||||
with open(spec_path, encoding="utf-8") as f:
|
||||
spec = json.load(f)
|
||||
|
||||
spec, fixed_schemas = add_missing_schemas(spec)
|
||||
fixed_ops = add_missing_operation_ids(spec)
|
||||
fixed_tags = add_tag_definitions(spec)
|
||||
fixed_servers = configure_servers(spec)
|
||||
|
||||
changes_made = False
|
||||
|
||||
if fixed_servers:
|
||||
print("Configured server URLs with variables for flexible API testing")
|
||||
changes_made = True
|
||||
|
||||
if fixed_samples := add_code_samples(spec):
|
||||
print(f"Added code samples to {fixed_samples} endpoints")
|
||||
changes_made = True
|
||||
|
||||
if fixed_examples := add_response_examples(spec):
|
||||
print(f"Added example JSON responses to {fixed_examples} response schemas")
|
||||
changes_made = True
|
||||
|
||||
if fixed_request_examples := add_request_body_examples(spec):
|
||||
print(f"Added example JSON to {fixed_request_examples} request bodies")
|
||||
changes_made = True
|
||||
|
||||
if fixed_schemas:
|
||||
print(f"Added missing schemas: {', '.join(fixed_schemas)}")
|
||||
changes_made = True
|
||||
|
||||
if fixed_ops:
|
||||
print(f"Added operationId/summary to {fixed_ops} operations")
|
||||
changes_made = True
|
||||
|
||||
if fixed_tags:
|
||||
print(f"Added {fixed_tags} tag definitions with descriptions")
|
||||
changes_made = True
|
||||
|
||||
if fixed_summaries := make_summaries_unique(spec):
|
||||
print(f"Made {fixed_summaries} duplicate summaries unique")
|
||||
changes_made = True
|
||||
|
||||
if changes_made:
|
||||
with open(spec_path, "w", encoding="utf-8") as f:
|
||||
json.dump(spec, f, indent=2)
|
||||
f.write("\n") # Ensure trailing newline for pre-commit
|
||||
|
||||
print(f"Updated {spec_path}")
|
||||
else:
|
||||
print("No fixes needed")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,277 +0,0 @@
|
||||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Generates a comprehensive API index MDX file from the OpenAPI spec.
|
||||
* This creates the api.mdx landing page with all endpoints organized by category.
|
||||
*
|
||||
* Uses the generated sidebar to get correct endpoint slugs (the plugin's
|
||||
* slug algorithm differs from a simple slugify, e.g. handling apostrophes
|
||||
* and camelCase differently).
|
||||
*/
|
||||
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { createRequire } from 'module';
|
||||
import { fileURLToPath } from 'url';
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
const SPEC_PATH = path.join(__dirname, '..', 'static', 'resources', 'openapi.json');
|
||||
const SIDEBAR_PATH = path.join(__dirname, '..', 'developer_docs', 'api', 'sidebar.js');
|
||||
const OUTPUT_PATH = path.join(__dirname, '..', 'developer_docs', 'api.mdx');
|
||||
|
||||
// Category groupings for better organization
|
||||
const CATEGORY_GROUPS = {
|
||||
'Authentication': ['Security'],
|
||||
'Core Resources': ['Dashboards', 'Charts', 'Datasets', 'Database'],
|
||||
'Data Exploration': ['Explore', 'SQL Lab', 'Queries', 'Datasources', 'Advanced Data Type'],
|
||||
'Organization & Customization': ['Tags', 'Annotation Layers', 'CSS Templates'],
|
||||
'Sharing & Embedding': [
|
||||
'Dashboard Permanent Link', 'Explore Permanent Link', 'SQL Lab Permanent Link',
|
||||
'Embedded Dashboard', 'Dashboard Filter State', 'Explore Form Data'
|
||||
],
|
||||
'Scheduling & Alerts': ['Report Schedules'],
|
||||
'Security & Access Control': [
|
||||
'Security Roles', 'Security Users', 'Security Permissions',
|
||||
'Security Resources (View Menus)', 'Security Permissions on Resources (View Menus)',
|
||||
'Row Level Security'
|
||||
],
|
||||
'Import/Export & Administration': ['Import/export', 'CacheRestApi', 'LogRestApi'],
|
||||
'User & System': ['Current User', 'User', 'Menu', 'Available Domains', 'AsyncEventsRestApi', 'OpenApi'],
|
||||
};
|
||||
|
||||
/**
|
||||
* Build a map from sidebar label → doc slug by reading the generated sidebar.
|
||||
* This ensures we use the exact same slugs that docusaurus-openapi-docs generated.
|
||||
*/
|
||||
function buildSlugMap() {
|
||||
const labelToSlug = {};
|
||||
|
||||
try {
|
||||
const sidebar = require(SIDEBAR_PATH);
|
||||
|
||||
const extractDocs = (items) => {
|
||||
for (const item of items) {
|
||||
if (item.type === 'doc' && item.label && item.id) {
|
||||
// id is like "api/create-security-login" → slug "create-security-login"
|
||||
const slug = item.id.replace(/^api\//, '');
|
||||
labelToSlug[item.label] = slug;
|
||||
}
|
||||
if (item.items) extractDocs(item.items);
|
||||
}
|
||||
};
|
||||
|
||||
extractDocs(sidebar);
|
||||
console.log(`Loaded ${Object.keys(labelToSlug).length} slug mappings from sidebar`);
|
||||
} catch {
|
||||
console.warn('Could not read sidebar, will use computed slugs');
|
||||
}
|
||||
|
||||
return labelToSlug;
|
||||
}
|
||||
|
||||
function slugify(text) {
|
||||
return text
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, '-')
|
||||
.replace(/(^-|-$)/g, '');
|
||||
}
|
||||
|
||||
function main() {
|
||||
console.log(`Reading OpenAPI spec from ${SPEC_PATH}`);
|
||||
const spec = JSON.parse(fs.readFileSync(SPEC_PATH, 'utf-8'));
|
||||
|
||||
// Build slug map from the generated sidebar
|
||||
const labelToSlug = buildSlugMap();
|
||||
|
||||
// Build a map of tag -> endpoints
|
||||
const tagEndpoints = {};
|
||||
const tagDescriptions = {};
|
||||
|
||||
// Get tag descriptions
|
||||
for (const tag of spec.tags || []) {
|
||||
tagDescriptions[tag.name] = tag.description || '';
|
||||
}
|
||||
|
||||
// Collect endpoints by tag
|
||||
for (const [pathUrl, methods] of Object.entries(spec.paths || {})) {
|
||||
for (const [method, details] of Object.entries(methods)) {
|
||||
if (!['get', 'post', 'put', 'delete', 'patch'].includes(method)) continue;
|
||||
|
||||
const tags = details.tags || ['Untagged'];
|
||||
const summary = details.summary || `${method.toUpperCase()} ${pathUrl}`;
|
||||
|
||||
// Use sidebar slug if available, fall back to computed slug
|
||||
const slug = labelToSlug[summary] || slugify(summary);
|
||||
|
||||
for (const tag of tags) {
|
||||
if (!tagEndpoints[tag]) {
|
||||
tagEndpoints[tag] = [];
|
||||
}
|
||||
tagEndpoints[tag].push({
|
||||
method: method.toUpperCase(),
|
||||
path: pathUrl,
|
||||
summary,
|
||||
slug,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Sort endpoints within each tag by path
|
||||
for (const tag of Object.keys(tagEndpoints)) {
|
||||
tagEndpoints[tag].sort((a, b) => a.path.localeCompare(b.path));
|
||||
}
|
||||
|
||||
// Generate MDX content
|
||||
let mdx = `---
|
||||
title: API Reference
|
||||
hide_title: true
|
||||
sidebar_position: 10
|
||||
---
|
||||
|
||||
import { Alert } from 'antd';
|
||||
|
||||
## REST API Reference
|
||||
|
||||
Superset exposes a comprehensive **REST API** that follows the [OpenAPI specification](https://swagger.io/specification/).
|
||||
You can use this API to programmatically interact with Superset for automation, integrations, and custom applications.
|
||||
|
||||
<Alert
|
||||
type="info"
|
||||
showIcon
|
||||
message="Code Samples & Schema Documentation"
|
||||
description={
|
||||
<span>
|
||||
Each endpoint includes ready-to-use code samples in <strong>cURL</strong>, <strong>Python</strong>, and <strong>JavaScript</strong>.
|
||||
The sidebar includes <strong>Schema definitions</strong> for detailed data model documentation.
|
||||
</span>
|
||||
}
|
||||
style={{ marginBottom: '24px' }}
|
||||
/>
|
||||
|
||||
---
|
||||
|
||||
`;
|
||||
|
||||
// Track which tags we've rendered
|
||||
const renderedTags = new Set();
|
||||
|
||||
// Render Authentication first (it's critical for using the API)
|
||||
mdx += `### Authentication
|
||||
|
||||
Most API endpoints require authentication via JWT tokens.
|
||||
|
||||
#### Quick Start
|
||||
|
||||
\`\`\`bash
|
||||
# 1. Get a JWT token
|
||||
curl -X POST http://localhost:8088/api/v1/security/login \\
|
||||
-H "Content-Type: application/json" \\
|
||||
-d '{"username": "admin", "password": "admin", "provider": "db"}'
|
||||
|
||||
# 2. Use the access_token from the response
|
||||
curl -H "Authorization: Bearer YOUR_ACCESS_TOKEN" \\
|
||||
http://localhost:8088/api/v1/dashboard/
|
||||
\`\`\`
|
||||
|
||||
#### Security Endpoints
|
||||
|
||||
`;
|
||||
|
||||
// Render Security tag endpoints
|
||||
if (tagEndpoints['Security']) {
|
||||
mdx += `| Method | Endpoint | Description |\n`;
|
||||
mdx += `|--------|----------|-------------|\n`;
|
||||
for (const ep of tagEndpoints['Security']) {
|
||||
mdx += `| \`${ep.method}\` | [${ep.summary}](/developer-docs/api/${ep.slug}) | \`${ep.path}\` |\n`;
|
||||
}
|
||||
mdx += '\n';
|
||||
renderedTags.add('Security');
|
||||
}
|
||||
|
||||
mdx += `---\n\n### API Endpoints\n\n`;
|
||||
|
||||
// Render each category group
|
||||
for (const [groupName, groupTags] of Object.entries(CATEGORY_GROUPS)) {
|
||||
if (groupName === 'Authentication') continue; // Already rendered
|
||||
|
||||
const tagsInGroup = groupTags.filter(tag => tagEndpoints[tag] && !renderedTags.has(tag));
|
||||
if (tagsInGroup.length === 0) continue;
|
||||
|
||||
mdx += `#### ${groupName}\n\n`;
|
||||
|
||||
for (const tag of tagsInGroup) {
|
||||
const description = tagDescriptions[tag] || '';
|
||||
const endpoints = tagEndpoints[tag];
|
||||
|
||||
mdx += `<details>\n`;
|
||||
mdx += `<summary><strong>${tag}</strong> (${endpoints.length} endpoints) — ${description}</summary>\n\n`;
|
||||
mdx += `| Method | Endpoint | Description |\n`;
|
||||
mdx += `|--------|----------|-------------|\n`;
|
||||
|
||||
for (const ep of endpoints) {
|
||||
mdx += `| \`${ep.method}\` | [${ep.summary}](/developer-docs/api/${ep.slug}) | \`${ep.path}\` |\n`;
|
||||
}
|
||||
|
||||
mdx += `\n</details>\n\n`;
|
||||
renderedTags.add(tag);
|
||||
}
|
||||
}
|
||||
|
||||
// Render any remaining tags not in a group
|
||||
const remainingTags = Object.keys(tagEndpoints).filter(tag => !renderedTags.has(tag));
|
||||
if (remainingTags.length > 0) {
|
||||
mdx += `#### Other\n\n`;
|
||||
|
||||
for (const tag of remainingTags.sort()) {
|
||||
const description = tagDescriptions[tag] || '';
|
||||
const endpoints = tagEndpoints[tag];
|
||||
|
||||
mdx += `<details>\n`;
|
||||
mdx += `<summary><strong>${tag}</strong> (${endpoints.length} endpoints) — ${description}</summary>\n\n`;
|
||||
mdx += `| Method | Endpoint | Description |\n`;
|
||||
mdx += `|--------|----------|-------------|\n`;
|
||||
|
||||
for (const ep of endpoints) {
|
||||
mdx += `| \`${ep.method}\` | [${ep.summary}](/developer-docs/api/${ep.slug}) | \`${ep.path}\` |\n`;
|
||||
}
|
||||
|
||||
mdx += `\n</details>\n\n`;
|
||||
}
|
||||
}
|
||||
|
||||
mdx += `---
|
||||
|
||||
### Additional Resources
|
||||
|
||||
- [Superset REST API Blog Post](https://preset.io/blog/2020-10-01-superset-api/)
|
||||
- [Accessing APIs with Superset](https://preset.io/blog/accessing-apis-with-superset/)
|
||||
`;
|
||||
|
||||
// Write output
|
||||
fs.writeFileSync(OUTPUT_PATH, mdx);
|
||||
console.log(`Generated API index at ${OUTPUT_PATH}`);
|
||||
console.log(`Total tags: ${Object.keys(tagEndpoints).length}`);
|
||||
console.log(`Total endpoints: ${Object.values(tagEndpoints).flat().length}`);
|
||||
}
|
||||
|
||||
main();
|
||||
@@ -1,176 +0,0 @@
|
||||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Replaces auto-generated tag pages (DocCardList cards) with endpoint tables
|
||||
* showing HTTP method, endpoint name, and URI path for each endpoint in the tag.
|
||||
*
|
||||
* Runs after `docusaurus gen-api-docs` and `convert-api-sidebar.mjs`.
|
||||
* Uses the generated sidebar to get correct endpoint slugs.
|
||||
*/
|
||||
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { createRequire } from 'module';
|
||||
import { fileURLToPath } from 'url';
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
const SPEC_PATH = path.join(__dirname, '..', 'static', 'resources', 'openapi.json');
|
||||
const API_DOCS_DIR = path.join(__dirname, '..', 'developer_docs', 'api');
|
||||
const SIDEBAR_PATH = path.join(API_DOCS_DIR, 'sidebar.js');
|
||||
|
||||
function slugify(text) {
|
||||
return text
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, '-')
|
||||
.replace(/(^-|-$)/g, '');
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a map from sidebar label → doc slug by reading the generated sidebar.
|
||||
*/
|
||||
function buildSlugMap() {
|
||||
const labelToSlug = {};
|
||||
|
||||
try {
|
||||
const sidebar = require(SIDEBAR_PATH);
|
||||
|
||||
const extractDocs = (items) => {
|
||||
for (const item of items) {
|
||||
if (item.type === 'doc' && item.label && item.id) {
|
||||
const slug = item.id.replace(/^api\//, '');
|
||||
labelToSlug[item.label] = slug;
|
||||
}
|
||||
if (item.items) extractDocs(item.items);
|
||||
}
|
||||
};
|
||||
|
||||
extractDocs(sidebar);
|
||||
} catch {
|
||||
console.warn('Could not read sidebar, will use computed slugs');
|
||||
}
|
||||
|
||||
return labelToSlug;
|
||||
}
|
||||
|
||||
function main() {
|
||||
console.log('Generating API tag pages with endpoint tables...');
|
||||
|
||||
const spec = JSON.parse(fs.readFileSync(SPEC_PATH, 'utf-8'));
|
||||
const labelToSlug = buildSlugMap();
|
||||
|
||||
// Build tag descriptions from the spec
|
||||
const tagDescriptions = {};
|
||||
for (const tag of spec.tags || []) {
|
||||
tagDescriptions[tag.name] = tag.description || '';
|
||||
}
|
||||
|
||||
// Build tag → endpoints map
|
||||
const tagEndpoints = {};
|
||||
for (const [pathUrl, methods] of Object.entries(spec.paths || {})) {
|
||||
for (const [method, details] of Object.entries(methods)) {
|
||||
if (!['get', 'post', 'put', 'delete', 'patch'].includes(method)) continue;
|
||||
|
||||
const tags = details.tags || ['Untagged'];
|
||||
const summary = details.summary || `${method.toUpperCase()} ${pathUrl}`;
|
||||
const slug = labelToSlug[summary] || slugify(summary);
|
||||
|
||||
for (const tag of tags) {
|
||||
if (!tagEndpoints[tag]) {
|
||||
tagEndpoints[tag] = [];
|
||||
}
|
||||
tagEndpoints[tag].push({
|
||||
method: method.toUpperCase(),
|
||||
path: pathUrl,
|
||||
summary,
|
||||
slug,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Sort endpoints within each tag by path then method
|
||||
for (const tag of Object.keys(tagEndpoints)) {
|
||||
tagEndpoints[tag].sort((a, b) =>
|
||||
a.path.localeCompare(b.path) || a.method.localeCompare(b.method)
|
||||
);
|
||||
}
|
||||
|
||||
// Scan existing .tag.mdx files and match by frontmatter title
|
||||
const tagFiles = fs.readdirSync(API_DOCS_DIR)
|
||||
.filter(f => f.endsWith('.tag.mdx'));
|
||||
|
||||
let updated = 0;
|
||||
for (const tagFile of tagFiles) {
|
||||
const tagFilePath = path.join(API_DOCS_DIR, tagFile);
|
||||
const existing = fs.readFileSync(tagFilePath, 'utf-8');
|
||||
|
||||
// Extract frontmatter
|
||||
const frontmatterMatch = existing.match(/^---\n([\s\S]*?)\n---/);
|
||||
if (!frontmatterMatch) {
|
||||
console.warn(` No frontmatter in ${tagFile}, skipping`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const frontmatter = frontmatterMatch[1];
|
||||
|
||||
// Extract the title from frontmatter (this matches the spec tag name)
|
||||
const titleMatch = frontmatter.match(/title:\s*"([^"]+)"/);
|
||||
if (!titleMatch) {
|
||||
console.warn(` No title in ${tagFile}, skipping`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const tagName = titleMatch[1];
|
||||
const endpoints = tagEndpoints[tagName];
|
||||
|
||||
if (!endpoints || endpoints.length === 0) {
|
||||
console.warn(` No endpoints found for tag "${tagName}" (${tagFile})`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const description = tagDescriptions[tagName] || '';
|
||||
|
||||
// Build the endpoint table
|
||||
let table = '| Method | Endpoint | Path |\n';
|
||||
table += '|--------|----------|------|\n';
|
||||
for (const ep of endpoints) {
|
||||
table += `| \`${ep.method}\` | [${ep.summary}](./${ep.slug}) | \`${ep.path}\` |\n`;
|
||||
}
|
||||
|
||||
// Generate the new MDX content
|
||||
const mdx = `---
|
||||
${frontmatter}
|
||||
---
|
||||
|
||||
${description}
|
||||
|
||||
${table}
|
||||
`;
|
||||
|
||||
fs.writeFileSync(tagFilePath, mdx);
|
||||
updated++;
|
||||
}
|
||||
|
||||
console.log(`Updated ${updated} tag pages with endpoint tables`);
|
||||
}
|
||||
|
||||
main();
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,307 +0,0 @@
|
||||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Smart generator wrapper: only runs generators whose input files have changed.
|
||||
*
|
||||
* Computes a hash of each generator's input files (stories, engine specs,
|
||||
* openapi.json, and the generator scripts themselves). Compares against a
|
||||
* stored cache. Skips generators whose inputs and outputs are unchanged.
|
||||
*
|
||||
* Usage:
|
||||
* node scripts/generate-if-changed.mjs # smart mode (default)
|
||||
* node scripts/generate-if-changed.mjs --force # force regenerate all
|
||||
*/
|
||||
|
||||
import { createHash } from 'crypto';
|
||||
import { execSync, spawn } from 'child_process';
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
const DOCS_DIR = path.resolve(__dirname, '..');
|
||||
const ROOT_DIR = path.resolve(DOCS_DIR, '..');
|
||||
const CACHE_FILE = path.join(DOCS_DIR, '.docusaurus', 'generator-hashes.json');
|
||||
|
||||
const FORCE = process.argv.includes('--force');
|
||||
|
||||
// Ensure local node_modules/.bin is on PATH (needed for docusaurus CLI)
|
||||
const localBin = path.join(DOCS_DIR, 'node_modules', '.bin');
|
||||
process.env.PATH = `${localBin}${path.delimiter}${process.env.PATH}`;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Generator definitions
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const GENERATORS = [
|
||||
{
|
||||
name: 'superset-components',
|
||||
command: 'node scripts/generate-superset-components.mjs',
|
||||
inputs: [
|
||||
{
|
||||
type: 'glob',
|
||||
base: path.join(ROOT_DIR, 'superset-frontend/packages/superset-ui-core/src/components'),
|
||||
pattern: '**/*.stories.tsx',
|
||||
},
|
||||
{
|
||||
type: 'glob',
|
||||
base: path.join(ROOT_DIR, 'superset-frontend/packages/superset-core/src'),
|
||||
pattern: '**/*.stories.tsx',
|
||||
},
|
||||
{ type: 'file', path: path.join(DOCS_DIR, 'scripts/generate-superset-components.mjs') },
|
||||
{ type: 'file', path: path.join(DOCS_DIR, 'src/components/StorybookWrapper.jsx') },
|
||||
],
|
||||
outputs: [
|
||||
path.join(DOCS_DIR, 'developer_docs/components/index.mdx'),
|
||||
path.join(DOCS_DIR, 'static/data/components.json'),
|
||||
path.join(DOCS_DIR, 'src/types/apache-superset-core/index.d.ts'),
|
||||
],
|
||||
},
|
||||
{
|
||||
name: 'database-docs',
|
||||
command: 'node scripts/generate-database-docs.mjs',
|
||||
inputs: [
|
||||
{
|
||||
type: 'glob',
|
||||
base: path.join(ROOT_DIR, 'superset/db_engine_specs'),
|
||||
pattern: '**/*.py',
|
||||
},
|
||||
{ type: 'file', path: path.join(DOCS_DIR, 'scripts/generate-database-docs.mjs') },
|
||||
],
|
||||
outputs: [
|
||||
path.join(DOCS_DIR, 'src/data/databases.json'),
|
||||
path.join(DOCS_DIR, 'docs/databases/supported'),
|
||||
],
|
||||
},
|
||||
{
|
||||
name: 'api-docs',
|
||||
command:
|
||||
'python3 scripts/fix-openapi-spec.py && docusaurus gen-api-docs superset && node scripts/convert-api-sidebar.mjs && node scripts/generate-api-index.mjs && node scripts/generate-api-tag-pages.mjs',
|
||||
inputs: [
|
||||
{ type: 'file', path: path.join(DOCS_DIR, 'static/resources/openapi.json') },
|
||||
{ type: 'file', path: path.join(DOCS_DIR, 'scripts/fix-openapi-spec.py') },
|
||||
{ type: 'file', path: path.join(DOCS_DIR, 'scripts/convert-api-sidebar.mjs') },
|
||||
{ type: 'file', path: path.join(DOCS_DIR, 'scripts/generate-api-index.mjs') },
|
||||
{ type: 'file', path: path.join(DOCS_DIR, 'scripts/generate-api-tag-pages.mjs') },
|
||||
],
|
||||
outputs: [
|
||||
path.join(DOCS_DIR, 'docs/api.mdx'),
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Hashing utilities
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function walkDir(dir, pattern) {
|
||||
const results = [];
|
||||
if (!fs.existsSync(dir)) return results;
|
||||
|
||||
const regex = globToRegex(pattern);
|
||||
function walk(currentDir) {
|
||||
for (const entry of fs.readdirSync(currentDir, { withFileTypes: true })) {
|
||||
const fullPath = path.join(currentDir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
if (entry.name === 'node_modules' || entry.name === '__pycache__') continue;
|
||||
walk(fullPath);
|
||||
} else {
|
||||
// Normalize to forward slashes so glob patterns work on all platforms
|
||||
const relativePath = path.relative(dir, fullPath).split(path.sep).join('/');
|
||||
if (regex.test(relativePath)) {
|
||||
results.push(fullPath);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
walk(dir);
|
||||
return results.sort();
|
||||
}
|
||||
|
||||
function globToRegex(pattern) {
|
||||
// Simple glob-to-regex: ** matches any path, * matches anything except /
|
||||
const escaped = pattern
|
||||
.replace(/[.+^${}()|[\]\\]/g, '\\$&')
|
||||
.replace(/\*\*/g, '<<<GLOBSTAR>>>')
|
||||
.replace(/\*/g, '[^/]*')
|
||||
.replace(/<<<GLOBSTAR>>>/g, '.*');
|
||||
return new RegExp(`^${escaped}$`);
|
||||
}
|
||||
|
||||
function hashFile(filePath) {
|
||||
if (!fs.existsSync(filePath)) return 'missing';
|
||||
const stat = fs.statSync(filePath);
|
||||
// Use mtime + size for speed (avoids reading file contents)
|
||||
return `${stat.mtimeMs}:${stat.size}`;
|
||||
}
|
||||
|
||||
function computeInputHash(inputs) {
|
||||
const hash = createHash('md5');
|
||||
for (const input of inputs) {
|
||||
if (input.type === 'file') {
|
||||
hash.update(`file:${input.path}:${hashFile(input.path)}\n`);
|
||||
} else if (input.type === 'glob') {
|
||||
const files = walkDir(input.base, input.pattern);
|
||||
hash.update(`glob:${input.base}:${input.pattern}:count=${files.length}\n`);
|
||||
for (const file of files) {
|
||||
hash.update(` ${path.relative(input.base, file)}:${hashFile(file)}\n`);
|
||||
}
|
||||
}
|
||||
}
|
||||
return hash.digest('hex');
|
||||
}
|
||||
|
||||
function outputsExist(outputs) {
|
||||
return outputs.every((p) => fs.existsSync(p));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Cache management
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function loadCache() {
|
||||
try {
|
||||
if (fs.existsSync(CACHE_FILE)) {
|
||||
return JSON.parse(fs.readFileSync(CACHE_FILE, 'utf8'));
|
||||
}
|
||||
} catch {
|
||||
// Corrupted cache — start fresh
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
function saveCache(cache) {
|
||||
const dir = path.dirname(CACHE_FILE);
|
||||
if (!fs.existsSync(dir)) {
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
}
|
||||
fs.writeFileSync(CACHE_FILE, JSON.stringify(cache, null, 2));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Main
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function main() {
|
||||
const cache = loadCache();
|
||||
const updatedCache = { ...cache };
|
||||
let skipped = 0;
|
||||
let ran = 0;
|
||||
|
||||
// First pass: determine which generators need to run
|
||||
// Run independent generators (superset-components, database-docs) in
|
||||
// parallel, then api-docs sequentially (it depends on docusaurus CLI
|
||||
// being available, not on other generators).
|
||||
|
||||
const independent = GENERATORS.filter((g) => g.name !== 'api-docs');
|
||||
const sequential = GENERATORS.filter((g) => g.name === 'api-docs');
|
||||
|
||||
// Check and run independent generators in parallel
|
||||
const parallelPromises = independent.map((gen) => {
|
||||
const currentHash = computeInputHash(gen.inputs);
|
||||
const cachedHash = cache[gen.name];
|
||||
const hasOutputs = outputsExist(gen.outputs);
|
||||
|
||||
if (!FORCE && currentHash === cachedHash && hasOutputs) {
|
||||
console.log(` ✓ ${gen.name} — no changes, skipping`);
|
||||
skipped++;
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
const reason = FORCE
|
||||
? 'forced'
|
||||
: !hasOutputs
|
||||
? 'output missing'
|
||||
: 'inputs changed';
|
||||
console.log(` ↻ ${gen.name} — ${reason}, regenerating...`);
|
||||
ran++;
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const child = spawn('sh', ['-c', gen.command], {
|
||||
cwd: DOCS_DIR,
|
||||
stdio: 'inherit',
|
||||
env: process.env,
|
||||
});
|
||||
child.on('close', (code) => {
|
||||
if (code === 0) {
|
||||
updatedCache[gen.name] = currentHash;
|
||||
resolve();
|
||||
} else {
|
||||
console.error(` ✗ ${gen.name} failed (exit ${code})`);
|
||||
reject(new Error(`${gen.name} failed with exit code ${code}`));
|
||||
}
|
||||
});
|
||||
child.on('error', (err) => {
|
||||
console.error(` ✗ ${gen.name} failed to start`);
|
||||
reject(err);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
await Promise.all(parallelPromises);
|
||||
|
||||
// Run sequential generators (api-docs)
|
||||
for (const gen of sequential) {
|
||||
const currentHash = computeInputHash(gen.inputs);
|
||||
const cachedHash = cache[gen.name];
|
||||
const hasOutputs = outputsExist(gen.outputs);
|
||||
|
||||
if (!FORCE && currentHash === cachedHash && hasOutputs) {
|
||||
console.log(` ✓ ${gen.name} — no changes, skipping`);
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
const reason = FORCE
|
||||
? 'forced'
|
||||
: !hasOutputs
|
||||
? 'output missing'
|
||||
: 'inputs changed';
|
||||
console.log(` ↻ ${gen.name} — ${reason}, regenerating...`);
|
||||
ran++;
|
||||
|
||||
try {
|
||||
execSync(gen.command, {
|
||||
cwd: DOCS_DIR,
|
||||
stdio: 'inherit',
|
||||
timeout: 300_000,
|
||||
});
|
||||
updatedCache[gen.name] = currentHash;
|
||||
} catch (err) {
|
||||
console.error(` ✗ ${gen.name} failed`);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
saveCache(updatedCache);
|
||||
|
||||
if (ran === 0) {
|
||||
console.log(`\nAll ${skipped} generators up-to-date — nothing to do!\n`);
|
||||
} else {
|
||||
console.log(`\nDone: ${ran} regenerated, ${skipped} skipped.\n`);
|
||||
}
|
||||
}
|
||||
|
||||
console.log('Checking generators for changes...\n');
|
||||
main().catch((err) => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,230 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
/**
|
||||
* lint-docs-links — source-level checks for internal markdown links.
|
||||
*
|
||||
* Catches three failure modes that combine to break SPA navigation in
|
||||
* a Docusaurus build:
|
||||
*
|
||||
* 1. BARE — `[X](../foo)` with no extension. Skips
|
||||
* Docusaurus's file resolver entirely. Emitted
|
||||
* as a raw href and resolved by the browser
|
||||
* against the current page URL — usually the
|
||||
* wrong directory for trailing-slash routes.
|
||||
* `onBrokenLinks: 'throw'` cannot catch this.
|
||||
*
|
||||
* 2. MISSING_TARGET — `[X](./gone.md)` with an extension, but no
|
||||
* file at that path. The Docusaurus build
|
||||
* catches this too (via
|
||||
* `onBrokenMarkdownLinks: 'throw'`) but only
|
||||
* after a multi-minute build. This script
|
||||
* flags it in ~1s.
|
||||
*
|
||||
* 3. WRONG_EXTENSION — `[X](./foo.md)` where the file is actually
|
||||
* `foo.mdx` (or vice versa). Same end result
|
||||
* as MISSING_TARGET, but the fix is one
|
||||
* character — so we report it as its own
|
||||
* category with the actual extension on disk.
|
||||
*
|
||||
* Skips: fenced code blocks, asset-style targets (.png/.json/etc.),
|
||||
* external URLs, in-page anchors, and the `versioned_docs/`
|
||||
* snapshots (those are frozen historical content).
|
||||
*
|
||||
* Run from `docs/`:
|
||||
* node scripts/lint-docs-links.mjs
|
||||
*
|
||||
* Exits 0 on clean, 1 on any finding.
|
||||
*/
|
||||
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
const docsRoot = path.join(__dirname, '..');
|
||||
|
||||
const ROOTS = ['docs', 'admin_docs', 'developer_docs', 'components'];
|
||||
|
||||
const NON_DOC_EXTENSIONS = new Set([
|
||||
'.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg', '.ico',
|
||||
'.json', '.yaml', '.yml', '.txt', '.csv',
|
||||
'.zip', '.tar', '.gz',
|
||||
'.pdf',
|
||||
'.mp4', '.webm', '.mov',
|
||||
]);
|
||||
|
||||
const LINK_RE = /\[[^\]\n]+?\]\((?<url>\.{1,2}\/[^)\s]+?)\)/g;
|
||||
|
||||
/**
|
||||
* Classify a single markdown link from a source file.
|
||||
* Returns one of: ok / bare / asset / missing-target / wrong-extension.
|
||||
*/
|
||||
function classifyLink(sourceFile, url) {
|
||||
const stripped = url.split('#', 1)[0].split('?', 1)[0];
|
||||
const ext = path.extname(stripped).toLowerCase();
|
||||
|
||||
// Non-doc assets — legit bare extensions, leave alone.
|
||||
if (ext && NON_DOC_EXTENSIONS.has(ext)) {
|
||||
return { kind: 'asset' };
|
||||
}
|
||||
|
||||
// Anything that doesn't end in .md/.mdx is a bare relative URL.
|
||||
if (ext !== '.md' && ext !== '.mdx') {
|
||||
return { kind: 'bare' };
|
||||
}
|
||||
|
||||
// Has a .md/.mdx extension — make sure the target exists.
|
||||
const target = path.normalize(path.join(path.dirname(sourceFile), stripped));
|
||||
if (fs.existsSync(target)) {
|
||||
return { kind: 'ok' };
|
||||
}
|
||||
|
||||
// Target doesn't exist — check if the OTHER extension does.
|
||||
const otherExt = ext === '.md' ? '.mdx' : '.md';
|
||||
const otherTarget = target.slice(0, -ext.length) + otherExt;
|
||||
if (fs.existsSync(otherTarget)) {
|
||||
return { kind: 'wrong-extension', actualExt: otherExt };
|
||||
}
|
||||
|
||||
return { kind: 'missing-target' };
|
||||
}
|
||||
|
||||
function* walk(dir) {
|
||||
const entries = fs.readdirSync(dir, { withFileTypes: true });
|
||||
for (const entry of entries) {
|
||||
const full = path.join(dir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
if (
|
||||
entry.name.startsWith('.') ||
|
||||
entry.name === 'node_modules' ||
|
||||
entry.name.endsWith('_versioned_docs') ||
|
||||
entry.name === 'versioned_docs'
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
yield* walk(full);
|
||||
} else if (entry.isFile()) {
|
||||
if (entry.name.endsWith('.md') || entry.name.endsWith('.mdx')) {
|
||||
yield full;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function lintFile(file) {
|
||||
const src = fs.readFileSync(file, 'utf8');
|
||||
const findings = [];
|
||||
let inFence = false;
|
||||
const lines = src.split('\n');
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const line = lines[i];
|
||||
if (line.trimStart().startsWith('```')) {
|
||||
inFence = !inFence;
|
||||
continue;
|
||||
}
|
||||
if (inFence) continue;
|
||||
for (const m of line.matchAll(LINK_RE)) {
|
||||
const url = m.groups.url;
|
||||
const result = classifyLink(file, url);
|
||||
if (result.kind !== 'ok' && result.kind !== 'asset') {
|
||||
findings.push({ line: i + 1, url, ...result });
|
||||
}
|
||||
}
|
||||
}
|
||||
return findings;
|
||||
}
|
||||
|
||||
const findings = [];
|
||||
for (const root of ROOTS) {
|
||||
const abs = path.join(docsRoot, root);
|
||||
if (!fs.existsSync(abs)) continue;
|
||||
for (const file of walk(abs)) {
|
||||
for (const f of lintFile(file)) {
|
||||
findings.push({ file: path.relative(docsRoot, file), ...f });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (findings.length === 0) {
|
||||
console.log('✓ lint-docs-links: no broken internal links found');
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// Group by kind for readable output.
|
||||
const groups = {
|
||||
bare: [],
|
||||
'wrong-extension': [],
|
||||
'missing-target': [],
|
||||
};
|
||||
for (const f of findings) {
|
||||
groups[f.kind].push(f);
|
||||
}
|
||||
|
||||
console.error(
|
||||
`✗ lint-docs-links: found ${findings.length} broken internal link(s)`
|
||||
);
|
||||
console.error('');
|
||||
|
||||
if (groups.bare.length) {
|
||||
console.error(
|
||||
` ${groups.bare.length} bare relative link(s) (no .md/.mdx extension)`
|
||||
);
|
||||
console.error(
|
||||
" Docusaurus's file resolver skips these; the browser resolves them"
|
||||
);
|
||||
console.error(
|
||||
' against the current page URL — wrong directory for trailing-slash routes.'
|
||||
);
|
||||
console.error(' Add the extension so the file resolver picks them up.');
|
||||
console.error('');
|
||||
for (const f of groups.bare) {
|
||||
console.error(` ${f.file}:${f.line} ${f.url}`);
|
||||
}
|
||||
console.error('');
|
||||
}
|
||||
|
||||
if (groups['wrong-extension'].length) {
|
||||
console.error(
|
||||
` ${groups['wrong-extension'].length} wrong-extension link(s) (.md vs .mdx mismatch)`
|
||||
);
|
||||
console.error(' The target file exists with the other extension on disk.');
|
||||
console.error('');
|
||||
for (const f of groups['wrong-extension']) {
|
||||
console.error(
|
||||
` ${f.file}:${f.line} ${f.url} → use ${f.actualExt}`
|
||||
);
|
||||
}
|
||||
console.error('');
|
||||
}
|
||||
|
||||
if (groups['missing-target'].length) {
|
||||
console.error(
|
||||
` ${groups['missing-target'].length} missing-target link(s) (file doesn't exist)`
|
||||
);
|
||||
console.error('');
|
||||
for (const f of groups['missing-target']) {
|
||||
console.error(` ${f.file}:${f.line} ${f.url}`);
|
||||
}
|
||||
console.error('');
|
||||
}
|
||||
|
||||
process.exit(1);
|
||||
@@ -1,407 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { execSync } from 'child_process';
|
||||
import { fileURLToPath } from 'url';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
|
||||
const CONFIG_FILE = path.join(__dirname, '..', 'versions-config.json');
|
||||
|
||||
// Parse command line arguments
|
||||
const rawArgs = process.argv.slice(2);
|
||||
const skipGenerate = rawArgs.includes('--skip-generate');
|
||||
const args = rawArgs.filter((a) => a !== '--skip-generate');
|
||||
const command = args[0]; // 'add' or 'remove'
|
||||
const section = args[1]; // 'docs', 'admin_docs', 'developer_docs', or 'components'
|
||||
const version = args[2]; // version string like '1.2.0'
|
||||
|
||||
function loadConfig() {
|
||||
return JSON.parse(fs.readFileSync(CONFIG_FILE, 'utf8'));
|
||||
}
|
||||
|
||||
function saveConfig(config) {
|
||||
fs.writeFileSync(CONFIG_FILE, JSON.stringify(config, null, 2) + '\n');
|
||||
}
|
||||
|
||||
function freezeDataImports(section, version) {
|
||||
// MDX files can `import` JSON/YAML data from outside the section, either
|
||||
// via escaping relative paths (e.g. country-map-tools.mdx imports
|
||||
// `../../data/countries.json`) or via the `@site/` alias (e.g.
|
||||
// feature-flags.mdx imports `@site/static/feature-flags.json`). Without
|
||||
// intervention the snapshot keeps reading the live file, so the
|
||||
// historical version's content silently changes whenever the data file
|
||||
// is updated. Copy each escaping data import into a snapshot-local
|
||||
// `_versioned_data/` dir and rewrite the import to point there.
|
||||
const sectionRoot = section === 'docs'
|
||||
? path.join(__dirname, '..', 'docs')
|
||||
: path.join(__dirname, '..', section);
|
||||
const docsRoot = path.join(__dirname, '..');
|
||||
const versionedDocsDir = section === 'docs'
|
||||
? `versioned_docs/version-${version}`
|
||||
: `${section}_versioned_docs/version-${version}`;
|
||||
const versionedDocsPath = path.join(__dirname, '..', versionedDocsDir);
|
||||
const frozenDataDir = path.join(versionedDocsPath, '_versioned_data');
|
||||
|
||||
if (!fs.existsSync(versionedDocsPath)) {
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(` Freezing data imports in ${versionedDocsDir}...`);
|
||||
|
||||
// Matches data file imports in two flavors:
|
||||
// `from '../../foo/bar.json'` (relative, must escape one or more dirs)
|
||||
// `from '@site/static/foo.json'` (Docusaurus site-root alias)
|
||||
const dataImportRe = /(from\s+['"])((?:\.\.\/)+|@site\/)([^'"\s]+\.(?:json|ya?ml))(['"])/g;
|
||||
|
||||
function freezeOne(fullPath, depth, prefix, pathSpec, importPath, suffix) {
|
||||
let resolvedSource;
|
||||
if (pathSpec === '@site/') {
|
||||
// `@site/...` always resolves relative to the docs root.
|
||||
resolvedSource = path.join(docsRoot, importPath);
|
||||
} else {
|
||||
// Relative path — must escape the file's depth within the section
|
||||
// to point at content outside the section. Imports that stay inside
|
||||
// are copied wholesale by Docusaurus, so we leave them alone.
|
||||
const upCount = pathSpec.match(/\.\.\//g).length;
|
||||
if (upCount <= depth) return null;
|
||||
const relativeFromVersioned = path.relative(versionedDocsPath, fullPath);
|
||||
const originalDir = path.dirname(path.join(sectionRoot, relativeFromVersioned));
|
||||
resolvedSource = path.resolve(originalDir, pathSpec + importPath);
|
||||
}
|
||||
// Skip imports that land inside the section root — those get copied
|
||||
// with the section snapshot already.
|
||||
const relFromSection = path.relative(sectionRoot, resolvedSource);
|
||||
if (!relFromSection.startsWith('..')) return null;
|
||||
const relFromDocsRoot = path.relative(docsRoot, resolvedSource);
|
||||
if (relFromDocsRoot.startsWith('..') || !fs.existsSync(resolvedSource)) {
|
||||
return null;
|
||||
}
|
||||
const destPath = path.join(frozenDataDir, relFromDocsRoot);
|
||||
fs.mkdirSync(path.dirname(destPath), { recursive: true });
|
||||
fs.copyFileSync(resolvedSource, destPath);
|
||||
const rewritten = path
|
||||
.relative(path.dirname(fullPath), destPath)
|
||||
.split(path.sep)
|
||||
.join('/');
|
||||
const finalImport = rewritten.startsWith('.') ? rewritten : `./${rewritten}`;
|
||||
return `${prefix}${finalImport}${suffix}`;
|
||||
}
|
||||
|
||||
function walk(dir, depth) {
|
||||
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
|
||||
const fullPath = path.join(dir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
if (entry.name.startsWith('_')) continue;
|
||||
walk(fullPath, depth + 1);
|
||||
} else if (entry.isFile() && /\.(md|mdx)$/.test(entry.name)) {
|
||||
const original = fs.readFileSync(fullPath, 'utf8');
|
||||
let inFence = false;
|
||||
let mutated = false;
|
||||
const updated = original.split('\n').map(line => {
|
||||
if (/^\s*(```|~~~)/.test(line)) {
|
||||
inFence = !inFence;
|
||||
return line;
|
||||
}
|
||||
if (inFence) return line;
|
||||
return line.replace(dataImportRe, (match, prefix, pathSpec, importPath, suffix) => {
|
||||
const rewritten = freezeOne(fullPath, depth, prefix, pathSpec, importPath, suffix);
|
||||
if (rewritten === null) return match;
|
||||
mutated = true;
|
||||
return rewritten;
|
||||
});
|
||||
}).join('\n');
|
||||
if (mutated) {
|
||||
fs.writeFileSync(fullPath, updated);
|
||||
const rel = path.relative(versionedDocsPath, fullPath);
|
||||
console.log(` Froze data imports in ${rel}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
walk(versionedDocsPath, 0);
|
||||
}
|
||||
|
||||
function fixVersionedImports(section, version) {
|
||||
// Versioned content lands one directory deeper than the source content,
|
||||
// so any `../../src/` or `../../data/` imports in .md/.mdx files need
|
||||
// an extra `../` to keep reaching docs/src and docs/data.
|
||||
const versionedDocsDir = section === 'docs'
|
||||
? `versioned_docs/version-${version}`
|
||||
: `${section}_versioned_docs/version-${version}`;
|
||||
const versionedDocsPath = path.join(__dirname, '..', versionedDocsDir);
|
||||
|
||||
if (!fs.existsSync(versionedDocsPath)) {
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(` Fixing relative imports in ${versionedDocsDir}...`);
|
||||
|
||||
// Imports whose `../` count exceeds the file's depth within the section
|
||||
// escape the section root, so they need one extra `../` once the file
|
||||
// lives one level deeper inside the snapshot dir. Imports that stay
|
||||
// inside the section are unaffected (the section copies wholesale).
|
||||
function walk(dir, depth) {
|
||||
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
|
||||
const fullPath = path.join(dir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
walk(fullPath, depth + 1);
|
||||
} else if (entry.isFile() && /\.(md|mdx)$/.test(entry.name)) {
|
||||
const original = fs.readFileSync(fullPath, 'utf8');
|
||||
// Track fenced code blocks so we don't rewrite import samples inside
|
||||
// ```ts / ```js (etc.) blocks that are documentation, not real imports.
|
||||
let inFence = false;
|
||||
const updated = original.split('\n').map(line => {
|
||||
if (/^\s*(```|~~~)/.test(line)) {
|
||||
inFence = !inFence;
|
||||
return line;
|
||||
}
|
||||
if (inFence) return line;
|
||||
return line.replace(
|
||||
/(from\s+['"])((?:\.\.\/)+)/g,
|
||||
(match, prefix, dots) => {
|
||||
const upCount = dots.match(/\.\.\//g).length;
|
||||
return upCount > depth ? `${prefix}../${dots}` : match;
|
||||
},
|
||||
);
|
||||
}).join('\n');
|
||||
if (updated !== original) {
|
||||
fs.writeFileSync(fullPath, updated);
|
||||
const rel = path.relative(versionedDocsPath, fullPath);
|
||||
console.log(` Fixed imports in ${rel}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
walk(versionedDocsPath, 0);
|
||||
}
|
||||
|
||||
function addVersion(section, version) {
|
||||
const config = loadConfig();
|
||||
|
||||
if (!config[section]) {
|
||||
console.error(`Section '${section}' not found in config`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Check if version already exists
|
||||
if (config[section].onlyIncludeVersions.includes(version)) {
|
||||
console.error(`Version ${version} already exists in ${section}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log(`Creating version ${version} for ${section}...`);
|
||||
|
||||
// Refresh auto-generated content (database pages, API reference,
|
||||
// component playground) so the snapshot captures the current state of
|
||||
// master rather than whatever happened to be on disk. `generate:smart`
|
||||
// hashes its inputs and skips unchanged generators, so this is cheap
|
||||
// when the dev already has fresh output.
|
||||
//
|
||||
// Use --skip-generate if you've placed a CI-artifact databases.json
|
||||
// (the `database-diagnostics` artifact from Python-Integration) and
|
||||
// want to preserve it instead of letting the local env regenerate it.
|
||||
// See docs/README.md "Before You Cut" for the canonical release flow.
|
||||
if (skipGenerate) {
|
||||
console.log(` Skipping auto-gen refresh (--skip-generate set)`);
|
||||
} else {
|
||||
console.log(` Refreshing auto-generated docs...`);
|
||||
try {
|
||||
execSync('yarn run generate:smart', { stdio: 'inherit' });
|
||||
} catch (error) {
|
||||
console.error(`Failed to refresh auto-generated docs: ${error.message}`);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// Run Docusaurus version command
|
||||
const docusaurusCommand = section === 'docs'
|
||||
? `yarn docusaurus docs:version ${version}`
|
||||
: `yarn docusaurus docs:version:${section} ${version}`;
|
||||
|
||||
try {
|
||||
execSync(docusaurusCommand, { stdio: 'inherit' });
|
||||
} catch (error) {
|
||||
console.error(`Failed to create version: ${error.message}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Freeze data imports BEFORE adjusting paths, so the depth-aware rewriter
|
||||
// doesn't process the now-local imports we just rewrote.
|
||||
freezeDataImports(section, version);
|
||||
|
||||
// Fix relative imports in versioned content
|
||||
fixVersionedImports(section, version);
|
||||
|
||||
// Update config
|
||||
// Add to onlyIncludeVersions array (after 'current')
|
||||
const versionIndex = config[section].onlyIncludeVersions.indexOf('current') + 1;
|
||||
config[section].onlyIncludeVersions.splice(versionIndex, 0, version);
|
||||
|
||||
// Add version metadata
|
||||
const versionPath = section === 'docs' ? version : version;
|
||||
config[section].versions[version] = {
|
||||
label: version,
|
||||
path: versionPath,
|
||||
banner: 'none'
|
||||
};
|
||||
|
||||
// Note: we deliberately do NOT auto-bump `lastVersion` to the new
|
||||
// version. Superset's docs site keeps `lastVersion: 'current'` so
|
||||
// the canonical URLs (`/user-docs/...`, `/admin-docs/...`,
|
||||
// `/developer-docs/...`, `/components/...`) always render master
|
||||
// content; cut versions are accessed only via their explicit version
|
||||
// segment. (`/docs/...` paths are legacy and handled via per-page
|
||||
// redirects in docusaurus.config.ts — not a current canonical
|
||||
// form.) If you want a different policy, edit versions-config.json
|
||||
// after cutting.
|
||||
|
||||
saveConfig(config);
|
||||
console.log(`✅ Version ${version} added successfully to ${section}`);
|
||||
console.log(`📝 Updated versions-config.json`);
|
||||
}
|
||||
|
||||
function removeVersion(section, version) {
|
||||
const config = loadConfig();
|
||||
|
||||
if (!config[section]) {
|
||||
console.error(`Section '${section}' not found in config`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (version === 'current') {
|
||||
console.error(`Cannot remove 'current' version`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (!config[section].onlyIncludeVersions.includes(version)) {
|
||||
console.error(`Version ${version} not found in ${section}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log(`Removing version ${version} from ${section}...`);
|
||||
|
||||
// Determine file paths based on section
|
||||
const versionedDocsDir = section === 'docs'
|
||||
? `versioned_docs/version-${version}`
|
||||
: `${section}_versioned_docs/version-${version}`;
|
||||
|
||||
const versionedSidebarsFile = section === 'docs'
|
||||
? `versioned_sidebars/version-${version}-sidebars.json`
|
||||
: `${section}_versioned_sidebars/version-${version}-sidebars.json`;
|
||||
|
||||
// Remove versioned files
|
||||
const docsPath = path.join(__dirname, '..', versionedDocsDir);
|
||||
const sidebarsPath = path.join(__dirname, '..', versionedSidebarsFile);
|
||||
|
||||
if (fs.existsSync(docsPath)) {
|
||||
fs.rmSync(docsPath, { recursive: true });
|
||||
console.log(` Removed ${versionedDocsDir}`);
|
||||
}
|
||||
|
||||
if (fs.existsSync(sidebarsPath)) {
|
||||
fs.unlinkSync(sidebarsPath);
|
||||
console.log(` Removed ${versionedSidebarsFile}`);
|
||||
}
|
||||
|
||||
// Update versions.json file
|
||||
const versionsJsonFile = section === 'docs'
|
||||
? 'versions.json'
|
||||
: `${section}_versions.json`;
|
||||
const versionsJsonPath = path.join(__dirname, '..', versionsJsonFile);
|
||||
|
||||
if (fs.existsSync(versionsJsonPath)) {
|
||||
const versions = JSON.parse(fs.readFileSync(versionsJsonPath, 'utf8'));
|
||||
const versionIndex = versions.indexOf(version);
|
||||
if (versionIndex > -1) {
|
||||
versions.splice(versionIndex, 1);
|
||||
if (versions.length === 0) {
|
||||
// Sections with no versions shouldn't carry an empty versions file
|
||||
// on disk — Docusaurus doesn't require it, and an empty `[]` file
|
||||
// gets picked up by `docusaurus version` and snapshotted into the
|
||||
// next cut.
|
||||
fs.unlinkSync(versionsJsonPath);
|
||||
console.log(` Removed empty ${versionsJsonFile}`);
|
||||
} else {
|
||||
fs.writeFileSync(versionsJsonPath, JSON.stringify(versions, null, 2) + '\n');
|
||||
console.log(` Updated ${versionsJsonFile}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Update config
|
||||
const versionIndex = config[section].onlyIncludeVersions.indexOf(version);
|
||||
config[section].onlyIncludeVersions.splice(versionIndex, 1);
|
||||
delete config[section].versions[version];
|
||||
|
||||
// Update lastVersion if needed
|
||||
if (config[section].lastVersion === version) {
|
||||
// Set to the next available version or 'current'
|
||||
const remainingVersions = config[section].onlyIncludeVersions.filter(v => v !== 'current');
|
||||
config[section].lastVersion = remainingVersions.length > 0 ? remainingVersions[0] : 'current';
|
||||
console.log(` Updated lastVersion to ${config[section].lastVersion}`);
|
||||
}
|
||||
|
||||
saveConfig(config);
|
||||
console.log(`✅ Version ${version} removed successfully from ${section}`);
|
||||
console.log(`📝 Updated versions-config.json`);
|
||||
}
|
||||
|
||||
function printUsage() {
|
||||
console.log(`
|
||||
Usage:
|
||||
node scripts/manage-versions.mjs add <section> <version> [--skip-generate]
|
||||
node scripts/manage-versions.mjs remove <section> <version>
|
||||
|
||||
Where:
|
||||
- section: 'docs', 'developer_docs', 'admin_docs', or 'components'
|
||||
- version: version string (e.g., '1.2.0', '2.0.0')
|
||||
- --skip-generate: skip refreshing auto-generated docs before snapshotting
|
||||
(use when you've already placed a fresh databases.json
|
||||
from CI and want to preserve it)
|
||||
|
||||
Examples:
|
||||
node scripts/manage-versions.mjs add docs 2.0.0
|
||||
node scripts/manage-versions.mjs add developer_docs 1.3.0
|
||||
node scripts/manage-versions.mjs remove components 1.0.0
|
||||
`);
|
||||
}
|
||||
|
||||
// Main execution
|
||||
if (!command || !section || !version) {
|
||||
printUsage();
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (command === 'add') {
|
||||
addVersion(section, version);
|
||||
} else if (command === 'remove') {
|
||||
removeVersion(section, version);
|
||||
} else {
|
||||
console.error(`Unknown command: ${command}`);
|
||||
printUsage();
|
||||
process.exit(1);
|
||||
}
|
||||
Reference in New Issue
Block a user