Merge branch 'master' into plavacquery-config-map-tiles

.gitignore

@@ -80,7 +80,7 @@ yarn-error.log
 *.min.js
 test-changelog.md
 *.tsbuildinfo
-
+.venv
 # Ignore package-lock in packages
 plugins/*/package-lock.json
 packages/*/package-lock.json

UPDATING.md

@@ -24,6 +24,7 @@ assists people when migrating to a new version.
 
 ## Next
 
+- [33603](https://github.com/apache/superset/pull/33603) OpenStreetView has been promoted as the new default for Deck.gl visualization since it can be enabled by default without requiring an API key. If you have Mapbox set up and want to disable OpenStreeView in your environment, please follow the steps documented here [https://superset.apache.org/docs/configuration/map-tiles].
 - [33116](https://github.com/apache/superset/pull/33116) In Echarts Series charts (e.g. Line, Area, Bar, etc.) charts, the `x_axis_sort_series` and `x_axis_sort_series_ascending` form data items have been renamed with `x_axis_sort` and `x_axis_sort_asc`.
   There's a migration added that can potentially affect a significant number of existing charts.
 - [32317](https://github.com/apache/superset/pull/32317) The horizontal filter bar feature is now out of testing/beta development and its feature flag `HORIZONTAL_FILTER_BAR` has been removed.
@@ -55,6 +56,7 @@ assists people when migrating to a new version.
 - [30284](https://github.com/apache/superset/pull/30284) Deprecated GLOBAL_ASYNC_QUERIES_REDIS_CONFIG in favor of the new GLOBAL_ASYNC_QUERIES_CACHE_BACKEND configuration. To leverage Redis Sentinel, set CACHE_TYPE to RedisSentinelCache, or use RedisCache for standalone Redis
 - [31961](https://github.com/apache/superset/pull/31961) Upgraded React from version 16.13.1 to 17.0.2. If you are using custom frontend extensions or plugins, you may need to update them to be compatible with React 17.
 - [31260](https://github.com/apache/superset/pull/31260) Docker images now use `uv pip install` instead of `pip install` to manage the python envrionment. Most docker-based deployments will be affected, whether you derive one of the published images, or have custom bootstrap script that install python libraries (drivers)
+- [32432](https://github.com/apache/superset/pull/31260) Moves the List Roles FAB view to the frontend and requires `FAB_ADD_SECURITY_API` to be enabled in the configuration and `superset init` to be executed.
 
 ### Potential Downtime
 

docs/docs/configuration/map-tiles.mdx

@@ -0,0 +1,78 @@
+---
+title: Map Tiles
+sidebar_position: 12
+version: 1
+---
+
+# Map tiles
+
+Superset uses OSM and Mapbox tiles by default. OSM is free but you still need setting your MAPBOX_API_KEY if you want to use mapbox maps.
+
+## Setting map tiles
+
+Map tiles can be set with `DECKGL_BASE_MAP` in your `superset_config.py` or `superset_config_docker.py`
+For adding your own map tiles, you can use the following format.
+
+```python
+DECKGL_BASE_MAP = [
+    ['tile://https://your_personal_url/{z}/{x}/{y}.png', 'MyTile']
+]
+```
+Openstreetmap tiles url can be added without prefix.
+```python
+DECKGL_BASE_MAP = [
+    ['https://c.tile.openstreetmap.org/{z}/{x}/{y}.png', 'OpenStreetMap']
+]
+```
+
+Default values are:
+```python
+DECKGL_BASE_MAP = [
+    ['https://tile.openstreetmap.org/{z}/{x}/{y}.png', 'Streets (OSM)'],
+    ['https://tile.osm.ch/osm-swiss-style/{z}/{x}/{y}.png', 'Topography (OSM)'],
+    ['mapbox://styles/mapbox/streets-v9', 'Streets'],
+    ['mapbox://styles/mapbox/dark-v9', 'Dark'],
+    ['mapbox://styles/mapbox/light-v9', 'Light'],
+    ['mapbox://styles/mapbox/satellite-streets-v9', 'Satellite Streets'],
+    ['mapbox://styles/mapbox/satellite-v9', 'Satellite'],
+    ['mapbox://styles/mapbox/outdoors-v9', 'Outdoors'],
+]
+```
+
+It is possible to set only mapbox by removing osm tiles and other way around.
+
+:::warning
+Setting `DECKGL_BASE_MAP` overwrite default values
+:::
+
+After defining your map tiles, set them in these variables:
+- `CORS_OPTIONS`
+- `connect-src` of `TALISMAN_CONFIG` and `TALISMAN_CONFIG_DEV` variables.
+
+```python
+ENABLE_CORS = True
+CORS_OPTIONS: dict[Any, Any] = {
+    "origins": [
+        "https://tile.openstreetmap.org",
+        "https://tile.osm.ch",
+        "https://your_personal_url/{z}/{x}/{y}.png",
+    ]
+}
+
+.
+.
+
+TALISMAN_CONFIG = {
+    "content_security_policy": {
+        ...
+        "connect-src": [
+            "'self'",
+            "https://api.mapbox.com",
+            "https://events.mapbox.com",
+            "https://tile.openstreetmap.org",
+            "https://tile.osm.ch",
+            "https://your_personal_url/{z}/{x}/{y}.png",
+        ],
+        ...
+}
+```

requirements/base.in

@@ -36,3 +36,6 @@ marshmallow-sqlalchemy>=1.3.0,<1.4.1
 
 # needed for python 3.12 support
 openapi-schema-validator>=0.6.3
+
+# needed when using the flask-cors extension
+.[cors]

requirements/base.txt

@@ -104,6 +104,7 @@ flask==2.3.3
     #   flask-babel
     #   flask-caching
     #   flask-compress
+    #   flask-cors
     #   flask-jwt-extended
     #   flask-limiter
     #   flask-login
@@ -119,6 +120,8 @@ flask-caching==2.3.1
     # via apache-superset (pyproject.toml)
 flask-compress==1.17
     # via apache-superset (pyproject.toml)
+flask-cors==4.0.2
+    # via apache-superset (pyproject.toml)
 flask-jwt-extended==4.7.1
     # via flask-appbuilder
 flask-limiter==3.12

superset-frontend/packages/superset-ui-core/src/validator/validateMapboxStylesUrl.ts

@@ -19,18 +19,28 @@
 
 import { t } from '../translation';
 
+const VALIDE_OSM_URLS = ['https://tile.osm', 'https://tile.openstreetmap'];
+
 /**
  * Validate a [Mapbox styles URL](https://docs.mapbox.com/help/glossary/style-url/)
+ * or a title server URL.
  * @param v
  */
 export default function validateMapboxStylesUrl(v: unknown) {
-  if (
-    typeof v === 'string' &&
-    v.trim().length > 0 &&
-    v.trim().startsWith('mapbox://styles/')
-  ) {
-    return false;
+  if (typeof v === 'string') {
+    const trimmed_v = v.trim();
+    if (
+      typeof v === 'string' &&
+      trimmed_v.length > 0 &&
+      (trimmed_v.startsWith('mapbox://styles/') ||
+        trimmed_v.startsWith('tile://http') ||
+        VALIDE_OSM_URLS.some(s => trimmed_v.startsWith(s)))
+    ) {
+      return false;
+    }
   }
 
-  return t('is expected to be a Mapbox URL');
+  return t(
+    'is expected to be a Mapbox/OSM URL (eg. mapbox://styles/...) or a tile server URL (eg. tile://http...)',
+  );
 }

superset-frontend/packages/superset-ui-core/test/validator/validateMapboxStylesUrl.test.ts

@@ -29,6 +29,11 @@ describe('validateMapboxStylesUrl', () => {
         'mapbox://styles/foobar/clp2dr5r4008a01pcg4ad45m8',
       ),
     ).toEqual(false);
+    expect(
+      validateMapboxStylesUrl(
+        'tile://https://c.tile.openstreetmap.org/{z}/{x}/{y}.png',
+      ),
+    ).toEqual(false);
   });
 
   [
@@ -40,7 +45,7 @@ describe('validateMapboxStylesUrl', () => {
   ].forEach(value => {
     it(`should not validate ${value}`, () => {
       expect(validateMapboxStylesUrl(value)).toEqual(
-        'is expected to be a Mapbox URL',
+        'is expected to be a Mapbox/OSM URL (eg. mapbox://styles/...) or a tile server URL (eg. tile://http...)',
       );
     });
   });

superset-frontend/plugins/legacy-preset-chart-deckgl/package.json

@@ -24,10 +24,11 @@
     "lib"
   ],
   "dependencies": {
-    "@deck.gl/aggregation-layers": "^9.0.38",
-    "@deck.gl/core": "^9.0.37",
-    "@deck.gl/layers": "^9.0.38",
-    "@deck.gl/react": "^9.1.4",
+    "@deck.gl/aggregation-layers": "^9.1.9",
+    "@deck.gl/core": "^9.1.9",
+    "@deck.gl/geo-layers": "^9.1.9",
+    "@deck.gl/layers": "^9.1.9",
+    "@deck.gl/react": "^9.1.9",
     "@mapbox/geojson-extent": "^1.0.1",
     "@math.gl/web-mercator": "^4.1.0",
     "@types/d3-array": "^2.0.0",

superset-frontend/plugins/legacy-preset-chart-deckgl/src/DeckGLContainer.tsx

@@ -37,6 +37,12 @@ import { JsonObject, JsonValue, styled, usePrevious } from '@superset-ui/core';
 import Tooltip, { TooltipProps } from './components/Tooltip';
 import 'mapbox-gl/dist/mapbox-gl.css';
 import { Viewport } from './utils/fitViewport';
+import {
+  MAPBOX_LAYER_PREFIX,
+  OSM_LAYER_KEYWORDS,
+  TILE_LAYER_PREFIX,
+  buildTileLayer,
+} from './utils';
 
 const TICK = 250; // milliseconds
 
@@ -92,6 +98,20 @@ export const DeckGLContainer = memo(
     );
 
     const layers = useCallback(() => {
+      if (
+        (props.mapStyle?.startsWith(TILE_LAYER_PREFIX) ||
+          OSM_LAYER_KEYWORDS.some(tilek => props.mapStyle?.includes(tilek))) &&
+        props.layers.some(
+          l => typeof l !== 'function' && l?.id === 'tile-layer',
+        ) === false
+      ) {
+        props.layers.unshift(
+          buildTileLayer(
+            (props.mapStyle ?? '').replace(TILE_LAYER_PREFIX, ''),
+            'tile-layer',
+          ),
+        );
+      }
       // Support for layer factory
       if (props.layers.some(l => typeof l === 'function')) {
         return props.layers.map(l =>
@@ -100,7 +120,7 @@ export const DeckGLContainer = memo(
       }
 
       return props.layers as Layer[];
-    }, [props.layers]);
+    }, [props.layers, props.mapStyle]);
 
     const { children = null, height, width } = props;
 
@@ -115,11 +135,13 @@ export const DeckGLContainer = memo(
             viewState={viewState}
             onViewStateChange={onViewStateChange}
           >
-            <StaticMap
-              preserveDrawingBuffer
-              mapStyle={props.mapStyle || 'light'}
-              mapboxApiAccessToken={props.mapboxApiAccessToken}
-            />
+            {props.mapStyle?.startsWith(MAPBOX_LAYER_PREFIX) && (
+              <StaticMap
+                preserveDrawingBuffer
+                mapStyle={props.mapStyle || 'light'}
+                mapboxApiAccessToken={props.mapboxApiAccessToken}
+              />
+            )}
           </DeckGL>
           {children}
         </div>

superset-frontend/plugins/legacy-preset-chart-deckgl/src/utilities/Shared_DeckGL.jsx

@@ -29,6 +29,30 @@ import {
 import { D3_FORMAT_OPTIONS, sharedControls } from '@superset-ui/chart-controls';
 import { columnChoices, PRIMARY_COLOR } from './controls';
 
+let deckglTiles;
+
+export const DEFAULT_DECKGL_TILES = [
+  ['https://tile.openstreetmap.org/{z}/{x}/{y}.png', 'Streets (OSM)'],
+  ['https://tile.osm.ch/osm-swiss-style/{z}/{x}/{y}.png', 'Topography (OSM)'],
+  ['mapbox://styles/mapbox/streets-v9', 'Streets (Mapbox)'],
+  ['mapbox://styles/mapbox/dark-v9', 'Dark (Mapbox)'],
+  ['mapbox://styles/mapbox/light-v9', 'Light (Mapbox)'],
+  ['mapbox://styles/mapbox/satellite-streets-v9', 'Satellite Streets (Mapbox)'],
+  ['mapbox://styles/mapbox/satellite-v9', 'Satellite (Mapbox)'],
+  ['mapbox://styles/mapbox/outdoors-v9', 'Outdoors (Mapbox)'],
+];
+
+const getDeckGLTiles = () => {
+  if (!deckglTiles) {
+    const appContainer = document.getElementById('app');
+    const { common } = JSON.parse(
+      appContainer?.getAttribute('data-bootstrap') || '{}',
+    );
+    deckglTiles = common?.deckgl_tiles ?? DEFAULT_DECKGL_TILES;
+  }
+  return deckglTiles;
+};
+
 const DEFAULT_VIEWPORT = {
   longitude: 6.85236157047845,
   latitude: 31.222656842808707,
@@ -372,17 +396,10 @@ export const mapboxStyle = {
     renderTrigger: true,
     freeForm: true,
     validators: [validateMapboxStylesUrl],
-    choices: [
-      ['mapbox://styles/mapbox/streets-v9', t('Streets')],
-      ['mapbox://styles/mapbox/dark-v9', t('Dark')],
-      ['mapbox://styles/mapbox/light-v9', t('Light')],
-      ['mapbox://styles/mapbox/satellite-streets-v9', t('Satellite Streets')],
-      ['mapbox://styles/mapbox/satellite-v9', t('Satellite')],
-      ['mapbox://styles/mapbox/outdoors-v9', t('Outdoors')],
-    ],
-    default: 'mapbox://styles/mapbox/light-v9',
+    choices: getDeckGLTiles(),
+    default: getDeckGLTiles()[0][0],
     description: t(
-      'Base layer map style. See Mapbox documentation: %s',
+      'Mapbox base layer map style (see Mapbox documentation: %s) or tile server URL.',
       'https://docs.mapbox.com/help/glossary/style-url/',
     ),
   },

superset-frontend/plugins/legacy-preset-chart-deckgl/src/utils.ts

@@ -25,10 +25,16 @@ import {
   QueryFormData,
   SequentialScheme,
 } from '@superset-ui/core';
+import { GeoBoundingBox, TileLayer } from '@deck.gl/geo-layers';
+import { BitmapLayer, PathLayer } from '@deck.gl/layers';
 import { hexToRGB } from './utils/colors';
 
 const DEFAULT_NUM_BUCKETS = 10;
 
+export const MAPBOX_LAYER_PREFIX = 'mapbox://';
+export const TILE_LAYER_PREFIX = 'tile://';
+export const OSM_LAYER_KEYWORDS = ['openstreetmap', 'osm'];
+
 export type Buckets = {
   break_points: string[];
   num_buckets: string;
@@ -190,3 +196,42 @@ export function getBuckets(
 
   return buckets;
 }
+
+export function buildTileLayer(url: string, id: string) {
+  interface TileLayerProps {
+    id: string;
+    data: string;
+    minZoom: number;
+    maxZoom: number;
+    tileSize: number;
+    renderSubLayers: (props: any) => (BitmapLayer | PathLayer)[];
+  }
+
+  interface RenderSubLayerProps {
+    tile: {
+      bbox: GeoBoundingBox;
+    };
+    data: any;
+  }
+
+  return new TileLayer({
+    data: url,
+    id,
+    minZoom: 0,
+    maxZoom: 19,
+    tileSize: 256,
+
+    renderSubLayers: (props: RenderSubLayerProps): BitmapLayer[] => {
+      const { west, north, east, south } = props.tile.bbox as GeoBoundingBox;
+
+      // Ajouter une BitmapLayer
+      const bitmapLayer = new BitmapLayer(props, {
+        data: undefined,
+        image: props.data,
+        bounds: [west, south, east, north],
+      });
+
+      return [bitmapLayer];
+    },
+  } as TileLayerProps);
+}

superset/config.py

@@ -422,6 +422,29 @@ class D3Format(TypedDict, total=False):
 D3_FORMAT: D3Format = {}
 
 
+# Override the default mapbox tiles
+# Default values are equivalent to
+# DECKGL_BASE_MAP = [
+#   ['https://tile.openstreetmap.org/{z}/{x}/{y}.png', 'Streets (OSM)'],
+#   ['https://tile.osm.ch/osm-swiss-style/{z}/{x}/{y}.png', 'Topography (OSM)'],
+#   ['mapbox://styles/mapbox/streets-v9', 'Streets'],
+#   ['mapbox://styles/mapbox/dark-v9', 'Dark'],
+#   ['mapbox://styles/mapbox/light-v9', 'Light'],
+#   ['mapbox://styles/mapbox/satellite-streets-v9', 'Satellite Streets'],
+#   ['mapbox://styles/mapbox/satellite-v9', 'Satellite'],
+#   ['mapbox://styles/mapbox/outdoors-v9', 'Outdoors'],
+# ]
+# for adding your own map tiles, you can use the following format:
+# - tile:// + your_personal_url or openstreetmap_url
+#   example:
+#   DECKGL_BASE_MAP = [
+#       ['tile://https://c.tile.openstreetmap.org/{z}/{x}/{y}.png', 'OpenStreetMap']
+#    ]
+# Enable CORS and set map url in origins option.
+# Add also map url in connect-src of TALISMAN_CONFIG variable
+DECKGL_BASE_MAP: list[list[str, str]] = None
+
+
 # Override the default d3 locale for time format
 # Default values are equivalent to
 # D3_TIME_FORMAT = {
@@ -830,8 +853,13 @@ STORE_CACHE_KEYS_IN_METADATA_DB = False
 # CORS Options
 # NOTE: enabling this requires installing the cors-related python dependencies
 # `pip install .[cors]` or `pip install apache_superset[cors]`, depending
-ENABLE_CORS = False
-CORS_OPTIONS: dict[Any, Any] = {}
+ENABLE_CORS = True
+CORS_OPTIONS: dict[Any, Any] = {
+    "origins": [
+        "https://tile.openstreetmap.org",
+        "https://tile.osm.ch",
+    ]
+}
 
 # Sanitizes the HTML content used in markdowns to allow its rendering in a safe manner.
 # Disabling this option is not recommended for security reasons. If you wish to allow
@@ -1660,6 +1688,8 @@ TALISMAN_CONFIG = {
             "'self'",
             "https://api.mapbox.com",
             "https://events.mapbox.com",
+            "https://tile.openstreetmap.org",
+            "https://tile.osm.ch",
         ],
         "object-src": "'none'",
         "style-src": [
@@ -1692,6 +1722,8 @@ TALISMAN_DEV_CONFIG = {
             "'self'",
             "https://api.mapbox.com",
             "https://events.mapbox.com",
+            "https://tile.openstreetmap.org",
+            "https://tile.osm.ch",
         ],
         "object-src": "'none'",
         "style-src": [

superset/examples/deck.py

@@ -194,7 +194,7 @@ def load_deck_dash() -> None:  # pylint: disable=too-many-statements
         "datasource": "5__table",
         "granularity_sqla": None,
         "groupby": [],
-        "mapbox_style": "mapbox://styles/mapbox/light-v9",
+        "mapbox_style": "https://tile.openstreetmap.org/{z}/{x}/{y}.png",
         "multiplier": 10,
         "point_radius_fixed": {"type": "metric", "value": "count"},
         "point_unit": "square_m",
@@ -229,7 +229,7 @@ def load_deck_dash() -> None:  # pylint: disable=too-many-statements
         "point_unit": "square_m",
         "row_limit": 5000,
         "spatial": {"type": "latlong", "lonCol": "LON", "latCol": "LAT"},
-        "mapbox_style": "mapbox://styles/mapbox/dark-v9",
+        "mapbox_style": "https://tile.openstreetmap.org/{z}/{x}/{y}.png",
         "granularity_sqla": None,
         "size": "count",
         "viz_type": "deck_screengrid",
@@ -263,7 +263,7 @@ def load_deck_dash() -> None:  # pylint: disable=too-many-statements
     slice_data = {
         "spatial": {"type": "latlong", "lonCol": "LON", "latCol": "LAT"},
         "row_limit": 5000,
-        "mapbox_style": "mapbox://styles/mapbox/streets-v9",
+        "mapbox_style": "https://tile.openstreetmap.org/{z}/{x}/{y}.png",
         "granularity_sqla": None,
         "size": "count",
         "viz_type": "deck_hex",
@@ -300,7 +300,7 @@ def load_deck_dash() -> None:  # pylint: disable=too-many-statements
         "autozoom": False,
         "spatial": {"type": "latlong", "lonCol": "LON", "latCol": "LAT"},
         "row_limit": 5000,
-        "mapbox_style": "mapbox://styles/mapbox/satellite-streets-v9",
+        "mapbox_style": "https://tile.openstreetmap.org/{z}/{x}/{y}.png",
         "granularity_sqla": None,
         "size": "count",
         "viz_type": "deck_grid",
@@ -367,7 +367,7 @@ def load_deck_dash() -> None:  # pylint: disable=too-many-statements
         },
         "line_type": "json",
         "linear_color_scheme": "oranges",
-        "mapbox_style": "mapbox://styles/mapbox/light-v9",
+        "mapbox_style": "https://tile.openstreetmap.org/{z}/{x}/{y}.png",
         "viewport": {
             "longitude": -122.43388541747726,
             "latitude": 37.752020331384834,
@@ -442,7 +442,7 @@ def load_deck_dash() -> None:  # pylint: disable=too-many-statements
             "lonCol": "LONGITUDE_DEST",
         },
         "row_limit": 5000,
-        "mapbox_style": "mapbox://styles/mapbox/light-v9",
+        "mapbox_style": "https://tile.openstreetmap.org/{z}/{x}/{y}.png",
         "viewport": {
             "altitude": 1.5,
             "bearing": 8.546256357301871,
@@ -486,7 +486,7 @@ def load_deck_dash() -> None:  # pylint: disable=too-many-statements
         "line_column": "path_json",
         "line_type": "json",
         "row_limit": 5000,
-        "mapbox_style": "mapbox://styles/mapbox/light-v9",
+        "mapbox_style": "https://tile.openstreetmap.org/{z}/{x}/{y}.png",
         "viewport": {
             "longitude": -122.18885402582598,
             "latitude": 37.73671752604488,

superset/examples/long_lat.py

@@ -107,18 +107,18 @@ def load_long_lat_data(only_metadata: bool = False, force: bool = False) -> None
         "granularity_sqla": "day",
         "since": "2014-01-01",
         "until": "now",
-        "viz_type": "mapbox",
+        "viz_type": "osm",
         "all_columns_x": "LON",
         "all_columns_y": "LAT",
-        "mapbox_style": "mapbox://styles/mapbox/light-v9",
+        "mapbox_style": "https://tile.openstreetmap.org/{z}/{x}/{y}.png",
         "all_columns": ["occupancy"],
         "row_limit": 500000,
     }
 
     logger.debug("Creating a slice")
     slc = Slice(
-        slice_name="Mapbox Long/Lat",
-        viz_type="mapbox",
+        slice_name="OSM Long/Lat",
+        viz_type="osm",
         datasource_type=DatasourceType.TABLE,
         datasource_id=tbl.id,
         params=get_slice_json(slice_data),

superset/examples/misc_dashboard.py

@@ -47,7 +47,7 @@ def load_misc_dashboard() -> None:
         "meta": {
             "chartId": 3969,
             "height": 69,
-            "sliceName": "Mapbox Long/Lat",
+            "sliceName": "OSM Long/Lat",
             "uuid": "164efe31-295b-4408-aaa6-2f4bfb58a212",
             "width": 4
         },

superset/views/base.py

@@ -368,6 +368,7 @@ def cached_common_bootstrap_data(  # pylint: disable=unused-argument
         "d3_format": conf.get("D3_FORMAT"),
         "d3_time_format": conf.get("D3_TIME_FORMAT"),
         "currencies": conf.get("CURRENCIES"),
+        "deckgl_tiles": conf.get("DECKGL_BASE_MAP"),
         "feature_flags": get_feature_flags(),
         "extra_sequential_color_schemes": conf["EXTRA_SEQUENTIAL_COLOR_SCHEMES"],
         "extra_categorical_color_schemes": conf["EXTRA_CATEGORICAL_COLOR_SCHEMES"],