diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000000..eb6274e58de --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,22 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +repos: + - repo: https://github.com/ambv/black + rev: stable + hooks: + - id: black + language_version: python3.6 diff --git a/.pylintrc b/.pylintrc index ec88ef0b50b..04a53e0d379 100644 --- a/.pylintrc +++ b/.pylintrc @@ -81,7 +81,7 @@ confidence= # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use"--disable=all --enable=classes # --disable=W" -disable=standarderror-builtin,long-builtin,dict-view-method,intern-builtin,suppressed-message,no-absolute-import,unpacking-in-except,apply-builtin,delslice-method,indexing-exception,old-raise-syntax,print-statement,cmp-builtin,reduce-builtin,useless-suppression,coerce-method,input-builtin,cmp-method,raw_input-builtin,nonzero-method,backtick,basestring-builtin,setslice-method,reload-builtin,oct-method,map-builtin-not-iterating,execfile-builtin,old-octal-literal,zip-builtin-not-iterating,buffer-builtin,getslice-method,metaclass-assignment,xrange-builtin,long-suffix,round-builtin,range-builtin-not-iterating,next-method-called,dict-iter-method,parameter-unpacking,unicode-builtin,unichr-builtin,import-star-module-level,raising-string,filter-builtin-not-iterating,old-ne-operator,using-cmp-argument,coerce-builtin,file-builtin,old-division,hex-method,invalid-unary-operand-type,missing-docstring,too-many-lines,duplicate-code +disable=standarderror-builtin,long-builtin,dict-view-method,intern-builtin,suppressed-message,no-absolute-import,unpacking-in-except,apply-builtin,delslice-method,indexing-exception,old-raise-syntax,print-statement,cmp-builtin,reduce-builtin,useless-suppression,coerce-method,input-builtin,cmp-method,raw_input-builtin,nonzero-method,backtick,basestring-builtin,setslice-method,reload-builtin,oct-method,map-builtin-not-iterating,execfile-builtin,old-octal-literal,zip-builtin-not-iterating,buffer-builtin,getslice-method,metaclass-assignment,xrange-builtin,long-suffix,round-builtin,range-builtin-not-iterating,next-method-called,dict-iter-method,parameter-unpacking,unicode-builtin,unichr-builtin,import-star-module-level,raising-string,filter-builtin-not-iterating,old-ne-operator,using-cmp-argument,coerce-builtin,file-builtin,old-division,hex-method,invalid-unary-operand-type,missing-docstring,too-many-lines,duplicate-code,bad-continuation [REPORTS] @@ -209,7 +209,7 @@ max-nested-blocks=5 [FORMAT] # Maximum number of characters on a single line. -max-line-length=90 +max-line-length=88 # Regexp for a line that is allowed to be longer than the limit. ignore-long-lines=^\s*(# )??$ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 569cc916a92..d165ef42874 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -296,9 +296,9 @@ python setup.py build_sphinx #### OS Dependencies -Make sure your machine meets the [OS dependencies](https://superset.incubator.apache.org/installation.html#os-dependencies) before following these steps. +Make sure your machine meets the [OS dependencies](https://superset.incubator.apache.org/installation.html#os-dependencies) before following these steps. -Developers should use a virtualenv. +Developers should use a virtualenv. ``` pip install virtualenv @@ -447,6 +447,15 @@ export enum FeatureFlag { those specified under FEATURE_FLAGS in `superset_config.py`. For example, `DEFAULT_FEATURE_FLAGS = { 'FOO': True, 'BAR': False }` in `superset/config.py` and `FEATURE_FLAGS = { 'BAR': True, 'BAZ': True }` in `superset_config.py` will result in combined feature flags of `{ 'FOO': True, 'BAR': True, 'BAZ': True }`. +## Git Hooks + +Superset uses Git pre-commit hooks courtesy of [pre-commit](https://pre-commit.com/). To install run the following: + +```bash +pip3 install -r requirements-dev.txt +pre-commit install +``` + ## Linting Lint the project with: @@ -461,6 +470,10 @@ npm ci npm run lint ``` +The Python code is auto-formatted using [Black](https://github.com/python/black) which +is configured as a pre-commit hook. There are also numerous [editor integrations](https://black.readthedocs.io/en/stable/editor_integration.html). + + ## Testing ### Python Testing @@ -736,7 +749,7 @@ to work on `async` related features. To do this, you'll need to: * Add an additional database entry. We recommend you copy the connection - string from the database labeled `main`, and then enable `SQL Lab` and the + string from the database labeled `main`, and then enable `SQL Lab` and the features you want to use. Don't forget to check the `Async` box * Configure a results backend, here's a local `FileSystemCache` example, not recommended for production, diff --git a/requirements-dev.txt b/requirements-dev.txt index 1b008ad2664..52b341ce5dc 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -14,17 +14,17 @@ # See the License for the specific language governing permissions and # limitations under the License. # +black==19.3b0 coverage==4.5.3 -flake8-commas==2.0.0 flake8-import-order==0.18.1 flake8-mypy==17.8.0 -flake8-quotes==2.0.1 flake8==3.7.7 flask-cors==3.0.7 ipdb==0.12 mypy==0.670 nose==1.3.7 pip-tools==3.7.0 +pre-commit==1.17.0 psycopg2-binary==2.7.5 pycodestyle==2.5.0 pyhive==0.6.1 diff --git a/setup.py b/setup.py index 8d3fcc6c9be..dc21c38f6aa 100644 --- a/setup.py +++ b/setup.py @@ -23,113 +23,100 @@ import sys from setuptools import find_packages, setup if sys.version_info < (3, 6): - sys.exit('Sorry, Python < 3.6 is not supported') + sys.exit("Sorry, Python < 3.6 is not supported") BASE_DIR = os.path.abspath(os.path.dirname(__file__)) -PACKAGE_DIR = os.path.join(BASE_DIR, 'superset', 'static', 'assets') -PACKAGE_FILE = os.path.join(PACKAGE_DIR, 'package.json') +PACKAGE_DIR = os.path.join(BASE_DIR, "superset", "static", "assets") +PACKAGE_FILE = os.path.join(PACKAGE_DIR, "package.json") with open(PACKAGE_FILE) as package_file: - version_string = json.load(package_file)['version'] + version_string = json.load(package_file)["version"] -with io.open('README.md', encoding='utf-8') as f: +with io.open("README.md", encoding="utf-8") as f: long_description = f.read() def get_git_sha(): try: - s = subprocess.check_output(['git', 'rev-parse', 'HEAD']) + s = subprocess.check_output(["git", "rev-parse", "HEAD"]) return s.decode().strip() except Exception: - return '' + return "" GIT_SHA = get_git_sha() -version_info = { - 'GIT_SHA': GIT_SHA, - 'version': version_string, -} -print('-==-' * 15) -print('VERSION: ' + version_string) -print('GIT SHA: ' + GIT_SHA) -print('-==-' * 15) +version_info = {"GIT_SHA": GIT_SHA, "version": version_string} +print("-==-" * 15) +print("VERSION: " + version_string) +print("GIT SHA: " + GIT_SHA) +print("-==-" * 15) -with open(os.path.join(PACKAGE_DIR, 'version_info.json'), 'w') as version_file: +with open(os.path.join(PACKAGE_DIR, "version_info.json"), "w") as version_file: json.dump(version_info, version_file) setup( - name='apache-superset', - description=( - 'A modern, enterprise-ready business intelligence web application'), + name="apache-superset", + description=("A modern, enterprise-ready business intelligence web application"), long_description=long_description, - long_description_content_type='text/markdown', + long_description_content_type="text/markdown", version=version_string, packages=find_packages(), include_package_data=True, zip_safe=False, - scripts=['superset/bin/superset'], + scripts=["superset/bin/superset"], install_requires=[ - 'bleach>=3.0.2, <4.0.0', - 'celery>=4.2.0, <5.0.0', - 'click>=6.0, <7.0.0', # `click`>=7 forces "-" instead of "_" - 'colorama', - 'contextlib2', - 'croniter>=0.3.28', - 'cryptography>=2.4.2', - 'flask>=1.0.0, <2.0.0', - 'flask-appbuilder>=2.1.5, <2.3.0', - 'flask-caching', - 'flask-compress', - 'flask-talisman', - 'flask-migrate', - 'flask-wtf', - 'geopy', - 'gunicorn', # deprecated - 'humanize', - 'idna', - 'isodate', - 'markdown>=3.0', - 'pandas>=0.18.0, <0.24.0', # `pandas`>=0.24.0 changes datetimelike API - 'parsedatetime', - 'pathlib2', - 'polyline', - 'pydruid>=0.5.2', - 'python-dateutil', - 'python-dotenv', - 'python-geohash', - 'pyyaml>=5.1', - 'requests>=2.22.0', - 'retry>=0.9.2', - 'selenium>=3.141.0', - 'simplejson>=3.15.0', - 'sqlalchemy>=1.3.5,<2.0', - 'sqlalchemy-utils>=0.33.2', - 'sqlparse', - 'wtforms-json', + "bleach>=3.0.2, <4.0.0", + "celery>=4.2.0, <5.0.0", + "click>=6.0, <7.0.0", # `click`>=7 forces "-" instead of "_" + "colorama", + "contextlib2", + "croniter>=0.3.28", + "cryptography>=2.4.2", + "flask>=1.0.0, <2.0.0", + "flask-appbuilder>=2.1.5, <2.3.0", + "flask-caching", + "flask-compress", + "flask-talisman", + "flask-migrate", + "flask-wtf", + "geopy", + "gunicorn", # deprecated + "humanize", + "idna", + "isodate", + "markdown>=3.0", + "pandas>=0.18.0, <0.24.0", # `pandas`>=0.24.0 changes datetimelike API + "parsedatetime", + "pathlib2", + "polyline", + "pydruid>=0.5.2", + "python-dateutil", + "python-dotenv", + "python-geohash", + "pyyaml>=5.1", + "requests>=2.22.0", + "retry>=0.9.2", + "selenium>=3.141.0", + "simplejson>=3.15.0", + "sqlalchemy>=1.3.5,<2.0", + "sqlalchemy-utils>=0.33.2", + "sqlparse", + "wtforms-json", ], extras_require={ - 'bigquery': [ - 'pybigquery>=0.4.10', - 'pandas_gbq>=0.10.0', - ], - 'cors': ['flask-cors>=2.0.0'], - 'gsheets': ['gsheetsdb>=0.1.9'], - 'hive': [ - 'pyhive[hive]>=0.6.1', - 'tableschema', - 'thrift>=0.11.0, <1.0.0', - ], - 'mysql': ['mysqlclient==1.4.2.post1'], - 'postgres': ['psycopg2-binary==2.7.5'], - 'presto': ['pyhive[presto]>=0.4.0'], + "bigquery": ["pybigquery>=0.4.10", "pandas_gbq>=0.10.0"], + "cors": ["flask-cors>=2.0.0"], + "gsheets": ["gsheetsdb>=0.1.9"], + "hive": ["pyhive[hive]>=0.6.1", "tableschema", "thrift>=0.11.0, <1.0.0"], + "mysql": ["mysqlclient==1.4.2.post1"], + "postgres": ["psycopg2-binary==2.7.5"], + "presto": ["pyhive[presto]>=0.4.0"], }, - author='Apache Software Foundation', - author_email='dev@superset.incubator.apache.org', - url='https://superset.apache.org/', + author="Apache Software Foundation", + author_email="dev@superset.incubator.apache.org", + url="https://superset.apache.org/", download_url=( - 'https://dist.apache.org/repos/dist/release/superset/' + version_string + "https://dist.apache.org/repos/dist/release/superset/" + version_string ), - classifiers=[ - 'Programming Language :: Python :: 3.6', - ], + classifiers=["Programming Language :: Python :: 3.6"], ) diff --git a/superset/__init__.py b/superset/__init__.py index d03f28e6277..2f13660e09d 100644 --- a/superset/__init__.py +++ b/superset/__init__.py @@ -40,7 +40,7 @@ from superset.utils.core import pessimistic_connection_handling, setup_cache wtforms_json.init() APP_DIR = os.path.dirname(__file__) -CONFIG_MODULE = os.environ.get('SUPERSET_CONFIG', 'superset.config') +CONFIG_MODULE = os.environ.get("SUPERSET_CONFIG", "superset.config") if not os.path.exists(config.DATA_DIR): os.makedirs(config.DATA_DIR) @@ -52,18 +52,18 @@ conf = app.config ################################################################# # Handling manifest file logic at app start ################################################################# -MANIFEST_FILE = APP_DIR + '/static/assets/dist/manifest.json' +MANIFEST_FILE = APP_DIR + "/static/assets/dist/manifest.json" manifest = {} def parse_manifest_json(): global manifest try: - with open(MANIFEST_FILE, 'r') as f: + with open(MANIFEST_FILE, "r") as f: # the manifest inclues non-entry files # we only need entries in templates full_manifest = json.load(f) - manifest = full_manifest.get('entrypoints', {}) + manifest = full_manifest.get("entrypoints", {}) except Exception: pass @@ -72,14 +72,14 @@ def get_js_manifest_files(filename): if app.debug: parse_manifest_json() entry_files = manifest.get(filename, {}) - return entry_files.get('js', []) + return entry_files.get("js", []) def get_css_manifest_files(filename): if app.debug: parse_manifest_json() entry_files = manifest.get(filename, {}) - return entry_files.get('css', []) + return entry_files.get("css", []) def get_unloaded_chunks(files, loaded_chunks): @@ -104,16 +104,16 @@ def get_manifest(): ################################################################# -for bp in conf.get('BLUEPRINTS'): +for bp in conf.get("BLUEPRINTS"): try: print("Registering blueprint: '{}'".format(bp.name)) app.register_blueprint(bp) except Exception as e: - print('blueprint registration failed') + print("blueprint registration failed") logging.exception(e) -if conf.get('SILENCE_FAB'): - logging.getLogger('flask_appbuilder').setLevel(logging.ERROR) +if conf.get("SILENCE_FAB"): + logging.getLogger("flask_appbuilder").setLevel(logging.ERROR) if app.debug: app.logger.setLevel(logging.DEBUG) # pylint: disable=no-member @@ -121,44 +121,46 @@ else: # In production mode, add log handler to sys.stderr. app.logger.addHandler(logging.StreamHandler()) # pylint: disable=no-member app.logger.setLevel(logging.INFO) # pylint: disable=no-member -logging.getLogger('pyhive.presto').setLevel(logging.INFO) +logging.getLogger("pyhive.presto").setLevel(logging.INFO) db = SQLA(app) -if conf.get('WTF_CSRF_ENABLED'): +if conf.get("WTF_CSRF_ENABLED"): csrf = CSRFProtect(app) - csrf_exempt_list = conf.get('WTF_CSRF_EXEMPT_LIST', []) + csrf_exempt_list = conf.get("WTF_CSRF_EXEMPT_LIST", []) for ex in csrf_exempt_list: csrf.exempt(ex) pessimistic_connection_handling(db.engine) -cache = setup_cache(app, conf.get('CACHE_CONFIG')) -tables_cache = setup_cache(app, conf.get('TABLE_NAMES_CACHE_CONFIG')) +cache = setup_cache(app, conf.get("CACHE_CONFIG")) +tables_cache = setup_cache(app, conf.get("TABLE_NAMES_CACHE_CONFIG")) -migrate = Migrate(app, db, directory=APP_DIR + '/migrations') +migrate = Migrate(app, db, directory=APP_DIR + "/migrations") # Logging configuration -logging.basicConfig(format=app.config.get('LOG_FORMAT')) -logging.getLogger().setLevel(app.config.get('LOG_LEVEL')) +logging.basicConfig(format=app.config.get("LOG_FORMAT")) +logging.getLogger().setLevel(app.config.get("LOG_LEVEL")) -if app.config.get('ENABLE_TIME_ROTATE'): - logging.getLogger().setLevel(app.config.get('TIME_ROTATE_LOG_LEVEL')) +if app.config.get("ENABLE_TIME_ROTATE"): + logging.getLogger().setLevel(app.config.get("TIME_ROTATE_LOG_LEVEL")) handler = TimedRotatingFileHandler( - app.config.get('FILENAME'), - when=app.config.get('ROLLOVER'), - interval=app.config.get('INTERVAL'), - backupCount=app.config.get('BACKUP_COUNT')) + app.config.get("FILENAME"), + when=app.config.get("ROLLOVER"), + interval=app.config.get("INTERVAL"), + backupCount=app.config.get("BACKUP_COUNT"), + ) logging.getLogger().addHandler(handler) -if app.config.get('ENABLE_CORS'): +if app.config.get("ENABLE_CORS"): from flask_cors import CORS - CORS(app, **app.config.get('CORS_OPTIONS')) -if app.config.get('ENABLE_PROXY_FIX'): + CORS(app, **app.config.get("CORS_OPTIONS")) + +if app.config.get("ENABLE_PROXY_FIX"): app.wsgi_app = ProxyFix(app.wsgi_app) -if app.config.get('ENABLE_CHUNK_ENCODING'): +if app.config.get("ENABLE_CHUNK_ENCODING"): class ChunkedEncodingFix(object): def __init__(self, app): @@ -167,40 +169,41 @@ if app.config.get('ENABLE_CHUNK_ENCODING'): def __call__(self, environ, start_response): # Setting wsgi.input_terminated tells werkzeug.wsgi to ignore # content-length and read the stream till the end. - if environ.get('HTTP_TRANSFER_ENCODING', '').lower() == u'chunked': - environ['wsgi.input_terminated'] = True + if environ.get("HTTP_TRANSFER_ENCODING", "").lower() == u"chunked": + environ["wsgi.input_terminated"] = True return self.app(environ, start_response) app.wsgi_app = ChunkedEncodingFix(app.wsgi_app) -if app.config.get('UPLOAD_FOLDER'): +if app.config.get("UPLOAD_FOLDER"): try: - os.makedirs(app.config.get('UPLOAD_FOLDER')) + os.makedirs(app.config.get("UPLOAD_FOLDER")) except OSError: pass -for middleware in app.config.get('ADDITIONAL_MIDDLEWARE'): +for middleware in app.config.get("ADDITIONAL_MIDDLEWARE"): app.wsgi_app = middleware(app.wsgi_app) class MyIndexView(IndexView): - @expose('/') + @expose("/") def index(self): - return redirect('/superset/welcome') + return redirect("/superset/welcome") -custom_sm = app.config.get('CUSTOM_SECURITY_MANAGER') or SupersetSecurityManager +custom_sm = app.config.get("CUSTOM_SECURITY_MANAGER") or SupersetSecurityManager if not issubclass(custom_sm, SupersetSecurityManager): raise Exception( """Your CUSTOM_SECURITY_MANAGER must now extend SupersetSecurityManager, not FAB's security manager. - See [4565] in UPDATING.md""") + See [4565] in UPDATING.md""" + ) with app.app_context(): appbuilder = AppBuilder( app, db.session, - base_template='superset/base.html', + base_template="superset/base.html", indexview=MyIndexView, security_manager_class=custom_sm, update_perms=False, # Run `superset init` to update FAB's perms @@ -208,15 +211,15 @@ with app.app_context(): security_manager = appbuilder.sm -results_backend = app.config.get('RESULTS_BACKEND') +results_backend = app.config.get("RESULTS_BACKEND") # Merge user defined feature flags with default feature flags -_feature_flags = app.config.get('DEFAULT_FEATURE_FLAGS') or {} -_feature_flags.update(app.config.get('FEATURE_FLAGS') or {}) +_feature_flags = app.config.get("DEFAULT_FEATURE_FLAGS") or {} +_feature_flags.update(app.config.get("FEATURE_FLAGS") or {}) def get_feature_flags(): - GET_FEATURE_FLAGS_FUNC = app.config.get('GET_FEATURE_FLAGS_FUNC') + GET_FEATURE_FLAGS_FUNC = app.config.get("GET_FEATURE_FLAGS_FUNC") if GET_FEATURE_FLAGS_FUNC: return GET_FEATURE_FLAGS_FUNC(deepcopy(_feature_flags)) return _feature_flags @@ -228,22 +231,22 @@ def is_feature_enabled(feature): # Flask-Compress -if conf.get('ENABLE_FLASK_COMPRESS'): +if conf.get("ENABLE_FLASK_COMPRESS"): Compress(app) -if app.config['TALISMAN_ENABLED']: - talisman_config = app.config.get('TALISMAN_CONFIG') +if app.config["TALISMAN_ENABLED"]: + talisman_config = app.config.get("TALISMAN_CONFIG") Talisman(app, **talisman_config) # Hook that provides administrators a handle on the Flask APP # after initialization -flask_app_mutator = app.config.get('FLASK_APP_MUTATOR') +flask_app_mutator = app.config.get("FLASK_APP_MUTATOR") if flask_app_mutator: flask_app_mutator(app) from superset import views # noqa # Registering sources -module_datasource_map = app.config.get('DEFAULT_MODULE_DS_MAP') -module_datasource_map.update(app.config.get('ADDITIONAL_MODULE_DS_MAP')) +module_datasource_map = app.config.get("DEFAULT_MODULE_DS_MAP") +module_datasource_map.update(app.config.get("ADDITIONAL_MODULE_DS_MAP")) ConnectorRegistry.register_sources(module_datasource_map) diff --git a/superset/cli.py b/superset/cli.py index 6691b0148f8..cb363c2b595 100755 --- a/superset/cli.py +++ b/superset/cli.py @@ -26,11 +26,8 @@ from colorama import Fore, Style from pathlib2 import Path import yaml -from superset import ( - app, appbuilder, data, db, security_manager, -) -from superset.utils import ( - core as utils, dashboard_import_export, dict_import_export) +from superset import app, appbuilder, data, db, security_manager +from superset.utils import core as utils, dashboard_import_export, dict_import_export config = app.config celery_app = utils.get_celery_app(config) @@ -54,114 +51,128 @@ def init(): @app.cli.command() -@click.option('--verbose', '-v', is_flag=True, help='Show extra information') +@click.option("--verbose", "-v", is_flag=True, help="Show extra information") def version(verbose): """Prints the current version number""" - print(Fore.BLUE + '-=' * 15) - print(Fore.YELLOW + 'Superset ' + Fore.CYAN + '{version}'.format( - version=config.get('VERSION_STRING'))) - print(Fore.BLUE + '-=' * 15) + print(Fore.BLUE + "-=" * 15) + print( + Fore.YELLOW + + "Superset " + + Fore.CYAN + + "{version}".format(version=config.get("VERSION_STRING")) + ) + print(Fore.BLUE + "-=" * 15) if verbose: - print('[DB] : ' + '{}'.format(db.engine)) + print("[DB] : " + "{}".format(db.engine)) print(Style.RESET_ALL) def load_examples_run(load_test_data): - print('Loading examples into {}'.format(db)) + print("Loading examples into {}".format(db)) data.load_css_templates() - print('Loading energy related dataset') + print("Loading energy related dataset") data.load_energy() print("Loading [World Bank's Health Nutrition and Population Stats]") data.load_world_bank_health_n_pop() - print('Loading [Birth names]') + print("Loading [Birth names]") data.load_birth_names() - print('Loading [Unicode test data]') + print("Loading [Unicode test data]") data.load_unicode_test_data() if not load_test_data: - print('Loading [Random time series data]') + print("Loading [Random time series data]") data.load_random_time_series_data() - print('Loading [Random long/lat data]') + print("Loading [Random long/lat data]") data.load_long_lat_data() - print('Loading [Country Map data]') + print("Loading [Country Map data]") data.load_country_map_data() - print('Loading [Multiformat time series]') + print("Loading [Multiformat time series]") data.load_multiformat_time_series() - print('Loading [Paris GeoJson]') + print("Loading [Paris GeoJson]") data.load_paris_iris_geojson() - print('Loading [San Francisco population polygons]') + print("Loading [San Francisco population polygons]") data.load_sf_population_polygons() - print('Loading [Flights data]') + print("Loading [Flights data]") data.load_flights() - print('Loading [BART lines]') + print("Loading [BART lines]") data.load_bart_lines() - print('Loading [Multi Line]') + print("Loading [Multi Line]") data.load_multi_line() - print('Loading [Misc Charts] dashboard') + print("Loading [Misc Charts] dashboard") data.load_misc_dashboard() - print('Loading DECK.gl demo') + print("Loading DECK.gl demo") data.load_deck_dash() - print('Loading [Tabbed dashboard]') + print("Loading [Tabbed dashboard]") data.load_tabbed_dashboard() @app.cli.command() -@click.option('--load-test-data', '-t', is_flag=True, help='Load additional test data') +@click.option("--load-test-data", "-t", is_flag=True, help="Load additional test data") def load_examples(load_test_data): """Loads a set of Slices and Dashboards and a supporting dataset """ load_examples_run(load_test_data) @app.cli.command() -@click.option('--datasource', '-d', help='Specify which datasource name to load, if ' - 'omitted, all datasources will be refreshed') -@click.option('--merge', '-m', is_flag=True, default=False, - help="Specify using 'merge' property during operation. " - 'Default value is False.') +@click.option( + "--datasource", + "-d", + help="Specify which datasource name to load, if " + "omitted, all datasources will be refreshed", +) +@click.option( + "--merge", + "-m", + is_flag=True, + default=False, + help="Specify using 'merge' property during operation. " "Default value is False.", +) def refresh_druid(datasource, merge): """Refresh druid datasources""" session = db.session() from superset.connectors.druid.models import DruidCluster + for cluster in session.query(DruidCluster).all(): try: - cluster.refresh_datasources(datasource_name=datasource, - merge_flag=merge) + cluster.refresh_datasources(datasource_name=datasource, merge_flag=merge) except Exception as e: - print( - "Error while processing cluster '{}'\n{}".format( - cluster, str(e))) + print("Error while processing cluster '{}'\n{}".format(cluster, str(e))) logging.exception(e) cluster.metadata_last_refreshed = datetime.now() - print( - 'Refreshed metadata from cluster ' - '[' + cluster.cluster_name + ']') + print("Refreshed metadata from cluster " "[" + cluster.cluster_name + "]") session.commit() @app.cli.command() @click.option( - '--path', '-p', - help='Path to a single JSON file or path containing multiple JSON files' - 'files to import (*.json)') + "--path", + "-p", + help="Path to a single JSON file or path containing multiple JSON files" + "files to import (*.json)", +) @click.option( - '--recursive', '-r', is_flag=True, default=False, - help='recursively search the path for json files') + "--recursive", + "-r", + is_flag=True, + default=False, + help="recursively search the path for json files", +) def import_dashboards(path, recursive): """Import dashboards from JSON""" p = Path(path) @@ -169,114 +180,135 @@ def import_dashboards(path, recursive): if p.is_file(): files.append(p) elif p.exists() and not recursive: - files.extend(p.glob('*.json')) + files.extend(p.glob("*.json")) elif p.exists() and recursive: - files.extend(p.rglob('*.json')) + files.extend(p.rglob("*.json")) for f in files: - logging.info('Importing dashboard from file %s', f) + logging.info("Importing dashboard from file %s", f) try: with f.open() as data_stream: - dashboard_import_export.import_dashboards( - db.session, data_stream) + dashboard_import_export.import_dashboards(db.session, data_stream) except Exception as e: - logging.error('Error when importing dashboard from file %s', f) + logging.error("Error when importing dashboard from file %s", f) logging.error(e) @app.cli.command() @click.option( - '--dashboard-file', '-f', default=None, - help='Specify the the file to export to') + "--dashboard-file", "-f", default=None, help="Specify the the file to export to" +) @click.option( - '--print_stdout', '-p', is_flag=True, default=False, - help='Print JSON to stdout') + "--print_stdout", "-p", is_flag=True, default=False, help="Print JSON to stdout" +) def export_dashboards(print_stdout, dashboard_file): """Export dashboards to JSON""" data = dashboard_import_export.export_dashboards(db.session) if print_stdout or not dashboard_file: print(data) if dashboard_file: - logging.info('Exporting dashboards to %s', dashboard_file) - with open(dashboard_file, 'w') as data_stream: + logging.info("Exporting dashboards to %s", dashboard_file) + with open(dashboard_file, "w") as data_stream: data_stream.write(data) @app.cli.command() @click.option( - '--path', '-p', - help='Path to a single YAML file or path containing multiple YAML ' - 'files to import (*.yaml or *.yml)') + "--path", + "-p", + help="Path to a single YAML file or path containing multiple YAML " + "files to import (*.yaml or *.yml)", +) @click.option( - '--sync', '-s', 'sync', default='', - help='comma seperated list of element types to synchronize ' - 'e.g. "metrics,columns" deletes metrics and columns in the DB ' - 'that are not specified in the YAML file') + "--sync", + "-s", + "sync", + default="", + help="comma seperated list of element types to synchronize " + 'e.g. "metrics,columns" deletes metrics and columns in the DB ' + "that are not specified in the YAML file", +) @click.option( - '--recursive', '-r', is_flag=True, default=False, - help='recursively search the path for yaml files') + "--recursive", + "-r", + is_flag=True, + default=False, + help="recursively search the path for yaml files", +) def import_datasources(path, sync, recursive): """Import datasources from YAML""" - sync_array = sync.split(',') + sync_array = sync.split(",") p = Path(path) files = [] if p.is_file(): files.append(p) elif p.exists() and not recursive: - files.extend(p.glob('*.yaml')) - files.extend(p.glob('*.yml')) + files.extend(p.glob("*.yaml")) + files.extend(p.glob("*.yml")) elif p.exists() and recursive: - files.extend(p.rglob('*.yaml')) - files.extend(p.rglob('*.yml')) + files.extend(p.rglob("*.yaml")) + files.extend(p.rglob("*.yml")) for f in files: - logging.info('Importing datasources from file %s', f) + logging.info("Importing datasources from file %s", f) try: with f.open() as data_stream: dict_import_export.import_from_dict( - db.session, - yaml.safe_load(data_stream), - sync=sync_array) + db.session, yaml.safe_load(data_stream), sync=sync_array + ) except Exception as e: - logging.error('Error when importing datasources from file %s', f) + logging.error("Error when importing datasources from file %s", f) logging.error(e) @app.cli.command() @click.option( - '--datasource-file', '-f', default=None, - help='Specify the the file to export to') + "--datasource-file", "-f", default=None, help="Specify the the file to export to" +) @click.option( - '--print_stdout', '-p', is_flag=True, default=False, - help='Print YAML to stdout') + "--print_stdout", "-p", is_flag=True, default=False, help="Print YAML to stdout" +) @click.option( - '--back-references', '-b', is_flag=True, default=False, - help='Include parent back references') + "--back-references", + "-b", + is_flag=True, + default=False, + help="Include parent back references", +) @click.option( - '--include-defaults', '-d', is_flag=True, default=False, - help='Include fields containing defaults') -def export_datasources(print_stdout, datasource_file, - back_references, include_defaults): + "--include-defaults", + "-d", + is_flag=True, + default=False, + help="Include fields containing defaults", +) +def export_datasources( + print_stdout, datasource_file, back_references, include_defaults +): """Export datasources to YAML""" data = dict_import_export.export_to_dict( session=db.session, recursive=True, back_references=back_references, - include_defaults=include_defaults) + include_defaults=include_defaults, + ) if print_stdout or not datasource_file: yaml.safe_dump(data, stdout, default_flow_style=False) if datasource_file: - logging.info('Exporting datasources to %s', datasource_file) - with open(datasource_file, 'w') as data_stream: + logging.info("Exporting datasources to %s", datasource_file) + with open(datasource_file, "w") as data_stream: yaml.safe_dump(data, data_stream, default_flow_style=False) @app.cli.command() @click.option( - '--back-references', '-b', is_flag=True, default=False, - help='Include parent back references') + "--back-references", + "-b", + is_flag=True, + default=False, + help="Include parent back references", +) def export_datasource_schema(back_references): """Export datasource YAML schema to stdout""" - data = dict_import_export.export_schema_to_dict( - back_references=back_references) + data = dict_import_export.export_schema_to_dict(back_references=back_references) yaml.safe_dump(data, stdout, default_flow_style=False) @@ -284,47 +316,49 @@ def export_datasource_schema(back_references): def update_datasources_cache(): """Refresh sqllab datasources cache""" from superset.models.core import Database + for database in db.session.query(Database).all(): if database.allow_multi_schema_metadata_fetch: - print('Fetching {} datasources ...'.format(database.name)) + print("Fetching {} datasources ...".format(database.name)) try: database.get_all_table_names_in_database( - force=True, cache=True, cache_timeout=24 * 60 * 60) + force=True, cache=True, cache_timeout=24 * 60 * 60 + ) database.get_all_view_names_in_database( - force=True, cache=True, cache_timeout=24 * 60 * 60) + force=True, cache=True, cache_timeout=24 * 60 * 60 + ) except Exception as e: - print('{}'.format(str(e))) + print("{}".format(str(e))) @app.cli.command() @click.option( - '--workers', '-w', - type=int, - help='Number of celery server workers to fire up') + "--workers", "-w", type=int, help="Number of celery server workers to fire up" +) def worker(workers): """Starts a Superset worker for async SQL query execution.""" logging.info( "The 'superset worker' command is deprecated. Please use the 'celery " - "worker' command instead.") + "worker' command instead." + ) if workers: celery_app.conf.update(CELERYD_CONCURRENCY=workers) - elif config.get('SUPERSET_CELERY_WORKERS'): + elif config.get("SUPERSET_CELERY_WORKERS"): celery_app.conf.update( - CELERYD_CONCURRENCY=config.get('SUPERSET_CELERY_WORKERS')) + CELERYD_CONCURRENCY=config.get("SUPERSET_CELERY_WORKERS") + ) - worker = celery_app.Worker(optimization='fair') + worker = celery_app.Worker(optimization="fair") worker.start() @app.cli.command() @click.option( - '-p', '--port', - default='5555', - help='Port on which to start the Flower process') + "-p", "--port", default="5555", help="Port on which to start the Flower process" +) @click.option( - '-a', '--address', - default='localhost', - help='Address on which to run the service') + "-a", "--address", default="localhost", help="Address on which to run the service" +) def flower(port, address): """Runs a Celery Flower web server @@ -332,18 +366,19 @@ def flower(port, address): broker""" BROKER_URL = celery_app.conf.BROKER_URL cmd = ( - 'celery flower ' - f'--broker={BROKER_URL} ' - f'--port={port} ' - f'--address={address} ' + "celery flower " + f"--broker={BROKER_URL} " + f"--port={port} " + f"--address={address} " ) logging.info( "The 'superset flower' command is deprecated. Please use the 'celery " - "flower' command instead.") - print(Fore.GREEN + 'Starting a Celery Flower instance') - print(Fore.BLUE + '-=' * 40) + "flower' command instead." + ) + print(Fore.GREEN + "Starting a Celery Flower instance") + print(Fore.BLUE + "-=" * 40) print(Fore.YELLOW + cmd) - print(Fore.BLUE + '-=' * 40) + print(Fore.BLUE + "-=" * 40) Popen(cmd, shell=True).wait() @@ -354,7 +389,7 @@ def load_test_users(): Syncs permissions for those users/roles """ - print(Fore.GREEN + 'Loading a set of users for unit tests') + print(Fore.GREEN + "Loading a set of users for unit tests") load_test_users_run() @@ -364,51 +399,73 @@ def load_test_users_run(): Syncs permissions for those users/roles """ - if config.get('TESTING'): + if config.get("TESTING"): security_manager.sync_role_definitions() - gamma_sqllab_role = security_manager.add_role('gamma_sqllab') - for perm in security_manager.find_role('Gamma').permissions: + gamma_sqllab_role = security_manager.add_role("gamma_sqllab") + for perm in security_manager.find_role("Gamma").permissions: security_manager.add_permission_role(gamma_sqllab_role, perm) utils.get_or_create_main_db() db_perm = utils.get_main_database(security_manager.get_session).perm - security_manager.add_permission_view_menu('database_access', db_perm) + security_manager.add_permission_view_menu("database_access", db_perm) db_pvm = security_manager.find_permission_view_menu( - view_menu_name=db_perm, permission_name='database_access') + view_menu_name=db_perm, permission_name="database_access" + ) gamma_sqllab_role.permissions.append(db_pvm) - for perm in security_manager.find_role('sql_lab').permissions: + for perm in security_manager.find_role("sql_lab").permissions: security_manager.add_permission_role(gamma_sqllab_role, perm) - admin = security_manager.find_user('admin') + admin = security_manager.find_user("admin") if not admin: security_manager.add_user( - 'admin', 'admin', ' user', 'admin@fab.org', - security_manager.find_role('Admin'), - password='general') + "admin", + "admin", + " user", + "admin@fab.org", + security_manager.find_role("Admin"), + password="general", + ) - gamma = security_manager.find_user('gamma') + gamma = security_manager.find_user("gamma") if not gamma: security_manager.add_user( - 'gamma', 'gamma', 'user', 'gamma@fab.org', - security_manager.find_role('Gamma'), - password='general') + "gamma", + "gamma", + "user", + "gamma@fab.org", + security_manager.find_role("Gamma"), + password="general", + ) - gamma2 = security_manager.find_user('gamma2') + gamma2 = security_manager.find_user("gamma2") if not gamma2: security_manager.add_user( - 'gamma2', 'gamma2', 'user', 'gamma2@fab.org', - security_manager.find_role('Gamma'), - password='general') + "gamma2", + "gamma2", + "user", + "gamma2@fab.org", + security_manager.find_role("Gamma"), + password="general", + ) - gamma_sqllab_user = security_manager.find_user('gamma_sqllab') + gamma_sqllab_user = security_manager.find_user("gamma_sqllab") if not gamma_sqllab_user: security_manager.add_user( - 'gamma_sqllab', 'gamma_sqllab', 'user', 'gamma_sqllab@fab.org', - gamma_sqllab_role, password='general') + "gamma_sqllab", + "gamma_sqllab", + "user", + "gamma_sqllab@fab.org", + gamma_sqllab_role, + password="general", + ) - alpha = security_manager.find_user('alpha') + alpha = security_manager.find_user("alpha") if not alpha: security_manager.add_user( - 'alpha', 'alpha', 'user', 'alpha@fab.org', - security_manager.find_role('Alpha'), - password='general') + "alpha", + "alpha", + "user", + "alpha@fab.org", + security_manager.find_role("Alpha"), + password="general", + ) security_manager.get_session.commit() diff --git a/superset/common/query_context.py b/superset/common/query_context.py index fd6298d4708..40e32911320 100644 --- a/superset/common/query_context.py +++ b/superset/common/query_context.py @@ -32,7 +32,7 @@ from superset.utils.core import DTTM_ALIAS from .query_object import QueryObject config = app.config -stats_logger = config.get('STATS_LOGGER') +stats_logger = config.get("STATS_LOGGER") class QueryContext: @@ -41,21 +41,21 @@ class QueryContext: to retrieve the data payload for a given viz. """ - cache_type = 'df' + cache_type = "df" enforce_numerical_metrics = True # TODO: Type datasource and query_object dictionary with TypedDict when it becomes # a vanilla python type https://github.com/python/mypy/issues/5288 def __init__( - self, - datasource: Dict, - queries: List[Dict], - force: bool = False, - custom_cache_timeout: int = None, + self, + datasource: Dict, + queries: List[Dict], + force: bool = False, + custom_cache_timeout: int = None, ): - self.datasource = ConnectorRegistry.get_datasource(datasource.get('type'), - int(datasource.get('id')), # noqa: E501, T400 - db.session) + self.datasource = ConnectorRegistry.get_datasource( + datasource.get("type"), int(datasource.get("id")), db.session # noqa: T400 + ) self.queries = list(map(lambda query_obj: QueryObject(**query_obj), queries)) self.force = force @@ -72,7 +72,7 @@ class QueryContext: # support multiple queries from different data source. timestamp_format = None - if self.datasource.type == 'table': + if self.datasource.type == "table": dttm_col = self.datasource.get_col(query_object.granularity) if dttm_col: timestamp_format = dttm_col.python_date_format @@ -88,12 +88,13 @@ class QueryContext: # parsing logic if df is not None and not df.empty: if DTTM_ALIAS in df.columns: - if timestamp_format in ('epoch_s', 'epoch_ms'): + if timestamp_format in ("epoch_s", "epoch_ms"): # Column has already been formatted as a timestamp. df[DTTM_ALIAS] = df[DTTM_ALIAS].apply(pd.Timestamp) else: df[DTTM_ALIAS] = pd.to_datetime( - df[DTTM_ALIAS], utc=False, format=timestamp_format) + df[DTTM_ALIAS], utc=False, format=timestamp_format + ) if self.datasource.offset: df[DTTM_ALIAS] += timedelta(hours=self.datasource.offset) df[DTTM_ALIAS] += query_object.time_shift @@ -103,10 +104,10 @@ class QueryContext: df.replace([np.inf, -np.inf], np.nan) return { - 'query': result.query, - 'status': result.status, - 'error_message': result.error_message, - 'df': df, + "query": result.query, + "status": result.status, + "error_message": result.error_message, + "df": df, } def df_metrics_to_num(self, df, query_object): @@ -114,23 +115,23 @@ class QueryContext: metrics = [metric for metric in query_object.metrics] for col, dtype in df.dtypes.items(): if dtype.type == np.object_ and col in metrics: - df[col] = pd.to_numeric(df[col], errors='coerce') + df[col] = pd.to_numeric(df[col], errors="coerce") def get_data(self, df): - return df.to_dict(orient='records') + return df.to_dict(orient="records") def get_single_payload(self, query_obj): """Returns a payload of metadata and data""" payload = self.get_df_payload(query_obj) - df = payload.get('df') - status = payload.get('status') + df = payload.get("df") + status = payload.get("status") if status != utils.QueryStatus.FAILED: if df is not None and df.empty: - payload['error'] = 'No data' + payload["error"] = "No data" else: - payload['data'] = self.get_data(df) - if 'df' in payload: - del payload['df'] + payload["data"] = self.get_data(df) + if "df" in payload: + del payload["df"] return payload def get_payload(self): @@ -144,94 +145,94 @@ class QueryContext: if self.datasource.cache_timeout is not None: return self.datasource.cache_timeout if ( - hasattr(self.datasource, 'database') and - self.datasource.database.cache_timeout) is not None: + hasattr(self.datasource, "database") + and self.datasource.database.cache_timeout + ) is not None: return self.datasource.database.cache_timeout - return config.get('CACHE_DEFAULT_TIMEOUT') + return config.get("CACHE_DEFAULT_TIMEOUT") def get_df_payload(self, query_obj, **kwargs): """Handles caching around the df paylod retrieval""" - cache_key = query_obj.cache_key( - datasource=self.datasource.uid, **kwargs) if query_obj else None - logging.info('Cache key: {}'.format(cache_key)) + cache_key = ( + query_obj.cache_key(datasource=self.datasource.uid, **kwargs) + if query_obj + else None + ) + logging.info("Cache key: {}".format(cache_key)) is_loaded = False stacktrace = None df = None - cached_dttm = datetime.utcnow().isoformat().split('.')[0] + cached_dttm = datetime.utcnow().isoformat().split(".")[0] cache_value = None status = None - query = '' + query = "" error_message = None if cache_key and cache and not self.force: cache_value = cache.get(cache_key) if cache_value: - stats_logger.incr('loaded_from_cache') + stats_logger.incr("loaded_from_cache") try: cache_value = pkl.loads(cache_value) - df = cache_value['df'] - query = cache_value['query'] + df = cache_value["df"] + query = cache_value["query"] status = utils.QueryStatus.SUCCESS is_loaded = True except Exception as e: logging.exception(e) - logging.error('Error reading cache: ' + - utils.error_msg_from_exception(e)) - logging.info('Serving from cache') + logging.error( + "Error reading cache: " + utils.error_msg_from_exception(e) + ) + logging.info("Serving from cache") if query_obj and not is_loaded: try: query_result = self.get_query_result(query_obj) - status = query_result['status'] - query = query_result['query'] - error_message = query_result['error_message'] - df = query_result['df'] + status = query_result["status"] + query = query_result["query"] + error_message = query_result["error_message"] + df = query_result["df"] if status != utils.QueryStatus.FAILED: - stats_logger.incr('loaded_from_source') + stats_logger.incr("loaded_from_source") is_loaded = True except Exception as e: logging.exception(e) if not error_message: - error_message = '{}'.format(e) + error_message = "{}".format(e) status = utils.QueryStatus.FAILED stacktrace = traceback.format_exc() - if ( - is_loaded and - cache_key and - cache and - status != utils.QueryStatus.FAILED): + if is_loaded and cache_key and cache and status != utils.QueryStatus.FAILED: try: cache_value = dict( - dttm=cached_dttm, - df=df if df is not None else None, - query=query, + dttm=cached_dttm, df=df if df is not None else None, query=query ) - cache_binary = pkl.dumps( - cache_value, protocol=pkl.HIGHEST_PROTOCOL) + cache_binary = pkl.dumps(cache_value, protocol=pkl.HIGHEST_PROTOCOL) - logging.info('Caching {} chars at key {}'.format( - len(cache_binary), cache_key)) + logging.info( + "Caching {} chars at key {}".format( + len(cache_binary), cache_key + ) + ) - stats_logger.incr('set_cache_key') + stats_logger.incr("set_cache_key") cache.set( - cache_key, - cache_value=cache_binary, - timeout=self.cache_timeout) + cache_key, cache_value=cache_binary, timeout=self.cache_timeout + ) except Exception as e: # cache.set call can fail if the backend is down or if # the key is too large or whatever other reasons - logging.warning('Could not cache key {}'.format(cache_key)) + logging.warning("Could not cache key {}".format(cache_key)) logging.exception(e) cache.delete(cache_key) return { - 'cache_key': cache_key, - 'cached_dttm': cache_value['dttm'] if cache_value is not None else None, - 'cache_timeout': self.cache_timeout, - 'df': df, - 'error': error_message, - 'is_cached': cache_key is not None, - 'query': query, - 'status': status, - 'stacktrace': stacktrace, - 'rowcount': len(df.index) if df is not None else 0, + "cache_key": cache_key, + "cached_dttm": cache_value["dttm"] if cache_value is not None else None, + "cache_timeout": self.cache_timeout, + "df": df, + "error": error_message, + "is_cached": cache_key is not None, + "query": query, + "status": status, + "stacktrace": stacktrace, + "rowcount": len(df.index) if df is not None else 0, } diff --git a/superset/common/query_object.py b/superset/common/query_object.py index 553c0b9ddcf..f4130319c24 100644 --- a/superset/common/query_object.py +++ b/superset/common/query_object.py @@ -27,6 +27,7 @@ from superset.utils import core as utils # TODO: Type Metrics dictionary with TypedDict when it becomes a vanilla python type # https://github.com/python/mypy/issues/5288 + class QueryObject: """ The query object's schema matches the interfaces of DB connectors like sqla @@ -34,25 +35,25 @@ class QueryObject: """ def __init__( - self, - granularity: str, - metrics: List[Union[Dict, str]], - groupby: List[str] = None, - filters: List[str] = None, - time_range: Optional[str] = None, - time_shift: Optional[str] = None, - is_timeseries: bool = False, - timeseries_limit: int = 0, - row_limit: int = app.config.get('ROW_LIMIT'), - timeseries_limit_metric: Optional[Dict] = None, - order_desc: bool = True, - extras: Optional[Dict] = None, - prequeries: Optional[List[Dict]] = None, - is_prequery: bool = False, - columns: List[str] = None, - orderby: List[List] = None, - relative_start: str = app.config.get('DEFAULT_RELATIVE_START_TIME', 'today'), - relative_end: str = app.config.get('DEFAULT_RELATIVE_END_TIME', 'today'), + self, + granularity: str, + metrics: List[Union[Dict, str]], + groupby: List[str] = None, + filters: List[str] = None, + time_range: Optional[str] = None, + time_shift: Optional[str] = None, + is_timeseries: bool = False, + timeseries_limit: int = 0, + row_limit: int = app.config.get("ROW_LIMIT"), + timeseries_limit_metric: Optional[Dict] = None, + order_desc: bool = True, + extras: Optional[Dict] = None, + prequeries: Optional[List[Dict]] = None, + is_prequery: bool = False, + columns: List[str] = None, + orderby: List[List] = None, + relative_start: str = app.config.get("DEFAULT_RELATIVE_START_TIME", "today"), + relative_end: str = app.config.get("DEFAULT_RELATIVE_END_TIME", "today"), ): self.granularity = granularity self.from_dttm, self.to_dttm = utils.get_since_until( @@ -69,7 +70,7 @@ class QueryObject: # Temporal solution for backward compatability issue # due the new format of non-ad-hoc metric. self.metrics = [ - metric if 'expressionType' in metric else metric['label'] # noqa: T484 + metric if "expressionType" in metric else metric["label"] # noqa: T484 for metric in metrics ] self.row_limit = row_limit @@ -85,22 +86,22 @@ class QueryObject: def to_dict(self): query_object_dict = { - 'granularity': self.granularity, - 'from_dttm': self.from_dttm, - 'to_dttm': self.to_dttm, - 'is_timeseries': self.is_timeseries, - 'groupby': self.groupby, - 'metrics': self.metrics, - 'row_limit': self.row_limit, - 'filter': self.filter, - 'timeseries_limit': self.timeseries_limit, - 'timeseries_limit_metric': self.timeseries_limit_metric, - 'order_desc': self.order_desc, - 'prequeries': self.prequeries, - 'is_prequery': self.is_prequery, - 'extras': self.extras, - 'columns': self.columns, - 'orderby': self.orderby, + "granularity": self.granularity, + "from_dttm": self.from_dttm, + "to_dttm": self.to_dttm, + "is_timeseries": self.is_timeseries, + "groupby": self.groupby, + "metrics": self.metrics, + "row_limit": self.row_limit, + "filter": self.filter, + "timeseries_limit": self.timeseries_limit, + "timeseries_limit_metric": self.timeseries_limit_metric, + "order_desc": self.order_desc, + "prequeries": self.prequeries, + "is_prequery": self.is_prequery, + "extras": self.extras, + "columns": self.columns, + "orderby": self.orderby, } return query_object_dict @@ -115,17 +116,14 @@ class QueryObject: cache_dict = self.to_dict() cache_dict.update(extra) - for k in ['from_dttm', 'to_dttm']: + for k in ["from_dttm", "to_dttm"]: del cache_dict[k] if self.time_range: - cache_dict['time_range'] = self.time_range + cache_dict["time_range"] = self.time_range json_data = self.json_dumps(cache_dict, sort_keys=True) - return hashlib.md5(json_data.encode('utf-8')).hexdigest() + return hashlib.md5(json_data.encode("utf-8")).hexdigest() def json_dumps(self, obj, sort_keys=False): return json.dumps( - obj, - default=utils.json_int_dttm_ser, - ignore_nan=True, - sort_keys=sort_keys, + obj, default=utils.json_int_dttm_ser, ignore_nan=True, sort_keys=sort_keys ) diff --git a/superset/config.py b/superset/config.py index 68ce8b5bf96..4261b5464f9 100644 --- a/superset/config.py +++ b/superset/config.py @@ -37,18 +37,18 @@ from superset.stats_logger import DummyStatsLogger STATS_LOGGER = DummyStatsLogger() BASE_DIR = os.path.abspath(os.path.dirname(__file__)) -if 'SUPERSET_HOME' in os.environ: - DATA_DIR = os.environ['SUPERSET_HOME'] +if "SUPERSET_HOME" in os.environ: + DATA_DIR = os.environ["SUPERSET_HOME"] else: - DATA_DIR = os.path.join(os.path.expanduser('~'), '.superset') + DATA_DIR = os.path.join(os.path.expanduser("~"), ".superset") # --------------------------------------------------------- # Superset specific config # --------------------------------------------------------- -PACKAGE_DIR = os.path.join(BASE_DIR, 'static', 'assets') -PACKAGE_FILE = os.path.join(PACKAGE_DIR, 'package.json') +PACKAGE_DIR = os.path.join(BASE_DIR, "static", "assets") +PACKAGE_FILE = os.path.join(PACKAGE_DIR, "package.json") with open(PACKAGE_FILE) as package_file: - VERSION_STRING = json.load(package_file)['version'] + VERSION_STRING = json.load(package_file)["version"] ROW_LIMIT = 50000 VIZ_ROW_LIMIT = 10000 @@ -57,7 +57,7 @@ FILTER_SELECT_ROW_LIMIT = 10000 SUPERSET_WORKERS = 2 # deprecated SUPERSET_CELERY_WORKERS = 32 # deprecated -SUPERSET_WEBSERVER_ADDRESS = '0.0.0.0' +SUPERSET_WEBSERVER_ADDRESS = "0.0.0.0" SUPERSET_WEBSERVER_PORT = 8088 # This is an important setting, and should be lower than your @@ -73,10 +73,10 @@ SQLALCHEMY_TRACK_MODIFICATIONS = False # --------------------------------------------------------- # Your App secret key -SECRET_KEY = '\2\1thisismyscretkey\1\2\e\y\y\h' # noqa +SECRET_KEY = "\2\1thisismyscretkey\1\2\e\y\y\h" # noqa # The SQLAlchemy connection string. -SQLALCHEMY_DATABASE_URI = 'sqlite:///' + os.path.join(DATA_DIR, 'superset.db') +SQLALCHEMY_DATABASE_URI = "sqlite:///" + os.path.join(DATA_DIR, "superset.db") # SQLALCHEMY_DATABASE_URI = 'mysql://myapp@localhost/myapp' # SQLALCHEMY_DATABASE_URI = 'postgresql://root:password@localhost/myapp' @@ -96,10 +96,10 @@ QUERY_SEARCH_LIMIT = 1000 WTF_CSRF_ENABLED = True # Add endpoints that need to be exempt from CSRF protection -WTF_CSRF_EXEMPT_LIST = ['superset.views.core.log'] +WTF_CSRF_EXEMPT_LIST = ["superset.views.core.log"] # Whether to run the web server in debug mode or not -DEBUG = os.environ.get('FLASK_ENV') == 'development' +DEBUG = os.environ.get("FLASK_ENV") == "development" FLASK_USE_RELOAD = True # Whether to show the stacktrace on 500 error @@ -112,10 +112,10 @@ ENABLE_PROXY_FIX = False # GLOBALS FOR APP Builder # ------------------------------ # Uncomment to setup Your App name -APP_NAME = 'Superset' +APP_NAME = "Superset" # Uncomment to setup an App icon -APP_ICON = '/static/assets/images/superset-logo@2x.png' +APP_ICON = "/static/assets/images/superset-logo@2x.png" APP_ICON_WIDTH = 126 # Uncomment to specify where clicking the logo would take the user @@ -131,7 +131,7 @@ LOGO_TARGET_PATH = None # other tz can be overridden by providing a local_config DRUID_IS_ACTIVE = True DRUID_TZ = tz.tzutc() -DRUID_ANALYSIS_TYPES = ['cardinality'] +DRUID_ANALYSIS_TYPES = ["cardinality"] # ---------------------------------------------------- # AUTHENTICATION CONFIG @@ -175,21 +175,21 @@ PUBLIC_ROLE_LIKE_GAMMA = False # Babel config for translations # --------------------------------------------------- # Setup default language -BABEL_DEFAULT_LOCALE = 'en' +BABEL_DEFAULT_LOCALE = "en" # Your application default translation path -BABEL_DEFAULT_FOLDER = 'superset/translations' +BABEL_DEFAULT_FOLDER = "superset/translations" # The allowed translation for you app LANGUAGES = { - 'en': {'flag': 'us', 'name': 'English'}, - 'it': {'flag': 'it', 'name': 'Italian'}, - 'fr': {'flag': 'fr', 'name': 'French'}, - 'zh': {'flag': 'cn', 'name': 'Chinese'}, - 'ja': {'flag': 'jp', 'name': 'Japanese'}, - 'de': {'flag': 'de', 'name': 'German'}, - 'pt': {'flag': 'pt', 'name': 'Portuguese'}, - 'pt_BR': {'flag': 'br', 'name': 'Brazilian Portuguese'}, - 'ru': {'flag': 'ru', 'name': 'Russian'}, - 'ko': {'flag': 'kr', 'name': 'Korean'}, + "en": {"flag": "us", "name": "English"}, + "it": {"flag": "it", "name": "Italian"}, + "fr": {"flag": "fr", "name": "French"}, + "zh": {"flag": "cn", "name": "Chinese"}, + "ja": {"flag": "jp", "name": "Japanese"}, + "de": {"flag": "de", "name": "German"}, + "pt": {"flag": "pt", "name": "Portuguese"}, + "pt_BR": {"flag": "br", "name": "Brazilian Portuguese"}, + "ru": {"flag": "ru", "name": "Russian"}, + "ko": {"flag": "kr", "name": "Korean"}, } # --------------------------------------------------- @@ -202,7 +202,7 @@ LANGUAGES = { # will result in combined feature flags of { 'FOO': True, 'BAR': True, 'BAZ': True } DEFAULT_FEATURE_FLAGS = { # Experimental feature introducing a client (browser) cache - 'CLIENT_CACHE': False, + "CLIENT_CACHE": False } # A function that receives a dict of all feature flags @@ -225,19 +225,19 @@ GET_FEATURE_FLAGS_FUNC = None # Image and file configuration # --------------------------------------------------- # The file upload folder, when using models with files -UPLOAD_FOLDER = BASE_DIR + '/app/static/uploads/' +UPLOAD_FOLDER = BASE_DIR + "/app/static/uploads/" # The image upload folder, when using models with images -IMG_UPLOAD_FOLDER = BASE_DIR + '/app/static/uploads/' +IMG_UPLOAD_FOLDER = BASE_DIR + "/app/static/uploads/" # The image upload url, when using models with images -IMG_UPLOAD_URL = '/static/uploads/' +IMG_UPLOAD_URL = "/static/uploads/" # Setup image size default is (300, 200, True) # IMG_SIZE = (300, 200, True) CACHE_DEFAULT_TIMEOUT = 60 * 60 * 24 -CACHE_CONFIG = {'CACHE_TYPE': 'null'} -TABLE_NAMES_CACHE_CONFIG = {'CACHE_TYPE': 'null'} +CACHE_CONFIG = {"CACHE_TYPE": "null"} +TABLE_NAMES_CACHE_CONFIG = {"CACHE_TYPE": "null"} # CORS Options ENABLE_CORS = False @@ -252,13 +252,12 @@ SUPERSET_WEBSERVER_DOMAINS = None # Allowed format types for upload on Database view # TODO: Add processing of other spreadsheet formats (xls, xlsx etc) -ALLOWED_EXTENSIONS = set(['csv']) +ALLOWED_EXTENSIONS = set(["csv"]) -# CSV Options: key/value pairs that will be passed as argument to DataFrame.to_csv method +# CSV Options: key/value pairs that will be passed as argument to DataFrame.to_csv +# method. # note: index option should not be overridden -CSV_EXPORT = { - 'encoding': 'utf-8', -} +CSV_EXPORT = {"encoding": "utf-8"} # --------------------------------------------------- # Time grain configurations @@ -301,10 +300,12 @@ DRUID_DATA_SOURCE_BLACKLIST = [] # -------------------------------------------------- # Modules, datasources and middleware to be registered # -------------------------------------------------- -DEFAULT_MODULE_DS_MAP = OrderedDict([ - ('superset.connectors.sqla.models', ['SqlaTable']), - ('superset.connectors.druid.models', ['DruidDatasource']), -]) +DEFAULT_MODULE_DS_MAP = OrderedDict( + [ + ("superset.connectors.sqla.models", ["SqlaTable"]), + ("superset.connectors.druid.models", ["DruidDatasource"]), + ] +) ADDITIONAL_MODULE_DS_MAP = {} ADDITIONAL_MIDDLEWARE = [] @@ -315,8 +316,8 @@ ADDITIONAL_MIDDLEWARE = [] # Console Log Settings -LOG_FORMAT = '%(asctime)s:%(levelname)s:%(name)s:%(message)s' -LOG_LEVEL = 'DEBUG' +LOG_FORMAT = "%(asctime)s:%(levelname)s:%(name)s:%(message)s" +LOG_LEVEL = "DEBUG" # --------------------------------------------------- # Enable Time Rotate Log Handler @@ -324,9 +325,9 @@ LOG_LEVEL = 'DEBUG' # LOG_LEVEL = DEBUG, INFO, WARNING, ERROR, CRITICAL ENABLE_TIME_ROTATE = False -TIME_ROTATE_LOG_LEVEL = 'DEBUG' -FILENAME = os.path.join(DATA_DIR, 'superset.log') -ROLLOVER = 'midnight' +TIME_ROTATE_LOG_LEVEL = "DEBUG" +FILENAME = os.path.join(DATA_DIR, "superset.log") +ROLLOVER = "midnight" INTERVAL = 1 BACKUP_COUNT = 30 @@ -344,7 +345,7 @@ BACKUP_COUNT = 30 # pass # Set this API key to enable Mapbox visualizations -MAPBOX_API_KEY = os.environ.get('MAPBOX_API_KEY', '') +MAPBOX_API_KEY = os.environ.get("MAPBOX_API_KEY", "") # Maximum number of rows returned from a database # in async mode, no more than SQL_MAX_ROW will be returned and stored @@ -378,31 +379,26 @@ WARNING_MSG = None class CeleryConfig(object): - BROKER_URL = 'sqla+sqlite:///celerydb.sqlite' - CELERY_IMPORTS = ( - 'superset.sql_lab', - 'superset.tasks', - ) - CELERY_RESULT_BACKEND = 'db+sqlite:///celery_results.sqlite' - CELERYD_LOG_LEVEL = 'DEBUG' + BROKER_URL = "sqla+sqlite:///celerydb.sqlite" + CELERY_IMPORTS = ("superset.sql_lab", "superset.tasks") + CELERY_RESULT_BACKEND = "db+sqlite:///celery_results.sqlite" + CELERYD_LOG_LEVEL = "DEBUG" CELERYD_PREFETCH_MULTIPLIER = 1 CELERY_ACKS_LATE = True CELERY_ANNOTATIONS = { - 'sql_lab.get_sql_results': { - 'rate_limit': '100/s', - }, - 'email_reports.send': { - 'rate_limit': '1/s', - 'time_limit': 120, - 'soft_time_limit': 150, - 'ignore_result': True, + "sql_lab.get_sql_results": {"rate_limit": "100/s"}, + "email_reports.send": { + "rate_limit": "1/s", + "time_limit": 120, + "soft_time_limit": 150, + "ignore_result": True, }, } CELERYBEAT_SCHEDULE = { - 'email_reports.schedule_hourly': { - 'task': 'email_reports.schedule_hourly', - 'schedule': crontab(minute=1, hour='*'), - }, + "email_reports.schedule_hourly": { + "task": "email_reports.schedule_hourly", + "schedule": crontab(minute=1, hour="*"), + } } @@ -444,7 +440,7 @@ CSV_TO_HIVE_UPLOAD_S3_BUCKET = None # The directory within the bucket specified above that will # contain all the external tables -CSV_TO_HIVE_UPLOAD_DIRECTORY = 'EXTERNAL_HIVE_TABLES/' +CSV_TO_HIVE_UPLOAD_DIRECTORY = "EXTERNAL_HIVE_TABLES/" # The namespace within hive where the tables created from # uploading CSVs will be stored. @@ -458,9 +454,9 @@ JINJA_CONTEXT_ADDONS = {} # Roles that are controlled by the API / Superset and should not be changes # by humans. -ROBOT_PERMISSION_ROLES = ['Public', 'Gamma', 'Alpha', 'Admin', 'sql_lab'] +ROBOT_PERMISSION_ROLES = ["Public", "Gamma", "Alpha", "Admin", "sql_lab"] -CONFIG_PATH_ENV_VAR = 'SUPERSET_CONFIG_PATH' +CONFIG_PATH_ENV_VAR = "SUPERSET_CONFIG_PATH" # If a callable is specified, it will be called at app startup while passing # a reference to the Flask app. This can be used to alter the Flask app @@ -474,16 +470,16 @@ ENABLE_ACCESS_REQUEST = False # smtp server configuration EMAIL_NOTIFICATIONS = False # all the emails are sent using dryrun -SMTP_HOST = 'localhost' +SMTP_HOST = "localhost" SMTP_STARTTLS = True SMTP_SSL = False -SMTP_USER = 'superset' +SMTP_USER = "superset" SMTP_PORT = 25 -SMTP_PASSWORD = 'superset' -SMTP_MAIL_FROM = 'superset@superset.com' +SMTP_PASSWORD = "superset" +SMTP_MAIL_FROM = "superset@superset.com" if not CACHE_DEFAULT_TIMEOUT: - CACHE_DEFAULT_TIMEOUT = CACHE_CONFIG.get('CACHE_DEFAULT_TIMEOUT') + CACHE_DEFAULT_TIMEOUT = CACHE_CONFIG.get("CACHE_DEFAULT_TIMEOUT") # Whether to bump the logging level to ERROR on the flask_appbuilder package # Set to False if/when debugging FAB related issues like @@ -492,14 +488,14 @@ SILENCE_FAB = True # The link to a page containing common errors and their resolutions # It will be appended at the bottom of sql_lab errors. -TROUBLESHOOTING_LINK = '' +TROUBLESHOOTING_LINK = "" # CSRF token timeout, set to None for a token that never expires WTF_CSRF_TIME_LIMIT = 60 * 60 * 24 * 7 # This link should lead to a page with instructions on how to gain access to a # Datasource. It will be placed at the bottom of permissions errors. -PERMISSION_INSTRUCTIONS_LINK = '' +PERMISSION_INSTRUCTIONS_LINK = "" # Integrate external Blueprints to the app by passing them to your # configuration. These blueprints will get integrated in the app @@ -565,7 +561,7 @@ EMAIL_REPORTS_CRON_RESOLUTION = 15 # Email report configuration # From address in emails -EMAIL_REPORT_FROM_ADDRESS = 'reports@superset.org' +EMAIL_REPORT_FROM_ADDRESS = "reports@superset.org" # Send bcc of all reports to this address. Set to None to disable. # This is useful for maintaining an audit trail of all email deliveries. @@ -575,8 +571,8 @@ EMAIL_REPORT_BCC_ADDRESS = None # This user should have permissions to browse all the dashboards and # slices. # TODO: In the future, login as the owner of the item to generate reports -EMAIL_REPORTS_USER = 'admin' -EMAIL_REPORTS_SUBJECT_PREFIX = '[Report] ' +EMAIL_REPORTS_USER = "admin" +EMAIL_REPORTS_SUBJECT_PREFIX = "[Report] " # The webdriver to use for generating reports. Use one of the following # firefox @@ -585,19 +581,16 @@ EMAIL_REPORTS_SUBJECT_PREFIX = '[Report] ' # chrome: # Requires: headless chrome # Limitations: unable to generate screenshots of elements -EMAIL_REPORTS_WEBDRIVER = 'firefox' +EMAIL_REPORTS_WEBDRIVER = "firefox" # Window size - this will impact the rendering of the data -WEBDRIVER_WINDOW = { - 'dashboard': (1600, 2000), - 'slice': (3000, 1200), -} +WEBDRIVER_WINDOW = {"dashboard": (1600, 2000), "slice": (3000, 1200)} # Any config options to be passed as-is to the webdriver WEBDRIVER_CONFIGURATION = {} # The base URL to query for accessing the user interface -WEBDRIVER_BASEURL = 'http://0.0.0.0:8080/' +WEBDRIVER_BASEURL = "http://0.0.0.0:8080/" # Send user to a link where they can report bugs BUG_REPORT_URL = None @@ -611,33 +604,34 @@ DOCUMENTATION_URL = None # filter a moving window. By only setting the end time to now, # start time will be set to midnight, while end will be relative to # the query issue time. -DEFAULT_RELATIVE_START_TIME = 'today' -DEFAULT_RELATIVE_END_TIME = 'today' +DEFAULT_RELATIVE_START_TIME = "today" +DEFAULT_RELATIVE_END_TIME = "today" # Configure which SQL validator to use for each engine -SQL_VALIDATORS_BY_ENGINE = { - 'presto': 'PrestoDBSQLValidator', -} +SQL_VALIDATORS_BY_ENGINE = {"presto": "PrestoDBSQLValidator"} # Do you want Talisman enabled? TALISMAN_ENABLED = False # If you want Talisman, how do you want it configured?? TALISMAN_CONFIG = { - 'content_security_policy': None, - 'force_https': True, - 'force_https_permanent': False, + "content_security_policy": None, + "force_https": True, + "force_https_permanent": False, } try: if CONFIG_PATH_ENV_VAR in os.environ: # Explicitly import config module that is not in pythonpath; useful # for case where app is being executed via pex. - print('Loaded your LOCAL configuration at [{}]'.format( - os.environ[CONFIG_PATH_ENV_VAR])) + print( + "Loaded your LOCAL configuration at [{}]".format( + os.environ[CONFIG_PATH_ENV_VAR] + ) + ) module = sys.modules[__name__] override_conf = imp.load_source( - 'superset_config', - os.environ[CONFIG_PATH_ENV_VAR]) + "superset_config", os.environ[CONFIG_PATH_ENV_VAR] + ) for key in dir(override_conf): if key.isupper(): setattr(module, key, getattr(override_conf, key)) @@ -645,7 +639,9 @@ try: else: from superset_config import * # noqa import superset_config - print('Loaded your LOCAL configuration at [{}]'.format( - superset_config.__file__)) + + print( + "Loaded your LOCAL configuration at [{}]".format(superset_config.__file__) + ) except ImportError: pass diff --git a/superset/connectors/base/models.py b/superset/connectors/base/models.py index 2e8bc25adff..2a5fbc7f239 100644 --- a/superset/connectors/base/models.py +++ b/superset/connectors/base/models.py @@ -17,9 +17,7 @@ # pylint: disable=C,R,W import json -from sqlalchemy import ( - and_, Boolean, Column, Integer, String, Text, -) +from sqlalchemy import and_, Boolean, Column, Integer, String, Text from sqlalchemy.ext.declarative import declared_attr from sqlalchemy.orm import foreign, relationship @@ -67,7 +65,7 @@ class BaseDatasource(AuditMixinNullable, ImportMixin): @declared_attr def slices(self): return relationship( - 'Slice', + "Slice", primaryjoin=lambda: and_( foreign(Slice.datasource_id) == self.id, foreign(Slice.datasource_type) == self.type, @@ -82,11 +80,11 @@ class BaseDatasource(AuditMixinNullable, ImportMixin): @property def uid(self): """Unique id across datasource types""" - return f'{self.id}__{self.type}' + return f"{self.id}__{self.type}" @property def column_names(self): - return sorted([c.column_name for c in self.columns], key=lambda x: x or '') + return sorted([c.column_name for c in self.columns], key=lambda x: x or "") @property def columns_types(self): @@ -94,7 +92,7 @@ class BaseDatasource(AuditMixinNullable, ImportMixin): @property def main_dttm_col(self): - return 'timestamp' + return "timestamp" @property def datasource_name(self): @@ -120,22 +118,18 @@ class BaseDatasource(AuditMixinNullable, ImportMixin): @property def url(self): - return '/{}/edit/{}'.format(self.baselink, self.id) + return "/{}/edit/{}".format(self.baselink, self.id) @property def explore_url(self): if self.default_endpoint: return self.default_endpoint else: - return '/superset/explore/{obj.type}/{obj.id}/'.format(obj=self) + return "/superset/explore/{obj.type}/{obj.id}/".format(obj=self) @property def column_formats(self): - return { - m.metric_name: m.d3format - for m in self.metrics - if m.d3format - } + return {m.metric_name: m.d3format for m in self.metrics if m.d3format} def add_missing_metrics(self, metrics): exisiting_metrics = {m.metric_name for m in self.metrics} @@ -148,14 +142,14 @@ class BaseDatasource(AuditMixinNullable, ImportMixin): def short_data(self): """Data representation of the datasource sent to the frontend""" return { - 'edit_url': self.url, - 'id': self.id, - 'uid': self.uid, - 'schema': self.schema, - 'name': self.name, - 'type': self.type, - 'connection': self.connection, - 'creator': str(self.created_by), + "edit_url": self.url, + "id": self.id, + "uid": self.uid, + "schema": self.schema, + "name": self.name, + "type": self.type, + "connection": self.connection, + "creator": str(self.created_by), } @property @@ -168,68 +162,65 @@ class BaseDatasource(AuditMixinNullable, ImportMixin): order_by_choices = [] # self.column_names return sorted column_names for s in self.column_names: - s = str(s or '') - order_by_choices.append((json.dumps([s, True]), s + ' [asc]')) - order_by_choices.append((json.dumps([s, False]), s + ' [desc]')) + s = str(s or "") + order_by_choices.append((json.dumps([s, True]), s + " [asc]")) + order_by_choices.append((json.dumps([s, False]), s + " [desc]")) - verbose_map = {'__timestamp': 'Time'} - verbose_map.update({ - o.metric_name: o.verbose_name or o.metric_name - for o in self.metrics - }) - verbose_map.update({ - o.column_name: o.verbose_name or o.column_name - for o in self.columns - }) + verbose_map = {"__timestamp": "Time"} + verbose_map.update( + {o.metric_name: o.verbose_name or o.metric_name for o in self.metrics} + ) + verbose_map.update( + {o.column_name: o.verbose_name or o.column_name for o in self.columns} + ) return { # simple fields - 'id': self.id, - 'column_formats': self.column_formats, - 'description': self.description, - 'database': self.database.data, # pylint: disable=no-member - 'default_endpoint': self.default_endpoint, - 'filter_select': self.filter_select_enabled, # TODO deprecate - 'filter_select_enabled': self.filter_select_enabled, - 'name': self.name, - 'datasource_name': self.datasource_name, - 'type': self.type, - 'schema': self.schema, - 'offset': self.offset, - 'cache_timeout': self.cache_timeout, - 'params': self.params, - 'perm': self.perm, - 'edit_url': self.url, - + "id": self.id, + "column_formats": self.column_formats, + "description": self.description, + "database": self.database.data, # pylint: disable=no-member + "default_endpoint": self.default_endpoint, + "filter_select": self.filter_select_enabled, # TODO deprecate + "filter_select_enabled": self.filter_select_enabled, + "name": self.name, + "datasource_name": self.datasource_name, + "type": self.type, + "schema": self.schema, + "offset": self.offset, + "cache_timeout": self.cache_timeout, + "params": self.params, + "perm": self.perm, + "edit_url": self.url, # sqla-specific - 'sql': self.sql, - + "sql": self.sql, # one to many - 'columns': [o.data for o in self.columns], - 'metrics': [o.data for o in self.metrics], - + "columns": [o.data for o in self.columns], + "metrics": [o.data for o in self.metrics], # TODO deprecate, move logic to JS - 'order_by_choices': order_by_choices, - 'owners': [owner.id for owner in self.owners], - 'verbose_map': verbose_map, - 'select_star': self.select_star, + "order_by_choices": order_by_choices, + "owners": [owner.id for owner in self.owners], + "verbose_map": verbose_map, + "select_star": self.select_star, } @staticmethod def filter_values_handler( - values, target_column_is_numeric=False, is_list_target=False): + values, target_column_is_numeric=False, is_list_target=False + ): def handle_single_value(v): # backward compatibility with previous